@thinkingai/ae-cli 6.1.28 → 6.1.30

package/dist/{data-integration-EXSUPUDZ.js → data-integration-QE37EQ3A.js}

@@ -8,20 +8,20 @@ import {
   utils,
   validateAndFix,
   validateDraft
-} from "./chunk-KHS64JZQ.js";
-import "./chunk-2IYIYSUO.js";
+} from "./chunk-72NJKS7O.js";
+import "./chunk-QVRIDN42.js";
 import {
   require_archiver
-} from "./chunk-QSAR3GHY.js";
-import "./chunk-T44M3PLZ.js";
+} from "./chunk-OAWPT7VP.js";
+import "./chunk-JEUXDY3B.js";
 import {
   getConfigDir
-} from "./chunk-LMCYSVVA.js";
+} from "./chunk-4PSTYEG5.js";
 import {
   CliValidationError,
   LocalDataUploadError,
   __toESM
-} from "./chunk-ZW4VMY5H.js";
+} from "./chunk-Z3Q2AEIN.js";
 
 // src/commands/data-integration/inspect.ts
 import { basename as basename3 } from "path";
@@ -35,7 +35,8 @@ var THROUGHPUT_BYTES_PER_SECOND = {
   json: 8 * 1024 * 1024,
   csv: 4 * 1024 * 1024,
   tsv: 4 * 1024 * 1024,
-  xlsx: 2 * 1024 * 1024
+  xlsx: 2 * 1024 * 1024,
+  parquet: 8 * 1024 * 1024
 };
 var SLOW_ENCODINGS = /* @__PURE__ */ new Set(["gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr"]);
 function estimateProcessingSeconds(format, sizeBytes, encoding) {
@@ -72,6 +73,9 @@ function largeFileMemoryWarning(format) {
   if (format === "json") {
     return "Memory risk: a JSON root object or a very large record may be materialized in memory during inspection, so peak memory can substantially exceed the file size.";
   }
+  if (format === "parquet") {
+    return "Memory risk: Parquet input is decoded whole-file into memory (the reader materializes the full table), so peak memory can be a multiple of the file size. Split large files by row group or partition.";
+  }
   return null;
 }
 function assessFileSize(fileName, format, sizeBytes, encoding) {
@@ -641,6 +645,150 @@ function isStrongDateTime(value) {
   return /^\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:[ T]\d{1,2}:\d{2}(?::\d{2}(?:\.\d{1,3})?)?(?:Z|[+-]\d{2}:?\d{2})?)?$/.test(value.trim());
 }
 
+// src/commands/data-integration/parquet.ts
+import { readFileSync as readFileSync2 } from "fs";
+var TypeId = {
+  Int: 2,
+  Float: 3,
+  Binary: 4,
+  Utf8: 5,
+  Bool: 6,
+  Decimal: 7,
+  Date: 8,
+  Timestamp: 10,
+  List: 12,
+  Struct: 13,
+  FixedSizeBinary: 15,
+  FixedSizeList: 16,
+  Map: 17,
+  LargeBinary: 19,
+  LargeUtf8: 20,
+  LargeList: 21
+};
+function toJsValue(value, type) {
+  if (value === null || value === void 0) return null;
+  switch (type.typeId) {
+    case TypeId.Int:
+      return typeof value === "bigint" ? String(value) : value;
+    case TypeId.Float:
+    case TypeId.Bool:
+    case TypeId.Utf8:
+    case TypeId.LargeUtf8:
+      return value;
+    case TypeId.Binary:
+    case TypeId.LargeBinary:
+    case TypeId.FixedSizeBinary:
+      return Buffer.from(value).toString("hex");
+    case TypeId.Timestamp:
+      return toTimestampValue(value, type);
+    case TypeId.Date:
+      return toWallClockString(new Date(value));
+    case TypeId.Decimal:
+      return toDecimalValue(String(value), type.scale);
+    case TypeId.List:
+    case TypeId.LargeList:
+    case TypeId.FixedSizeList: {
+      const childType = type.children[0].type;
+      return Array.from(value, (item) => toJsValue(item, childType));
+    }
+    case TypeId.Struct: {
+      const row = value;
+      const out = {};
+      for (const child of type.children) out[child.name] = toJsValue(row[child.name], child.type);
+      return out;
+    }
+    case TypeId.Map: {
+      const valueType = type.children[0].type.children[1].type;
+      const out = {};
+      for (const [key, val] of value) {
+        out[String(key)] = toJsValue(val, valueType);
+      }
+      return out;
+    }
+    default:
+      return value;
+  }
+}
+function toTimestampValue(ms, type) {
+  const date = new Date(ms);
+  if (type.timezone) return date;
+  return toWallClockString(date);
+}
+function toWallClockString(date) {
+  const pad = (part, width = 2) => String(part).padStart(width, "0");
+  const ymd = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`;
+  const hours = date.getUTCHours();
+  const minutes = date.getUTCMinutes();
+  const seconds = date.getUTCSeconds();
+  const ms = date.getUTCMilliseconds();
+  if (hours === 0 && minutes === 0 && seconds === 0 && ms === 0) return ymd;
+  const time = `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  return ms === 0 ? `${ymd} ${time}` : `${ymd} ${time}.${pad(ms, 3)}`;
+}
+function toDecimalValue(unscaled, scale) {
+  if (scale <= 0) return unscaled;
+  const negative = unscaled.startsWith("-");
+  const digits = negative ? unscaled.slice(1) : unscaled;
+  const padded = digits.padStart(scale + 1, "0");
+  const whole = padded.slice(0, padded.length - scale);
+  const fraction = padded.slice(padded.length - scale).replace(/0+$/, "");
+  const sign = negative ? "-" : "";
+  return fraction ? `${sign}${whole}.${fraction}` : `${sign}${whole}`;
+}
+async function readParquetSchema(filePath) {
+  const { readSchema, readMetadata } = await import("parquet-wasm");
+  const { tableFromIPC } = await import("apache-arrow");
+  try {
+    const bytes = new Uint8Array(readFileSync2(filePath));
+    const metadata = readMetadata(bytes);
+    const numRows = metadata.fileMetadata().numRows();
+    metadata.free();
+    const table = tableFromIPC(readSchema(bytes).intoIPCStream());
+    const columns = table.schema.fields.map((field) => ({ name: field.name, type: field.type.toString() }));
+    return {
+      columns,
+      signature: columns.map((column) => `${column.name}:${column.type}`).join("|"),
+      numRows
+    };
+  } catch {
+    throw new CliValidationError("The file could not be read as Parquet.", {
+      code: "LOCAL_DATA_PARQUET_INVALID",
+      hint: "Verify the file is a valid Parquet file and retry without changing the source file.",
+      location: { field: "input-file" }
+    });
+  }
+}
+async function streamParquet(filePath, onRow) {
+  const { readParquet } = await import("parquet-wasm");
+  const { tableFromIPC } = await import("apache-arrow");
+  let table;
+  try {
+    const bytes = new Uint8Array(readFileSync2(filePath));
+    const wasmTable = readParquet(bytes);
+    table = tableFromIPC(wasmTable.intoIPCStream());
+  } catch {
+    throw new CliValidationError("The file could not be read as Parquet.", {
+      code: "LOCAL_DATA_PARQUET_INVALID",
+      hint: "Verify the file is a valid Parquet file and retry without changing the source file.",
+      location: { field: "input-file" }
+    });
+  }
+  const fields = table.schema.fields;
+  const columns = fields.map((_, index) => table.getChildAt(index));
+  let count = 0;
+  for (let rowIndex = 0; rowIndex < table.numRows; rowIndex += 1) {
+    const row = {};
+    for (let columnIndex = 0; columnIndex < fields.length; columnIndex += 1) {
+      const column = columns[columnIndex];
+      if (!column) continue;
+      row[fields[columnIndex].name] = toJsValue(column.get(rowIndex), fields[columnIndex].type);
+    }
+    count += 1;
+    await onRow(row, count);
+  }
+  return count;
+}
+
 // src/commands/data-integration/input.ts
 set_fs(nodeFs);
 function resolveLocalDataInputMeta(filePath) {
@@ -664,7 +812,7 @@ function resolveLocalDataInputMeta(filePath) {
       location: { field: "input-file" }
     });
   }
-  if (format !== "xls" && format !== "xlsx" && !encoding) {
+  if (format !== "xls" && format !== "xlsx" && format !== "parquet" && !encoding) {
     encoding = detectEncoding(filePath);
   }
   return {
@@ -788,6 +936,8 @@ async function streamLocalDataRows(input, dataSet, onRow, options = {}) {
         return await streamXlsx(input.filePath, dataSet.label, wrappedRow, opts);
       case "xls":
         return await streamXls(input.filePath, dataSet.label, wrappedRow, opts);
+      case "parquet":
+        return await streamParquet(input.filePath, wrappedRow);
     }
   } catch (error) {
     if (error instanceof CliValidationError) throw error;
@@ -810,6 +960,7 @@ function resolveFormat(filePath) {
   if (extension === "ndjson") return "jsonl";
   if (extension === "xls") return "xls";
   if (extension === "xlsx" || extension === "xlsm") return "xlsx";
+  if (extension === "parquet" || extension === "pq") return "parquet";
   return "txt";
 }
 function sniffDelimiter(filePath, encoding) {
@@ -924,7 +1075,7 @@ function isNumeric(value) {
   return !Number.isNaN(Number(trimmed));
 }
 async function discoverDataSets(filePath, format) {
-  if (format === "csv" || format === "tsv" || format === "jsonl") {
+  if (format === "csv" || format === "tsv" || format === "jsonl" || format === "parquet") {
     return { visible: [{ id: "$", kind: "file", label: basename(filePath), selector: "$" }], hidden: [] };
   }
   if (format === "xls") {
@@ -1685,7 +1836,7 @@ async function firstNonWhitespaceCharacter(filePath, encoding) {
 }
 
 // src/commands/data-integration/mapping.ts
-import { readFileSync as readFileSync2 } from "fs";
+import { readFileSync as readFileSync3 } from "fs";
 
 // src/commands/data-integration/types.ts
 var MAPPING_VERSION = "ae-data-integration-mapping/v1";
@@ -1706,7 +1857,7 @@ function readLocalDataMapping(raw, options) {
       text = trimmed;
     } else {
       const filePath = trimmed.startsWith("@") ? trimmed.slice(1) : trimmed;
-      text = readFileSync2(filePath, "utf8");
+      text = readFileSync3(filePath, "utf8");
     }
   } catch {
     throw new CliValidationError("The local-data mapping could not be read.", {
@@ -1731,7 +1882,7 @@ function validateMapping(value, options) {
     throw mappingError(`Mapping version must be ${MAPPING_VERSION}.`);
   }
   const sha256Valid = typeof value.source?.sha256 === "string" && (options?.sourceWildcard ? value.source.sha256 === "*" || /^[a-f0-9]{64}$/i.test(value.source.sha256) : /^[a-f0-9]{64}$/i.test(value.source.sha256));
-  if (!isRecord(value.source) || !sha256Valid || !["csv", "tsv", "json", "jsonl", "xls", "xlsx"].includes(String(value.source.format)) || typeof value.source.data_set !== "string" || !value.source.data_set) {
+  if (!isRecord(value.source) || !sha256Valid || !["csv", "tsv", "json", "jsonl", "xls", "xlsx", "parquet"].includes(String(value.source.format)) || typeof value.source.data_set !== "string" || !value.source.data_set) {
     throw mappingError("Mapping source fingerprint and data set are required.");
   }
   if (value.mode !== "track" && value.mode !== "user_set" && value.mode !== "mixed") {
@@ -2949,9 +3100,9 @@ var dataIntegrationInspect = {
   service: "data-integration",
   command: "inspect",
   usesAeHost: false,
-  description: "Stream one or more local CSV, TSV, TXT, JSON, JSONL, XLS, or XLSX files and recommend a UE mapping.",
+  description: "Stream one or more local CSV, TSV, TXT, JSON, JSONL, XLS, XLSX, or Parquet files and recommend a UE mapping.",
   flags: [
-    { name: "input-file", type: "string", required: true, sensitive: true, variadic: true, desc: "Local CSV, TSV, TXT, JSON, JSONL, XLS, or XLSX file. Repeat for multiple files." },
+    { name: "input-file", type: "string", required: true, sensitive: true, variadic: true, desc: "Local CSV, TSV, TXT, JSON, JSONL, XLS, XLSX, or Parquet file. Repeat for multiple files." },
     { name: "data-set", type: "string", sensitive: true, desc: "Sheet or JSON Path ID returned by a discovery-only inspection." },
     { name: "source-timezone", type: "string", default: "Asia/Shanghai", desc: "IANA timezone used to interpret source times without an offset." },
     { name: "headers", type: "string", sensitive: true, desc: "Comma-separated explicit column names for a headerless file; the first row is data." },
@@ -3039,6 +3190,10 @@ var dataIntegrationInspect = {
       };
     }
     const files = [];
+    if (inputFiles.every((file) => resolveLocalDataInputMeta(file).format === "parquet")) {
+      const checked = await inspectParquetFiles(inputFiles, sourceTimezone);
+      if (checked) return checked;
+    }
     for (const inputFile of inputFiles) {
       const input = await inspectLocalDataInput(inputFile);
       const dataSet = selectDataSet(input);
@@ -3073,6 +3228,34 @@ var dataIntegrationInspect = {
     };
   }
 };
+async function inspectParquetFiles(inputFiles, sourceTimezone) {
+  const firstPath = inputFiles[0];
+  const firstSchema = await readParquetSchema(firstPath);
+  const first = await inspectLocalDataInput(firstPath);
+  const profile = await profileLocalData(first, selectDataSet(first), sourceTimezone, {
+    collectSamples: true,
+    collectNestedTree: true
+  });
+  const files = [];
+  let totalRowCount = 0;
+  let totalSizeBytes = 0;
+  for (const inputFile of inputFiles) {
+    const meta = resolveLocalDataInputMeta(inputFile);
+    const schema = inputFile === firstPath ? firstSchema : await readParquetSchema(inputFile);
+    if (schema.signature !== firstSchema.signature) return void 0;
+    files.push({ file: basename3(inputFile), row_count: schema.numRows, size_bytes: meta.sizeBytes });
+    totalRowCount += schema.numRows;
+    totalSizeBytes += meta.sizeBytes;
+  }
+  return {
+    version: "ae-local-data-profile/v1",
+    schema_checked: true,
+    representative: { file: basename3(firstPath), profile },
+    files,
+    total_row_count: totalRowCount,
+    total_size_bytes: totalSizeBytes
+  };
+}
 function summarizeExcludedSheets(input) {
   const excluded = input.excludedDataSets ?? [];
   if (excluded.length === 0) return {};
@@ -3331,7 +3514,7 @@ import {
   existsSync,
   mkdirSync,
   readdirSync,
-  readFileSync as readFileSync3,
+  readFileSync as readFileSync4,
   renameSync,
   statSync as statSync2,
   unlinkSync,
@@ -3699,7 +3882,7 @@ function readZoneOffset(row, field) {
 function readSalvageRowNumbers(path) {
   let text;
   try {
-    text = readFileSync3(path, "utf8");
+    text = readFileSync4(path, "utf8");
   } catch {
     throw new CliValidationError("The salvage file could not be read.", {
       code: "LOCAL_DATA_SALVAGE_INVALID",
@@ -4101,7 +4284,7 @@ var dataIntegrationConvert = {
 };
 
 // src/commands/data-integration/upload.ts
-import { createReadStream as createReadStream4, readFileSync as readFileSync4, statSync as statSync3 } from "fs";
+import { createReadStream as createReadStream4, readFileSync as readFileSync5, statSync as statSync3 } from "fs";
 import { basename as basename5, dirname, resolve as resolve2 } from "path";
 import { createInterface as createInterface3 } from "readline";
 var DEFAULT_BATCH_SIZE = 500;
@@ -4335,7 +4518,7 @@ async function prepareUpload(ctx, enforceReady) {
 function readManifest(path) {
   let value;
   try {
-    value = JSON.parse(readFileSync4(path, "utf8"));
+    value = JSON.parse(readFileSync5(path, "utf8"));
   } catch {
     throw new CliValidationError("The local-data manifest could not be read as JSON.", {
       code: "LOCAL_DATA_MANIFEST_INVALID",
@@ -4447,7 +4630,7 @@ function isRecord3(value) {
 
 // src/commands/data-integration/handoff.ts
 import { createHash as createHash3 } from "crypto";
-import { chmodSync as chmodSync2, mkdirSync as mkdirSync2, mkdtempSync, readFileSync as readFileSync5, renameSync as renameSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
+import { chmodSync as chmodSync2, mkdirSync as mkdirSync2, mkdtempSync, readFileSync as readFileSync6, renameSync as renameSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
 import { dirname as dirname2, join as join3, resolve as resolve3 } from "path";
 import { tmpdir } from "os";
 
@@ -5379,7 +5562,7 @@ function buildHandoffPackage(outDir, mapping, planFile) {
 function readHandoffIndex(path) {
   let raw;
   try {
-    raw = readFileSync5(path, "utf8");
+    raw = readFileSync6(path, "utf8");
   } catch {
     return { version: HANDOFF_INDEX_VERSION, entries: [] };
   }
@@ -5403,7 +5586,7 @@ function readHandoffIndex(path) {
 function readPlanFile(path) {
   let raw;
   try {
-    raw = readFileSync5(path, "utf8");
+    raw = readFileSync6(path, "utf8");
   } catch {
     throw new CliValidationError("The tracking-plan file could not be read.", {
       code: "LOCAL_DATA_HANDOFF_PLAN_NOT_FOUND",
@@ -5611,7 +5794,7 @@ function stageScopedPackage(outDir, mappingDirs, entries, relayFiles) {
     for (const rel of mappingDirFiles(dirName, planFile)) {
       const dest = join3(staging, rel);
       mkdirSync2(dirname2(dest), { recursive: true, mode: 448 });
-      writeFileSync2(dest, readFileSync5(join3(outDir, rel), "utf8"), { encoding: "utf8", mode: 384 });
+      writeFileSync2(dest, readFileSync6(join3(outDir, rel), "utf8"), { encoding: "utf8", mode: 384 });
       chmodSync2(dest, 384);
     }
   }
@@ -5619,7 +5802,7 @@ function stageScopedPackage(outDir, mappingDirs, entries, relayFiles) {
 }
 
 // src/commands/data-integration/reuse.ts
-import { readFileSync as readFileSync6 } from "fs";
+import { readFileSync as readFileSync7 } from "fs";
 import { dirname as dirname4, join as join5, resolve as resolve5 } from "path";
 
 // src/commands/data-integration/handoff-root.ts
@@ -5678,7 +5861,7 @@ function detectReuse(mapping, outDir) {
 function readFrozenEventName(path) {
   let value;
   try {
-    value = JSON.parse(readFileSync6(path, "utf8"));
+    value = JSON.parse(readFileSync7(path, "utf8"));
   } catch {
     return {};
   }

package/dist/{evaluation-5YT4QLCL.js → evaluation-LR4YDYWL.js}

@@ -4,17 +4,17 @@ import {
   evalDownloadFile,
   evalRequest,
   resolveEvaluationHost
-} from "./chunk-X7DRLE62.js";
+} from "./chunk-YVG6QD3V.js";
 import {
   getCliToken
-} from "./chunk-P7KL7IOH.js";
+} from "./chunk-SOSFRU3O.js";
 import {
   isAeSandboxRuntime
-} from "./chunk-LMCYSVVA.js";
+} from "./chunk-4PSTYEG5.js";
 import {
   CliValidationError,
   externalCliTokenRequestOptions
-} from "./chunk-ZW4VMY5H.js";
+} from "./chunk-Z3Q2AEIN.js";
 
 // src/core/eval-session.ts
 var SESSION_PATH = "/api/sandbox/agent/evaluation/session";