@thinkingai/ae-cli 6.1.16 → 6.1.18

package/dist/{data-integration-2MYMANJI.js → data-integration-QEKDWQDY.js}

@@ -1,13 +1,108 @@
+import {
+  validateAndFix,
+  validateDraft
+} from "./chunk-RJDU7NYP.js";
+import {
+  getConfigDir
+} from "./chunk-3FY3RJ26.js";
 import {
   CliValidationError,
   LocalDataUploadError
 } from "./chunk-UW5UN47B.js";
-import "./chunk-QGM4M3NI.js";
+import "./chunk-JHENBQ5B.js";
 
-// src/commands/data-integration/local-data/inspect.ts
+// src/commands/data-integration/inspect.ts
 import { basename as basename3 } from "path";
 
-// src/commands/data-integration/local-data/input.ts
+// src/commands/data-integration/estimate.ts
+var XLS_SIZE_WARN_BYTES = 100 * 1024 * 1024;
+var LARGE_FILE_WARN_BYTES = 1024 * 1024 * 1024;
+var XLS_HARD_LIMIT_BYTES = 1024 * 1024 * 1024;
+var THROUGHPUT_BYTES_PER_SECOND = {
+  jsonl: 10 * 1024 * 1024,
+  json: 8 * 1024 * 1024,
+  csv: 4 * 1024 * 1024,
+  tsv: 4 * 1024 * 1024,
+  xlsx: 2 * 1024 * 1024
+};
+var SLOW_ENCODINGS = /* @__PURE__ */ new Set(["gbk", "gb2312", "gb18030", "big5", "shift_jis", "euc-jp", "euc-kr"]);
+function estimateProcessingSeconds(format, sizeBytes, encoding) {
+  if (format === "xls") return 0;
+  let throughput = THROUGHPUT_BYTES_PER_SECOND[format];
+  if (encoding && SLOW_ENCODINGS.has(encoding.toLowerCase())) throughput /= 2;
+  return sizeBytes / throughput;
+}
+function formatDuration(seconds) {
+  if (seconds < 60) {
+    const low2 = Math.max(1, Math.round(seconds * 0.5));
+    const high2 = Math.max(low2 + 1, Math.round(seconds * 1.5));
+    return `roughly ${low2} to ${high2} seconds`;
+  }
+  const low = Math.max(1, Math.round(seconds / 60 * 0.5));
+  const high = Math.max(low + 1, Math.round(seconds / 60 * 1.5));
+  return `roughly ${low} to ${high} minutes`;
+}
+function humanSize(bytes) {
+  if (bytes >= 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`;
+  return `${Math.round(bytes / (1024 * 1024))} MB`;
+}
+function xlsMemoryWarning(fileName, sizeBytes) {
+  return `Warning: ${fileName} is ${humanSize(sizeBytes)} (XLS). The legacy XLS parser loads the entire workbook into memory, roughly 5-10x the file size. Prefer converting to XLSX or splitting the workbook first.`;
+}
+function largeFileTimeWarning(fileName, sizeBytes, format, encoding) {
+  const duration = formatDuration(estimateProcessingSeconds(format, sizeBytes, encoding));
+  return `Warning: ${fileName} is ${humanSize(sizeBytes)}; processing is estimated to take ${duration}. Consider splitting the file if that is too long.`;
+}
+function largeFileMemoryWarning(format) {
+  if (format === "xlsx") {
+    return "Memory risk: XLSX processing keeps the workbook's shared string table in memory, so peak memory can substantially exceed the file size.";
+  }
+  if (format === "json") {
+    return "Memory risk: a JSON root object or a very large record may be materialized in memory during inspection, so peak memory can substantially exceed the file size.";
+  }
+  return null;
+}
+function assessFileSize(fileName, format, sizeBytes, encoding) {
+  const baseAssessment = {
+    size: humanSize(sizeBytes),
+    estimatedDuration: null,
+    warning: null,
+    reason: null,
+    memoryRisk: false,
+    rejected: false
+  };
+  if (format === "xls") {
+    if (sizeBytes > XLS_HARD_LIMIT_BYTES) {
+      return {
+        ...baseAssessment,
+        reason: "XLS files larger than 1 GB are not supported; convert the workbook to XLSX or split it first.",
+        memoryRisk: true,
+        rejected: true
+      };
+    }
+    if (sizeBytes > XLS_SIZE_WARN_BYTES) {
+      return {
+        ...baseAssessment,
+        warning: xlsMemoryWarning(fileName, sizeBytes),
+        memoryRisk: true
+      };
+    }
+    return baseAssessment;
+  }
+  if (sizeBytes > LARGE_FILE_WARN_BYTES) {
+    const memoryWarning = largeFileMemoryWarning(format);
+    const timeWarning = largeFileTimeWarning(fileName, sizeBytes, format, encoding);
+    return {
+      ...baseAssessment,
+      estimatedDuration: formatDuration(estimateProcessingSeconds(format, sizeBytes, encoding)),
+      warning: memoryWarning ? `${timeWarning} ${memoryWarning}` : timeWarning,
+      memoryRisk: Boolean(memoryWarning)
+    };
+  }
+  return baseAssessment;
+}
+
+// src/commands/data-integration/input.ts
 import { createHash } from "crypto";
 import { createReadStream as createReadStream2, statSync } from "fs";
 import { createRequire as createRequire2 } from "module";
@@ -24,7 +119,7 @@ import StreamValues from "stream-json/streamers/StreamValues.js";
 import * as unzipper from "unzipper";
 import { SaxesParser } from "saxes";
 
-// src/commands/data-integration/local-data/encoding.ts
+// src/commands/data-integration/encoding.ts
 import { createReadStream } from "fs";
 import { createRequire } from "module";
 import { openSync, readSync, closeSync } from "fs";
@@ -68,7 +163,7 @@ function readSample(filePath, maxBytes) {
   }
 }
 
-// src/commands/data-integration/local-data/time.ts
+// src/commands/data-integration/time.ts
 var TIME_FORMATS = [
   // Standard AE format
   "yyyy-MM-dd HH:mm:ss.SSS",
@@ -312,7 +407,7 @@ function tokenizeFormat(format) {
   return tokens;
 }
 
-// src/commands/data-integration/local-data/flatten.ts
+// src/commands/data-integration/flatten.ts
 var NDJSON_MAX_DEPTH = 1;
 var NESTED_NODE_SAMPLE_LIMIT = 5;
 var NESTED_SAMPLE_TRUNCATE = 40;
@@ -345,7 +440,7 @@ function flattenJSON(obj, prefix = "", depth = 0, maxDepth = NDJSON_MAX_DEPTH) {
   }
   return result;
 }
-function buildRowWithFlatten(obj, flattenRules) {
+function buildRowWithFlatten(obj, flattenRules, misses) {
   const row = {};
   const coveredRoots = new Set(Object.values(flattenRules).map((path) => path.split(".")[0]));
   const base = flattenJSON(obj);
@@ -354,43 +449,77 @@ function buildRowWithFlatten(obj, flattenRules) {
   }
   for (const [outColumn, sourcePath] of Object.entries(flattenRules)) {
     const value = getNestedValue(obj, sourcePath);
-    row[outColumn] = value == null ? "" : typeof value === "object" ? JSON.stringify(value) : String(value);
+    if (value == null) {
+      recordFlattenMiss(misses, outColumn);
+      row[outColumn] = "";
+    } else {
+      row[outColumn] = typeof value === "object" ? JSON.stringify(value) : String(value);
+    }
   }
   return row;
 }
-function flattenLocalDataRow(value, flattenRules) {
+function flattenLocalDataRow(value, flattenRules, misses) {
   if (value !== null && typeof value === "object" && !Array.isArray(value)) {
     if (flattenRules && Object.keys(flattenRules).length > 0) {
-      return buildRowWithFlatten(value, flattenRules);
+      return buildRowWithFlatten(value, flattenRules, misses);
     }
     return value;
   }
   return { value };
 }
-function flattenDelimitedRow(row, flattenRules) {
+function flattenDelimitedRow(row, flattenRules, misses) {
   const result = { ...row };
   for (const [outColumn, path] of Object.entries(flattenRules)) {
     const dot = path.indexOf(".");
-    if (dot <= 0) continue;
+    if (dot <= 0) {
+      recordFlattenMiss(misses, outColumn);
+      continue;
+    }
     const column = path.slice(0, dot);
     const cell = row[column];
-    if (typeof cell !== "string") continue;
+    if (typeof cell !== "string") {
+      recordFlattenMiss(misses, outColumn);
+      continue;
+    }
     let parsed;
     try {
       parsed = JSON.parse(cell);
     } catch {
+      recordFlattenMiss(misses, outColumn);
       continue;
     }
     const value = getNestedValue(parsed, path.slice(dot + 1));
-    if (value === null || value === void 0) continue;
+    if (value === null || value === void 0) {
+      recordFlattenMiss(misses, outColumn);
+      continue;
+    }
     result[outColumn] = typeof value === "object" ? JSON.stringify(value) : String(value);
   }
   return result;
 }
+function recordFlattenMiss(misses, outColumn) {
+  if (!misses) return;
+  misses[outColumn] = (misses[outColumn] ?? 0) + 1;
+}
 function buildNestedTree(rows) {
   const objects = rows.filter((row) => row !== null && typeof row === "object" && !Array.isArray(row));
   return buildObjectChildren(objects, "");
 }
+function buildColumnNestedTree(cells) {
+  const objects = cells.filter((cell) => cell !== null && typeof cell === "object" && !Array.isArray(cell));
+  const arrays = cells.filter(Array.isArray);
+  if (objects.length >= arrays.length) {
+    return objects.length > 0 ? buildObjectChildren(objects, "") : [];
+  }
+  const elements = arrays.flat();
+  const elementObjects = elements.filter(
+    (value) => value !== null && typeof value === "object" && !Array.isArray(value)
+  );
+  const elementKind = elementObjects.length > 0 ? "object" : "primitive";
+  const node = { path: "", name: "", kind: "array", elementKind, nonEmpty: elements.length > 0 };
+  if (elementKind === "object") node.children = buildObjectChildren(elementObjects, "");
+  return [node];
+}
 function buildObjectChildren(objects, parentPath) {
   const keys = [];
   const seen = /* @__PURE__ */ new Set();
@@ -416,8 +545,18 @@ function buildNode(path, name, values, nonEmpty) {
   if (structural > 0 && structural >= values.length - structural) {
     if (arrays.length > objects.length) {
       const elementValues = arrays.flat();
-      const elementKind = elementValues.some((value) => value !== null && typeof value === "object") ? "object" : "primitive";
-      return { path, name, kind: "array", elementKind, nonEmpty };
+      const elementObjects = elementValues.filter(
+        (value) => value !== null && typeof value === "object" && !Array.isArray(value)
+      );
+      const elementKind = elementObjects.length > 0 ? "object" : "primitive";
+      return {
+        path,
+        name,
+        kind: "array",
+        elementKind,
+        nonEmpty,
+        ...elementKind === "object" ? { children: buildObjectChildren(elementObjects, path) } : {}
+      };
     }
     return { path, name, kind: "object", children: buildObjectChildren(objects, path), nonEmpty };
   }
@@ -485,13 +624,11 @@ function isStrongDateTime(value) {
   return /^\d{4}[-/]\d{1,2}[-/]\d{1,2}(?:[ T]\d{1,2}:\d{2}(?::\d{2}(?:\.\d{1,3})?)?(?:Z|[+-]\d{2}:?\d{2})?)?$/.test(value.trim());
 }
 
-// src/commands/data-integration/local-data/input.ts
-var MAX_STREAMING_FILE_BYTES = 200 * 1024 * 1024;
-var MAX_XLS_FILE_BYTES = 50 * 1024 * 1024;
+// src/commands/data-integration/input.ts
 var XLSX = XLSXMod.default ?? XLSXMod;
 var require3 = createRequire2(import.meta.url);
 var ExcelWorksheetReader = require3("exceljs/lib/stream/xlsx/worksheet-reader");
-async function inspectLocalDataInput(filePath) {
+function resolveLocalDataInputMeta(filePath) {
   let format = resolveFormat(filePath);
   let delimiter;
   let encoding;
@@ -512,33 +649,43 @@ async function inspectLocalDataInput(filePath) {
       location: { field: "input-file" }
     });
   }
-  const maxBytes = format === "xls" ? MAX_XLS_FILE_BYTES : MAX_STREAMING_FILE_BYTES;
-  if (sizeBytes > maxBytes) {
-    throw new CliValidationError(
-      `${format.toUpperCase()} input exceeds the supported file size limit.`,
-      {
-        code: "LOCAL_DATA_FILE_TOO_LARGE",
-        hint: `Split the file below ${format === "xls" ? "50" : "200"} MB and retry.`,
-        location: { field: "input-file" }
-      }
-    );
-  }
   if (format !== "xls" && format !== "xlsx" && !encoding) {
     encoding = detectEncoding(filePath);
   }
+  return {
+    filePath,
+    format,
+    sizeBytes,
+    ...delimiter ? { delimiter } : {},
+    ...encoding ? { encoding } : {}
+  };
+}
+async function inspectLocalDataInput(filePath) {
+  const meta = resolveLocalDataInputMeta(filePath);
+  emitSizeWarning(meta.filePath, meta.format, meta.sizeBytes, meta.encoding);
   try {
     return {
-      filePath,
-      format,
-      sizeBytes,
+      ...meta,
       sha256: await sha256File(filePath),
-      dataSets: await discoverDataSets(filePath, format),
-      ...delimiter ? { delimiter } : {},
-      ...encoding ? { encoding } : {}
+      dataSets: await discoverDataSets(filePath, meta.format)
     };
   } catch (error) {
     if (error instanceof CliValidationError) throw error;
-    throw localDataParseError(format);
+    throw localDataParseError(meta.format);
+  }
+}
+function emitSizeWarning(filePath, format, sizeBytes, encoding) {
+  const assessment = assessFileSize(basename(filePath), format, sizeBytes, encoding);
+  if (assessment.rejected) {
+    throw new CliValidationError("XLS input exceeds the supported file size limit.", {
+      code: "LOCAL_DATA_FILE_TOO_LARGE",
+      hint: "XLS files larger than 1 GB are not supported; convert the workbook to XLSX or split it first.",
+      location: { field: "input-file" }
+    });
+  }
+  if (assessment.warning) {
+    process.stderr.write(`${assessment.warning}
+`);
   }
 }
 function selectDataSet(input, requested) {
@@ -562,6 +709,24 @@ function selectDataSet(input, requested) {
   }
   return input.dataSets[0];
 }
+var LocalDataRowCallbackError = class extends Error {
+  constructor(cause) {
+    super(cause instanceof Error ? cause.message : String(cause));
+    this.cause = cause;
+    this.name = "LocalDataRowCallbackError";
+  }
+  cause;
+};
+function wrapRowCallback(onRow) {
+  return async (row, rowNumber) => {
+    try {
+      await onRow(row, rowNumber);
+    } catch (error) {
+      if (error instanceof CliValidationError) throw error;
+      throw new LocalDataRowCallbackError(error);
+    }
+  };
+}
 async function streamLocalDataRows(input, dataSet, onRow, options = {}) {
   const opts = {
     delimiter: options.delimiter ?? input.delimiter ?? (input.format === "tsv" ? "	" : ","),
@@ -569,24 +734,30 @@ async function streamLocalDataRows(input, dataSet, onRow, options = {}) {
     headerNames: options.headerNames,
     noHeader: options.noHeader,
     flattenRules: options.flattenRules,
-    mergeSheets: options.mergeSheets
+    flattenMisses: options.flattenMisses,
+    mergeSheets: options.mergeSheets,
+    warnRagged: options.warnRagged
   };
+  const wrappedRow = wrapRowCallback(onRow);
   try {
     switch (input.format) {
       case "csv":
       case "tsv":
-        return await streamDelimited(input.filePath, onRow, opts);
+        return await streamDelimited(input.filePath, wrappedRow, opts);
       case "jsonl":
-        return await streamJsonLines(input.filePath, onRow, opts);
+        return await streamJsonLines(input.filePath, wrappedRow, opts);
       case "json":
-        return await streamJson(input.filePath, dataSet.selector ?? "$", onRow, opts);
+        return await streamJson(input.filePath, dataSet.selector ?? "$", wrappedRow, opts);
       case "xlsx":
-        return await streamXlsx(input.filePath, dataSet.label, onRow, opts);
+        return await streamXlsx(input.filePath, dataSet.label, wrappedRow, opts);
       case "xls":
-        return await streamXls(input.filePath, dataSet.label, onRow, opts);
+        return await streamXls(input.filePath, dataSet.label, wrappedRow, opts);
     }
   } catch (error) {
     if (error instanceof CliValidationError) throw error;
+    if (error instanceof LocalDataRowCallbackError) {
+      throw error.cause instanceof Error ? error.cause : error;
+    }
     throw localDataParseError(input.format);
   }
 }
@@ -774,29 +945,36 @@ async function streamDelimited(filePath, onRow, options) {
     quote: delimiter === "	" ? null : '"',
     relax_column_count: true,
     skip_empty_lines: true,
-    trim: true,
-    columns: headerNames ? headerNames : (headers) => dedupeHeaders(headers.map((header) => String(header).trim()))
+    trim: true
   });
   decodeTextStream(filePath, encoding).pipe(parser);
   let count = 0;
-  for await (const value of parser) {
+  let widthMismatches = 0;
+  let resolvedHeaders = headerNames;
+  for await (const raw of parser) {
+    const values = raw;
+    if (!resolvedHeaders) {
+      resolvedHeaders = dedupeHeaders(values.map((header) => String(header).trim()));
+      continue;
+    }
+    if (values.length !== resolvedHeaders.length) widthMismatches += 1;
     count += 1;
-    const row = normalizeDelimitedRow(value, headerNames);
+    const row = {};
+    for (let index = 0; index < resolvedHeaders.length; index += 1) {
+      row[resolvedHeaders[index]] = values[index] ?? null;
+    }
     await onRow(
-      options.flattenRules && Object.keys(options.flattenRules).length > 0 ? flattenDelimitedRow(row, options.flattenRules) : row,
+      options.flattenRules && Object.keys(options.flattenRules).length > 0 ? flattenDelimitedRow(row, options.flattenRules, options.flattenMisses) : row,
       count
     );
   }
-  return count;
-}
-function normalizeDelimitedRow(value, headerNames) {
-  if (!headerNames || value === null || typeof value !== "object" || Array.isArray(value)) {
-    return normalizeRow(value);
+  if (widthMismatches > 0 && options.warnRagged !== false) {
+    process.stderr.write(
+      `Warning: ${widthMismatches} record(s) had a column count different from the header row; extra fields were dropped and missing fields were treated as empty.
+`
+    );
   }
-  const source = value;
-  const row = {};
-  for (const header of headerNames) row[header] = source[header] ?? null;
-  return row;
+  return count;
 }
 async function streamJsonLines(filePath, onRow, options) {
   let count = 0;
@@ -813,7 +991,7 @@ async function streamJsonLines(filePath, onRow, options) {
         location: { record: count }
       });
     }
-    await onRow(flattenLocalDataRow(value, options.flattenRules), count);
+    await onRow(flattenLocalDataRow(value, options.flattenRules, options.flattenMisses), count);
   }
   return count;
 }
@@ -825,7 +1003,7 @@ async function streamJson(filePath, selector, onRow, options) {
   const chain = selector === "$" || selector === "$object" ? source.pipe(parser).pipe(streamer) : source.pipe(parser).pipe(Pick.pick({ filter: selector })).pipe(streamer);
   for await (const item of chain) {
     count += 1;
-    await onRow(flattenLocalDataRow(item.value, options.flattenRules), count);
+    await onRow(flattenLocalDataRow(item.value, options.flattenRules, options.flattenMisses), count);
   }
   return count;
 }
@@ -869,7 +1047,11 @@ async function streamXlsx(filePath, sheetName, onRow, options) {
       }
       if (values.every(isMissing)) continue;
       count += 1;
-      await onRow(Object.fromEntries(headers.map((header, index) => [header, values[index] ?? null])), count);
+      const row = Object.fromEntries(headers.map((header, index) => [header, values[index] ?? null]));
+      await onRow(
+        options.flattenRules && Object.keys(options.flattenRules).length > 0 ? flattenDelimitedRow(row, options.flattenRules) : row,
+        count
+      );
     }
   }
   return count;
@@ -1021,17 +1203,15 @@ async function streamXls(filePath, sheetName, onRow, options) {
     for (const values of rows.slice(start)) {
       if (values.every(isMissing)) continue;
       count += 1;
-      await onRow(Object.fromEntries(headers.map((header, index) => [header, values[index] ?? null])), count);
+      const row = Object.fromEntries(headers.map((header, index) => [header, values[index] ?? null]));
+      await onRow(
+        options.flattenRules && Object.keys(options.flattenRules).length > 0 ? flattenDelimitedRow(row, options.flattenRules) : row,
+        count
+      );
     }
   }
   return count;
 }
-function normalizeRow(value) {
-  if (value !== null && typeof value === "object" && !Array.isArray(value)) {
-    return value;
-  }
-  return { value };
-}
 function normalizeExcelValue(value) {
   if (value instanceof Date) return value;
   if (value && typeof value === "object") {
@@ -1074,9 +1254,16 @@ async function firstNonWhitespaceCharacter(filePath, encoding) {
   return void 0;
 }
 
-// src/commands/data-integration/local-data/mapping.ts
+// src/commands/data-integration/mapping.ts
 import { readFileSync } from "fs";
+
+// src/commands/data-integration/types.ts
+var MAPPING_VERSION = "ae-data-integration-mapping/v1";
+
+// src/commands/data-integration/mapping.ts
 var VALID_PROPERTY_NAME = /^[a-z][a-z0-9_]{0,49}$/;
+var VALID_CHILD_PROPERTY_NAME = /^[a-z][a-z0-9_]{0,49}(\.[a-z][a-z0-9_]{0,49})+$/;
+var FLATTEN_SUPPORTED_FORMATS = ["csv", "tsv", "json", "jsonl", "xls", "xlsx"];
 function readLocalDataMapping(raw, options) {
   const trimmed = raw.trim();
   let text;
@@ -1106,8 +1293,8 @@ function readLocalDataMapping(raw, options) {
   return value;
 }
 function validateMapping(value, options) {
-  if (!isRecord(value) || value.version !== "ae-local-data-mapping/v1") {
-    throw mappingError("Mapping version must be ae-local-data-mapping/v1.");
+  if (!isRecord(value) || value.version !== MAPPING_VERSION) {
+    throw mappingError(`Mapping version must be ${MAPPING_VERSION}.`);
   }
   const sha256Valid = typeof value.source?.sha256 === "string" && (options?.sourceWildcard ? value.source.sha256 === "*" || /^[a-f0-9]{64}$/i.test(value.source.sha256) : /^[a-f0-9]{64}$/i.test(value.source.sha256));
   if (!isRecord(value.source) || !sha256Valid || !["csv", "tsv", "json", "jsonl", "xls", "xlsx"].includes(String(value.source.format)) || typeof value.source.data_set !== "string" || !value.source.data_set) {
@@ -1137,12 +1324,28 @@ function validateMapping(value, options) {
   }
   if (!Array.isArray(value.properties)) throw mappingError("Mapping properties must be an array.");
   const targets = /* @__PURE__ */ new Set();
+  const propertyTypes = /* @__PURE__ */ new Map();
   for (const property of value.properties) {
-    if (!isRecord(property) || typeof property.source !== "string" || typeof property.target !== "string" || !VALID_PROPERTY_NAME.test(property.target) || !["number", "string", "boolean", "datetime", "list", "object"].includes(String(property.type)) || property.transform !== void 0 && !["stringify", "number", "boolean", "json"].includes(String(property.transform)) || property.value_mapping !== void 0 && !isStringMap(property.value_mapping) || property.time_format !== void 0 && (typeof property.time_format !== "string" || !property.time_format.trim() || property.time_format.length > 64) || property.desc !== void 0 && (typeof property.desc !== "string" || !property.desc.trim() || property.desc.length > 200)) {
+    if (!isRecord(property) || typeof property.source !== "string" || typeof property.target !== "string" || !(VALID_PROPERTY_NAME.test(property.target) || VALID_CHILD_PROPERTY_NAME.test(property.target)) || !["number", "string", "boolean", "datetime", "list", "object", "array_row"].includes(String(property.type)) || property.transform !== void 0 && !["stringify", "number", "boolean", "json"].includes(String(property.transform)) || property.value_mapping !== void 0 && !isStringMap(property.value_mapping) || property.time_format !== void 0 && (typeof property.time_format !== "string" || !property.time_format.trim() || property.time_format.length > 64) || property.desc !== void 0 && (typeof property.desc !== "string" || !property.desc.trim() || property.desc.length > 200)) {
       throw mappingError("Every property mapping needs a source, legal AE target name, and supported type.");
     }
     if (targets.has(property.target)) throw mappingError("Property target names must be unique.");
     targets.add(property.target);
+    propertyTypes.set(property.target, { type: String(property.type), source: property.source });
+  }
+  for (const property of value.properties) {
+    if (!property.target.includes(".")) continue;
+    const parentName = property.target.split(".")[0];
+    const parent = propertyTypes.get(parentName);
+    if (!parent || parent.type !== "object" && parent.type !== "array_row") {
+      throw mappingError(`The sub-property "${property.target}" references "${parentName}", which is not an object/array_row property in this mapping.`);
+    }
+    if (property.type === "object" || property.type === "array_row") {
+      throw mappingError(`The sub-property "${property.target}" must be scalar or list, not ${property.type}.`);
+    }
+    if (property.source !== parent.source) {
+      throw mappingError(`The sub-property "${property.target}" must read from the same source column as its parent "${parentName}".`);
+    }
   }
   if (value.time_format !== void 0 && (typeof value.time_format !== "string" || !value.time_format.trim() || value.time_format.length > 64)) {
     throw mappingError("time_format must be a non-empty string of at most 64 characters.");
@@ -1184,6 +1387,9 @@ function validateMapping(value, options) {
   }
   if (value.flatten_rules !== void 0) {
     if (!isRecord(value.flatten_rules)) throw mappingError("flatten_rules must be an object of { column: dot.path }.");
+    if (!FLATTEN_SUPPORTED_FORMATS.includes(String(value.source.format))) {
+      throw mappingError(`flatten_rules are not supported for ${String(value.source.format)} input.`);
+    }
     for (const [column, path] of Object.entries(value.flatten_rules)) {
       if (!VALID_PROPERTY_NAME.test(column) || typeof path !== "string" || !path.trim()) {
         throw mappingError("flatten_rules keys must be legal AE property names and values must be non-empty dot paths.");
@@ -1227,6 +1433,31 @@ function validateMapping(value, options) {
 function isValidAeName(value) {
   return VALID_PROPERTY_NAME.test(value);
 }
+function sourceColumns(mapping) {
+  const columns = /* @__PURE__ */ new Set();
+  const flattenOut = new Set(Object.keys(mapping.flatten_rules ?? {}));
+  for (const property of mapping.properties) {
+    if (!flattenOut.has(property.source)) columns.add(property.source);
+  }
+  for (const path of Object.values(mapping.flatten_rules ?? {})) {
+    const root = path.split(".")[0];
+    if (root) columns.add(root);
+  }
+  const systemFields = [
+    mapping.account_id_field,
+    mapping.distinct_id_field,
+    mapping.record_type_field,
+    mapping.event_name_field,
+    mapping.time.field,
+    mapping.ip_field,
+    mapping.uuid_field,
+    mapping.zone_offset_field
+  ];
+  for (const field of systemFields) if (field) columns.add(field);
+  for (const column of mapping.exclude_columns ?? []) columns.add(column);
+  for (const header of mapping.headers ?? []) columns.add(header);
+  return [...columns].sort();
+}
 function mappingError(message) {
   return new CliValidationError(message, {
     code: "LOCAL_DATA_MAPPING_INVALID",
@@ -1253,7 +1484,7 @@ function isRecord(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 
-// src/commands/data-integration/local-data/multi.ts
+// src/commands/data-integration/multi.ts
 var PROPERTY_TYPES = /* @__PURE__ */ new Set([
   "number",
   "string",
@@ -1398,9 +1629,46 @@ function isRecord2(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 
-// src/commands/data-integration/local-data/profile.ts
+// src/commands/data-integration/profile.ts
 import { createHash as createHash2, randomInt } from "crypto";
 import { basename as basename2, extname as extname2 } from "path";
+
+// src/commands/data-integration/field-spec.ts
+import { isIP } from "net";
+function stripQuotes(value) {
+  if (typeof value !== "string") return value;
+  const text = value;
+  if (text.length >= 2 && (text.startsWith('"') && text.endsWith('"') || text.startsWith("'") && text.endsWith("'"))) {
+    return text.slice(1, -1).trim();
+  }
+  return text.trim();
+}
+var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+function isValidUuid(value) {
+  if (typeof value !== "string") return false;
+  return UUID_PATTERN.test(stripQuotes(value));
+}
+function isValidIp(value) {
+  if (typeof value !== "string") return false;
+  return isIP(stripQuotes(value)) !== 0;
+}
+function isPrivateIp(value) {
+  if (typeof value !== "string") return false;
+  const text = stripQuotes(value);
+  const version = isIP(text);
+  if (version === 6) {
+    const lower = text.toLowerCase();
+    if (lower === "::1") return true;
+    const first2 = lower.split(":")[0];
+    return first2.startsWith("fc") || first2.startsWith("fd") || /^fe[89ab]/.test(lower);
+  }
+  if (version !== 4) return false;
+  const octets = text.split(".").map((part) => Number(part));
+  const [first, second] = octets;
+  return first === 10 || first === 172 && second >= 16 && second <= 31 || first === 192 && second === 168 || first === 127 || first === 169 && second === 254;
+}
+
+// src/commands/data-integration/profile.ts
 var UNIQUE_SAMPLE_LIMIT = 1e4;
 var GLOBAL_UNIQUE_SAMPLE_LIMIT = 2e5;
 var IDENTITY_MAX_LENGTH = 128;
@@ -1424,6 +1692,8 @@ var DISTINCT_NAMES = ["#distinct_id", "distinct_id", "distinctid", "device_id",
 var TIME_NAMES = ["#time", "time", "timestamp", "event_time", "created_at", "occurred_at", "datetime", "date", "\u65F6\u95F4", "\u4E8B\u4EF6\u65F6\u95F4", "\u53D1\u751F\u65F6\u95F4", "\u521B\u5EFA\u65F6\u95F4", "\u4E0B\u5355\u65F6\u95F4", "\u8BA2\u5355\u65F6\u95F4"];
 var EVENT_NAMES = ["#event_name", "event_name", "event", "action", "activity", "\u4E8B\u4EF6\u540D", "\u4E8B\u4EF6\u540D\u79F0", "\u4E8B\u4EF6", "\u884C\u4E3A", "\u52A8\u4F5C"];
 var TYPE_NAMES = ["#type", "record_type", "data_type", "\u64CD\u4F5C\u7C7B\u578B"];
+var IP_NAMES = ["#ip", "ip", "ip_address", "ipaddress", "client_ip", "clientip", "remote_addr", "remoteaddr", "ip\u5730\u5740", "\u5BA2\u6237\u7AEFip"];
+var UUID_NAMES = ["#uuid", "uuid", "event_uuid", "eventuuid", "request_uuid", "requestuuid", "\u552F\u4E00\u6807\u8BC6", "\u552F\u4E00id"];
 async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai", options = {}) {
   const columns = /* @__PURE__ */ new Map();
   const recognizedRecordTypes = /* @__PURE__ */ new Set();
@@ -1433,7 +1703,7 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
     options.collectNestedTree && (input.format === "json" || input.format === "jsonl")
   );
   const collectDelimitedTree = Boolean(
-    options.collectNestedTree && (input.format === "csv" || input.format === "tsv")
+    options.collectNestedTree && (input.format === "csv" || input.format === "tsv" || input.format === "xlsx" || input.format === "xls")
   );
   let nestedObjects = [];
   let nestedSeen = 0;
@@ -1464,6 +1734,9 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
             uniqueOverflow: false,
             timeParseCount: 0,
             timeFormatCounts: /* @__PURE__ */ new Map(),
+            uuidValidCount: 0,
+            ipValidCount: 0,
+            lanIpCount: 0,
             samples: [],
             sampleSet: /* @__PURE__ */ new Set()
           };
@@ -1486,24 +1759,27 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
           accumulator.uniqueOverflow = true;
         }
         if (options.collectSamples) recordSample(accumulator, value);
-        if (collectDelimitedTree && typeof value === "string" && value.trim().startsWith("{")) {
-          try {
-            const parsed = JSON.parse(value);
-            if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
-              let sampler = delimitedNested.get(name);
-              if (!sampler) {
-                sampler = { seen: 0, objects: [] };
-                delimitedNested.set(name, sampler);
-              }
-              sampler.seen += 1;
-              if (sampler.objects.length < NESTED_TREE_SAMPLE_LIMIT) {
-                sampler.objects.push(parsed);
-              } else {
-                const slot = randomInt(sampler.seen);
-                if (slot < NESTED_TREE_SAMPLE_LIMIT) sampler.objects[slot] = parsed;
+        if (collectDelimitedTree && typeof value === "string") {
+          const trimmed = value.trim();
+          if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+            try {
+              const parsed = JSON.parse(value);
+              if (parsed !== null && typeof parsed === "object") {
+                let sampler = delimitedNested.get(name);
+                if (!sampler) {
+                  sampler = { seen: 0, values: [] };
+                  delimitedNested.set(name, sampler);
+                }
+                sampler.seen += 1;
+                if (sampler.values.length < NESTED_TREE_SAMPLE_LIMIT) {
+                  sampler.values.push(parsed);
+                } else {
+                  const slot = randomInt(sampler.seen);
+                  if (slot < NESTED_TREE_SAMPLE_LIMIT) sampler.values[slot] = parsed;
+                }
               }
+            } catch {
             }
-          } catch {
           }
         }
         if (isParseableTime(value, matchesName(name, TIME_NAMES))) {
@@ -1515,6 +1791,11 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
             }
           }
         }
+        if (matchesName(name, UUID_NAMES) && isValidUuid(value)) accumulator.uuidValidCount += 1;
+        if (matchesName(name, IP_NAMES) && isValidIp(value)) {
+          accumulator.ipValidCount += 1;
+          if (isPrivateIp(value)) accumulator.lanIpCount += 1;
+        }
         if (matchesName(name, TYPE_NAMES)) {
           const normalized = normalizeRecordType(value);
           if (normalized) recognizedRecordTypes.add(normalized);
@@ -1527,23 +1808,25 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
       headerNames: options.headerNames,
       noHeader: options.noHeader,
       flattenRules: options.flattenRules,
-      mergeSheets: options.mergeSheets
+      mergeSheets: options.mergeSheets,
+      warnRagged: options.warnRagged
     }
   );
   const delimitedNestedTree = /* @__PURE__ */ new Map();
   for (const [columnName, sampler] of delimitedNested) {
-    if (sampler.objects.length > 0) delimitedNestedTree.set(columnName, buildNestedTree(sampler.objects));
+    if (sampler.values.length > 0) delimitedNestedTree.set(columnName, buildColumnNestedTree(sampler.values));
   }
   const columnProfiles = [];
   const timeFormatByColumn = /* @__PURE__ */ new Map();
   for (const column of columns.values()) {
     const profile = formatColumnProfile(column, rowCount, options.collectSamples ?? false);
-    const nestedTree = delimitedNestedTree.get(column.name);
-    if (nestedTree && nestedTree.length > 0) profile.nested_tree = nestedTree;
+    const nestedTree2 = delimitedNestedTree.get(column.name);
+    if (nestedTree2 && nestedTree2.length > 0) profile.nested_tree = nestedTree2;
     columnProfiles.push(profile);
     const dominantFormat = dominantTimeFormat(column);
     if (dominantFormat) timeFormatByColumn.set(column.name, dominantFormat);
   }
+  const nestedTree = collectNestedTree && nestedObjects.length > 0 ? buildNestedTree(nestedObjects) : void 0;
   const identityCandidates = findIdentityCandidates(columnProfiles);
   const recommendedMapping = recommendMapping({
     input,
@@ -1553,9 +1836,21 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
     recognizedRecordTypes,
     sourceTimezone,
     headerNames: options.headerNames,
-    timeFormatByColumn
+    timeFormatByColumn,
+    nestedTree
   });
   const warnings = [...recommendedMapping.warnings ?? []];
+  for (const column of columns.values()) {
+    if (matchesName(column.name, UUID_NAMES) && column.nonMissing > 0) {
+      const invalid = column.nonMissing - column.uuidValidCount;
+      if (invalid > 0) warnings.push(`${column.name}: ${invalid} non-empty value(s) are not standard 36-character UUIDs; #uuid requires the standard UUID format.`);
+    }
+    if (matchesName(column.name, IP_NAMES) && column.nonMissing > 0) {
+      const invalid = column.nonMissing - column.ipValidCount;
+      if (invalid > 0) warnings.push(`${column.name}: ${invalid} non-empty value(s) are not valid IPv4 or IPv6 addresses.`);
+      if (column.lanIpCount > 0) warnings.push(`${column.name}: ${column.lanIpCount} value(s) are private/LAN IP addresses; AE cannot resolve geo information for them.`);
+    }
+  }
   if (rowCount === 0) warnings.push("The selected data set contains no data rows.");
   return {
     version: "ae-local-data-profile/v1",
@@ -1574,7 +1869,7 @@ async function profileLocalData(input, dataSet, sourceTimezone = "Asia/Shanghai"
       (recommendedMapping.account_id_field || recommendedMapping.distinct_id_field) && recommendedMapping.time.field
     ),
     warnings,
-    ...collectNestedTree && nestedObjects.length > 0 ? { nested_tree: buildNestedTree(nestedObjects) } : {}
+    ...nestedTree ? { nested_tree: nestedTree } : {}
   };
 }
 function normalizeAeName(input, fallback) {
@@ -1647,24 +1942,12 @@ function recommendMapping(input) {
     warnings.push("No event-name column was found; review the generated default event name.");
   }
   const reserved = new Set([account?.name, distinct?.name, time?.name, event?.name, recordType?.name].filter(Boolean));
-  const usedTargets = /* @__PURE__ */ new Set();
-  const properties = input.columns.filter((column) => !reserved.has(column.name)).map((column, index) => {
-    const baseTarget = normalizeAeName(column.name, `field_${index + 1}`);
-    let target = baseTarget;
-    let suffix = 2;
-    while (usedTargets.has(target)) target = `${baseTarget.slice(0, 46)}_${suffix++}`;
-    usedTargets.add(target);
-    const type = mappingType(column.inferred_type);
-    return {
-      source: column.name,
-      target,
-      type,
-      ...type === "object" || type === "list" ? { transform: "json" } : {}
-    };
-  });
+  const recordRoots = /* @__PURE__ */ new Map();
+  for (const node of input.nestedTree ?? []) recordRoots.set(node.name, node);
+  const { properties, flattenRules } = recommendProperties(input.columns, reserved, recordRoots, warnings);
   const defaultEventSource = input.dataSet.kind === "sheet" ? input.dataSet.label : basename2(input.input.filePath, extname2(input.input.filePath));
   return {
-    version: "ae-local-data-mapping/v1",
+    version: MAPPING_VERSION,
     source: {
       sha256: input.input.sha256,
       format: input.input.format,
@@ -1684,6 +1967,7 @@ function recommendMapping(input) {
     ...recordType ? { record_type_field: recordType.name } : {},
     ...event ? { event_name_field: event.name } : {},
     ...mode !== "user_set" && !event ? { default_event_name: normalizeAeName(defaultEventSource, "local_event") } : {},
+    ...Object.keys(flattenRules).length > 0 ? { flatten_rules: flattenRules } : {},
     properties,
     ...warnings.length > 0 ? { warnings } : {}
   };
@@ -1767,9 +2051,120 @@ function normalizeRecordType(value) {
   };
   return aliases[normalized];
 }
+function recommendProperties(columns, reserved, recordRoots, warnings) {
+  const properties = [];
+  const flattenRules = {};
+  const usedTargets = /* @__PURE__ */ new Set();
+  const claim = (desired) => {
+    let target = desired;
+    let suffix = 2;
+    while (usedTargets.has(target)) target = `${desired.slice(0, 46)}_${suffix++}`;
+    usedTargets.add(target);
+    return target;
+  };
+  const state = { properties, flattenRules, usedTargets, warnings, claim };
+  columns.filter((column) => !reserved.has(column.name)).forEach((column, index) => {
+    const baseName = normalizeAeName(column.name, `field_${index + 1}`);
+    const valueNode = columnValueNode(column, recordRoots);
+    if (!valueNode) {
+      const type = mappingType(column.inferred_type);
+      const target = claim(baseName);
+      properties.push({ source: column.name, target, type, ...isContainerType(type) ? { transform: "json" } : {} });
+      return;
+    }
+    if (valueNode.kind === "primitive") {
+      const type = mappingType(valueNode.inferredType ?? column.inferred_type);
+      const target = claim(baseName);
+      properties.push({ source: column.name, target, type });
+      return;
+    }
+    if (valueNode.kind === "object") {
+      recommendObject(state, baseName, column.name, valueNode.children ?? [], column.name);
+      return;
+    }
+    recommendArray(state, baseName, column.name, valueNode, column.name);
+  });
+  return { properties, flattenRules };
+}
+function columnValueNode(column, recordRoots) {
+  const recordNode = recordRoots.get(column.name);
+  if (recordNode) return recordNode;
+  const tree = column.nested_tree;
+  if (!tree || tree.length === 0) return void 0;
+  if (tree.length === 1 && tree[0].kind === "array") return tree[0];
+  return { path: "", name: column.name, kind: "object", children: tree, nonEmpty: true };
+}
+function isScalarNode(node) {
+  return node.kind === "primitive" || node.kind === "array" && node.elementKind === "primitive";
+}
+function isContainerType(type) {
+  return type === "object" || type === "list" || type === "array_row";
+}
+function scalarPropType(node) {
+  if (node.kind === "array") return "list";
+  switch (node.inferredType) {
+    case "number":
+      return "number";
+    case "boolean":
+      return "boolean";
+    case "datetime":
+      return "datetime";
+    default:
+      return "string";
+  }
+}
+function snakeSegment(name) {
+  return normalizeAeName(name, "field");
+}
+function recommendObject(state, prefix, dotPath, children, columnName) {
+  const hasComposite = children.some((child) => child.kind === "object" || child.kind === "array" && child.elementKind === "object");
+  if (!hasComposite) {
+    const parentTarget = state.claim(prefix);
+    const source = dotPath === columnName ? columnName : parentTarget;
+    if (source !== columnName) state.flattenRules[parentTarget] = dotPath;
+    state.properties.push({ source, target: parentTarget, type: "object", transform: "json" });
+    for (const child of children) {
+      state.properties.push({ source: parentTarget, target: `${parentTarget}.${snakeSegment(child.name)}`, type: scalarPropType(child) });
+    }
+    return;
+  }
+  for (const child of children) {
+    const childPrefix = `${prefix}_${snakeSegment(child.name)}`;
+    const childDotPath = `${dotPath}.${child.name}`;
+    if (isScalarNode(child)) {
+      const target = state.claim(childPrefix);
+      state.flattenRules[target] = childDotPath;
+      state.properties.push({ source: target, target, type: scalarPropType(child) });
+    } else if (child.kind === "object") {
+      recommendObject(state, childPrefix, childDotPath, child.children ?? [], columnName);
+    } else {
+      recommendArray(state, childPrefix, childDotPath, child, columnName);
+    }
+  }
+}
+function recommendArray(state, prefix, dotPath, node, columnName) {
+  if (node.elementKind !== "object") {
+    const target = state.claim(prefix);
+    const source2 = dotPath === columnName ? columnName : target;
+    if (source2 !== columnName) state.flattenRules[target] = dotPath;
+    state.properties.push({ source: source2, target, type: "list", transform: "json" });
+    return;
+  }
+  const parentTarget = state.claim(prefix);
+  const source = dotPath === columnName ? columnName : parentTarget;
+  if (source !== columnName) state.flattenRules[parentTarget] = dotPath;
+  state.properties.push({ source, target: parentTarget, type: "array_row", transform: "json" });
+  for (const field of node.children ?? []) {
+    if (isScalarNode(field)) {
+      state.properties.push({ source: parentTarget, target: `${parentTarget}.${snakeSegment(field.name)}`, type: scalarPropType(field) });
+    } else {
+      state.warnings.push(`${dotPath} element field "${field.name}" is nested; it stays inside the ${parentTarget} array data and is not declared as a sub-property (array-element flattening is not yet supported).`);
+    }
+  }
+}
 function mappingType(type) {
   if (type === "datetime") return "datetime";
-  if (type === "number" || type === "boolean" || type === "list" || type === "object") return type;
+  if (type === "number" || type === "boolean" || type === "object" || type === "list") return type;
   return "string";
 }
 function compatibleTypes(types) {
@@ -1832,7 +2227,7 @@ function dominantTimeFormat(column) {
   return best && bestCount >= total * 0.9 ? best : void 0;
 }
 
-// src/commands/data-integration/local-data/inspect.ts
+// src/commands/data-integration/inspect.ts
 var HEADERLESS_WARNING = "The first row appears to be data, not a header; columns were auto-named col_1..col_N. Re-run with --headers to supply explicit names.";
 var dataIntegrationInspect = {
   service: "data-integration",
@@ -1847,6 +2242,30 @@ var dataIntegrationInspect = {
     { name: "headerless", type: "boolean", default: false, desc: "Treat the first row as data and auto-generate col_1..col_N names." }
   ],
   risk: "read",
+  // Fast size/time pre-check (stat + format/encoding sniff only — no sha256, no profile).
+  // Lets agents surface the estimate before committing to a multi-minute full inspection.
+  dryRun: async (ctx) => {
+    const files = ctx.list("input-file").map((filePath) => {
+      const meta = resolveLocalDataInputMeta(filePath);
+      const assessment = assessFileSize(basename3(filePath), meta.format, meta.sizeBytes, meta.encoding);
+      return {
+        file: basename3(filePath),
+        format: meta.format,
+        size_bytes: meta.sizeBytes,
+        size: assessment.size,
+        ...assessment.estimatedDuration ? { estimated_duration: assessment.estimatedDuration } : {},
+        ...assessment.warning ? { warning: assessment.warning } : {},
+        ...assessment.reason ? { reason: assessment.reason } : {},
+        ...assessment.memoryRisk ? { memory_risk: true } : {},
+        ...assessment.rejected ? { rejected: true } : {}
+      };
+    });
+    return {
+      version: "ae-local-data-estimate/v1",
+      files,
+      has_large_file: files.some((file) => Boolean(file.warning) || file.rejected)
+    };
+  },
   execute: async (ctx) => {
     const inputFiles = ctx.list("input-file");
     const headerNames = splitHeaders(ctx.str("headers"));
@@ -1940,7 +2359,7 @@ function splitHeaders(raw) {
   return headers.length > 0 ? headers : void 0;
 }
 
-// src/commands/data-integration/local-data/plan.ts
+// src/commands/data-integration/plan.ts
 import { writeFile } from "fs/promises";
 var VALID_LOCALES = /* @__PURE__ */ new Set(["zh", "en", "ja", "ko"]);
 var EVENT_NAME_RE = /^[a-z][a-z0-9_]*$/;
@@ -1956,6 +2375,8 @@ function toPropType(type) {
       return "array_string";
     case "object":
       return "object";
+    case "array_row":
+      return "array_row";
     case "string":
       return "string";
   }
@@ -1963,13 +2384,16 @@ function toPropType(type) {
 function buildDraftFromMapping(options) {
   const { mapping } = options;
   const excluded = new Set(mapping.exclude_columns ?? []);
-  const properties = mapping.properties.filter((property) => !excluded.has(property.source)).map((property) => ({
-    name: property.target,
-    display_name: property.source,
-    desc: property.desc ?? property.source,
-    type: toPropType(property.type),
-    source: "data"
-  }));
+  const properties = mapping.properties.filter((property) => !excluded.has(property.source)).map((property) => {
+    const leaf = property.target.includes(".") ? property.target.split(".").pop() : void 0;
+    return {
+      name: property.target,
+      display_name: leaf ?? property.source,
+      desc: property.desc ?? leaf ?? property.source,
+      type: toPropType(property.type),
+      source: "data"
+    };
+  });
   const propNames = properties.map((property) => property.name);
   const eventNames = resolveEventNames(options);
   const events = eventNames.map((eventName) => {
@@ -2047,7 +2471,7 @@ function buildPlanDraft(ctx) {
       location: { field: "lang" }
     });
   }
-  return buildDraftFromMapping({
+  const draft = buildDraftFromMapping({
     mapping,
     planName: ctx.str("plan-name").trim() || mapping.default_event_name || "local-data",
     eventNames: ctx.list("event-name"),
@@ -2055,6 +2479,17 @@ function buildPlanDraft(ctx) {
     lang,
     projectId: ctx.optionalNum("project-id")
   });
+  validateAndFix(draft);
+  try {
+    validateDraft(draft);
+  } catch (error) {
+    throw new CliValidationError("The tracking-plan draft is invalid.", {
+      code: "LOCAL_DATA_PLAN_INVALID_DRAFT",
+      hint: error instanceof Error ? error.message : String(error),
+      location: { field: "mapping" }
+    });
+  }
+  return draft;
 }
 var dataIntegrationPlan = {
   service: "data-integration",
@@ -2062,7 +2497,7 @@ var dataIntegrationPlan = {
   usesAeHost: false,
   description: "Convert a confirmed local-data mapping into a tracking-plan draft.json (source_type=data, sdk_integration_mode=none).",
   flags: [
-    { name: "mapping", type: "string", required: true, sensitive: true, desc: "Confirmed ae-local-data-mapping/v1 JSON, file path, or @file." },
+    { name: "mapping", type: "string", required: true, sensitive: true, desc: `Confirmed ${MAPPING_VERSION} JSON, file path, or @file.` },
     { name: "event-name", type: "string", variadic: true, desc: "Concrete event name (track/mixed without default_event_name). Repeat for multiple events." },
     { name: "plan-name", type: "string", desc: 'Plan name. Default: the mapping default_event_name, else "local-data".' },
     { name: "app-type", type: "string", default: "unknown", desc: "Application type recorded in draft meta (informational)." },
@@ -2093,7 +2528,7 @@ var dataIntegrationPlan = {
   }
 };
 
-// src/commands/data-integration/local-data/conversion.ts
+// src/commands/data-integration/conversion.ts
 import { randomInt as randomInt2, randomUUID } from "crypto";
 import {
   chmodSync,
@@ -2114,14 +2549,6 @@ import { createInterface as createInterface2 } from "readline";
 var SORT_CHUNK_SIZE = 1e4;
 var THREE_YEARS_MS = 3 * 365 * 24 * 60 * 60 * 1e3;
 var THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1e3;
-function stripQuotes(value) {
-  if (typeof value !== "string") return value;
-  const text = value;
-  if (text.length >= 2 && (text.startsWith('"') && text.endsWith('"') || text.startsWith("'") && text.endsWith("'"))) {
-    return text.slice(1, -1).trim();
-  }
-  return text.trim();
-}
 async function convertLocalData(options) {
   const input = await inspectLocalDataInput(options.inputFile);
   if (input.format !== options.mapping.source.format) {
@@ -2141,12 +2568,15 @@ async function convertLocalData(options) {
   const streamOptions = {
     headerNames: options.mapping.headers,
     flattenRules: options.mapping.flatten_rules,
-    mergeSheets: options.mergeSheets
+    mergeSheets: options.mergeSheets,
+    // The profile pass inside convert is internal (it writes profile.json); the ragged-row
+    // warning is surfaced by the conversion pass below instead, so suppress it here.
+    warnRagged: false
   };
   const salvageSet = options.salvageFrom ? readSalvageRowNumbers(options.salvageFrom) : void 0;
   let salvageMatched = 0;
   const runId = `${formatRunTimestamp(/* @__PURE__ */ new Date())}-${randomUUID().slice(0, 8)}`;
-  const outputDir = resolve(options.outputDir || join(".ae-cli", "data-integration", runId));
+  const outputDir = resolve(options.outputDir || join(".ae-cli", "data-integration", "runs", runId));
   prepareOutputDirectory(outputDir);
   const profile = await profileLocalData(input, dataSet, options.mapping.time.source_timezone, streamOptions);
   const profilePath = join(outputDir, "profile.json");
@@ -2163,7 +2593,10 @@ async function convertLocalData(options) {
   let validRecords = 0;
   let invalidRecords = 0;
   const recordTypes = {};
-  await streamLocalDataRows(
+  const skippedFields = {};
+  let lanIpRecords = 0;
+  const flattenMisses = {};
+  const rowCount = await streamLocalDataRows(
     input,
     dataSet,
     async (row, rowNumber) => {
@@ -2177,6 +2610,10 @@ async function convertLocalData(options) {
       }
       validRecords += 1;
       recordTypes[result.recordType] = (recordTypes[result.recordType] ?? 0) + 1;
+      if (result.lanIp) lanIpRecords += 1;
+      for (const skip of result.skips) {
+        skippedFields[skip.code] = (skippedFields[skip.code] ?? 0) + 1;
+      }
       const line = JSON.stringify(result.record);
       if (isUserProfileType(result.recordType)) {
         userSetBuffer.push({ key: result.sortKey, line });
@@ -2188,13 +2625,16 @@ async function convertLocalData(options) {
     {
       headerNames: streamOptions.headerNames,
       flattenRules: streamOptions.flattenRules,
+      flattenMisses,
       mergeSheets: streamOptions.mergeSheets
     }
   );
   if (userSetBuffer.length > 0) flushSortChunk(outputDir, userSetBuffer, sortChunks);
-  trackStream.end();
-  invalidStream.end();
-  await Promise.all([once(trackStream, "finish"), once(invalidStream, "finish")]);
+  await Promise.all([finishStream(trackStream), finishStream(invalidStream)]);
+  for (const [outColumn, count] of Object.entries(flattenMisses)) {
+    process.stderr.write(`Warning: flatten rule "${outColumn}" did not materialize for ${count} row(s).
+`);
+  }
   if (salvageSet && salvageMatched === 0) {
     throw new CliValidationError("The salvage file lists no rows from this source.", {
       code: "LOCAL_DATA_SALVAGE_NO_MATCH",
@@ -2205,12 +2645,11 @@ async function convertLocalData(options) {
   const validStream = secureWriteStream(validPath);
   if (existsSync(trackTempPath)) {
     for await (const chunk of createReadStream3(trackTempPath)) {
-      if (!validStream.write(chunk)) await once(validStream, "drain");
+      await writeRaw(validStream, chunk);
     }
   }
   await mergeSortChunks(sortChunks, validStream);
-  validStream.end();
-  await once(validStream, "finish");
+  await finishStream(validStream);
   if (existsSync(trackTempPath)) unlinkSync(trackTempPath);
   for (const path of sortChunks) if (existsSync(path)) unlinkSync(path);
   writeSecureJson(profilePath, profile);
@@ -2218,8 +2657,9 @@ async function convertLocalData(options) {
   writeSecureText(transformPath, createTransformScript(options.inputFile, mappingPath));
   const validBytes = statSize(validPath);
   const blockedReasons = [
-    ...invalidRecords > 0 ? ["Some source rows failed UE validation."] : [],
-    ...validRecords === 0 ? ["No valid UE records were generated."] : []
+    ...rowCount === 0 ? ["The source contained no data rows."] : [],
+    ...rowCount > 0 && validRecords === 0 ? ["No valid UE records were generated."] : [],
+    ...invalidRecords > 0 ? ["Some source rows failed UE validation."] : []
   ];
   const manifest = {
     version: "ae-local-data-manifest/v1",
@@ -2240,7 +2680,10 @@ async function convertLocalData(options) {
       valid_records: validRecords,
       invalid_records: invalidRecords,
       valid_bytes: validBytes,
-      record_types: recordTypes
+      record_types: recordTypes,
+      ...Object.keys(skippedFields).length > 0 ? { skipped_fields: skippedFields } : {},
+      ...lanIpRecords > 0 ? { lan_ip_records: lanIpRecords } : {},
+      ...Object.keys(flattenMisses).length > 0 ? { flatten_misses: flattenMisses } : {}
     },
     blocked_reasons: blockedReasons
   };
@@ -2258,7 +2701,8 @@ async function convertLocalDataMulti(options) {
     const profile = await profileLocalData(input, dataSet, sourceTimezone, {
       collectSamples: true,
       headerNames: options.mapping.headers,
-      flattenRules: options.mapping.flatten_rules
+      flattenRules: options.mapping.flatten_rules,
+      warnRagged: false
     });
     profiled.push({ file: basename4(inputFile), profile });
   }
@@ -2274,7 +2718,7 @@ async function convertLocalDataMulti(options) {
   validateTypeResolutions(resolutions, profiled.map((entry) => entry.file));
   const overrides = applyTypeResolutions(resolutions, profiled);
   const parent = resolve(
-    options.outputDir || join(".ae-cli", "data-integration", `${formatRunTimestamp(/* @__PURE__ */ new Date())}-${randomUUID().slice(0, 8)}`)
+    options.outputDir || join(".ae-cli", "data-integration", "runs", `${formatRunTimestamp(/* @__PURE__ */ new Date())}-${randomUUID().slice(0, 8)}`)
   );
   prepareOutputDirectory(parent);
   const files = [];
@@ -2332,11 +2776,32 @@ function convertRow(row, rowNumber, mapping, now) {
       errors.push({ code: "INVALID_EVENT_NAME", field: mapping.event_name_field });
     }
   }
-  const ip = readOptionalField(row, mapping.ip_field);
-  const uuid = readOptionalField(row, mapping.uuid_field);
+  const skips = [];
+  let lanIp = false;
+  let ip;
+  if (recordType === "track") {
+    const rawIp = readOptionalField(row, mapping.ip_field);
+    if (rawIp) {
+      if (isValidIp(rawIp)) {
+        ip = rawIp;
+        lanIp = isPrivateIp(rawIp);
+      } else {
+        skips.push({ code: "INVALID_IP", field: mapping.ip_field });
+      }
+    }
+  }
+  let uuid;
+  {
+    const rawUuid = readOptionalField(row, mapping.uuid_field);
+    if (rawUuid) {
+      if (isValidUuid(rawUuid)) uuid = rawUuid;
+      else skips.push({ code: "INVALID_UUID", field: mapping.uuid_field });
+    }
+  }
   const excluded = new Set(mapping.exclude_columns ?? []);
   const properties = {};
   for (const property of mapping.properties) {
+    if (property.target.includes(".")) continue;
     if (excluded.has(property.source)) continue;
     let value = stripQuotes(row[property.source]);
     if (isMissing2(value)) continue;
@@ -2350,8 +2815,8 @@ function convertRow(row, rowNumber, mapping, now) {
       properties[property.target] = converted.value;
     }
   }
-  const zoneOffset = mapping.zone_offset_value !== void 0 ? resolveZoneOffsetValue(mapping.zone_offset_value, now) : mapping.zone_offset_field ? readZoneOffset(row, mapping.zone_offset_field) : void 0;
-  if (mapping.zone_offset_field && zoneOffset === void 0) {
+  const zoneOffset = recordType === "track" && mapping.zone_offset_value !== void 0 ? resolveZoneOffsetValue(mapping.zone_offset_value, now) : recordType === "track" && mapping.zone_offset_field ? readZoneOffset(row, mapping.zone_offset_field) : void 0;
+  if (recordType === "track" && mapping.zone_offset_field && zoneOffset === void 0) {
     errors.push({ code: "INVALID_ZONE_OFFSET", field: mapping.zone_offset_field });
   }
   if (errors.length > 0 || !recordType || !normalizedTime) return { ok: false, errors };
@@ -2369,7 +2834,9 @@ function convertRow(row, rowNumber, mapping, now) {
     ok: true,
     recordType,
     record,
-    sortKey: `${accountId ?? distinctId ?? ""}\0${normalizedTime.instant.toISOString()}\0${String(rowNumber).padStart(12, "0")}`
+    sortKey: `${accountId ?? distinctId ?? ""}\0${normalizedTime.instant.toISOString()}\0${String(rowNumber).padStart(12, "0")}`,
+    skips,
+    lanIp
   };
 }
 function readOptionalField(row, field) {
@@ -2477,7 +2944,7 @@ function convertProperty(value, type, transform, timeZone = "UTC", timeFormat) {
       const normalized = normalizeTime(value, timeZone, timeFormat);
       return normalized ? { ok: true, value: normalized.formatted } : { ok: false, code: "PROPERTY_TYPE_CONFLICT" };
     }
-    if (type === "list") {
+    if (type === "list" || type === "array_row") {
       if (!Array.isArray(value)) return { ok: false, code: "PROPERTY_TYPE_CONFLICT" };
       return isPropertyWithinLimits(value, type) ? { ok: true, value } : { ok: false, code: "PROPERTY_LIMIT_EXCEEDED" };
     }
@@ -2679,11 +3146,40 @@ function prepareOutputDirectory(path) {
   chmodSync(path, 448);
 }
 function secureWriteStream(path) {
-  return createWriteStream(path, { encoding: "utf8", mode: 384, flags: "wx" });
+  const stream = createWriteStream(path, { encoding: "utf8", mode: 384, flags: "wx" });
+  let fail;
+  const errorPromise = new Promise((_, reject) => {
+    fail = reject;
+  });
+  errorPromise.catch(() => {
+  });
+  stream.on("error", (error) => fail?.(error));
+  return { stream, path, errorPromise };
 }
-async function writeLine(stream, line) {
-  if (!stream.write(`${line}
-`)) await once(stream, "drain");
+function writeFailure(path, error) {
+  const detail = error instanceof Error ? error.message : String(error);
+  return new Error(`Failed to write "${path}": ${detail}. Check available disk space and directory permissions.`, { cause: error });
+}
+async function writeRaw(target, data) {
+  try {
+    if (!target.stream.write(data)) {
+      await Promise.race([once(target.stream, "drain"), target.errorPromise]);
+    }
+  } catch (error) {
+    throw writeFailure(target.path, error);
+  }
+}
+async function writeLine(target, line) {
+  await writeRaw(target, `${line}
+`);
+}
+async function finishStream(target) {
+  try {
+    target.stream.end();
+    await Promise.race([once(target.stream, "finish"), target.errorPromise]);
+  } catch (error) {
+    throw writeFailure(target.path, error);
+  }
 }
 function writeSecureJson(path, value) {
   writeSecureText(path, `${JSON.stringify(value, null, 2)}
@@ -2710,7 +3206,7 @@ function formatRunTimestamp(value) {
   return value.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
 }
 
-// src/commands/data-integration/local-data/convert.ts
+// src/commands/data-integration/convert.ts
 var dataIntegrationConvert = {
   service: "data-integration",
   command: "convert",
@@ -2718,8 +3214,8 @@ var dataIntegrationConvert = {
   description: "Convert one or more local data sets into validated UE JSONL and quarantine invalid rows.",
   flags: [
     { name: "input-file", type: "string", required: true, sensitive: true, variadic: true, desc: "Source local data file. Repeat for multiple files (requires a wildcard mapping). The source is never modified." },
-    { name: "mapping", type: "string", required: true, sensitive: true, desc: "ae-local-data-mapping/v1 JSON, file path, or @file." },
-    { name: "output-dir", type: "string", sensitive: true, desc: "New or empty output directory. Default: .ae-cli/data-integration/<run-id>." },
+    { name: "mapping", type: "string", required: true, sensitive: true, desc: `${MAPPING_VERSION} JSON, file path, or @file.` },
+    { name: "output-dir", type: "string", sensitive: true, desc: "New or empty output directory. Default: .ae-cli/data-integration/runs/<run-id>." },
     { name: "type-resolutions", type: "json", sensitive: true, desc: "JSON object resolving cross-file column type conflicts (unify, split, or skip)." },
     { name: "merge-sheets", type: "boolean", default: false, desc: "Stream every worksheet in file order instead of a single selected sheet." },
     { name: "salvage-from", type: "string", sensitive: true, desc: "Re-process only the rows listed in a previous run's invalid.rows.jsonl, against the current (fixed) mapping. Single-file only." }
@@ -2771,7 +3267,7 @@ var dataIntegrationConvert = {
   }
 };
 
-// src/commands/data-integration/local-data/upload.ts
+// src/commands/data-integration/upload.ts
 import { createReadStream as createReadStream4, readFileSync as readFileSync3, statSync as statSync3 } from "fs";
 import { basename as basename5, dirname, resolve as resolve2 } from "path";
 import { createInterface as createInterface3 } from "readline";
@@ -3116,22 +3612,900 @@ function isRecord3(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 
-// src/commands/data-integration/local-data/handoff.ts
+// src/commands/data-integration/handoff.ts
 import { createHash as createHash3 } from "crypto";
-import { chmodSync as chmodSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync4, renameSync as renameSync2, writeFileSync as writeFileSync2 } from "fs";
-import { join as join2, resolve as resolve3 } from "path";
+import { chmodSync as chmodSync2, mkdirSync as mkdirSync2, mkdtempSync, readFileSync as readFileSync4, renameSync as renameSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
+import { dirname as dirname2, join as join3, resolve as resolve3 } from "path";
+import { tmpdir } from "os";
+
+// src/commands/data-integration/archive.ts
+import archiver from "archiver";
+import { createWriteStream as createWriteStream2, readdirSync as readdirSync2, statSync as statSync4 } from "fs";
+import { join as join2, relative, sep } from "path";
+async function zipPackage(dir, zipPath) {
+  await new Promise((resolvePromise, rejectPromise) => {
+    const output = createWriteStream2(zipPath, { mode: 384 });
+    const zip = archiver("zip", { zlib: { level: 9 } });
+    output.on("close", resolvePromise);
+    output.on("error", rejectPromise);
+    zip.on("error", rejectPromise);
+    zip.on("warning", (error) => {
+      if (error.code !== "ENOENT") rejectPromise(error);
+    });
+    zip.pipe(output);
+    const walk = (current) => {
+      for (const name of readdirSync2(current)) {
+        if (name === ".DS_Store") continue;
+        const full = join2(current, name);
+        const stats = statSync4(full);
+        if (stats.isDirectory()) {
+          walk(full);
+        } else {
+          const rel = relative(dir, full).split(sep).join("/");
+          zip.file(full, { name: rel, mode: stats.mode & 511 });
+        }
+      }
+    };
+    walk(dir);
+    void zip.finalize();
+  });
+}
+
+// src/commands/data-integration/relay.ts
+var PIPELINE_VERSION = "ae-data-integration-pipeline/v1";
+var SHAPE_VERSION = "ae-data-integration-shape/v1";
+var DEFAULT_BATCH_SIZE2 = 500;
+var ENV_FILE = ".local/target.env";
+function buildPipelineDescriptor(entries, target = {}) {
+  const first = entries[0];
+  return {
+    version: PIPELINE_VERSION,
+    created_at: first?.created_at ?? (/* @__PURE__ */ new Date()).toISOString(),
+    source: { type: "local_file", params: { format: first?.format ?? "csv" } },
+    transform: { type: MAPPING_VERSION, refs: entries.map((entry) => entry.mapping_file) },
+    sink: {
+      type: "restful_sync_json",
+      params: {
+        batch_size: DEFAULT_BATCH_SIZE2,
+        env_file: ENV_FILE,
+        ...target.pushurl ? { pushurl: target.pushurl } : {},
+        ...target.project_id ? { project_id: target.project_id } : {}
+      }
+    }
+  };
+}
+function buildShapeBaseline(items) {
+  return {
+    version: SHAPE_VERSION,
+    entries: items.map(({ mapping, fingerprint }) => ({
+      fingerprint,
+      mode: mapping.mode,
+      data_set: mapping.source.data_set,
+      format: mapping.source.format,
+      columns: sourceColumns(mapping)
+    }))
+  };
+}
+function sh(...lines) {
+  return `${lines.join("\n")}
+`;
+}
+function generateBinScripts() {
+  return [
+    { relPath: "bin/run.sh", content: runSh(), mode: 448 },
+    { relPath: "bin/upload.sh", content: uploadSh(), mode: 448 },
+    { relPath: "bin/bind_mapping.py", content: bindMappingPy(), mode: 448 },
+    { relPath: "bin/summarize.py", content: summarizePy(), mode: 448 },
+    { relPath: "bin/plan_check.py", content: planCheckPy(), mode: 448 },
+    { relPath: "bin/verify.py", content: verifyPy(), mode: 448 },
+    { relPath: "bin/resolve_appid.py", content: resolveAppidPy(), mode: 448 }
+  ];
+}
+function runSh() {
+  return sh(
+    "#!/usr/bin/env bash",
+    "# Generic pipeline executor: source -> transform -> plan. Never uploads (see upload.sh).",
+    "# Reads pipeline.json and dispatches each stage by its `type` to ae-cli subcommands.",
+    "set -euo pipefail",
+    'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"',
+    'PKG_ROOT="$(dirname "$SCRIPT_DIR")"',
+    'cd "$PKG_ROOT"',
+    "",
+    `SRC_TYPE="$(python3 -c 'import json; print(json.load(open("pipeline.json"))["source"]["type"])')"`,
+    'case "$SRC_TYPE" in',
+    "  local_file) ;;",
+    "  *)",
+    '    echo "unsupported source type: $SRC_TYPE" >&2',
+    '    echo "re-run the full ae-data-integration pipeline (or upgrade ae-cli) for this source." >&2',
+    "    exit 64",
+    "    ;;",
+    "esac",
+    "",
+    'INPUT="${1:-}"',
+    'if [ -z "$INPUT" ]; then',
+    "  shopt -s nullglob; FILES=(inbox/*); shopt -u nullglob",
+    '  if [ "${#FILES[@]}" -ne 1 ]; then',
+    '    echo "usage: bin/run.sh <input-file>" >&2',
+    '    echo "       (or put exactly one file in inbox/)" >&2',
+    "    exit 2",
+    "  fi",
+    '  INPUT="${FILES[0]}"',
+    "fi",
+    "",
+    'RUN_DIR="runs/$(date +%Y%m%d-%H%M%S)"',
+    'mkdir -p "$RUN_DIR"',
+    'echo "run: $RUN_DIR"',
+    "",
+    'python3 bin/bind_mapping.py "$INPUT" "$RUN_DIR"',
+    "",
+    "while IFS= read -r ref; do",
+    '  ref_dir="$(dirname "$ref")"',
+    '  echo "convert: $ref_dir"',
+    "  ae-cli data-integration convert \\",
+    '    --input-file "$INPUT" \\',
+    '    --mapping "$RUN_DIR/bound/$ref_dir/mapping.json" \\',
+    '    --output-dir "$RUN_DIR/$ref_dir" || exit $?',
+    `done < <(python3 -c 'import json,sys; print("\\n".join(json.load(open("pipeline.json"))["transform"]["refs"]))')`,
+    "",
+    'python3 bin/summarize.py "$RUN_DIR"',
+    'python3 bin/plan_check.py "$RUN_DIR"',
+    "",
+    "# Salvage hint: quarantined rows are never silently dropped. Re-process only",
+    "# them against the fixed mapping instead of re-uploading the whole file.",
+    "while IFS= read -r ref; do",
+    '  ref_dir="$(dirname "$ref")"',
+    '  inv="$RUN_DIR/$ref_dir/invalid.rows.jsonl"',
+    '  if [ -s "$inv" ]; then',
+    '    echo "note: $inv has quarantined rows \u2014 salvage them with:"',
+    '    echo "  ae-cli data-integration convert --input-file "$INPUT" --mapping "$RUN_DIR/bound/$ref_dir/mapping.json" --salvage-from "$inv" --output-dir "$RUN_DIR-salvage""',
+    "  fi",
+    `done < <(python3 -c 'import json,sys; print("\\n".join(json.load(open("pipeline.json"))["transform"]["refs"]))')`,
+    "",
+    'echo "done. review the summary and plan check, then run: bin/upload.sh $RUN_DIR"'
+  );
+}
+function uploadSh() {
+  return sh(
+    "#!/usr/bin/env bash",
+    "# Sink executor. Dry-run by default; --confirm actually uploads.",
+    "# Resolves the recorded target from pipeline.json (pushurl + project_id), derives",
+    "# the APPID via `ae-cli project info get` (bin/resolve_appid.py), and falls back to",
+    "# .local/target.env for explicit APPID / endpoint / project-id overrides.",
+    "set -euo pipefail",
+    'SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"',
+    'PKG_ROOT="$(dirname "$SCRIPT_DIR")"',
+    'cd "$PKG_ROOT"',
+    "",
+    `SINK_TYPE="$(python3 -c 'import json; print(json.load(open("pipeline.json"))["sink"]["type"])')"`,
+    'case "$SINK_TYPE" in',
+    "  restful_sync_json) ;;",
+    "  *)",
+    '    echo "unsupported sink type: $SINK_TYPE" >&2',
+    '    echo "re-run the full ae-data-integration pipeline (or upgrade ae-cli) for this sink." >&2',
+    "    exit 64",
+    "    ;;",
+    "esac",
+    "",
+    `SINK_PARAMS="$(python3 -c 'import json; print(json.dumps(json.load(open("pipeline.json"))["sink"]["params"]))')"`,
+    `BATCH_SIZE="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1]).get("batch_size", 500))' "$SINK_PARAMS")"`,
+    `ENV_FILE="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1]).get("env_file", ".local/target.env"))' "$SINK_PARAMS")"`,
+    `PUSHURL="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1]).get("pushurl") or "")' "$SINK_PARAMS")"`,
+    `PROJECT_ID="$(python3 -c 'import json,sys; print(json.loads(sys.argv[1]).get("project_id") or "")' "$SINK_PARAMS")"`,
+    "",
+    "CONFIRM=0",
+    'RUN_DIR=""',
+    'for arg in "$@"; do',
+    '  case "$arg" in',
+    "    --confirm) CONFIRM=1 ;;",
+    '    -*) echo "unknown flag: $arg" >&2; exit 2 ;;',
+    '    *) RUN_DIR="$arg" ;;',
+    "  esac",
+    "done",
+    "",
+    'if [ -z "$RUN_DIR" ]; then',
+    '  echo "usage: bin/upload.sh [--confirm] <runs/<run-id>>" >&2',
+    "  exit 2",
+    "fi",
+    "",
+    "# .local/target.env is optional: the package may record the target itself.",
+    "# Env values still win as explicit overrides (the documented fallback).",
+    'if [ -f "$ENV_FILE" ]; then set -a; . "$ENV_FILE"; set +a; fi',
+    '[ -z "$PROJECT_ID" ] && PROJECT_ID="${AE_PROJECT_ID:-}"',
+    "",
+    "# Endpoint: the recorded pushurl (a receiver base URL; append /sync_json), else AE_ENDPOINT.",
+    'if [ -n "$PUSHURL" ]; then',
+    '  case "$PUSHURL" in',
+    '    */sync_json) ENDPOINT="$PUSHURL" ;;',
+    '    *) ENDPOINT="${PUSHURL%/}/sync_json" ;;',
+    "  esac",
+    "else",
+    '  : "${AE_ENDPOINT:?set AE_ENDPOINT in .local/target.env (or record pushurl at handoff)}"',
+    '  ENDPOINT="$AE_ENDPOINT"',
+    "fi",
+    "",
+    "# APPID: explicit env wins, else derive from the recorded project via project info get.",
+    'APPID="${AE_APPID:-}"',
+    'if [ -z "$APPID" ] && [ -n "$PROJECT_ID" ]; then',
+    '  APPID="$(python3 bin/resolve_appid.py "$PROJECT_ID")"',
+    "fi",
+    ': "${APPID:?set AE_APPID in .local/target.env (or record project_id at handoff)}"',
+    "",
+    "# Mask the APPID in anything this script prints \u2014 it must never land in logs.",
+    "mask() {",
+    '  local a="$1"',
+    '  if [ "${#a}" -le 4 ]; then printf "%s" "****"; return; fi',
+    '  printf "%s%s" "$(printf "%*s" "$(( ${#a} - 4 ))" "" | tr " " "*")" "${a: -4}"',
+    "}",
+    "display_args() {",
+    '  local args=("$@") out=() i',
+    "  for ((i=0; i<${#args[@]}; i++)); do",
+    '    if [ "${args[$i]}" = "--appid" ] && [ -n "${args[$((i+1))]:-}" ]; then',
+    '      out+=("--appid" "$(mask "${args[$((i+1))]}")")',
+    "      i=$((i+1))",
+    "    else",
+    '      out+=("${args[$i]}")',
+    "    fi",
+    "  done",
+    '  printf "%s\\n" "${out[*]}"',
+    "}",
+    "",
+    'echo "target: project_id=${PROJECT_ID:-<unset>}"',
+    'echo "        endpoint=$ENDPOINT"',
+    'echo "        appid=$(mask "$APPID")"',
+    'if [ "$CONFIRM" -eq 0 ]; then',
+    '  echo "dry-run \u2014 re-run with --confirm to upload to this address and project."',
+    "else",
+    '  echo "confirmed: uploading to the address and project shown above."',
+    "fi",
+    "",
+    'FLAGS=(--endpoint "$ENDPOINT" --appid "$APPID" --batch-size "$BATCH_SIZE")',
+    'if [ "$CONFIRM" -eq 0 ]; then FLAGS+=(--dry-run); fi',
+    "",
+    "while IFS= read -r ref; do",
+    '  ref_dir="$(dirname "$ref")"',
+    '  ue="$RUN_DIR/$ref_dir/valid.ue.jsonl"',
+    '  manifest="$RUN_DIR/$ref_dir/manifest.json"',
+    '  if [ ! -f "$ue" ]; then',
+    '    echo "missing $ue (run bin/run.sh first)" >&2',
+    "    exit 2",
+    "  fi",
+    `  status="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["status"])' "$manifest")"`,
+    '  args=("${FLAGS[@]}" --ue-file "$ue" --manifest "$manifest")',
+    '  if [ "$status" = "blocked" ]; then',
+    '    echo "manifest $manifest is blocked (rows were quarantined)."',
+    '    echo "Uploading passes only the valid subset (--allow-clean-subset); quarantined rows stay in invalid.rows.jsonl."',
+    '    if [ "$CONFIRM" -eq 1 ]; then args+=(--allow-clean-subset); else echo "  (dry-run) re-run with --confirm to accept the clean subset."; fi',
+    "  fi",
+    '  echo "> ae-cli data-integration upload $(display_args "${args[@]}")"',
+    '  ae-cli data-integration upload "${args[@]}" || exit $?',
+    `done < <(python3 -c 'import json,sys; print("\\n".join(json.load(open("pipeline.json"))["transform"]["refs"]))')`
+  );
+}
+function bindMappingPy() {
+  return `#!/usr/bin/env python3
+"""Source stage: rebind the frozen mappings to a new same-shape file.
+
+Runs \`ae-cli data-integration inspect\` once, then for every mapping the
+pipeline references (pipeline.json's transform.refs) re-binds the frozen
+mapping's \`source.sha256\` and \`source.data_set\` to the new file (identity
+fields only \u2014 business logic is untouched), after checking the column set
+against shape.json. Historical index entries the pipeline does not run are
+left alone \u2014 the index accumulates across handoffs in the same directory.
+
+Usage: bin/bind_mapping.py <input-file> <run-dir>
+"""
+import json
+import os
+import subprocess
+import sys
+
+
+def fail(message):
+    print(f"bind_mapping: {message}", file=sys.stderr)
+    sys.exit(1)
+
+
+def pkg_root():
+    return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def load_json(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def write_json(path, value):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(value, f, ensure_ascii=False, indent=2)
+        f.write("\\n")
+
+
+def dataset_key(dataset):
+    return dataset.get("id") or dataset.get("label") or ""
+
+
+def main():
+    if len(sys.argv) != 3:
+        fail("usage: bind_mapping.py <input-file> <run-dir>")
+    input_file, run_dir = sys.argv[1], sys.argv[2]
+    root = pkg_root()
+
+    pipeline = load_json(os.path.join(root, "pipeline.json"))
+    if pipeline["source"]["type"] != "local_file":
+        fail(f"unsupported source type: {pipeline['source']['type']}")
+
+    index = load_json(os.path.join(root, "index.json"))
+    shape = load_json(os.path.join(root, "shape.json"))
+    shape_by_fp = {entry["fingerprint"]: entry for entry in shape["entries"]}
+    index_by_ref = {entry["mapping_file"]: entry for entry in index["entries"]}
+
+    inspect = run_inspect(input_file)
+    datasets = extract_datasets(inspect)
+    headers_by_dataset = extract_headers(inspect, datasets)
+    sha = (inspect.get("source") or {}).get("sha256")
+
+    # Rebind only the mappings this pipeline runs (transform.refs). The index
+    # accumulates entries across handoffs in the same directory; earlier entries
+    # may have no shape baseline here and are never converted by run.sh, so
+    # walking the whole index would fail on them.
+    for ref in pipeline["transform"]["refs"]:
+        entry = index_by_ref.get(ref)
+        if entry is None:
+            fail(f"index entry missing for {ref}; re-run the full pipeline")
+        fingerprint = entry["fingerprint"]
+        baseline = shape_by_fp.get(fingerprint)
+        if baseline is None:
+            fail(f"shape baseline missing for {fingerprint}; re-run the full pipeline")
+        frozen = load_json(os.path.join(root, ref))
+        data_set_id = match_dataset(frozen, baseline, datasets, headers_by_dataset)
+        validate_headers(data_set_id, baseline, headers_by_dataset)
+        frozen["source"]["sha256"] = sha
+        frozen["source"]["data_set"] = data_set_id
+        out = os.path.join(root, run_dir, "bound", os.path.dirname(ref), "mapping.json")
+        write_json(out, frozen)
+        print(f"rebound {os.path.dirname(ref)} -> data_set {data_set_id!r}")
+    print("shape check passed")
+
+
+def run_inspect(input_file):
+    proc = subprocess.run(
+        ["ae-cli", "data-integration", "inspect", "--input-file", input_file],
+        capture_output=True, text=True,
+    )
+    if proc.returncode != 0:
+        fail(f"inspect failed: {proc.stderr.strip()}")
+    try:
+        parsed = json.loads(proc.stdout)
+    except json.JSONDecodeError:
+        fail("inspect returned non-JSON output")
+    # ae-cli wraps every command result in { ok, data, error }; unwrap it.
+    data = parsed.get("data") if isinstance(parsed, dict) else None
+    if not isinstance(data, dict):
+        fail("inspect returned no data payload")
+    return data
+
+
+def extract_datasets(inspect):
+    if inspect.get("selection_required"):
+        return inspect.get("data_sets") or []
+    data_set = inspect.get("data_set")
+    return [data_set] if data_set else []
+
+
+def extract_headers(inspect, datasets):
+    result = {}
+    details = inspect.get("header_details")
+    if details:
+        for dataset in datasets:
+            names = (dataset.get("label"), dataset.get("id"), dataset.get("selector"))
+            for sheet in details:
+                if sheet.get("name") in names:
+                    result[dataset_key(dataset)] = sheet.get("headers") or []
+                    break
+        return result
+    columns = inspect.get("columns")
+    if columns and datasets:
+        result[dataset_key(datasets[0])] = [column["name"] for column in columns]
+    return result
+
+
+def match_dataset(frozen, baseline, datasets, headers_by_dataset):
+    if not datasets:
+        fail("inspect reported no data sets")
+    wanted = frozen["source"]["data_set"]
+    for dataset in datasets:
+        if dataset.get("id") == wanted:
+            return dataset.get("id")
+    for dataset in datasets:
+        if dataset.get("label") == wanted:
+            return dataset.get("id")
+    baseline_cols = set(baseline.get("columns") or [])
+    if baseline_cols:
+        for dataset in datasets:
+            headers = headers_by_dataset.get(dataset_key(dataset))
+            if headers and set(headers) == baseline_cols:
+                return dataset.get("id")
+    if len(datasets) == 1:
+        return datasets[0].get("id")
+    fail(f"cannot rebind data_set {wanted!r}: no exact or header match; re-run the full pipeline")
+
+
+def validate_headers(data_set_id, baseline, headers_by_dataset):
+    baseline_cols = set(baseline.get("columns") or [])
+    if not baseline_cols:
+        return
+    headers = headers_by_dataset.get(data_set_id)
+    if headers is None:
+        return
+    if set(headers) != baseline_cols:
+        missing = sorted(baseline_cols - set(headers))
+        extra = sorted(set(headers) - baseline_cols)
+        fail(
+            f"shape mismatch for {data_set_id!r}: missing={missing} extra={extra} \u2014 "
+            "re-run the full pipeline; do not edit the frozen mapping"
+        )
+
+
+if __name__ == "__main__":
+    main()
+`;
+}
+function summarizePy() {
+  return `#!/usr/bin/env python3
+"""Transform stage summary: print valid/quarantined counts per data set."""
+import json
+import os
+import sys
+
+
+def pkg_root():
+    return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: summarize.py <run-dir>", file=sys.stderr)
+        sys.exit(2)
+    run_dir = sys.argv[1]
+    root = pkg_root()
+    with open(os.path.join(root, "pipeline.json"), "r", encoding="utf-8") as f:
+        pipeline = json.load(f)
+    total_valid = 0
+    total_invalid = 0
+    for ref in pipeline["transform"]["refs"]:
+        ref_dir = os.path.dirname(ref)
+        manifest_path = os.path.join(root, run_dir, ref_dir, "manifest.json")
+        if not os.path.exists(manifest_path):
+            continue
+        with open(manifest_path, "r", encoding="utf-8") as f:
+            manifest = json.load(f)
+        output = manifest["output"]
+        total_valid += output["valid_records"]
+        total_invalid += output["invalid_records"]
+        print(f"{ref_dir}: {output['valid_records']} valid / {output['invalid_records']} quarantined")
+        for reason in manifest.get("blocked_reasons") or []:
+            print(f"  - {reason}")
+    print(f"total: {total_valid} valid / {total_invalid} quarantined")
+
+
+if __name__ == "__main__":
+    main()
+`;
+}
+function planCheckPy() {
+  return `#!/usr/bin/env python3
+"""Plan gate: every event and property produced must already exist in the frozen tracking plan."""
+import json
+import os
+import sys
+
+
+def pkg_root():
+    return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def load_json(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: plan_check.py <run-dir>", file=sys.stderr)
+        sys.exit(2)
+    run_dir = sys.argv[1]
+    root = pkg_root()
+    pipeline = load_json(os.path.join(root, "pipeline.json"))
+    new_events = []
+    new_properties = []
+    missing_plans = []
+    for ref in pipeline["transform"]["refs"]:
+        ref_dir = os.path.dirname(ref)
+        plan_path = os.path.join(root, ref_dir, "plan.json")
+        ue_path = os.path.join(root, run_dir, ref_dir, "valid.ue.jsonl")
+        if not os.path.exists(plan_path):
+            missing_plans.append(ref_dir)
+            continue
+        plan = load_json(plan_path)
+        plan_events = {event["event_name"] for event in plan.get("events", [])}
+        plan_properties = {prop["name"] for prop in plan.get("event_properties", [])}
+        plan_properties |= {prop["name"] for prop in plan.get("common_event_properties", [])}
+        plan_properties |= {prop["name"] for prop in plan.get("user_properties", [])}
+        produced_events = set()
+        produced_properties = set()
+        if os.path.exists(ue_path):
+            with open(ue_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    record = json.loads(line)
+                    if record.get("#event_name"):
+                        produced_events.add(record["#event_name"])
+                    props = record.get("properties")
+                    if isinstance(props, dict):
+                        for key in props:
+                            if key.startswith("#"):
+                                continue
+                            produced_properties.add(key)
+        new_events.extend(sorted(produced_events - plan_events))
+        new_properties.extend(sorted(produced_properties - plan_properties))
+    if missing_plans:
+        print("no plan.json in package for: " + ", ".join(missing_plans), file=sys.stderr)
+        print("run the Tracking plan step first \u2014 the plan gate cannot be skipped", file=sys.stderr)
+        sys.exit(3)
+    if new_events:
+        print("new events not in the plan: " + ", ".join(new_events), file=sys.stderr)
+        sys.exit(3)
+    if new_properties:
+        print("new properties not in the plan: " + ", ".join(new_properties), file=sys.stderr)
+        sys.exit(3)
+    print("plan coverage ok")
+
+
+if __name__ == "__main__":
+    main()
+`;
+}
+function verifyPy() {
+  return `#!/usr/bin/env python3
+"""Persistence consistency check: submit-window counts vs the platform summary.
+
+A soft check, not a hard gate. It computes what this run submitted from the local
+UE output (knowable), snapshots \`ae-cli tracking ingest summary\` over the submit
+window before and after upload, and prints both next to the expected counts. It
+does NOT parse the summary payload into per-event numbers: the capability's data
+shape is server-defined and not a stable CLI contract, and a shared project cannot
+attribute the window delta to this import alone. For a hard per-event SQL judge,
+overlay a project custom layer (see custom-layer.md in the ae-data-integration skill).
+
+  verify.py <run-dir> --baseline    snapshot the summary before upload
+  verify.py <run-dir> --check       snapshot again and diff against the baseline
+"""
+import json
+import os
+import subprocess
+import sys
+
+
+def pkg_root():
+    return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def load_json(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def visit_window(record, lo, hi):
+    raw = record.get("#time")
+    if not isinstance(raw, str) or len(raw) < 19:
+        return lo, hi
+    stamp = raw[:19]  # YYYY-MM-DD HH:mm:ss
+    if lo is None or stamp < lo:
+        lo = stamp
+    if hi is None or stamp > hi:
+        hi = stamp
+    return lo, hi
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("usage: verify.py <run-dir> [--baseline|--check]", file=sys.stderr)
+        sys.exit(2)
+    run_dir = sys.argv[1]
+    mode = sys.argv[2] if len(sys.argv) > 2 else "--check"
+    if mode not in ("--baseline", "--check"):
+        print("usage: verify.py <run-dir> [--baseline|--check]", file=sys.stderr)
+        sys.exit(2)
+    root = pkg_root()
+    pipeline = load_json(os.path.join(root, "pipeline.json"))
+    project_id = pipeline["sink"]["params"].get("project_id") or os.environ.get("AE_PROJECT_ID") or "1"
+
+    expected_events = {}
+    expected_users = 0
+    lo = hi = None
+    for ref in pipeline["transform"]["refs"]:
+        ue = os.path.join(root, run_dir, os.path.dirname(ref), "valid.ue.jsonl")
+        if not os.path.exists(ue):
+            continue
+        with open(ue, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                record = json.loads(line)
+                if record.get("#type") == "track" and record.get("#event_name"):
+                    expected_events[record["#event_name"]] = expected_events.get(record["#event_name"], 0) + 1
+                else:
+                    expected_users += 1
+                lo, hi = visit_window(record, lo, hi)
+
+    if lo is None:
+        print("verify: no UE records found in " + run_dir, file=sys.stderr)
+        sys.exit(2)
+
+    def run_summary():
+        proc = subprocess.run(
+            ["ae-cli", "tracking", "ingest", "summary",
+             "-p", str(project_id), "--start-time", lo, "--end-time", hi],
+            capture_output=True, text=True,
+        )
+        if proc.returncode != 0:
+            return {"error": (proc.stderr or proc.stdout).strip()[:500]}
+        try:
+            return json.loads(proc.stdout)
+        except json.JSONDecodeError:
+            return {"raw": proc.stdout.strip()[:500]}
+
+    print("window " + lo + " .. " + hi + "  project_id=" + str(project_id))
+    print("expected (this run):")
+    for event, count in sorted(expected_events.items()):
+        print("  {:<20}{:>10,}".format(event, count))
+    print("  {:<20}{:>10,}".format("<user rows>", expected_users))
+    print()
+
+    baseline_path = os.path.join(root, run_dir, "baseline.json")
+    if mode == "--baseline":
+        payload = run_summary()
+        with open(baseline_path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, ensure_ascii=False, indent=2)
+        print("baseline recorded: " + baseline_path)
+        print("platform summary (before):")
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+        sys.exit(0)
+
+    if not os.path.exists(baseline_path):
+        print("no baseline.json \u2014 run \`verify.py <run-dir> --baseline\` before upload.", file=sys.stderr)
+        print("falling back to a single after-upload summary:", file=sys.stderr)
+        print(json.dumps(run_summary(), ensure_ascii=False, indent=2))
+        sys.exit(1)
+
+    with open(baseline_path, "r", encoding="utf-8") as f:
+        baseline = json.load(f)
+    after = run_summary()
+    print("platform summary before vs after:")
+    print(json.dumps(baseline, ensure_ascii=False, indent=2))
+    print("---")
+    print(json.dumps(after, ensure_ascii=False, indent=2))
+    print()
+    print("summary changed: " + ("yes" if baseline != after else "no"))
+    print()
+    print("Boundary: the summary payload is server-defined and may under-report even")
+    print("landed data; this check surfaces it for comparison, it does not auto-verify")
+    print("per-event counts, and a shared project's window delta is not attributed to")
+    print("this import. Cross-check with:")
+    print("  ae-cli tracking live-data list -p " + str(project_id))
+    print("For a hard SQL judge, add a project custom layer (custom-layer.md).")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()
+`;
+}
+function resolveAppidPy() {
+  return `#!/usr/bin/env python3
+"""Derive the destination APPID from \`ae-cli project info get --project-id <id>\`.
+
+\`project info get\` returns \`data.appid\` at the top level (verified against the
+AE demo host), so this helper reads that exact field and prints it to stdout. It
+prints nothing to stdout and reports the payload when the field is absent or not
+a non-empty string \u2014 the caller then falls back to AE_APPID.
+
+Usage: bin/resolve_appid.py <project-id>
+"""
+import json
+import subprocess
+import sys
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: resolve_appid.py <project-id>", file=sys.stderr)
+        sys.exit(2)
+    project_id = sys.argv[1]
+    proc = subprocess.run(
+        ["ae-cli", "project", "info", "get", "--project-id", project_id],
+        capture_output=True, text=True,
+    )
+    if proc.returncode != 0:
+        print("resolve_appid: project info get failed: " + (proc.stderr or proc.stdout).strip()[:300], file=sys.stderr)
+        sys.exit(0)
+    try:
+        parsed = json.loads(proc.stdout)
+    except json.JSONDecodeError:
+        print("resolve_appid: project info get returned non-JSON output", file=sys.stderr)
+        sys.exit(0)
+    data = parsed.get("data") if isinstance(parsed, dict) else None
+    appid = data.get("appid") if isinstance(data, dict) else None
+    if not isinstance(appid, str) or not appid:
+        print("resolve_appid: project info get returned no appid; set AE_APPID", file=sys.stderr)
+        print(json.dumps(data, ensure_ascii=False, indent=2) if data is not None else "{}", file=sys.stderr)
+        sys.exit(0)
+    masked = appid if len(appid) <= 4 else ("*" * (len(appid) - 4)) + appid[-4:]
+    print("resolve_appid: resolved APPID data.appid = " + masked, file=sys.stderr)
+    print(appid)
+
+
+if __name__ == "__main__":
+    main()
+`;
+}
+function generateReadme() {
+  return `# AE Data Integration \u2014 handoff package
+
+A frozen, reusable pipeline for importing a **same-shape** local file into AE,
+generated by \`ae-cli data-integration handoff\`. Source and Transform are frozen
+(the confirmed business logic); only the Tracking-plan and Sink gates still
+require human confirmation.
+
+## Quick start
+
+\`\`\`bash
+cp <today's file> inbox/
+bin/run.sh                    # shape check -> convert -> plan check (no upload)
+bin/upload.sh runs/<latest>   # dry-run
+bin/upload.sh runs/<latest> --confirm
+\`\`\`
+
+Read [RUNBOOK.md](RUNBOOK.md) for the full flow, the four confirmation gates,
+and how to verify persistence.
+
+## Handing off to an agent
+
+Import \`inbox/<today's file>\` through the ae-data-integration skill using this
+package. Tell it to follow RUNBOOK.md and stop for confirmation before uploading.
+
+## Package layout
+
+| Path | Purpose |
+| --- | --- |
+| \`pipeline.json\` | Declarative source -> transform -> sink descriptor |
+| \`index.json\` | Structure-fingerprint index (reuse matching) |
+| \`shape.json\` | Column baseline used by the shape gate |
+| \`<fingerprint16>/\` | Frozen mapping + tracking plan + transform wrapper |
+| \`bin/run.sh\` | Source + Transform + Plan executor (never uploads) |
+| \`bin/upload.sh\` | Sink executor (dry-run by default; resolves the recorded target) |
+| \`bin/verify.py\` | Soft persistence check (submit window vs ingest summary) |
+| \`bin/resolve_appid.py\` | APPID derivation helper (project info get) |
+| \`.local/target.env\` | Upload target overrides (APPID / endpoint); only the template ships |
+| \`inbox/\` \`runs/\` | Daily input / per-run outputs |
+
+## Safety
+
+The package records at most a destination \`pushurl\` and \`project_id\` (no APPID,
+tokens, or raw data values). \`bin/upload.sh\` always requires \`--confirm\` before
+sending, so the operator re-confirms the address and project on every reuse. Copy
+\`.local/target.env.example\` to \`.local/target.env\` for explicit overrides and
+never commit it.
+`;
+}
+function generateRunbook() {
+  return `# RUNBOOK \u2014 same-shape file import
+
+Run this when a file of the **same shape** arrives again (same sheets and
+headers as \`shape.json\`). If the headers changed, stop: re-run the full
+ae-data-integration pipeline instead of editing the frozen mapping.
+
+## Gates
+
+1. **Shape gate** \u2014 \`bin/run.sh\` rebinds the frozen mappings to the new file
+   and compares the column set against \`shape.json\`. A mismatch fails fast on
+   purpose: a different shape means the frozen business logic was never reviewed
+   for it.
+2. **Transform** \u2014 each frozen mapping runs through
+   \`ae-cli data-integration convert\`. Quarantined rows land in
+   \`invalid.rows.jsonl\`; they are never silently dropped.
+3. **Tracking-plan gate** \u2014 \`bin/run.sh\` verifies every produced event and
+   property already exists in the frozen \`plan.json\`. New events or properties
+   make it exit with code 3: merge them into the project tracking plan first,
+   then return here to upload.
+4. **Sink gate** \u2014 \`bin/upload.sh\` is dry-run by default. Read
+   \`record_count\`, \`batch_count\`, and \`manifest_status\` before adding
+   \`--confirm\`. A \`blocked\` manifest means rows were quarantined;
+   \`--confirm\` then uploads only the valid subset (\`--allow-clean-subset\`).
+
+## Destination
+
+The package records the destination it was handed off for when \`handoff\` was run
+with \`--pushurl\` / \`--project-id\` (see \`pipeline.json\` \u2192 \`sink.params\`). Reuse
+defaults to that target, but \`bin/upload.sh\` never sends without \`--confirm\`, so
+the operator re-confirms the address and project every time.
+
+Resolution order at upload time:
+
+- endpoint: recorded \`pushurl\` (+ \`/sync_json\`), else \`AE_ENDPOINT\`.
+- APPID: \`AE_APPID\`, else derived via \`ae-cli project info get --project-id <id>\`
+  (see \`bin/resolve_appid.py\`; it reads the \`data.appid\` field \u2014 set \`AE_APPID\`
+  when that field is absent).
+- project id: recorded \`project_id\`, else \`AE_PROJECT_ID\`.
+
+\`.local/target.env\` remains the explicit override for all three:
+\`AE_ENDPOINT\` (full receiver URL ending in \`/sync_json\`), \`AE_APPID\`, \`AE_PROJECT_ID\`.
+
+## Verify persistence
+
+\`receiver_accepted\` is not persistence. About a minute after upload, confirm
+the data landed with ae-cli. The package ships a soft check that automates the
+before/after comparison:
+
+\`\`\`bash
+bin/verify.py runs/<run-id> --baseline   # before upload
+bin/upload.sh runs/<run-id> --confirm
+bin/verify.py runs/<run-id> --check      # after upload
+\`\`\`
+
+\`bin/verify.py\` prints the submit window and expected counts, then shows the
+\`tracking ingest summary\` payload before and after for comparison. It does not
+auto-verify per-event counts \u2014 the summary shape is server-defined, and a shared
+project's window delta is not attributed to this import. Cross-check with:
+
+\`\`\`bash
+ae-cli tracking live-data list -p <AE_PROJECT_ID>
+ae-cli tracking ingest-error list -p <AE_PROJECT_ID> --data-name <name>
+\`\`\`
+
+For a hard per-event SQL judge, overlay a project custom layer instead of editing
+this package (see \`custom-layer.md\` in the ae-data-integration skill).
+
+## Interrupted uploads
+
+If a batch times out or loses the network, that batch's state is unknown. Stop:
+verify what actually landed, then follow the ae-data-integration skill to resume
+from the verified offset. Never re-run the whole upload blindly.
+
+## Files
+
+See \`README.md\` for the package layout.
+`;
+}
+function generateEnvTemplate() {
+  return [
+    "# Upload target overrides. Copy this file to .local/target.env and fill only",
+    "# what the package does not already record (pipeline.json sink.params).",
+    "# AE_ENDPOINT: a full receiver URL ending in /sync_json (used when no pushurl is recorded).",
+    "# AE_APPID: the destination project APPID (overrides the project info get derivation).",
+    "# AE_PROJECT_ID: the destination project ID, used when no project_id is recorded.",
+    "AE_ENDPOINT=",
+    "AE_APPID=",
+    "AE_PROJECT_ID=",
+    ""
+  ].join("\n");
+}
+function generateGitignore() {
+  return ["inbox/", "runs/", ".local/target.env", ""].join("\n");
+}
+
+// src/commands/data-integration/handoff.ts
 var HANDOFF_INDEX_VERSION = "ae-data-integration-index/v1";
 var HANDOFF_DIR_LEN = 16;
 function structureFingerprint(mapping) {
   const canonical = {
     mode: mapping.mode,
-    time_field: mapping.time.field,
-    account_id_field: mapping.account_id_field ?? null,
-    distinct_id_field: mapping.distinct_id_field ?? null,
-    record_type_field: mapping.record_type_field ?? null,
-    event_name_field: mapping.event_name_field ?? null,
-    columns: mapping.properties.map((property) => ({ source: property.source, type: property.type })).sort((left, right) => left.source.localeCompare(right.source)),
-    excluded: [...mapping.exclude_columns ?? []].sort()
+    format: mapping.source.format,
+    columns: sourceColumns(mapping)
   };
   return createHash3("sha256").update(JSON.stringify(canonical)).digest("hex");
 }
@@ -3142,17 +4516,17 @@ function upsertIndexEntry(index, entry) {
 function buildHandoffPackage(outDir, mapping, planFile) {
   const fingerprint = structureFingerprint(mapping);
   const dirName = fingerprint.slice(0, HANDOFF_DIR_LEN);
-  const handoffDir = join2(outDir, dirName);
-  const indexPath = join2(outDir, "index.json");
+  const handoffDir = join3(outDir, dirName);
+  const indexPath = join3(outDir, "index.json");
   const index = readHandoffIndex(indexPath);
   const reusedExisting = index.entries.some((item) => item.fingerprint === fingerprint);
   mkdirSync2(handoffDir, { recursive: true, mode: 448 });
   chmodSync2(handoffDir, 448);
-  writeSecureJson2(join2(handoffDir, "mapping.json"), mapping);
-  writeSecureText2(join2(handoffDir, "transform.mjs"), createHandoffScript());
+  writeSecureJson2(join3(handoffDir, "mapping.json"), mapping);
+  writeSecureText2(join3(handoffDir, "transform.mjs"), createHandoffScript());
   let planFileRel;
   if (planFile) {
-    writeSecureJson2(join2(handoffDir, "plan.json"), readPlanFile(planFile));
+    writeSecureJson2(join3(handoffDir, "plan.json"), readPlanFile(planFile));
     planFileRel = `${dirName}/plan.json`;
   }
   const entry = {
@@ -3167,7 +4541,7 @@ function buildHandoffPackage(outDir, mapping, planFile) {
     ...planFileRel ? { plan_file: planFileRel } : {}
   };
   writeAtomicJson(indexPath, upsertIndexEntry(index, entry));
-  return { outDir, handoffDir, dirName, fingerprint, reusedExisting, planFile: planFileRel };
+  return { outDir, handoffDir, dirName, fingerprint, reusedExisting, planFile: planFileRel, entry };
 }
 function readHandoffIndex(path) {
   let raw;
@@ -3262,56 +4636,199 @@ var dataIntegrationHandoff = {
   service: "data-integration",
   command: "handoff",
   usesAeHost: false,
-  description: "Export a reusable handoff package (frozen mapping + transform script + plan reference) under .ae-data-integration/.",
+  description: "Export a reusable handoff package (pipeline descriptor + frozen mappings + stage executors + docs) and a shareable zip.",
   flags: [
-    { name: "mapping", type: "string", required: true, sensitive: true, desc: "Confirmed ae-local-data-mapping/v1 JSON, file path, or @file." },
+    { name: "mapping", type: "string", required: true, variadic: true, sensitive: true, desc: `Confirmed ${MAPPING_VERSION} JSON, file path, or @file. Repeat for multiple data sets (one sheet each).` },
     { name: "plan-file", type: "string", sensitive: true, desc: "Tracking-plan draft.json to reference inside the handoff package." },
-    { name: "out-dir", type: "string", sensitive: true, desc: "Handoff root directory. Default: .ae-data-integration." }
+    { name: "out-dir", type: "string", sensitive: true, desc: "Handoff root directory. Default: .ae-cli/data-integration (project workspace)." },
+    { name: "pushurl", type: "string", sensitive: true, desc: "Receiver base URL to record as the reuse upload target (sink endpoint = pushurl + /sync_json). Reuse still requires --confirm." },
+    { name: "project-id", type: "string", sensitive: true, desc: "Numeric destination project ID to record; upload derives the APPID from it via project info get." }
   ],
   risk: "write",
   dryRun: async (ctx) => {
-    const mapping = readLocalDataMapping(ctx.str("mapping"));
-    const outDir = resolve3(ctx.str("out-dir").trim() || ".ae-data-integration");
+    const mappings = ctx.list("mapping").map((source) => readLocalDataMapping(source));
+    const outDir = resolve3(ctx.str("out-dir").trim() || join3(".ae-cli", "data-integration"));
     const planFile = ctx.str("plan-file").trim() || void 0;
+    const pushurl = ctx.str("pushurl").trim() || void 0;
+    const projectId = ctx.str("project-id").trim() || void 0;
+    const fingerprints = mappings.map(structureFingerprint);
     return {
       action: "handoff_local_data",
       out_dir: outDir,
-      fingerprint: structureFingerprint(mapping),
-      source_sha256: mapping.source.sha256,
-      mode: mapping.mode,
-      property_count: mapping.properties.length,
-      files: ["mapping.json", "transform.mjs", ...planFile ? ["plan.json"] : []],
-      index_file: join2(outDir, "index.json")
+      mapping_count: mappings.length,
+      fingerprints,
+      target: { pushurl, project_id: projectId },
+      files: handoffFileList(fingerprints, planFile),
+      index_file: join3(outDir, "index.json"),
+      pipeline_file: join3(outDir, "pipeline.json"),
+      shape_file: join3(outDir, "shape.json"),
+      zip_path: zipPathFor(outDir, fingerprints[0])
     };
   },
   execute: async (ctx) => {
-    const mapping = readLocalDataMapping(ctx.str("mapping"));
-    const outDir = resolve3(ctx.str("out-dir").trim() || ".ae-data-integration");
+    const mappings = ctx.list("mapping").map((source) => readLocalDataMapping(source));
+    const outDir = resolve3(ctx.str("out-dir").trim() || join3(".ae-cli", "data-integration"));
     const planFile = ctx.str("plan-file").trim() || void 0;
-    const build = buildHandoffPackage(outDir, mapping, planFile);
+    const target = {
+      pushurl: ctx.str("pushurl").trim() || void 0,
+      project_id: ctx.str("project-id").trim() || void 0
+    };
+    const items = mappings.map((mapping) => ({
+      mapping,
+      build: buildHandoffPackage(outDir, mapping, planFile)
+    }));
+    const entries = items.map((item) => item.build.entry);
+    const relayFiles = [
+      { relPath: "pipeline.json", content: `${JSON.stringify(buildPipelineDescriptor(entries, target), null, 2)}
+`, mode: 384 },
+      { relPath: "shape.json", content: `${JSON.stringify(buildShapeBaseline(items.map((item) => ({ mapping: item.mapping, fingerprint: item.build.fingerprint }))), null, 2)}
+`, mode: 384 },
+      ...generateBinScripts(),
+      { relPath: "README.md", content: generateReadme(), mode: 384 },
+      { relPath: "RUNBOOK.md", content: generateRunbook(), mode: 384 },
+      { relPath: ".local/target.env.example", content: generateEnvTemplate(), mode: 384 },
+      { relPath: ".gitignore", content: generateGitignore(), mode: 384 },
+      { relPath: "inbox/.gitkeep", content: "", mode: 384 },
+      { relPath: "runs/.gitkeep", content: "", mode: 384 }
+    ];
+    for (const file of relayFiles) writeRelayFile(outDir, file);
+    const zipPath = zipPathFor(outDir, entries[0].fingerprint);
+    const stagingDir = stageScopedPackage(outDir, items.map((item) => item.build), entries, relayFiles);
+    try {
+      await zipPackage(stagingDir, zipPath);
+      chmodSync2(zipPath, 384);
+    } finally {
+      rmSync(stagingDir, { recursive: true, force: true });
+    }
     return {
-      out_dir: build.outDir,
-      handoff_dir: build.handoffDir,
-      fingerprint: build.fingerprint,
-      mapping_file: `${build.dirName}/mapping.json`,
-      ...build.planFile ? { plan_file: build.planFile } : {},
-      run: `node ${join2(build.handoffDir, "transform.mjs")} <new-input-file> [<output-dir>]`,
-      index_file: join2(outDir, "index.json"),
-      reused_existing: build.reusedExisting
+      out_dir: outDir,
+      zip_path: zipPath,
+      pipeline_file: join3(outDir, "pipeline.json"),
+      shape_file: join3(outDir, "shape.json"),
+      index_file: join3(outDir, "index.json"),
+      handoff_dirs: items.map((item) => item.build.dirName),
+      deliverables: buildDeliverables(outDir, relayFiles, items),
+      next_steps: [
+        `Review the pipeline descriptor and four confirmation gates: ${join3(outDir, "RUNBOOK.md")}.`,
+        `Share the archive: ${zipPath}.`,
+        `Next same-shape file: cd ${outDir} && bin/run.sh <new-file>, then bin/upload.sh runs/<run-id> --confirm.`
+      ],
+      reused_existing: items.some((item) => item.build.reusedExisting)
     };
   }
 };
+function zipPathFor(outDir, fingerprint) {
+  if (!fingerprint) return void 0;
+  return join3(dirname2(resolve3(outDir)), `ae-data-integration-handoff-${fingerprint.slice(0, 8)}.zip`);
+}
+var RELAY_FILE_PATHS = [
+  "pipeline.json",
+  "shape.json",
+  "index.json",
+  "README.md",
+  "RUNBOOK.md",
+  ".local/target.env.example",
+  ".gitignore",
+  "bin/run.sh",
+  "bin/upload.sh",
+  "bin/bind_mapping.py",
+  "bin/summarize.py",
+  "bin/plan_check.py",
+  "bin/verify.py",
+  "bin/resolve_appid.py",
+  "inbox/.gitkeep",
+  "runs/.gitkeep"
+];
+function mappingDirFiles(dirName, planFile) {
+  return [
+    `${dirName}/mapping.json`,
+    `${dirName}/transform.mjs`,
+    ...planFile ? [`${dirName}/plan.json`] : []
+  ];
+}
+function handoffFileList(fingerprints, planFile) {
+  const perMapping = fingerprints.flatMap(
+    (fingerprint) => mappingDirFiles(fingerprint.slice(0, HANDOFF_DIR_LEN), planFile)
+  );
+  return [...RELAY_FILE_PATHS, ...perMapping];
+}
+function buildDeliverables(outDir, relayFiles, items) {
+  const relay = [
+    { rel_path: "index.json", abs_path: join3(outDir, "index.json") },
+    ...relayFiles.map((file) => ({ rel_path: file.relPath, abs_path: join3(outDir, file.relPath) }))
+  ];
+  const mappings = items.flatMap(
+    (item) => mappingDirFiles(item.build.dirName, item.build.planFile).map((rel) => ({
+      rel_path: rel,
+      abs_path: join3(outDir, rel)
+    }))
+  );
+  return [...relay, ...mappings];
+}
+function writeRelayFile(outDir, file) {
+  const abs = join3(outDir, file.relPath);
+  mkdirSync2(dirname2(abs), { recursive: true, mode: 448 });
+  writeFileSync2(abs, file.content, { encoding: "utf8", mode: file.mode });
+  chmodSync2(abs, file.mode);
+}
+function stageScopedPackage(outDir, mappingDirs, entries, relayFiles) {
+  const staging = mkdtempSync(join3(tmpdir(), "ae-handoff-"));
+  writeSecureJson2(join3(staging, "index.json"), { version: HANDOFF_INDEX_VERSION, entries });
+  for (const file of relayFiles) writeRelayFile(staging, file);
+  for (const { dirName, planFile } of mappingDirs) {
+    for (const rel of mappingDirFiles(dirName, planFile)) {
+      const dest = join3(staging, rel);
+      mkdirSync2(dirname2(dest), { recursive: true, mode: 448 });
+      writeFileSync2(dest, readFileSync4(join3(outDir, rel), "utf8"), { encoding: "utf8", mode: 384 });
+      chmodSync2(dest, 384);
+    }
+  }
+  return staging;
+}
 
-// src/commands/data-integration/local-data/reuse.ts
+// src/commands/data-integration/reuse.ts
 import { readFileSync as readFileSync5 } from "fs";
-import { dirname as dirname2, join as join3, resolve as resolve4 } from "path";
+import { dirname as dirname4, join as join5, resolve as resolve5 } from "path";
+
+// src/commands/data-integration/handoff-root.ts
+import { existsSync as existsSync2 } from "fs";
+import { dirname as dirname3, join as join4, resolve as resolve4 } from "path";
+function globalHandoffDir() {
+  const home = process.env.HOME;
+  if (!home) return void 0;
+  return join4(getConfigDir(), "data-integration");
+}
+function upwardHandoffDirs(startDir) {
+  const dirs = [];
+  let current = resolve4(startDir);
+  for (; ; ) {
+    const candidate = join4(current, ".ae-cli", "data-integration");
+    if (!dirs.includes(candidate)) dirs.push(candidate);
+    const parent = dirname3(current);
+    if (parent === current) break;
+    current = parent;
+  }
+  return dirs;
+}
+function reuseSearchPaths(startDir = process.cwd()) {
+  const globalDir = globalHandoffDir();
+  return globalDir ? [...upwardHandoffDirs(startDir), globalDir] : upwardHandoffDirs(startDir);
+}
+function findReuseRoot(startDir = process.cwd()) {
+  for (const dir of upwardHandoffDirs(startDir)) {
+    if (existsSync2(join4(dir, "index.json"))) return dir;
+  }
+  return globalHandoffDir();
+}
+
+// src/commands/data-integration/reuse.ts
 function detectReuse(mapping, outDir) {
   const fingerprint = structureFingerprint(mapping);
-  const indexPath = join3(outDir, "index.json");
+  const indexPath = join5(outDir, "index.json");
   const index = readHandoffIndex(indexPath);
   const entry = index.entries.find((item) => item.fingerprint === fingerprint);
   if (!entry) return { matched: false, fingerprint, index_file: indexPath };
-  const mappingPath = join3(outDir, entry.mapping_file);
+  const mappingPath = join5(outDir, entry.mapping_file);
   const match = {
     fingerprint: entry.fingerprint,
     created_at: entry.created_at,
@@ -3322,7 +4839,7 @@ function detectReuse(mapping, outDir) {
     mapping_file: entry.mapping_file,
     ...entry.plan_file ? { plan_file: entry.plan_file } : {},
     ...readFrozenEventName(mappingPath),
-    run: `node ${join3(outDir, dirname2(entry.mapping_file), "transform.mjs")} <new-input-file> [<output-dir>]`
+    run: `node ${join5(outDir, dirname4(entry.mapping_file), "transform.mjs")} <new-input-file> [<output-dir>]`
   };
   return { matched: true, fingerprint, index_file: indexPath, match };
 }
@@ -3345,29 +4862,41 @@ var dataIntegrationReuse = {
   service: "data-integration",
   command: "reuse",
   usesAeHost: false,
-  description: "Match a candidate mapping against the .ae-data-integration/ handoff index and propose a reusable package.",
+  description: "Match a candidate mapping against the .ae-cli/data-integration/ handoff index and propose a reusable package.",
   flags: [
-    { name: "mapping", type: "string", required: true, sensitive: true, desc: "Candidate ae-local-data-mapping/v1 JSON, file path, or @file (typically inspect recommended_mapping)." },
-    { name: "out-dir", type: "string", sensitive: true, desc: "Handoff root directory. Default: .ae-data-integration." }
+    { name: "mapping", type: "string", required: true, sensitive: true, desc: `Candidate ${MAPPING_VERSION} JSON, file path, or @file (typically inspect recommended_mapping).` },
+    { name: "out-dir", type: "string", sensitive: true, desc: "Handoff root directory. Default: nearest .ae-cli/data-integration/ upward from cwd, then ~/.ae-cli/data-integration/." }
   ],
   risk: "read",
   dryRun: async (ctx) => {
     const mapping = readLocalDataMapping(ctx.str("mapping"));
-    const outDir = resolve4(ctx.str("out-dir").trim() || ".ae-data-integration");
+    const outDir = resolveReuseOutDir(ctx);
     const result = detectReuse(mapping, outDir);
     return {
       action: "reuse_detect",
       fingerprint: result.fingerprint,
       matched: result.matched,
-      index_file: result.index_file
+      index_file: result.index_file,
+      searched_paths: searchedIndexPaths(ctx)
     };
   },
   execute: async (ctx) => {
     const mapping = readLocalDataMapping(ctx.str("mapping"));
-    const outDir = resolve4(ctx.str("out-dir").trim() || ".ae-data-integration");
-    return detectReuse(mapping, outDir);
+    const outDir = resolveReuseOutDir(ctx);
+    const result = detectReuse(mapping, outDir);
+    return { ...result, searched_paths: searchedIndexPaths(ctx) };
   }
 };
+function resolveReuseOutDir(ctx) {
+  const explicit = ctx.str("out-dir").trim();
+  if (explicit) return resolve5(explicit);
+  return findReuseRoot() ?? resolve5(join5(".ae-cli", "data-integration"));
+}
+function searchedIndexPaths(ctx) {
+  const explicit = ctx.str("out-dir").trim();
+  const dirs = explicit ? [resolve5(explicit)] : reuseSearchPaths();
+  return dirs.map((dir) => join5(dir, "index.json"));
+}
 
 // src/commands/data-integration/index.ts
 var commands = [dataIntegrationInspect, dataIntegrationPlan, dataIntegrationConvert, dataIntegrationUpload, dataIntegrationHandoff, dataIntegrationReuse];