@meshxdata/fops 0.1.32 → 0.1.35

package/src/plugins/bundled/fops-plugin-dai-ttyd/lib/client.js

@@ -0,0 +1,164 @@
+import http from "node:http";
+import https from "node:https";
+
+const DEFAULT_BASE_URL = "https://api.dashboards.ai";
+
+function resolveBaseUrl(config) {
+  return (
+    config?.baseUrl?.trim() ||
+    process.env.DAI_TTYD_URL?.trim() ||
+    DEFAULT_BASE_URL
+  ).replace(/\/+$/, "");
+}
+
+function resolveAuthToken(config) {
+  const raw = config?.authToken || process.env.DAI_AUTH_TOKEN || "";
+  return raw.replace(/\s+/g, ""); // strip whitespace/newlines from env or shell wrapping
+}
+
+export class DaiTTYDClient {
+  constructor(config = {}) {
+    this.baseUrl = resolveBaseUrl(config);
+    this.authToken = resolveAuthToken(config);
+  }
+
+  _headers() {
+    return {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Bearer ${this.authToken}`,
+    };
+  }
+
+  /** POST /ttyd/threads → thread id (number) */
+  async createThread(title = "fops session") {
+    const url = `${this.baseUrl}/ttyd/threads`;
+    let { status, data } = await request("POST", url, this._headers(), { title });
+    // Backend may return 400 "User with email already exists" on the first call
+    // when auto-provisioning the user — retry once, it should succeed
+    if (status === 400 && data.includes("already exists")) {
+      ({ status, data } = await request("POST", url, this._headers(), { title }));
+    }
+    if (status >= 400) throw new Error(`Failed to create thread (HTTP ${status}): ${data}`);
+    const body = JSON.parse(data);
+    const id = body.id ?? body.thread_id;
+    if (id == null) throw new Error(`No thread id in response: ${data}`);
+    return id;
+  }
+
+  /**
+   * POST /ttyd/answer/stream — async generator yielding SSE event objects.
+   * Each event: { type: "sql"|"answer"|"done"|"error", content?: string, message?: string }
+   */
+  async *askStream(question, threadId, { dashboardId, tileId } = {}) {
+    const url = `${this.baseUrl}/ttyd/answer/stream`;
+    const payload = { question, thread_id: threadId };
+    if (dashboardId != null) payload.dashboard_id = Number(dashboardId);
+    if (tileId != null) payload.tile_id = Number(tileId);
+    yield* streamSSE("POST", url, this._headers(), payload);
+  }
+
+  /** Convenience: create a thread, ask one question, collect full sql + summary. */
+  async ask(question, { threadId, dashboardId, tileId } = {}) {
+    const id = threadId ?? (await this.createThread());
+    let sql = "";
+    const summaryParts = [];
+    for await (const event of this.askStream(question, id, { dashboardId, tileId })) {
+      if (event.type === "sql") sql = event.content ?? "";
+      else if (event.type === "answer") summaryParts.push(event.content ?? "");
+      else if (event.type === "error") throw new Error(event.message ?? "TTYD error");
+      else if (event.type === "done") break;
+    }
+    return { threadId: id, sql, summary: summaryParts.join("") };
+  }
+
+  /** Verify connectivity — returns true/false. */
+  async ping() {
+    try {
+      const url = `${this.baseUrl}/ttyd/threads`;
+      const { status } = await request("POST", url, this._headers(), { title: "ping" });
+      return status < 500;
+    } catch {
+      return false;
+    }
+  }
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function request(method, url, headers, body, timeoutMs = 15_000) {
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const lib = parsed.protocol === "https:" ? https : http;
+    const encoded = JSON.stringify(body);
+    const options = {
+      hostname: parsed.hostname,
+      port: parsed.port || (parsed.protocol === "https:" ? 443 : 80),
+      path: parsed.pathname + parsed.search,
+      method,
+      headers: { ...headers, "Content-Length": Buffer.byteLength(encoded) },
+      timeout: timeoutMs,
+    };
+    const req = lib.request(options, (res) => {
+      let data = "";
+      res.on("data", (c) => { data += c; });
+      res.on("end", () => resolve({ status: res.statusCode, data }));
+    });
+    req.on("error", reject);
+    req.on("timeout", () => { req.destroy(); reject(new Error("Request timeout")); });
+    req.write(encoded);
+    req.end();
+  });
+}
+
+/**
+ * Async generator that streams SSE from an HTTP endpoint.
+ * Yields parsed JSON objects from each "data: {...}" line.
+ */
+async function* streamSSE(method, url, headers, body, timeoutMs = 300_000) {
+  const parsed = new URL(url);
+  const lib = parsed.protocol === "https:" ? https : http;
+  const encoded = JSON.stringify(body);
+  const options = {
+    hostname: parsed.hostname,
+    port: parsed.port || (parsed.protocol === "https:" ? 443 : 80),
+    path: parsed.pathname + parsed.search,
+    method,
+    headers: {
+      ...headers,
+      "Content-Length": Buffer.byteLength(encoded),
+      Accept: "text/event-stream",
+    },
+    timeout: timeoutMs,
+  };
+
+  const res = await new Promise((resolve, reject) => {
+    const req = lib.request(options, resolve);
+    req.on("error", reject);
+    req.on("timeout", () => { req.destroy(); reject(new Error("Stream timeout")); });
+    req.write(encoded);
+    req.end();
+  });
+
+  if (res.statusCode >= 400) {
+    let errData = "";
+    for await (const chunk of res) errData += chunk;
+    throw new Error(`HTTP ${res.statusCode}: ${errData}`);
+  }
+
+  let buffer = "";
+  for await (const chunk of res) {
+    buffer += chunk.toString();
+    const lines = buffer.split("\n");
+    buffer = lines.pop(); // keep any incomplete line
+    for (const line of lines) {
+      if (!line.startsWith("data:")) continue;
+      const raw = line.slice("data:".length).trim();
+      if (!raw) continue;
+      let data;
+      try { data = JSON.parse(raw); } catch { continue; }
+      yield data;
+      if (data?.type === "done" || data?.type === "error") return;
+    }
+  }
+}

package/src/plugins/bundled/fops-plugin-dai-ttyd/package.json

@@ -0,0 +1 @@
+{ "type": "module" }

package/src/plugins/bundled/fops-plugin-embeddings/index.js

@@ -264,7 +264,9 @@ export default {
         if (!texts || texts.length === 0) return [];
         if (!embeddingClient.isModelCached()) return [];
         try {
-          return await embeddingClient.embed(texts);
+          await embeddingClient.isReady();
+          markOnnxLoaded();
+          return await embeddingClient.embedBatch(texts);
         } catch { return []; }
       },
     });

package/src/plugins/bundled/fops-plugin-embeddings/lib/indexer.js

@@ -1047,7 +1047,7 @@ export async function runIndex({ source, sources: explicitSources, force, onProg
     // If the stored model differs from current NL_MODEL, force full re-index
     const currentStoreModel = nlStore.load().model;
     if (currentStoreModel && currentStoreModel !== NL_MODEL) {
-      log(WARN(`  Model changed (${currentStoreModel} → ${NL_MODEL}), forcing full re-index`));
+      log(WARN(`  Model changed (${currentStoreModel} → ${NL_MODEL}) — one-time re-index to rebuild vectors with the new model`));
       force = true;
       nlStore.clear();
     }

package/src/plugins/bundled/fops-plugin-file/demo/landscape.yaml

@@ -0,0 +1,67 @@
+mesh:
+  ecommerce:
+    label: EC
+    description: E-commerce analytics mesh (file plugin demo)
+    purpose: Demonstrate fops embed file smart matching
+
+    data_system:
+      datalake:
+        label: DL
+        description: Central data lake
+
+    data_source:
+      orders_bucket:
+        label: ORD
+        description: Orders data in S3
+        system: data_system.datalake
+        connection:
+          type: s3
+          url: http://foundation-storage-engine:8080
+          access_key: S3_ACCESS_KEY
+          access_secret: S3_SECRET_KEY
+        secret:
+          S3_ACCESS_KEY: minio
+          S3_SECRET_KEY: minio123
+
+    data_object:
+      orders:
+        label: ORD
+        description: Raw orders dataset
+        source: data_source.orders_bucket
+        config:
+          data_object_type: csv
+          path: /raw/orders/orders.csv
+          has_header: true
+          delimiter: ","
+
+    data_product:
+      orders_sadp:
+        label: ORD
+        description: Source-aligned orders data product
+        object: data_object.orders
+        template: sadp_passthrough
+        select_columns: [order_id, customer_id, amount, currency, order_date, status, region]
+        cast_changes:
+          - column: amount
+            data_type: decimal
+            kwargs:
+              precision: 10
+              scale: 2
+        schema:
+          - name: order_id
+            type: integer
+            primary: true
+          - name: customer_id
+            type: integer
+          - name: amount
+            type: decimal
+            precision: 10
+            scale: 2
+          - name: currency
+            type: varchar
+          - name: order_date
+            type: date
+          - name: status
+            type: varchar
+          - name: region
+            type: varchar

package/src/plugins/bundled/fops-plugin-file/demo/orders_bad.csv

@@ -0,0 +1,6 @@
+order_id,customer_id,total_amount,currency,order_date,region
+3001,7001,120.00,USD,2024-03-01,north_america
+3002,7002,not_a_number,EUR,2024-03-02,europe
+3003,7003,88.50,USD,bad-date,asia_pacific
+3004,7004,-50.00,GBP,2024-03-04,europe
+3005,7005,405.00,USD,2024-03-05,north_america

package/src/plugins/bundled/fops-plugin-file/demo/orders_good.csv

@@ -0,0 +1,7 @@
+order_id,customer_id,amount,currency,order_date,status,region
+2001,6001,99.00,USD,2024-03-01,completed,north_america
+2002,6002,175.50,EUR,2024-03-02,pending,europe
+2003,6003,420.00,USD,2024-03-03,completed,asia_pacific
+2004,6004,58.25,GBP,2024-03-04,completed,europe
+2005,6005,200.00,USD,2024-03-05,pending,north_america
+2006,6006,315.00,USD,2024-03-06,completed,north_america

package/src/plugins/bundled/fops-plugin-file/demo/orders_reference.csv

@@ -0,0 +1,6 @@
+order_id,customer_id,amount,currency,order_date,status,region
+1001,5001,149.99,USD,2024-01-03,completed,north_america
+1002,5002,89.50,EUR,2024-01-04,pending,europe
+1003,5003,230.00,USD,2024-01-05,completed,north_america
+1004,5004,45.00,GBP,2024-01-06,cancelled,europe
+1005,5005,310.75,USD,2024-01-07,completed,asia_pacific

package/src/plugins/bundled/fops-plugin-file/demo/orders_renamed.aligned.csv

@@ -0,0 +1,6 @@
+id,customer_id,price,ccy,date,order_status,geo_region
+4001,8001,99.00,USD,2024-03-01,completed,north_america
+4002,8002,145.75,EUR,2024-03-02,pending,europe
+4003,8003,280.00,USD,2024-03-03,completed,asia_pacific
+4004,8004,62.50,GBP,2024-03-04,completed,europe
+4005,8005,195.00,USD,2024-03-05,pending,north_america

package/src/plugins/bundled/fops-plugin-file/demo/orders_renamed.csv

@@ -0,0 +1,6 @@
+id,cust_id,price,ccy,date,order_status,geo_region
+4001,8001,99.00,USD,2024-03-01,completed,north_america
+4002,8002,145.75,EUR,2024-03-02,pending,europe
+4003,8003,280.00,USD,2024-03-03,completed,asia_pacific
+4004,8004,62.50,GBP,2024-03-04,completed,europe
+4005,8005,195.00,USD,2024-03-05,pending,north_america

package/src/plugins/bundled/fops-plugin-file/demo/rules.json

@@ -0,0 +1,8 @@
+{
+  "columns": {
+    "amount": { "min": 0, "type": "decimal" },
+    "order_id": { "type": "integer", "unique": true },
+    "currency": { "allowed": ["USD", "EUR", "GBP"] },
+    "status": { "allowed": ["pending", "completed", "cancelled"] }
+  }
+}

package/src/plugins/bundled/fops-plugin-file/demo/run.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# fops embed file — demo script
+#
+# Showcases the full pipeline:
+#   Steps 1-5: pure schema / type comparison (no index needed)
+#   Step 6:    MiniLM + RRF smart matching (requires Foundation + `fops embed file index`)
+#   Step 7:    Fix CSV — rename columns to match reference (fops foundation align)
+#              In interactive mode, step 6 prompts to fix CSV or update the Data Product automatically.
+#
+# Files:
+#   orders_reference.csv  — expected schema (7 cols: order_id, customer_id, amount, currency, order_date, status, region)
+#   orders_good.csv       — clean file, matches reference exactly      → PASS
+#   orders_bad.csv        — renamed column (total_amount), missing status → FAIL
+#   orders_renamed.csv    — same data, all column names abbreviated (id, cust_id, price...) → no Jaccard match → MiniLM matches
+
+set -e
+DEMO_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+section() {
+  echo
+  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+  echo "  $1"
+  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+}
+
+# ── 1. Preview: inspect schema + inferred types ───────────────────────────────
+section "1 · Preview — inspect schema + inferred types"
+fops embed file preview "$DEMO_DIR/orders_good.csv"
+
+# ── 2. Validate: clean file against reference → PASS ─────────────────────────
+section "2 · Validate — clean file (expect PASS)"
+fops embed file validate "$DEMO_DIR/orders_good.csv" \
+  --reference "$DEMO_DIR/orders_reference.csv"
+
+# ── 3. Validate: bad file → FAIL (missing column, renamed column) ─────────────
+section "3 · Validate — bad file (expect FAIL)"
+fops embed file validate "$DEMO_DIR/orders_bad.csv" \
+  --reference "$DEMO_DIR/orders_reference.csv" || true
+
+# ── 4. Preview with reference diff ────────────────────────────────────────────
+section "4 · Preview with diff — visualise column drift vs reference"
+fops embed file preview "$DEMO_DIR/orders_bad.csv" \
+  --reference "$DEMO_DIR/orders_reference.csv"
+
+# ── 5. Smart match (Jaccard only, no index) ───────────────────────────────────
+# orders_renamed.csv has fully abbreviated column names — zero Jaccard overlap.
+# --no-semantic forces Jaccard-only mode (skips MiniLM, no ONNX load).
+section "5 · Smart match — Jaccard only (no index): expect no strong match"
+fops embed file "$DEMO_DIR/orders_renamed.csv" --local --no-semantic || true
+
+# ── 6. Smart match (MiniLM + RRF, requires index) ────────────────────────────
+# When the SQLite schema index exists (built by `fops embed file index`), the
+# matching pipeline upgrades to:
+#   - Signal 1: column-level MiniLM cosine + greedy bipartite matching
+#   - Signal 2: Jaccard overlap on exact column names
+#   - Signal 3: type compatibility score on matched column pairs
+#   - Signal 4: MiniLM cosine on table/entity names
+#   - Signal 5: fraction of candidate columns with sim > 0.75
+#   All fused via Reciprocal Rank Fusion (RRF k=60).
+#
+# With MiniLM, "id"→"order_id", "price"→"amount", "ccy"→"currency" etc. all
+# match semantically even with zero Jaccard overlap.
+section "6 · Smart match — MiniLM + RRF (requires Foundation + index)"
+
+SCHEMA_DB="$HOME/.fops/file/schema-index.db"
+if [ -f "$SCHEMA_DB" ]; then
+  echo "  Index found — running full semantic match..."
+  fops embed file "$DEMO_DIR/orders_renamed.csv"
+else
+  echo "  No schema index found. To enable MiniLM + RRF matching, run:"
+  echo
+  echo "    # 1. Start Foundation"
+  echo "    fops up"
+  echo
+  echo "    # 2. Register your data objects (or use the demo landscape)"
+  echo "    fops apply $DEMO_DIR/landscape.yaml"
+  echo
+  echo "    # 3. Build the SQLite schema index (embeds column names with MiniLM)"
+  echo "    fops embed file index"
+  echo
+  echo "    # 4. Re-run this step — orders_renamed.csv will now match via"
+  echo "    #    semantic similarity (id→order_id, price→amount, ccy→currency)"
+  echo "    fops embed file $DEMO_DIR/orders_renamed.csv"
+  echo
+  echo "  Matching signals when index is present:"
+  echo "    [1] MiniLM column cosine  — greedy bipartite match per column pair"
+  echo "    [2] Jaccard overlap       — exact column name set overlap"
+  echo "    [3] Type compatibility    — integer/decimal/date match on mapped cols"
+  echo "    [4] Name semantic         — MiniLM cosine on table/entity names"
+  echo "    [5] Mapping confidence    — fraction of cols with sim > 0.75"
+  echo "        ↳ all fused via Reciprocal Rank Fusion (RRF k=60)"
+fi
+
+# ── 7. Fix CSV — rename abbreviated columns to reference names ────────────────
+# After a smart match, fops embed file (interactive) prompts:
+#   ▸ Fix CSV — rename columns to match reference (→ orders_renamed.aligned.csv)
+#     Fix Data Product — update schema to match this file
+#     Nothing — done
+#
+# This step demonstrates the "Fix CSV" path explicitly via fops foundation align,
+# which uses the same MiniLM + Levenshtein column alignment under the hood.
+section "7 · Fix CSV — rename columns to match reference"
+# Uses MiniLM semantic matching to map abbreviated column names to reference schema.
+# The interactive Fix CSV prompt inside `fops embed file` (step 6) runs this automatically.
+# || true: ONNX native handles trigger a SIGKILL on exit — this is expected and harmless.
+fops foundation align "$DEMO_DIR/orders_renamed.csv" \
+  "order_id,customer_id,amount,currency,order_date,status,region" \
+  --output "$DEMO_DIR/orders_renamed.aligned.csv" || true
+
+echo

package/src/plugins/bundled/fops-plugin-file/index.js

@@ -35,6 +35,7 @@ export function register(api) {
       .option("--json", "Output matches as JSON (for scripting)")
       .option("--no-auto", "Always show picker, never auto-select")
       .option("--local", "Match against local directory files instead of Foundation landscape")
+      .option("--no-semantic", "Jaccard-only matching — skip MiniLM embeddings (useful for testing without the model)")
       .option("--sheet <name>", "Sheet name or index for XLSX files")
       .option("--data-source <id>", "Data source identifier for attempt logging")
       .option("--file-family <name>", "Logical file family name for logging")
@@ -71,7 +72,7 @@ export function register(api) {
         }
 
         let embeddingClient = null;
-        if (typeof api.getService === "function") {
+        if (!opts.noSemantic && typeof api.getService === "function") {
           try { embeddingClient = api.getService("embeddings"); } catch { /* not loaded */ }
         }
 
@@ -338,17 +339,24 @@ export function register(api) {
             columnEmbeddings: candResult.columnEmbeddings,
             nameEmbedding: candResult.nameEmbedding,
           };
-          const ranked = rankMatches(candidateData, references, rankOpts)
-            .filter((r) => r.score >= threshold)
-            .filter((r) => r.overlapCount >= 2 || r.overlapCount >= r.overlapTotal);
+          const allRanked = rankMatches(candidateData, references, rankOpts)
+            .filter((r) => r.score >= threshold);
+
+          // In Jaccard-only mode, require at least 2 overlapping column names to avoid false positives.
+          // When embeddings are active (v2 or v3), semantic matching works without exact name overlap.
+          const ranked = (hasEmbeddings || hasColumnEmbeddings)
+            ? allRanked
+            : allRanked.filter((r) => r.overlapCount >= 2 || r.overlapCount >= r.overlapTotal);
 
           spinner.stop();
 
           if (ranked.length === 0) {
-            console.log(chalk.yellow("  No strong matches found (need at least 2 overlapping columns)."));
+            console.log(chalk.yellow("  No strong matches found."));
             console.log(chalk.dim("  Use `fops embed file validate <file> --reference <ref>` for explicit validation."));
-            if (!hasEmbeddings && foundationAvailable) {
-              console.log(chalk.dim("  Tip: run `fops embed index` to enable semantic matching."));
+            if (!hasEmbeddings && !hasColumnEmbeddings && foundationAvailable) {
+              console.log(chalk.dim("  Tip: run `fops embed index` then `fops embed file index` to enable semantic matching."));
+            } else if (hasEmbeddings && !hasColumnEmbeddings && foundationAvailable) {
+              console.log(chalk.dim("  Tip: run `fops embed file index` (model cached) for column-level semantic matching."));
             }
             return;
           }
@@ -469,24 +477,52 @@ export function register(api) {
             rows: null,
           };
 
+          // When semantic column mappings are available, translate candidate column
+          // names to their matched reference names so schema/type checks are meaningful.
+          let candHeadersForValidation = candResult.headers;
+          let candTypesForValidation = candResult.types;
+          const colMap = selectedRef.columnMappings;
+          if (colMap?.length > 0) {
+            const translatedHeaders = [];
+            const translatedTypes = {};
+            for (const m of colMap) {
+              translatedHeaders.push(m.referenceCol);
+              if (candResult.types?.[m.candidateCol] !== undefined) {
+                translatedTypes[m.referenceCol] = candResult.types[m.candidateCol];
+              }
+            }
+            candHeadersForValidation = translatedHeaders;
+            candTypesForValidation = translatedTypes;
+          }
+
           const schemaResult = compareSchemas(
             { headers: refResult.headers },
-            { headers: candResult.headers },
+            { headers: candHeadersForValidation },
             { candidateCentric: true },
           );
 
           let typeResult = null;
-          const commonCols = candResult.headers.filter((h) => refResult.headers.includes(h));
-          if (commonCols.length > 0 && refResult.types && candResult.types) {
-            typeResult = compareTypes(refResult.types, candResult.types, commonCols);
+          const commonCols = candHeadersForValidation.filter((h) => refResult.headers.includes(h));
+          if (commonCols.length > 0 && refResult.types && candTypesForValidation) {
+            typeResult = compareTypes(refResult.types, candTypesForValidation, commonCols);
           }
 
           // Auto-derive value rules from schema metadata (sample up to 10K rows)
           const autoRules = deriveRules(refResult.types, commonCols);
           let valuesResult = null;
           if (autoRules && candResult.rows?.length > 0) {
+            // Map row values to reference column names for value checks
             const sampleRows = candResult.rows.length > 10000 ? candResult.rows.slice(0, 10000) : candResult.rows;
-            valuesResult = checkValues(sampleRows, candResult.headers, autoRules);
+            if (colMap?.length > 0) {
+              // Re-key rows from candidate names to reference names
+              const keyMap = Object.fromEntries(colMap.map((m) => [m.candidateCol, m.referenceCol]));
+              const remappedRows = sampleRows.map((row) =>
+                Object.fromEntries(Object.entries(row).map(([k, v]) => [keyMap[k] ?? k, v])),
+              );
+              valuesResult = checkValues(remappedRows, candHeadersForValidation, autoRules);
+            } else {
+              valuesResult = checkValues(sampleRows, candResult.headers, autoRules);
+            }
           }
 
           const report = buildReport(schemaResult, typeResult, valuesResult, null);
@@ -503,6 +539,82 @@ export function register(api) {
           if (report.verdict === "FAIL") {
             process.exitCode = 1;
           }
+
+          // ── 11. Post-match action prompt ────────────────────────────────
+          if (isTTY && !opts.json) {
+            const ext = pathMod.extname(candidate).toLowerCase();
+            const canFixCsv = ext === ".csv" && colMap?.length > 0;
+            const canFixDp = selectedRef.entityType === "data_product" && !!foundationClient;
+
+            if (canFixCsv || canFixDp) {
+              const baseName = pathMod.basename(candidate, ext);
+              const alignedPath = pathMod.join(pathMod.dirname(candidate), `${baseName}.aligned${ext}`);
+
+              const actionOptions = [];
+              if (canFixCsv) {
+                actionOptions.push({
+                  label: `Fix CSV — rename columns to match reference (→ ${pathMod.basename(alignedPath)})`,
+                  value: "fix-csv",
+                });
+              }
+              if (canFixDp) {
+                actionOptions.push({
+                  label: `Fix Data Product — update "${selectedRef.name}" schema to match this file`,
+                  value: "fix-dp",
+                });
+              }
+              actionOptions.push({ label: "Nothing — done", value: null });
+
+              let selectActionFn;
+              const confirmPath2 = api.getCliPath?.("src", "ui", "confirm.js");
+              if (confirmPath2) {
+                const { pathToFileURL: p2u } = await import("node:url");
+                ({ selectOption: selectActionFn } = await import(p2u(confirmPath2).href));
+              } else {
+                ({ selectOption: selectActionFn } = await import("../../../ui/confirm.js"));
+              }
+
+              console.log("");
+              const action = await selectActionFn("Next step:", actionOptions);
+
+              if (action === "fix-csv") {
+                const Papa = (await import("papaparse")).default;
+                const renameMap = Object.fromEntries(colMap.map((m) => [m.candidateCol, m.referenceCol]));
+                const newHeaders = candResult.headers.map((h) => renameMap[h] ?? h);
+                const newRows = (candResult.rows || []).map((row) =>
+                  Object.fromEntries(Object.entries(row).map(([k, v]) => [renameMap[k] ?? k, v])),
+                );
+                const csvText = Papa.unparse({ fields: newHeaders, data: newRows });
+                (await import("node:fs")).writeFileSync(alignedPath, csvText, "utf8");
+                console.log(chalk.green(`\n  ✓ Written: ${alignedPath}`));
+                const renamed = colMap.filter((m) => m.candidateCol !== m.referenceCol);
+                if (renamed.length > 0) {
+                  console.log(chalk.dim(`\n  Renamed ${renamed.length} column(s):`));
+                  for (const m of renamed) {
+                    console.log(chalk.dim(`    ${m.candidateCol.padEnd(22)} → ${m.referenceCol}`));
+                  }
+                }
+              } else if (action === "fix-dp") {
+                const typeToApiCol = (t) => ({ integer: "INTEGER", decimal: "DECIMAL", date: "DATE", timestamp: "TIMESTAMP", boolean: "BOOLEAN" })[t] || "VARCHAR";
+                const fields = candResult.headers.map((h) => {
+                  const ti = candResult.types?.[h];
+                  const col = { name: h, data_type: { column_type: typeToApiCol(ti?.type || "string") } };
+                  if (ti?.type === "decimal" && ti.precision != null) {
+                    col.data_type.meta = { precision: String(ti.precision), scale: String(ti.scale ?? 0) };
+                  }
+                  return col;
+                });
+                const schemaBody = { details: { data_product_type: "user", fields } };
+                try {
+                  await foundationClient.put(`/data/data_product/schema?identifier=${selectedRef.id}`, schemaBody);
+                  console.log(chalk.green(`\n  ✓ Schema updated for "${selectedRef.name}"`));
+                  console.log(chalk.dim(`    ${fields.length} columns from ${pathMod.basename(candidate)}`));
+                } catch (err) {
+                  console.error(chalk.red(`\n  ✗ Schema update failed: ${err.message}`));
+                }
+              }
+            }
+          }
         } catch (err) {
           spinner.stop();
           console.error(chalk.red(`\n  ✗ ${err.message}\n`));
@@ -777,21 +889,23 @@ export function register(api) {
           let refTypes = null;   // { colName: { type } }
           let refLabel = null;
 
-          if (opts.reference) {
+          // opts.reference may be consumed by the parent fileCmd (Commander option inheritance)
+          const referenceArg = opts.reference || fileCmd.opts().reference;
+          if (referenceArg) {
             const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-            if (UUID_RE.test(opts.reference)) {
+            if (UUID_RE.test(referenceArg)) {
               spinner.text = "Fetching reference schema...";
-              const refResult = await resolveSchemaFromApi(opts.reference, foundationClient, mapApiTypeToInferred);
+              const refResult = await resolveSchemaFromApi(referenceArg, foundationClient, mapApiTypeToInferred);
               refHeaders = new Set(refResult.headers);
               refTypes = refResult.types;
-              refLabel = `data_object:${opts.reference.slice(0, 8)}`;
+              refLabel = `data_object:${referenceArg.slice(0, 8)}`;
             } else {
               spinner.text = "Parsing reference...";
-              const refInput = await resolveInput(opts.reference, storage, fs, pathMod);
+              const refInput = await resolveInput(referenceArg, storage, fs, pathMod);
               const refResult = await parseFile(refInput.buffer, { format: refInput.format, maxRows: 1 });
               refHeaders = new Set(refResult.headers);
               refTypes = refResult.types;
-              refLabel = pathMod.basename(opts.reference);
+              refLabel = pathMod.basename(referenceArg);
             }
           }
 
@@ -1003,16 +1117,18 @@ export function register(api) {
           let refResult = null;
           let refLabel = "(none)";
 
-          if (opts.reference) {
-            if (isDataObjectUuid(opts.reference)) {
+          // opts.reference may be consumed by the parent fileCmd (Commander option inheritance)
+          const referenceArg = opts.reference || fileCmd.opts().reference;
+          if (referenceArg) {
+            if (isDataObjectUuid(referenceArg)) {
               spinner.text = "Fetching reference schema from Foundation API...";
-              refResult = await resolveSchemaFromApi(opts.reference, foundationClient, mapApiTypeToInferred);
-              refLabel = `data_object:${opts.reference.slice(0, 8)}`;
+              refResult = await resolveSchemaFromApi(referenceArg, foundationClient, mapApiTypeToInferred);
+              refLabel = `data_object:${referenceArg.slice(0, 8)}`;
             } else {
               spinner.text = "Parsing reference file...";
-              const refInput = await resolveInput(opts.reference, storage, fs, path);
+              const refInput = await resolveInput(referenceArg, storage, fs, path);
               refResult = await parseFile(refInput.buffer, { format: refInput.format, sheet: opts.sheet });
-              refLabel = path.basename(opts.reference);
+              refLabel = path.basename(referenceArg);
             }
           }