@meshxdata/fops 0.1.31 → 0.1.34

package/src/plugins/bundled/fops-plugin-file/demo/run.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# fops embed file — demo script
+#
+# Showcases the full pipeline:
+#   Steps 1-5: pure schema / type comparison (no index needed)
+#   Step 6:    MiniLM + RRF smart matching (requires Foundation + `fops embed file index`)
+#   Step 7:    Fix CSV — rename columns to match reference (fops foundation align)
+#              In interactive mode, step 6 prompts to fix CSV or update the Data Product automatically.
+#
+# Files:
+#   orders_reference.csv  — expected schema (7 cols: order_id, customer_id, amount, currency, order_date, status, region)
+#   orders_good.csv       — clean file, matches reference exactly      → PASS
+#   orders_bad.csv        — renamed column (total_amount), missing status → FAIL
+#   orders_renamed.csv    — same data, all column names abbreviated (id, cust_id, price...) → no Jaccard match → MiniLM matches
+
+set -e
+DEMO_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+section() {
+  echo
+  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+  echo "  $1"
+  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+}
+
+# ── 1. Preview: inspect schema + inferred types ───────────────────────────────
+section "1 · Preview — inspect schema + inferred types"
+fops embed file preview "$DEMO_DIR/orders_good.csv"
+
+# ── 2. Validate: clean file against reference → PASS ─────────────────────────
+section "2 · Validate — clean file (expect PASS)"
+fops embed file validate "$DEMO_DIR/orders_good.csv" \
+  --reference "$DEMO_DIR/orders_reference.csv"
+
+# ── 3. Validate: bad file → FAIL (missing column, renamed column) ─────────────
+section "3 · Validate — bad file (expect FAIL)"
+fops embed file validate "$DEMO_DIR/orders_bad.csv" \
+  --reference "$DEMO_DIR/orders_reference.csv" || true
+
+# ── 4. Preview with reference diff ────────────────────────────────────────────
+section "4 · Preview with diff — visualise column drift vs reference"
+fops embed file preview "$DEMO_DIR/orders_bad.csv" \
+  --reference "$DEMO_DIR/orders_reference.csv"
+
+# ── 5. Smart match (Jaccard only, no index) ───────────────────────────────────
+# orders_renamed.csv has fully abbreviated column names — zero Jaccard overlap.
+# --no-semantic forces Jaccard-only mode (skips MiniLM, no ONNX load).
+section "5 · Smart match — Jaccard only (no index): expect no strong match"
+fops embed file "$DEMO_DIR/orders_renamed.csv" --local --no-semantic || true
+
+# ── 6. Smart match (MiniLM + RRF, requires index) ────────────────────────────
+# When the SQLite schema index exists (built by `fops embed file index`), the
+# matching pipeline upgrades to:
+#   - Signal 1: column-level MiniLM cosine + greedy bipartite matching
+#   - Signal 2: Jaccard overlap on exact column names
+#   - Signal 3: type compatibility score on matched column pairs
+#   - Signal 4: MiniLM cosine on table/entity names
+#   - Signal 5: fraction of candidate columns with sim > 0.75
+#   All fused via Reciprocal Rank Fusion (RRF k=60).
+#
+# With MiniLM, "id"→"order_id", "price"→"amount", "ccy"→"currency" etc. all
+# match semantically even with zero Jaccard overlap.
+section "6 · Smart match — MiniLM + RRF (requires Foundation + index)"
+
+SCHEMA_DB="$HOME/.fops/file/schema-index.db"
+if [ -f "$SCHEMA_DB" ]; then
+  echo "  Index found — running full semantic match..."
+  fops embed file "$DEMO_DIR/orders_renamed.csv"
+else
+  echo "  No schema index found. To enable MiniLM + RRF matching, run:"
+  echo
+  echo "    # 1. Start Foundation"
+  echo "    fops up"
+  echo
+  echo "    # 2. Register your data objects (or use the demo landscape)"
+  echo "    fops apply $DEMO_DIR/landscape.yaml"
+  echo
+  echo "    # 3. Build the SQLite schema index (embeds column names with MiniLM)"
+  echo "    fops embed file index"
+  echo
+  echo "    # 4. Re-run this step — orders_renamed.csv will now match via"
+  echo "    #    semantic similarity (id→order_id, price→amount, ccy→currency)"
+  echo "    fops embed file $DEMO_DIR/orders_renamed.csv"
+  echo
+  echo "  Matching signals when index is present:"
+  echo "    [1] MiniLM column cosine  — greedy bipartite match per column pair"
+  echo "    [2] Jaccard overlap       — exact column name set overlap"
+  echo "    [3] Type compatibility    — integer/decimal/date match on mapped cols"
+  echo "    [4] Name semantic         — MiniLM cosine on table/entity names"
+  echo "    [5] Mapping confidence    — fraction of cols with sim > 0.75"
+  echo "        ↳ all fused via Reciprocal Rank Fusion (RRF k=60)"
+fi
+
+# ── 7. Fix CSV — rename abbreviated columns to reference names ────────────────
+# After a smart match, fops embed file (interactive) prompts:
+#   ▸ Fix CSV — rename columns to match reference (→ orders_renamed.aligned.csv)
+#     Fix Data Product — update schema to match this file
+#     Nothing — done
+#
+# This step demonstrates the "Fix CSV" path explicitly via fops foundation align,
+# which uses the same MiniLM + Levenshtein column alignment under the hood.
+section "7 · Fix CSV — rename columns to match reference"
+# Uses MiniLM semantic matching to map abbreviated column names to reference schema.
+# The interactive Fix CSV prompt inside `fops embed file` (step 6) runs this automatically.
+# || true: ONNX native handles trigger a SIGKILL on exit — this is expected and harmless.
+fops foundation align "$DEMO_DIR/orders_renamed.csv" \
+  "order_id,customer_id,amount,currency,order_date,status,region" \
+  --output "$DEMO_DIR/orders_renamed.aligned.csv" || true
+
+echo

package/src/plugins/bundled/fops-plugin-file/index.js

@@ -35,6 +35,7 @@ export function register(api) {
       .option("--json", "Output matches as JSON (for scripting)")
       .option("--no-auto", "Always show picker, never auto-select")
       .option("--local", "Match against local directory files instead of Foundation landscape")
+      .option("--no-semantic", "Jaccard-only matching — skip MiniLM embeddings (useful for testing without the model)")
       .option("--sheet <name>", "Sheet name or index for XLSX files")
       .option("--data-source <id>", "Data source identifier for attempt logging")
       .option("--file-family <name>", "Logical file family name for logging")
@@ -71,7 +72,7 @@ export function register(api) {
         }
 
         let embeddingClient = null;
-        if (typeof api.getService === "function") {
+        if (!opts.noSemantic && typeof api.getService === "function") {
           try { embeddingClient = api.getService("embeddings"); } catch { /* not loaded */ }
         }
 
@@ -338,17 +339,24 @@ export function register(api) {
             columnEmbeddings: candResult.columnEmbeddings,
             nameEmbedding: candResult.nameEmbedding,
           };
-          const ranked = rankMatches(candidateData, references, rankOpts)
-            .filter((r) => r.score >= threshold)
-            .filter((r) => r.overlapCount >= 2 || r.overlapCount >= r.overlapTotal);
+          const allRanked = rankMatches(candidateData, references, rankOpts)
+            .filter((r) => r.score >= threshold);
+
+          // In Jaccard-only mode, require at least 2 overlapping column names to avoid false positives.
+          // When embeddings are active (v2 or v3), semantic matching works without exact name overlap.
+          const ranked = (hasEmbeddings || hasColumnEmbeddings)
+            ? allRanked
+            : allRanked.filter((r) => r.overlapCount >= 2 || r.overlapCount >= r.overlapTotal);
 
           spinner.stop();
 
           if (ranked.length === 0) {
-            console.log(chalk.yellow("  No strong matches found (need at least 2 overlapping columns)."));
+            console.log(chalk.yellow("  No strong matches found."));
             console.log(chalk.dim("  Use `fops embed file validate <file> --reference <ref>` for explicit validation."));
-            if (!hasEmbeddings && foundationAvailable) {
-              console.log(chalk.dim("  Tip: run `fops embed index` to enable semantic matching."));
+            if (!hasEmbeddings && !hasColumnEmbeddings && foundationAvailable) {
+              console.log(chalk.dim("  Tip: run `fops embed index` then `fops embed file index` to enable semantic matching."));
+            } else if (hasEmbeddings && !hasColumnEmbeddings && foundationAvailable) {
+              console.log(chalk.dim("  Tip: run `fops embed file index` (model cached) for column-level semantic matching."));
             }
             return;
           }
@@ -469,24 +477,52 @@ export function register(api) {
             rows: null,
           };
 
+          // When semantic column mappings are available, translate candidate column
+          // names to their matched reference names so schema/type checks are meaningful.
+          let candHeadersForValidation = candResult.headers;
+          let candTypesForValidation = candResult.types;
+          const colMap = selectedRef.columnMappings;
+          if (colMap?.length > 0) {
+            const translatedHeaders = [];
+            const translatedTypes = {};
+            for (const m of colMap) {
+              translatedHeaders.push(m.referenceCol);
+              if (candResult.types?.[m.candidateCol] !== undefined) {
+                translatedTypes[m.referenceCol] = candResult.types[m.candidateCol];
+              }
+            }
+            candHeadersForValidation = translatedHeaders;
+            candTypesForValidation = translatedTypes;
+          }
+
           const schemaResult = compareSchemas(
             { headers: refResult.headers },
-            { headers: candResult.headers },
+            { headers: candHeadersForValidation },
             { candidateCentric: true },
           );
 
           let typeResult = null;
-          const commonCols = candResult.headers.filter((h) => refResult.headers.includes(h));
-          if (commonCols.length > 0 && refResult.types && candResult.types) {
-            typeResult = compareTypes(refResult.types, candResult.types, commonCols);
+          const commonCols = candHeadersForValidation.filter((h) => refResult.headers.includes(h));
+          if (commonCols.length > 0 && refResult.types && candTypesForValidation) {
+            typeResult = compareTypes(refResult.types, candTypesForValidation, commonCols);
           }
 
           // Auto-derive value rules from schema metadata (sample up to 10K rows)
           const autoRules = deriveRules(refResult.types, commonCols);
           let valuesResult = null;
           if (autoRules && candResult.rows?.length > 0) {
+            // Map row values to reference column names for value checks
             const sampleRows = candResult.rows.length > 10000 ? candResult.rows.slice(0, 10000) : candResult.rows;
-            valuesResult = checkValues(sampleRows, candResult.headers, autoRules);
+            if (colMap?.length > 0) {
+              // Re-key rows from candidate names to reference names
+              const keyMap = Object.fromEntries(colMap.map((m) => [m.candidateCol, m.referenceCol]));
+              const remappedRows = sampleRows.map((row) =>
+                Object.fromEntries(Object.entries(row).map(([k, v]) => [keyMap[k] ?? k, v])),
+              );
+              valuesResult = checkValues(remappedRows, candHeadersForValidation, autoRules);
+            } else {
+              valuesResult = checkValues(sampleRows, candResult.headers, autoRules);
+            }
           }
 
           const report = buildReport(schemaResult, typeResult, valuesResult, null);
@@ -503,6 +539,82 @@ export function register(api) {
           if (report.verdict === "FAIL") {
             process.exitCode = 1;
           }
+
+          // ── 11. Post-match action prompt ────────────────────────────────
+          if (isTTY && !opts.json) {
+            const ext = pathMod.extname(candidate).toLowerCase();
+            const canFixCsv = ext === ".csv" && colMap?.length > 0;
+            const canFixDp = selectedRef.entityType === "data_product" && !!foundationClient;
+
+            if (canFixCsv || canFixDp) {
+              const baseName = pathMod.basename(candidate, ext);
+              const alignedPath = pathMod.join(pathMod.dirname(candidate), `${baseName}.aligned${ext}`);
+
+              const actionOptions = [];
+              if (canFixCsv) {
+                actionOptions.push({
+                  label: `Fix CSV — rename columns to match reference (→ ${pathMod.basename(alignedPath)})`,
+                  value: "fix-csv",
+                });
+              }
+              if (canFixDp) {
+                actionOptions.push({
+                  label: `Fix Data Product — update "${selectedRef.name}" schema to match this file`,
+                  value: "fix-dp",
+                });
+              }
+              actionOptions.push({ label: "Nothing — done", value: null });
+
+              let selectActionFn;
+              const confirmPath2 = api.getCliPath?.("src", "ui", "confirm.js");
+              if (confirmPath2) {
+                const { pathToFileURL: p2u } = await import("node:url");
+                ({ selectOption: selectActionFn } = await import(p2u(confirmPath2).href));
+              } else {
+                ({ selectOption: selectActionFn } = await import("../../../ui/confirm.js"));
+              }
+
+              console.log("");
+              const action = await selectActionFn("Next step:", actionOptions);
+
+              if (action === "fix-csv") {
+                const Papa = (await import("papaparse")).default;
+                const renameMap = Object.fromEntries(colMap.map((m) => [m.candidateCol, m.referenceCol]));
+                const newHeaders = candResult.headers.map((h) => renameMap[h] ?? h);
+                const newRows = (candResult.rows || []).map((row) =>
+                  Object.fromEntries(Object.entries(row).map(([k, v]) => [renameMap[k] ?? k, v])),
+                );
+                const csvText = Papa.unparse({ fields: newHeaders, data: newRows });
+                (await import("node:fs")).writeFileSync(alignedPath, csvText, "utf8");
+                console.log(chalk.green(`\n  ✓ Written: ${alignedPath}`));
+                const renamed = colMap.filter((m) => m.candidateCol !== m.referenceCol);
+                if (renamed.length > 0) {
+                  console.log(chalk.dim(`\n  Renamed ${renamed.length} column(s):`));
+                  for (const m of renamed) {
+                    console.log(chalk.dim(`    ${m.candidateCol.padEnd(22)} → ${m.referenceCol}`));
+                  }
+                }
+              } else if (action === "fix-dp") {
+                const typeToApiCol = (t) => ({ integer: "INTEGER", decimal: "DECIMAL", date: "DATE", timestamp: "TIMESTAMP", boolean: "BOOLEAN" })[t] || "VARCHAR";
+                const fields = candResult.headers.map((h) => {
+                  const ti = candResult.types?.[h];
+                  const col = { name: h, data_type: { column_type: typeToApiCol(ti?.type || "string") } };
+                  if (ti?.type === "decimal" && ti.precision != null) {
+                    col.data_type.meta = { precision: String(ti.precision), scale: String(ti.scale ?? 0) };
+                  }
+                  return col;
+                });
+                const schemaBody = { details: { data_product_type: "user", fields } };
+                try {
+                  await foundationClient.put(`/data/data_product/schema?identifier=${selectedRef.id}`, schemaBody);
+                  console.log(chalk.green(`\n  ✓ Schema updated for "${selectedRef.name}"`));
+                  console.log(chalk.dim(`    ${fields.length} columns from ${pathMod.basename(candidate)}`));
+                } catch (err) {
+                  console.error(chalk.red(`\n  ✗ Schema update failed: ${err.message}`));
+                }
+              }
+            }
+          }
         } catch (err) {
           spinner.stop();
           console.error(chalk.red(`\n  ✗ ${err.message}\n`));
@@ -777,21 +889,23 @@ export function register(api) {
           let refTypes = null;   // { colName: { type } }
           let refLabel = null;
 
-          if (opts.reference) {
+          // opts.reference may be consumed by the parent fileCmd (Commander option inheritance)
+          const referenceArg = opts.reference || fileCmd.opts().reference;
+          if (referenceArg) {
             const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-            if (UUID_RE.test(opts.reference)) {
+            if (UUID_RE.test(referenceArg)) {
               spinner.text = "Fetching reference schema...";
-              const refResult = await resolveSchemaFromApi(opts.reference, foundationClient, mapApiTypeToInferred);
+              const refResult = await resolveSchemaFromApi(referenceArg, foundationClient, mapApiTypeToInferred);
               refHeaders = new Set(refResult.headers);
               refTypes = refResult.types;
-              refLabel = `data_object:${opts.reference.slice(0, 8)}`;
+              refLabel = `data_object:${referenceArg.slice(0, 8)}`;
             } else {
               spinner.text = "Parsing reference...";
-              const refInput = await resolveInput(opts.reference, storage, fs, pathMod);
+              const refInput = await resolveInput(referenceArg, storage, fs, pathMod);
               const refResult = await parseFile(refInput.buffer, { format: refInput.format, maxRows: 1 });
               refHeaders = new Set(refResult.headers);
               refTypes = refResult.types;
-              refLabel = pathMod.basename(opts.reference);
+              refLabel = pathMod.basename(referenceArg);
             }
           }
 
@@ -1003,16 +1117,18 @@ export function register(api) {
           let refResult = null;
           let refLabel = "(none)";
 
-          if (opts.reference) {
-            if (isDataObjectUuid(opts.reference)) {
+          // opts.reference may be consumed by the parent fileCmd (Commander option inheritance)
+          const referenceArg = opts.reference || fileCmd.opts().reference;
+          if (referenceArg) {
+            if (isDataObjectUuid(referenceArg)) {
               spinner.text = "Fetching reference schema from Foundation API...";
-              refResult = await resolveSchemaFromApi(opts.reference, foundationClient, mapApiTypeToInferred);
-              refLabel = `data_object:${opts.reference.slice(0, 8)}`;
+              refResult = await resolveSchemaFromApi(referenceArg, foundationClient, mapApiTypeToInferred);
+              refLabel = `data_object:${referenceArg.slice(0, 8)}`;
             } else {
               spinner.text = "Parsing reference file...";
-              const refInput = await resolveInput(opts.reference, storage, fs, path);
+              const refInput = await resolveInput(referenceArg, storage, fs, path);
               refResult = await parseFile(refInput.buffer, { format: refInput.format, sheet: opts.sheet });
-              refLabel = path.basename(opts.reference);
+              refLabel = path.basename(referenceArg);
             }
           }
 

package/src/plugins/bundled/fops-plugin-file/lib/embed-index.js

@@ -387,7 +387,14 @@ export async function runEmbedIndex(api, opts) {
           store.setMeta("embedding_dim", "384");
         } catch {
           // Embeddings failed — index still usable without them
+          spinner.stop();
+          console.log(WARN("  ⚠ Column embeddings skipped — MiniLM model not ready."));
+          console.log(DIM("    Run `fops embed index` first to download the model, then re-run `fops embed file index`."));
+          console.log(DIM("    Without embeddings, matching falls back to Jaccard (exact column name overlap only)."));
         }
+      } else {
+        console.log(WARN("  ⚠ Column embeddings skipped — embeddings plugin not available."));
+        console.log(DIM("    Run `fops embed index` first to download the MiniLM model, then re-run `fops embed file index`."));
       }
 
       store.setMeta("version", "3");

package/src/plugins/bundled/fops-plugin-file/lib/match.js

@@ -155,13 +155,20 @@ export async function embedSchemaEntries(entries, embeddingClient, opts = {}) {
 
 /**
  * Normalize a column name for embedding: split on separators, lowercase.
+ * For compound names (e.g. order_status), the suffix is repeated to boost its
+ * weight in the embedding so "order_status" stays close to "status" and doesn't
+ * drift toward "order_id" due to a shared prefix.
  */
 function normalizeColumnName(name) {
-  return (name || "")
-    .replace(/[_\-./]/g, " ")
+  const parts = (name || "")
     .replace(/([a-z])([A-Z])/g, "$1 $2") // camelCase split
-    .toLowerCase()
-    .trim() || name;
+    .split(/[_\-./\s]+/)
+    .filter(Boolean)
+    .map((p) => p.toLowerCase());
+  if (parts.length === 0) return name || "";
+  if (parts.length === 1) return parts[0];
+  // Repeat the last token to emphasise the semantic role over the entity prefix
+  return parts.join(" ") + " " + parts[parts.length - 1];
 }
 
 /**