@meshxdata/fops 0.1.32 → 0.1.34

package/src/plugins/bundled/fops-plugin-foundation/lib/align.js

@@ -21,7 +21,7 @@ function cosineSimilarity(a, b) {
   return denom === 0 ? 0 : dot / denom;
 }
 
-function levenshteinRatio(a, b) {
+export function levenshteinRatio(a, b) {
   const x = String(a || "");
   const y = String(b || "");
   if (x === y) return 1;
@@ -65,12 +65,50 @@ export async function alignColumns(sourceColumns, expectedColumns, { embedTexts,
     }
   }
 
+  // Helpers shared by embedding and Levenshtein passes
+  // Expand common opaque abbreviations so the embedding sees real words
+  const ABBREV = {
+    ccy: "currency code", cust: "customer", amt: "amount", qty: "quantity",
+    dt: "date", desc: "description", num: "number", addr: "address",
+    prod: "product", yr: "year", mo: "month", cnt: "count",
+    avg: "average", pct: "percent", dept: "department",
+  };
+  // Suffixes that strongly constrain a column's semantic role
+  const SEMANTIC_SUFFIXES = new Set([
+    "id", "date", "status", "name", "type", "code", "key",
+    "region", "currency", "amount", "count", "number",
+  ]);
+  const toHuman = (s) => {
+    const parts = normalizeColName(s).split("_").filter(Boolean);
+    return parts.map((p) => ABBREV[p] || p).join(" ");
+  };
+  const getSuffix = (s) => {
+    const parts = normalizeColName(s).split("_").filter(Boolean);
+    const last = parts[parts.length - 1];
+    return SEMANTIC_SUFFIXES.has(last) ? last : null;
+  };
+  // Adjust a raw similarity score by structural compatibility:
+  //   containment:    source name IS the last token of expected → +0.30 (e.g. "id" → "order_id")
+  //   same suffix:    both end with same known suffix → +0.10 (confirms alignment)
+  //   suffix mismatch: both have different known suffixes → -0.15 (strong signal of mismatch)
+  const adjustScore = (raw, src, exp) => {
+    let score = raw;
+    const ss = getSuffix(src), es = getSuffix(exp);
+    if (ss && es) score = ss === es ? Math.min(1, score + 0.1) : Math.max(0, score - 0.15);
+    // Containment: "id" IS the last token of "order_id" → strong structural match
+    const normSrc = normalizeColName(src);
+    const expParts = normalizeColName(exp).split("_").filter(Boolean);
+    if (expParts.length > 1 && expParts[expParts.length - 1] === normSrc) {
+      score = Math.min(1, score + 0.3);
+    }
+    return score;
+  };
+
   // Pass 2 — Embedding (if embedTexts provided)
   if (typeof embedTexts === "function" && unmappedSrc.size > 0 && unmappedExp.size > 0) {
     try {
       const srcList = [...unmappedSrc];
       const expList = [...unmappedExp];
-      const toHuman = (s) => normalizeColName(s).replace(/_/g, " ");
       const srcTexts = srcList.map(toHuman);
       const expTexts = expList.map(toHuman);
 
@@ -83,7 +121,7 @@ export async function alignColumns(sourceColumns, expectedColumns, { embedTexts,
         const candidates = [];
         for (let i = 0; i < srcList.length; i++) {
           for (let j = 0; j < expList.length; j++) {
-            const score = cosineSimilarity(srcVecs[i], expVecs[j]);
+            const score = adjustScore(cosineSimilarity(srcVecs[i], expVecs[j]), srcList[i], expList[j]);
             if (score >= threshold) candidates.push({ i, j, score });
           }
         }
@@ -109,7 +147,7 @@ export async function alignColumns(sourceColumns, expectedColumns, { embedTexts,
     const candidates = [];
     for (const src of srcList) {
       for (const exp of expList) {
-        const score = levenshteinRatio(normalizeColName(src), normalizeColName(exp));
+        const score = adjustScore(levenshteinRatio(normalizeColName(src), normalizeColName(exp)), src, exp);
         if (score >= threshold) candidates.push({ src, exp, score });
       }
     }

package/src/plugins/bundled/fops-plugin-foundation/lib/apply.js

@@ -324,8 +324,9 @@ export function parseYamlContent(content) {
       throw err;
     }
   }
-  if (!parsed || !parsed.mesh) {
-    throw new Error("No mesh block found in YAML content");
+  const hasRootEntities = ENTITY_ORDER.slice(1).some((t) => parsed?.[t]);
+  if (!parsed || (!parsed.mesh && !hasRootEntities)) {
+    throw new Error("No mesh block or entity blocks found in YAML content");
   }
   return { operations: parseYamlOperations(parsed), fileName: "inline.yaml" };
 }
@@ -348,8 +349,9 @@ export function parseYamlLandscape(filePath) {
     }
   }
 
-  if (!parsed || !parsed.mesh) {
-    throw new Error("No mesh block found in YAML file");
+  const hasRootEntities = ENTITY_ORDER.slice(1).some((t) => parsed?.[t]);
+  if (!parsed || (!parsed.mesh && !hasRootEntities)) {
+    throw new Error("No mesh block or entity blocks found in YAML file");
   }
 
   return { operations: parseYamlOperations(parsed), fileName: path.basename(filePath) };
@@ -377,6 +379,43 @@ function parseYamlOperations(parsed) {
     ENTITY_ORDER.slice(1).map((type) => [type, parsed[type]]),
   );
 
+  function parseEntityBlock(entityType, entityName, eBlock, meshName) {
+    const props = {};
+    const refs = {};
+    const op = { type: entityType, name: entityName, props, mesh: meshName || null };
+
+    for (const [k, v] of Object.entries(eBlock)) {
+      if (k === "connection" && typeof v === "object" && !Array.isArray(v)) {
+        op.connection = v;
+      } else if (k === "config" && typeof v === "object" && !Array.isArray(v)) {
+        op.config = v;
+      } else if (k === "template") {
+        op.template = v;
+      } else if (k === "select_columns") {
+        op.selectColumns = v;
+      } else if (k === "cast_changes") {
+        op.castChanges = Array.isArray(v) ? v : [v];
+      } else if (k === "schema" && Array.isArray(v)) {
+        op.schema = v;
+      } else if (k === "pipeline" && typeof v === "object" && !Array.isArray(v)) {
+        op.pipeline = parseYamlPipeline(v);
+      } else if (k === "secret" && typeof v === "object" && !Array.isArray(v)) {
+        op.secret = v;
+      } else if (REF_TO_TYPE[k]) {
+        const ref = parseYamlRef(v);
+        if (ref) refs[k] = ref;
+      } else if (!SPECIAL_KEYS.has(k)) {
+        props[k] = v;
+      }
+    }
+
+    if (refs.system) op.parent = refs.system;
+    else if (refs.source) op.parent = refs.source;
+    else if (refs.object) op.parent = refs.object;
+
+    return op;
+  }
+
   for (const [meshName, meshBlock] of meshEntries) {
     if (!meshBlock || typeof meshBlock !== "object" || Array.isArray(meshBlock)) continue;
     const meshProps = {};
@@ -397,40 +436,19 @@ function parseYamlOperations(parsed) {
 
       for (const [entityName, eBlock] of Object.entries(group)) {
         if (!eBlock || typeof eBlock !== "object" || Array.isArray(eBlock)) continue;
-        const props = {};
-        const refs = {};
-        const op = { type: entityType, name: entityName, props, mesh: meshName };
-
-        for (const [k, v] of Object.entries(eBlock)) {
-          if (k === "connection" && typeof v === "object" && !Array.isArray(v)) {
-            op.connection = v;
-          } else if (k === "config" && typeof v === "object" && !Array.isArray(v)) {
-            op.config = v;
-          } else if (k === "template") {
-            op.template = v;
-          } else if (k === "select_columns") {
-            op.selectColumns = v;
-          } else if (k === "cast_changes") {
-            op.castChanges = Array.isArray(v) ? v : [v];
-          } else if (k === "schema" && Array.isArray(v)) {
-            op.schema = v;
-          } else if (k === "pipeline" && typeof v === "object" && !Array.isArray(v)) {
-            op.pipeline = parseYamlPipeline(v);
-          } else if (k === "secret" && typeof v === "object" && !Array.isArray(v)) {
-            op.secret = v;
-          } else if (REF_TO_TYPE[k]) {
-            const ref = parseYamlRef(v);
-            if (ref) refs[k] = ref;
-          } else if (!SPECIAL_KEYS.has(k)) {
-            props[k] = v;
-          }
-        }
-
-        if (refs.system) op.parent = refs.system;
-        else if (refs.source) op.parent = refs.source;
-        else if (refs.object) op.parent = refs.object;
+        operations.push(parseEntityBlock(entityType, entityName, eBlock, meshName));
+      }
+    }
+  }
 
-        operations.push(op);
+  // Handle root-level entity blocks with no mesh context (e.g. UUID-keyed updates)
+  if (meshEntries.length === 0) {
+    for (const entityType of ENTITY_ORDER.slice(1)) {
+      const group = rootEntityGroups[entityType];
+      if (!group || typeof group !== "object") continue;
+      for (const [entityName, eBlock] of Object.entries(group)) {
+        if (!eBlock || typeof eBlock !== "object" || Array.isArray(eBlock)) continue;
+        operations.push(parseEntityBlock(entityType, entityName, eBlock, null));
       }
     }
   }
@@ -591,12 +609,21 @@ async function deleteEntity(client, type, identifier) {
 /**
  * Search for an existing entity by type and name. Returns its identifier or null.
  */
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
 async function findEntity(client, type, name) {
+  // If name is a UUID, look up directly by identifier
+  if (UUID_RE.test(name)) {
+    try {
+      const res = await client.get(`/data/${type}/${name}`);
+      if (res?.identifier) return res.identifier;
+    } catch { /* fall through */ }
+  }
   try {
     // Use type-specific list endpoint for reliable lookup
     const res = await client.get(`/data/${type}/list?per_page=100`);
     const entities = parseListResponse(res);
-    const match = entities.find((e) => e.name === name);
+    const match = entities.find((e) => e.name === name || (UUID_RE.test(name) && e.identifier === name));
     if (match) return match.identifier;
   } catch { /* fall through */ }
   try {
@@ -891,7 +918,8 @@ async function runSubActions(client, op, uuid, refs) {
  */
 async function applyPipelineOp(client, op, uuid, refs) {
   const pipelineInputs = op.pipeline.inputs.map((inp) => {
-    const inputUuid = refs[inp.ref];
+    // Allow raw UUIDs as refs (e.g. from FCL patch) — no lookup needed
+    const inputUuid = refs[inp.ref] ?? (UUID_RE.test(inp.ref) ? inp.ref : null);
     if (!inputUuid) throw new Error(`unresolved pipeline input ref: ${inp.ref}`);
     return { input_type: inp.type, identifier: inputUuid };
   });
@@ -899,7 +927,7 @@ async function applyPipelineOp(client, op, uuid, refs) {
   // Build refToKey map for resolving transform "other" refs
   const refToKey = {};
   for (const inp of op.pipeline.inputs) {
-    const inputUuid = refs[inp.ref];
+    const inputUuid = refs[inp.ref] ?? (UUID_RE.test(inp.ref) ? inp.ref : null);
     if (inputUuid) refToKey[inp.ref] = `input_${inputUuid.replace(/-/g, "_")}`;
   }
 
@@ -931,6 +959,18 @@ async function applyPipelineOp(client, op, uuid, refs) {
     if (cols && cols.length > 0) slotCols[slot] = cols;
   }
   if (Object.keys(slotCols).length > 0) {
+    // Prune rename_column steps: remove entries whose source column doesn't exist
+    // in any slot (data is already named correctly — mapping was generated from a
+    // raw file whose column names differ from the ingested data object schema).
+    const allSlotCols = new Set(Object.values(slotCols).flat());
+    builderBody.transformations = (builderBody.transformations || []).flatMap((t) => {
+      if (t.transform !== "rename_column") return [t];
+      const changes = Object.fromEntries(
+        Object.entries(t.changes || {}).filter(([from]) => allSlotCols.has(from))
+      );
+      if (Object.keys(changes).length === 0) return []; // step is a no-op — drop it
+      return [{ ...t, changes }];
+    });
     const flowErr = validateTransformColumnFlow(slotCols, builderBody.transformations || []);
     if (flowErr) throw new Error(flowErr);
   }
@@ -998,7 +1038,9 @@ export async function applyLandscape(client, operations, opts = {}) {
     const existing = await findEntity(client, op.type, op.name);
     if (existing) {
       let recreateFallThrough = false;
-      if (recreateBroken && (op.type === "data_product" || op.type === "data_object" || op.type === "data_source")) {
+      // When op.name is a UUID, we're targeting an existing entity by identifier — never delete/recreate.
+      const opNameIsUuid = UUID_RE.test(op.name);
+      if (!opNameIsUuid && recreateBroken && (op.type === "data_product" || op.type === "data_object" || op.type === "data_source")) {
         const broken = await isEntityBroken(client, op.type, existing, op);
         if (broken) {
           const deleted = await deleteEntity(client, op.type, existing);

package/src/plugins/bundled/fops-plugin-foundation/lib/stack-apply.js

@@ -70,10 +70,13 @@ export function parseStackFile(filePath) {
   const parsed = yaml.load(content);
   if (!parsed) return { stack: null, hasStack: false, hasMesh: false };
 
+  const ENTITY_TYPES = ["data_system", "data_source", "data_object", "data_product", "application"];
+  const hasMesh = !!parsed.mesh || ENTITY_TYPES.some((t) => !!parsed[t]);
+
   return {
     stack: parsed.stack ? parseStackBlock(parsed.stack) : null,
     hasStack: !!parsed.stack,
-    hasMesh: !!parsed.mesh,
+    hasMesh,
   };
 }
 

package/src/plugins/bundled/fops-plugin-foundation-graphql/index.js

@@ -81,7 +81,7 @@ export function register(api) {
     },
   });
 
-  // ─── CLI Command: fops foundation query ──────────────────────────────────
+  // ─── CLI Commands: fops foundation graphql / query ───────────────────────
   api.registerCommand((program) => {
     // Find or create the 'foundation' parent command
     let foundation = program.commands.find((c) => c.name() === "foundation");
@@ -89,6 +89,44 @@ export function register(api) {
       foundation = program.command("foundation").description("Foundation platform operations");
     }
 
+    foundation
+      .command("graphql")
+      .description("Start a local GraphiQL explorer for the Foundation GraphQL API")
+      .option("--port <port>", "Port to listen on (default: random)", "0")
+      .action(async (opts) => {
+        try {
+          const facade = await getFacade();
+          const { createGraphQLRoute } = await import("./lib/graphql/hono-route.js");
+          const { serve } = await import("@hono/node-server");
+          const { exec } = await import("node:child_process");
+
+          const route = createGraphQLRoute(facade);
+          const port = parseInt(opts.port, 10) || 0;
+
+          const server = serve({ fetch: route.fetch, port }, (info) => {
+            const url = `http://127.0.0.1:${info.port}`;
+            console.log(chalk.cyan(`\n  ── Foundation GraphQL Explorer ${"─".repeat(18)}`));
+            console.log(`  ${chalk.green("✓")} Listening on ${chalk.bold(url)}`);
+            console.log(chalk.dim("  Press Ctrl+C to stop\n"));
+
+            // Open browser with platform-specific command
+            const opener =
+              process.platform === "darwin" ? "open" :
+              process.platform === "win32" ? "start" : "xdg-open";
+            exec(`${opener} ${url}`);
+          });
+
+          // Keep alive until Ctrl+C
+          await new Promise((resolve) => {
+            process.on("SIGINT", () => { server.close(); resolve(); });
+            process.on("SIGTERM", () => { server.close(); resolve(); });
+          });
+        } catch (err) {
+          console.error(chalk.red(`  ${err.message}`));
+          process.exitCode = 1;
+        }
+      });
+
     foundation
       .command("query <graphql>")
       .description("Execute an ad-hoc GraphQL query against the Foundation API")

package/src/plugins/bundled/fops-plugin-foundation-graphql/lib/graphql/resolvers/data-object.js

@@ -28,12 +28,15 @@ export const dataObjectResolvers = {
         // Normalize: the API may return { fields: [...] } or { schema: { fields: [...] } }
         const fields = res?.fields || res?.schema?.fields || [];
         return {
-          fields: fields.map((f) => ({
-            name: f.name || f.column_name,
-            dataType: f.data_type || f.dataType || f.type,
-            primary: f.primary ?? f.is_primary ?? false,
-            nullable: f.nullable ?? f.is_nullable ?? true,
-          })),
+          fields: fields.map((f) => {
+            const raw = f.data_type || f.dataType || f.type;
+            return {
+              name: f.name || f.column_name,
+              dataType: raw && typeof raw === "object" ? raw.column_type ?? null : raw ?? null,
+              primary: f.primary ?? f.is_primary ?? false,
+              nullable: f.nullable ?? f.is_nullable ?? true,
+            };
+          }),
         };
       } catch { return null; }
     },

package/src/plugins/bundled/fops-plugin-foundation-graphql/lib/graphql/resolvers/data-product.js

@@ -29,12 +29,15 @@ export const dataProductResolvers = {
         const res = await loaders.dataProductSchema.load(id);
         if (res instanceof Error) return null;
         const details = res?.details || res?.schema || res;
-        const columns = (res?.columns || res?.fields || []).map((f) => ({
-          name: f.name || f.column_name,
-          dataType: f.data_type || f.dataType || f.type,
-          primary: f.primary ?? f.is_primary ?? false,
-          nullable: f.nullable ?? f.is_nullable ?? true,
-        }));
+        const columns = (res?.columns || res?.fields || []).map((f) => {
+          const raw = f.data_type || f.dataType || f.type;
+          return {
+            name: f.name || f.column_name,
+            dataType: raw && typeof raw === "object" ? raw.column_type ?? null : raw ?? null,
+            primary: f.primary ?? f.is_primary ?? false,
+            nullable: f.nullable ?? f.is_nullable ?? true,
+          };
+        });
         return { details, columns };
       } catch { return null; }
     },