@meshxdata/fops 0.1.26 → 0.1.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meshxdata/fops",
-  "version": "0.1.26",
+  "version": "0.1.28",
   "description": "CLI to install and manage data mesh platforms",
   "keywords": [
     "fops",
@@ -32,6 +32,7 @@
     "changelog:bootstrap": "node scripts/generate-changelog.js --bootstrap",
     "prepublishOnly": "npm run changelog",
     "postpublish": "node scripts/pin-checksum.js",
+    "preinstall": "node -e \"if(+process.versions.node.split('.')[0]<20){console.error('\\n\\x1b[31mError: @meshxdata/fops requires Node.js >= 20.\\nYou are running Node.js '+process.version+'.\\nPlease upgrade: https://nodejs.org\\x1b[0m\\n');process.exit(1)}\"",
     "prepare": "cd .. && husky operator-cli/.husky"
   },
   "dependencies": {
@@ -72,7 +73,7 @@
     "access": "public"
   },
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "devDependencies": {
     "@tailwindcss/vite": "^4.1.18",

package/src/commands/lifecycle.js

@@ -22,8 +22,8 @@ function resolveServiceName(root, input) {
 export function registerLifecycleCommands(program, registry) {
   program
     .command("up")
-    .description("Start all Foundation services (docker compose up)")
-    .argument("[component]", "Component to run from a specific branch (backend, frontend, watcher, processor, scheduler, storage, hive, traefik, data)")
+    .description("Start Foundation services (all or specific component)")
+    .argument("[component]", "Component to start (backend, frontend, watcher, processor, scheduler, storage, hive, traefik, data). Add [branch] to checkout first.")
     .argument("[branch]", "Git branch to checkout in that component (e.g. FOU-2072)")
     .option("-d, --detach", "Run in background", true)
     .option("--k3s", "Include k3s Kubernetes services (default: true for local)", true)
@@ -51,13 +51,28 @@ export function registerLifecycleCommands(program, registry) {
 
   program
     .command("down")
-    .description("Stop all Foundation services")
+    .description("Stop Foundation services (all or specific)")
+    .argument("[services...]", "Optional: stop only these services (e.g. watcher, backend)")
     .option("--clean", "Remove volumes and orphans (make clean)")
-    .action(async (opts) => {
+    .action(async (services, opts) => {
       const root = requireRoot(program);
       const { execa } = await import("execa");
       await runHook(registry, "before:down", { root });
-      if (opts.clean) {
+
+      // Resolve short service names to docker compose service names
+      const serviceNames = (services || []).filter(Boolean).flatMap((s) => {
+        const sub = COMPONENT_SUBMODULES[s];
+        if (sub) return sub.restart;
+        return [s.startsWith("foundation-") ? s : `foundation-${s}`];
+      });
+
+      if (serviceNames.length) {
+        // Stop specific services only
+        console.log(chalk.cyan(`  Stopping ${serviceNames.join(", ")}...`));
+        await dockerCompose(root, ["stop", "-t", "5", ...serviceNames], { timeout: 60000 });
+        // Remove the stopped containers so they don't show as "exited"
+        await dockerCompose(root, ["rm", "-f", ...serviceNames], { timeout: 30000 });
+      } else if (opts.clean) {
         const result = await dockerCompose(root, ["down", "-v", "--remove-orphans", "-t", "5"], { timeout: 120000 });
         if (result?.timedOut) {
           console.log(chalk.yellow("  Compose down timed out — force-killing containers..."));
@@ -403,6 +418,11 @@ async function runUp(program, registry, opts) {
 
   // Optional: checkout a component submodule to a specific branch before starting (e.g. fops up backend FOU-2072)
   const comp = opts.component && COMPONENT_SUBMODULES[opts.component];
+  if (opts.component && !comp) {
+    console.error(chalk.red(`\n  Unknown component: ${opts.component}`));
+    console.error(chalk.dim(`  Available: ${Object.keys(COMPONENT_SUBMODULES).join(", ")}\n`));
+    process.exit(1);
+  }
   if (comp && opts.branch) {
     const componentDir = path.join(root, comp.dir);
     if (!fs.existsSync(path.join(componentDir, ".git"))) {
@@ -902,8 +922,8 @@ async function runUp(program, registry, opts) {
     startSpinner();
   }
 
-  // When fops up <component> <branch>, only pull and up that component (and deps) — avoid full stack rebuild
-  const componentOnlyUp = comp && opts.branch;
+  // When fops up <component> [branch], only pull and up that component — avoid full stack rebuild
+  const componentOnlyUp = !!comp;
 
   // Pull images only when --pull is passed (or use `fops pull` separately)
   if (opts.pull) {

package/src/doctor.js

@@ -558,7 +558,7 @@ export async function runDoctor(opts = {}, registry = null) {
     } else {
       fail("Claude CLI not found", "install globally: npm install -g @anthropic-ai/claude-code", async () => {
         console.log(chalk.cyan("  ▶ npm install -g @anthropic-ai/claude-code"));
-        await execa("npm", ["install", "-g", "@anthropic-ai/claude-code"], { stdio: "inherit", timeout: 300_000 });
+        await execa("npm", ["install", "-g", "--loglevel=error", "@anthropic-ai/claude-code"], { stdio: "inherit", timeout: 300_000 });
       });
     }
   }

package/src/plugins/bundled/fops-plugin-embeddings/index.js

@@ -483,6 +483,7 @@ export default {
             let landscapeResults;
             let trinoTables = [];
             let trinoTableInfo = new Map(); // table name → { catalog, schema }
+            const productTableMap = new Map(); // data_product identifier → trino table name (explicit from API)
 
             // Helper: load trino table info from cache into trinoTables + trinoTableInfo
             const loadTrinoFromCache = () => {
@@ -657,6 +658,43 @@ export default {
                 const tEdges = Date.now();
                 if (meshes.length > 0) process.stderr.write(`\r${chalk.dim(`Fetching edges... ${landscapeDone}/${meshes.length} (${((tEdges - tEdgesStart) / 1000).toFixed(1)}s)`)}\n`);
 
+                // Fetch explicit trino table mappings from data product details.
+                // The landscape endpoint doesn't include table info, so we fetch each product's detail.
+                {
+                  const dpIdentifiers = new Set();
+                  for (const { edges } of landscapeResults) {
+                    for (const e of edges) {
+                      if (e.parent?.entity_type === "data_product") dpIdentifiers.add(e.parent.identifier);
+                      if (e.child?.entity_type === "data_product") dpIdentifiers.add(e.child.identifier);
+                    }
+                  }
+                  if (dpIdentifiers.size > 0) {
+                    const DP_CONCURRENCY = 10;
+                    const dpIds = [...dpIdentifiers];
+                    for (let i = 0; i < dpIds.length; i += DP_CONCURRENCY) {
+                      const batch = dpIds.slice(i, i + DP_CONCURRENCY);
+                      const results = await Promise.all(
+                        batch.map((id) =>
+                          client.get(`/data/data_product/${id}`)
+                            .then((res) => ({ id, table: res?.table }))
+                            .catch(() => ({ id, table: null })),
+                        ),
+                      );
+                      for (const { id, table } of results) {
+                        if (table?.table_name) {
+                          // table_name from API is the fully-qualified name (schema.table)
+                          const parts = table.table_name.split(".");
+                          const shortName = parts[parts.length - 1];
+                          productTableMap.set(id, shortName);
+                        }
+                      }
+                    }
+                    if (productTableMap.size > 0) {
+                      process.stderr.write(chalk.dim(`Resolved ${productTableMap.size}/${dpIdentifiers.size} product→table mappings\n`));
+                    }
+                  }
+                }
+
                 // Best-effort Trino table discovery so landscape can show product->table relationships.
                 // Supports --trino-url / TRINO_URL for HTTP-based discovery, falls back to docker compose exec.
                 // Auto-discovers ALL schemas in the catalog unless TRINO_SCHEMA is explicitly set.
@@ -787,8 +825,20 @@ export default {
               .replace(/[^a-z0-9]+/g, "_")
               .replace(/^_+|_+$/g, "");
             const preMatchedTrino = new Set();
+            // Build reverse map: normalized trino table name → original name (for explicit matching)
+            const explicitlyLinked = new Map(); // entity identifier → trino table name
             if (trinoTables.length > 0) {
-              // Collect entity names from landscape edges (products, objects, sources)
+              // 1. Explicit matches from API-fetched product→table mappings
+              for (const [dpId, tableName] of productTableMap) {
+                const tn = normName(tableName);
+                const match = trinoTables.find((t) => normName(t) === tn);
+                if (match) {
+                  preMatchedTrino.add(match);
+                  explicitlyLinked.set(dpId, match);
+                }
+              }
+
+              // 2. Name-based fallback for entities without explicit mapping
               const PRE_LINKABLE = new Set(["data_product", "data_object", "data_source"]);
               const allEntityNames = new Set();
               for (const { edges } of landscapeResults) {
@@ -958,15 +1008,40 @@ export default {
                 return true;
               });
 
-              // Add synthetic entity -> trino_table edges by name similarity.
-              // Match against data products, data objects, and data sources.
+              // Add synthetic entity -> trino_table edges.
+              // 1. Explicit mappings from API (productTableMap) take priority.
+              // 2. Fall back to name-based similarity for remaining entities.
               const LINKABLE_TYPES = new Set(["data_product", "data_object", "data_source"]);
               const linkableNodes = new Map();
               for (const e of unique) {
                 if (LINKABLE_TYPES.has(e.parent?.entity_type)) linkableNodes.set(e.parent.identifier, e.parent);
                 if (LINKABLE_TYPES.has(e.child?.entity_type)) linkableNodes.set(e.child.identifier, e.child);
               }
+              const addTrinoEdge = (node, table) => {
+                matchedTrinoTables.add(table);
+                const tNode = {
+                  identifier: `trino:${table}`,
+                  name: table,
+                  entity_type: "trino_table",
+                  state: { healthy: true },
+                };
+                const edge = { parent: node, child: tNode };
+                const key = `${edge.parent.identifier}:${edge.child.identifier}`;
+                if (!seen.has(key)) {
+                  seen.add(key);
+                  unique.push(edge);
+                }
+              };
+              const explicitlyLinkedNodes = new Set();
               for (const node of linkableNodes.values()) {
+                // Check explicit API mapping first
+                const explicitTable = explicitlyLinked.get(node.identifier);
+                if (explicitTable) {
+                  addTrinoEdge(node, explicitTable);
+                  explicitlyLinkedNodes.add(node.identifier);
+                  continue;
+                }
+                // Fall back to name-based matching
                 const nNorm = normName(node.name);
                 if (!nNorm) continue;
                 const matches = trinoTables.filter((t) => {
@@ -974,19 +1049,7 @@ export default {
                   return tn === nNorm || tn.startsWith(`${nNorm}_`) || nNorm.startsWith(`${tn}_`);
                 });
                 for (const table of matches) {
-                  matchedTrinoTables.add(table);
-                  const tNode = {
-                    identifier: `trino:${table}`,
-                    name: table,
-                    entity_type: "trino_table",
-                    state: { healthy: true },
-                  };
-                  const edge = { parent: node, child: tNode };
-                  const key = `${edge.parent.identifier}:${edge.child.identifier}`;
-                  if (!seen.has(key)) {
-                    seen.add(key);
-                    unique.push(edge);
-                  }
+                  addTrinoEdge(node, table);
                 }
               }
               totalEdges += unique.length;
@@ -1037,13 +1100,32 @@ export default {
 
                 // --deep: show columns and sample under trino_table nodes
                 if (meta && meta.columns.length > 0) {
-                  const colLine = meta.columns.map((c) => `${c.name}${chalk.dim(`:${c.type}`)}`).join(", ");
-                  console.log(`${childPrefix}${chalk.dim("columns: " + colLine)}`);
-                  if (meta.sample && meta.sample.rows.length > 0) {
-                    for (let r = 0; r < meta.sample.rows.length; r++) {
-                      const row = meta.sample.rows[r];
-                      const pairs = meta.sample.columns.map((col, ci) => `${col}=${chalk.white(row[ci] ?? "NULL")}`).join(", ");
-                      console.log(`${childPrefix}${chalk.dim(`  row ${r + 1}: ${pairs}`)}`);
+                  // Detect Foundation meta tables (_*_expectations, _*_profiling, _*_validations)
+                  const isMeta = e.entity_type === "trino_table" &&
+                    /_(expectations|profiling|validations)$/.test(e.name);
+
+                  if (isMeta) {
+                    // Meta tables: just show column count inline, no sample data
+                    const colNames = meta.columns.slice(0, 4).map((c) => c.name).join(", ");
+                    const more = meta.columns.length > 4 ? `, +${meta.columns.length - 4}` : "";
+                    console.log(`${childPrefix}${chalk.dim(`${colNames}${more}`)}`);
+                  } else {
+                    // Main data tables: show columns and sample rows
+                    const shortType = (t) => t.replace(/timestamp\(\d+\) with time zone/g, "timestamp(tz)")
+                      .replace(/character varying/g, "varchar");
+                    const colLine = meta.columns.map((c) => `${c.name}${chalk.dim(`:${shortType(c.type)}`)}`).join(", ");
+                    console.log(`${childPrefix}${chalk.dim("cols: " + colLine)}`);
+                    if (meta.sample && meta.sample.rows.length > 0) {
+                      const truncVal = (v, max = 40) => {
+                        if (v == null) return "NULL";
+                        const s = String(v);
+                        return s.length > max ? s.slice(0, max - 1) + "…" : s;
+                      };
+                      for (let r = 0; r < meta.sample.rows.length; r++) {
+                        const row = meta.sample.rows[r];
+                        const pairs = meta.sample.columns.map((col, ci) => `${col}=${chalk.white(truncVal(row[ci]))}`).join("  ");
+                        console.log(`${childPrefix}${chalk.dim(`  row ${r + 1}:`)} ${pairs}`);
+                      }
                     }
                   }
                 }

package/src/setup/wizard.js

@@ -67,9 +67,9 @@ async function installTool(name, { brew, brewCask, winget, apt, npm: npmPkg } =
       const cmd = needsSudo ? `sudo npm install -g ${pkg}` : `npm install -g ${pkg}`;
       console.log(ACCENT(`  ▶ ${cmd}`));
       if (needsSudo) {
-        await execa("sudo", ["npm", "install", "-g", pkg], { stdio: "inherit", timeout: 300_000 });
+        await execa("sudo", ["npm", "install", "-g", "--loglevel=error", pkg], { stdio: "inherit", timeout: 300_000 });
       } else {
-        await execa("npm", ["install", "-g", pkg], { stdio: "inherit", timeout: 300_000 });
+        await execa("npm", ["install", "-g", "--loglevel=error", pkg], { stdio: "inherit", timeout: 300_000 });
       }
       return true;
     }