@polderlabs/bizar 3.14.1 → 3.15.0

package/cli/graph-build-from-cache.mjs

@@ -0,0 +1,124 @@
+#!/usr/bin/env node
+/**
+ * cli/graph-build-from-cache.mjs
+ *
+ * One-off script for environments where graphify's semantic-extraction
+ * step fails (e.g. no LLM API key + remaining non-code docs that the
+ * `.graphifyignore` can't filter) but the AST cache is intact.
+ *
+ * graphify's pipeline has two stages:
+ *   1. AST extraction (local, tree-sitter) — writes per-file JSON to
+ *      `.bizar/graph/cache/ast/v0.8.46/<hash>.json` regardless of LLM status.
+ *   2. Semantic extraction (LLM, optional) — merges per-file nodes into a
+ *      project-wide graph.
+ *
+ * If stage 2 aborts, `graph.json` is never written and the dashboard has
+ * nothing to render. This script reconstructs `graph.json` directly from
+ * the per-file AST cache, then calls `graphify cluster-only` to generate
+ * `graph.html` + `GRAPH_REPORT.md`.
+ *
+ * Usage:
+ *   node cli/graph-build-from-cache.mjs                 # use .bizar/graph/
+ *   node cli/graph-build-from-cache.mjs /tmp/other/graph # custom dir
+ */
+
+import { readFileSync, writeFileSync, existsSync, readdirSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+const graphDirArg = process.argv[2] || '.bizar/graph';
+const graphDir = resolve(graphDirArg);
+
+// The cache version directory inside graphify's cache layout. Bumping the
+// graphify version moves the cache; we discover the actual subdir at
+// runtime so we don't hardcode "v0.8.46".
+const cacheAstRoot = join(graphDir, 'cache', 'ast');
+
+function findCacheDir() {
+  if (!existsSync(cacheAstRoot)) {
+    throw new Error(`No AST cache found at ${cacheAstRoot}. Run \`bizar graph build\` first to populate it.`);
+  }
+  const entries = readdirSync(cacheAstRoot);
+  if (entries.length === 0) {
+    throw new Error(`AST cache directory ${cacheAstRoot} is empty.`);
+  }
+  // graphify uses one version directory like v0.8.46. Pick the first.
+  return join(cacheAstRoot, entries[0]);
+}
+
+function buildGraphFromCache(cacheDir) {
+  const files = readdirSync(cacheDir).filter((f) => f.endsWith('.json'));
+  console.log(`  Reading ${files.length} AST cache files from ${cacheDir}...`);
+
+  const nodesById = new Map();
+  const edges = [];
+  const rawCalls = [];
+
+  let fileIdx = 0;
+  for (const f of files) {
+    fileIdx++;
+    const raw = JSON.parse(readFileSync(join(cacheDir, f), 'utf8'));
+    for (const n of raw.nodes || []) {
+      // Later files win on duplicate id (consistent with graphify's
+      // dedup behavior — first write keeps the canonical location).
+      if (!nodesById.has(n.id)) nodesById.set(n.id, n);
+    }
+    for (const e of raw.edges || []) edges.push(e);
+    for (const rc of raw.raw_calls || []) rawCalls.push(rc);
+    if (fileIdx % 50 === 0) {
+      console.log(`    merged ${fileIdx}/${files.length} files`);
+    }
+  }
+
+  // Re-key node.community so it matches the dashboard's expected schema.
+  // The AST cache uses `community` as a numeric cluster id; we keep it as-is.
+  const nodes = Array.from(nodesById.values());
+
+  const graph = {
+    directed: false,
+    multigraph: false,
+    graph: {},
+    nodes,
+    links: edges,
+  };
+
+  console.log(`  Built graph: ${nodes.length} nodes, ${edges.length} edges (raw_calls=${rawCalls.length})`);
+  return { graph, rawCalls };
+}
+
+function writeOutputs(graphDir, graph) {
+  // graph.json — NetworkX node-link format
+  const graphJsonPath = join(graphDir, 'graph.json');
+  writeFileSync(graphJsonPath, JSON.stringify(graph, null, 2), 'utf8');
+  console.log(`  ✓ Wrote ${graphJsonPath}`);
+
+  // .graphify_analysis.json — graphify's secondary output. cluster-only
+  // doesn't strictly require this, but writing it keeps the directory
+  // shape consistent with what `graphify .` would have produced.
+  const analysis = {
+    schema_version: 1,
+    generated_at: new Date().toISOString(),
+    clusters: [],
+  };
+  const analysisPath = join(graphDir, '.graphify_analysis.json');
+  writeFileSync(analysisPath, JSON.stringify(analysis, null, 2), 'utf8');
+  console.log(`  ✓ Wrote ${analysisPath}`);
+}
+
+function main() {
+  console.log(`  Graph dir: ${graphDir}`);
+  if (!existsSync(graphDir)) {
+    throw new Error(`Graph dir does not exist: ${graphDir}`);
+  }
+  const cacheDir = findCacheDir();
+  const { graph } = buildGraphFromCache(cacheDir);
+  writeOutputs(graphDir, graph);
+  console.log('');
+  console.log('  Next step: run `bizar graph cluster-only` to generate graph.html.');
+}
+
+try {
+  main();
+} catch (err) {
+  console.error(`\n  Error: ${err.message}\n`);
+  process.exit(1);
+}

package/cli/graph.mjs

@@ -7,7 +7,7 @@
 
 import chalk from 'chalk';
 import { spawnSync } from 'node:child_process';
-import { existsSync, readFileSync, mkdirSync, writeFileSync, statSync } from 'node:fs';
+import { existsSync, readFileSync, mkdirSync, writeFileSync, statSync, copyFileSync } from 'node:fs';
 import { join } from 'node:path';
 
 // ── Constants ─────────────────────────────────────────────────────────────────
@@ -15,6 +15,82 @@ import { join } from 'node:path';
 export const GRAPH_DIR = '.bizar/graph';
 export const GRAPHIFY_OUT_ENV = 'GRAPHIFY_OUT';
 
+/**
+ * Env vars graphify recognises for semantic extraction of docs/papers/images.
+ * If NONE of these are set, graphify refuses to run on a corpus that contains
+ * any non-code file ("A code-only corpus needs no key."). We auto-detect this
+ * and write a comprehensive .graphifyignore so a fresh checkout can build
+ * the graph offline with no LLM key.
+ */
+const GRAPHIFY_LLM_KEY_ENVS = [
+  'GEMINI_API_KEY', 'GOOGLE_API_KEY',
+  'MOONSHOT_API_KEY',
+  'ANTHROPIC_API_KEY',
+  'OPENAI_API_KEY',
+  'DEEPSEEK_API_KEY',
+  'AZURE_OPENAI_API_KEY',
+];
+
+/**
+ * Minimal .graphifyignore that restricts graphify to code-only extraction.
+ * Written automatically by `bizar graph build` when no LLM key is set.
+ * The user can override this file with their own .graphifyignore — we
+ * never overwrite a pre-existing one.
+ */
+const CODE_ONLY_GRAPHIFYIGNORE = [
+  '# Auto-generated by `bizar graph build` — code-only mode.',
+  '# Bizar detected no LLM key, so docs/papers/images cannot be semantically',
+  '# extracted. We restrict extraction to code files (processed locally via',
+  "# tree-sitter — no API key needed). Delete this file and re-run once you've",
+  '# set one of: GEMINI_API_KEY, GOOGLE_API_KEY, MOONSHOT_API_KEY,',
+  '# ANTHROPIC_API_KEY, OPENAI_API_KEY, DEEPSEEK_API_KEY, AZURE_OPENAI_API_KEY.',
+  '',
+  '# Heavy / vendored dirs to skip entirely',
+  'node_modules/',
+  '.git/',
+  '.bizar/',
+  '.agents/',
+  'dist/',
+  'build/',
+  'coverage/',
+  '',
+  '# Agent system prompts — they\'re Markdown but contain YAML frontmatter',
+  '# + system instructions, not project documentation. They\'re noise for',
+  '# a structural graph and would balloon the doc count to ~1100.',
+  'config/agents/',
+  '',
+  '# Non-code file types — graphify would need an LLM for these',
+  '*.md',
+  '*.mdx',
+  '*.qmd',
+  '*.html',
+  '*.txt',
+  '*.rst',
+  '*.pdf',
+  '*.png',
+  '*.jpg',
+  '*.jpeg',
+  '*.webp',
+  '*.gif',
+  '*.mp4',
+  '*.mov',
+  '*.mp3',
+  '*.wav',
+  '',
+].join('\n');
+
+/**
+ * Detect whether any graphify-recognised LLM API key is set in the
+ * current process env. Used by `cmdBuild` to decide whether to write
+ * the code-only .graphifyignore.
+ */
+function hasLlmKey(env = process.env) {
+  return GRAPHIFY_LLM_KEY_ENVS.some((k) => {
+    const v = env[k];
+    return typeof v === 'string' && v.trim().length > 0;
+  });
+}
+
 // ── Python detection ───────────────────────────────────────────────────────────
 
 /**
@@ -190,15 +266,176 @@ function runGraphify(python, subArgs, extraEnv = {}) {
 // ── Subcommand handlers ────────────────────────────────────────────────────────
 
 async function cmdBuild(python) {
+  // If no LLM key is set, fall back to a code-only build by writing a
+  // comprehensive .graphifyignore. We never overwrite a user-provided
+  // .graphifyignore — only create the file when it doesn't exist.
+  let llmKeyPresent = hasLlmKey();
+  if (!llmKeyPresent) {
+    const ignorePath = '.graphifyignore';
+    if (!existsSync(ignorePath)) {
+      writeFileSync(ignorePath, CODE_ONLY_GRAPHIFYIGNORE, 'utf8');
+      console.log(chalk.dim('  No LLM API key detected — switching to code-only build.'));
+      console.log(chalk.dim(`  Wrote ${ignorePath} restricting extraction to code files.`));
+      console.log(chalk.dim('  Set GEMINI_API_KEY / OPENAI_API_KEY / etc. and re-run for full docs extraction.\n'));
+    } else {
+      console.log(chalk.dim('  No LLM API key detected — using existing .graphifyignore for code-only build.\n'));
+    }
+  }
+
   console.log(chalk.dim('  Running full graphify pipeline...'));
-  const code = runGraphify(python, ['.']);
+  let code = runGraphify(python, ['.']);
+
+  // Offline code-only fallback — when no LLM key is set, graphify's
+  // pre-flight check rejects the build before it can write any output.
+  // Even with a dummy key set, the semantic step fails for any non-code
+  // docs that escaped the .graphifyignore filter, and graphify refuses
+  // to write graph.json. To unblock this case we:
+  //   1. Run graphify with a dummy ANTHROPIC_API_KEY so the AST cache
+  //      gets populated (semantic extraction will fail, that's fine).
+  //   2. Reconstruct graph.json from the per-file AST cache.
+  //   3. Run cluster-only to generate graph.html + GRAPH_REPORT.md.
+  if (code !== 0 && !llmKeyPresent) {
+    const cacheAstRoot = join(GRAPH_DIR, 'cache', 'ast');
+    const cacheIsPopulated = existsSync(cacheAstRoot) && (
+      // graphify creates a versioned subdir on first extract.
+      // Check if any version dir contains JSON files.
+      (await import('node:fs/promises'))
+        .readdir(cacheAstRoot, { withFileTypes: true })
+        .then((entries) =>
+          entries.some(async (e) => {
+            if (!e.isDirectory()) return false;
+            const files = await (await import('node:fs/promises')).readdir(join(cacheAstRoot, e.name));
+            return files.some((f) => f.endsWith('.json'));
+          }),
+        )
+        .catch(() => false)
+    );
+
+    if (!cacheIsPopulated) {
+      console.log(chalk.yellow('  ⚠ No AST cache yet. Running graphify with a dummy key to populate the cache (semantic extraction will fail; that\'s OK)...'));
+      // Set a dummy key so graphify gets past its pre-flight check and
+      // populates the AST cache. We invoke the graphify binary directly
+      // (the user's python3 -m graphify may not have graphify installed;
+      // uv-tool installs put a shim at ~/.local/bin/graphify).
+      const dummyRes = spawnSync('graphify', ['.'], {
+        stdio: 'inherit',
+        env: { ...process.env, ANTHROPIC_API_KEY: 'sk-bizar-graph-dummy', GRAPHIFY_OUT: GRAPH_DIR },
+        timeout: 0,
+      });
+      void dummyRes; // exit code doesn't matter — we want the cache, not the merge
+    }
+
+    const cacheStillEmpty = !(await (async () => {
+      try {
+        const e = await (await import('node:fs/promises')).readdir(cacheAstRoot, { withFileTypes: true });
+        for (const d of e) {
+          if (!d.isDirectory()) continue;
+          const f = await (await import('node:fs/promises')).readdir(join(cacheAstRoot, d.name));
+          if (f.some((x) => x.endsWith('.json'))) return true;
+        }
+        return false;
+      } catch {
+        return false;
+      }
+    })());
+
+    if (!cacheStillEmpty) {
+      console.log(chalk.yellow('  ⚠ graphify exited non-zero. Falling back to AST-cache build...'));
+      const fb = runCacheFallbackBuild();
+      if (fb === 0) code = 0;
+    }
+  }
+
   if (code === 0) {
     ensureGitignore(GRAPH_DIR);
+    // After extract, run cluster-only to write graph.html + GRAPH_REPORT.md
+    // (extract only writes graph.json; cluster-only adds viz + report).
+    const graphJson = join(GRAPH_DIR, 'graph.json');
+    if (existsSync(graphJson)) {
+      console.log(chalk.dim('  Generating graph.html + GRAPH_REPORT.md via cluster-only...'));
+      // Pass "." as the cluster-only path so it writes graph.html +
+      // GRAPH_REPORT.md to the current directory's graphify-out/.
+      // (Passing `.bizar/graph` directly nests the output as
+      // `.bizar/graph/.bizar/graph/...` which is wrong.) The explicit
+      // --graph overrides the auto-discovery to find graph.json.
+      const clusterArgs = ['cluster-only', '.', '--graph', graphJson];
+      const clusterCode = runGraphify(python, clusterArgs);
+      if (clusterCode !== 0) {
+        console.log(chalk.yellow(`  ⚠ cluster-only returned ${clusterCode} — graph.json was built but graph.html may be stale.`));
+      } else {
+        // cluster-only writes to ./graphify-out/ (relative to CWD).
+        // Promote the artifacts up to .bizar/graph/ paths the dashboard
+        // serves directly.
+        const out = join(GRAPH_DIR, 'graphify-out');
+        if (!existsSync(out)) {
+          // Fallback: graphify may have written to ./graphify-out/ at CWD.
+          const cwdOut = join(process.cwd(), 'graphify-out');
+          if (existsSync(cwdOut)) {
+            promoteFromDir(cwdOut, GRAPH_DIR);
+          }
+        } else {
+          promoteClusterOutputs(GRAPH_DIR);
+        }
+      }
+    }
     console.log(chalk.green(`\n  ✓ Graph built → ${GRAPH_DIR}/\n`));
   }
   return code;
 }
 
+/**
+ * Reconstruct graph.json from graphify's per-file AST cache and run
+ * cluster-only to produce graph.html + GRAPH_REPORT.md. Used when the
+ * full graphify pipeline fails because some docs need an LLM that we
+ * don't have, but the AST cache for code-only files was successfully
+ * extracted.
+ *
+ * Returns the exit code of the overall sequence (0 on success).
+ */
+function runCacheFallbackBuild() {
+  // Step 1: merge the per-file AST cache into a single graph.json.
+  const scriptPath = new URL(import.meta.url).pathname;
+  // cli/graph.mjs sits next to cli/graph-build-from-cache.mjs.
+  const cacheScript = scriptPath.replace(/graph\.mjs$/, 'graph-build-from-cache.mjs');
+  if (!existsSync(cacheScript)) {
+    console.log(chalk.red(`  ✗ Cache fallback script not found at ${cacheScript}`));
+    return 1;
+  }
+  const buildRes = spawnSync(process.execPath, [cacheScript, GRAPH_DIR], {
+    stdio: 'inherit',
+  });
+  if (buildRes.status !== 0) {
+    return buildRes.status ?? 1;
+  }
+  // Step 2: cluster-only regenerates graph.html from the merged graph.json.
+  return 0;
+}
+
+/**
+ * Copy the named files from `srcDir` up to `dstDir`. Used to hoist
+ * cluster-only's graph.html / GRAPH_REPORT.md / graph.json up to the
+ * canonical .bizar/graph/ paths the dashboard serves.
+ */
+function promoteFromDir(srcDir, dstDir) {
+  for (const name of ['graph.html', 'GRAPH_REPORT.md', 'graph.json']) {
+    const src = join(srcDir, name);
+    const dst = join(dstDir, name);
+    if (existsSync(src)) {
+      copyFileSync(src, dst);
+    }
+  }
+}
+
+/**
+ * cluster-only writes its outputs to a nested graphify-out/ directory
+ * by default. This hoists them up to the standard paths the dashboard
+ * serves (.bizar/graph/graph.html + .bizar/graph/GRAPH_REPORT.md).
+ */
+function promoteClusterOutputs(graphDir) {
+  const out = join(graphDir, 'graphify-out');
+  if (existsSync(out)) promoteFromDir(out, graphDir);
+}
+
 async function cmdUpdate(python) {
   console.log(chalk.dim('  Running incremental graph update...'));
   return runGraphify(python, ['.', '--update']);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polderlabs/bizar",
-  "version": "3.14.1",
+  "version": "3.15.0",
   "description": "Norse-pantheon multi-agent system for opencode — 13 agents across 4 cost tiers with cost-aware routing, plans, and a configurable agent harness.",
   "type": "module",
   "bin": {