@harness-engineering/graph 0.7.1 → 0.8.0

package/dist/index.mjs

@@ -91,7 +91,7 @@ var EDGE_TYPES = [
   "caches"
 ];
 var OBSERVABILITY_TYPES = /* @__PURE__ */ new Set(["span", "metric", "log"]);
-var CURRENT_SCHEMA_VERSION = 1;
+var CURRENT_SCHEMA_VERSION = 2;
 var NODE_STABILITY = {
   File: "session",
   Function: "session",
@@ -129,39 +129,100 @@ var GraphEdgeSchema = z.object({
 
 // src/store/Serializer.ts
 import { readFile, writeFile, mkdir, access } from "fs/promises";
-import { createWriteStream } from "fs";
+import { createReadStream, createWriteStream } from "fs";
+import { createInterface } from "readline";
 import { join } from "path";
-function streamGraphJson(filePath, nodes, edges) {
+function streamGraphNdjson(filePath, nodes, edges) {
   return new Promise((resolve, reject) => {
     const stream = createWriteStream(filePath, { encoding: "utf-8" });
     stream.on("error", reject);
-    stream.write('{"nodes":[');
-    for (let i = 0; i < nodes.length; i++) {
-      if (i > 0) stream.write(",");
-      stream.write(JSON.stringify(nodes[i]));
+    for (const node of nodes) {
+      stream.write(JSON.stringify({ kind: "node", ...node }));
+      stream.write("\n");
     }
-    stream.write('],"edges":[');
-    for (let i = 0; i < edges.length; i++) {
-      if (i > 0) stream.write(",");
-      stream.write(JSON.stringify(edges[i]));
+    for (const edge of edges) {
+      stream.write(JSON.stringify({ kind: "edge", ...edge }));
+      stream.write("\n");
     }
-    stream.write("]}");
     stream.end(() => resolve());
   });
 }
+function countNodesByType(nodes) {
+  const counts = {};
+  for (const node of nodes) {
+    counts[node.type] = (counts[node.type] ?? 0) + 1;
+  }
+  return counts;
+}
 async function saveGraph(dirPath, nodes, edges) {
   await mkdir(dirPath, { recursive: true });
   const metadata = {
     schemaVersion: CURRENT_SCHEMA_VERSION,
     lastScanTimestamp: (/* @__PURE__ */ new Date()).toISOString(),
     nodeCount: nodes.length,
-    edgeCount: edges.length
+    edgeCount: edges.length,
+    nodesByType: countNodesByType(nodes)
   };
   await Promise.all([
-    streamGraphJson(join(dirPath, "graph.json"), nodes, edges),
+    streamGraphNdjson(join(dirPath, "graph.json"), nodes, edges),
     writeFile(join(dirPath, "metadata.json"), JSON.stringify(metadata, null, 2), "utf-8")
   ]);
 }
+async function loadGraphMetadata(dirPath) {
+  const metaPath = join(dirPath, "metadata.json");
+  try {
+    await access(metaPath);
+  } catch {
+    return { status: "not_found" };
+  }
+  let metadata;
+  try {
+    const content = await readFile(metaPath, "utf-8");
+    metadata = JSON.parse(content);
+  } catch {
+    return { status: "not_found" };
+  }
+  if (metadata.schemaVersion !== CURRENT_SCHEMA_VERSION) {
+    return {
+      status: "schema_mismatch",
+      found: metadata.schemaVersion,
+      expected: CURRENT_SCHEMA_VERSION
+    };
+  }
+  return { status: "loaded", metadata };
+}
+async function streamReadGraph(filePath) {
+  const stream = createReadStream(filePath, { encoding: "utf-8" });
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  const nodes = [];
+  const edges = [];
+  try {
+    for await (const rawLine of rl) {
+      const line = rawLine.trim();
+      if (line === "") continue;
+      const record = JSON.parse(line);
+      if (record.kind === "node") {
+        const { kind: _kind, ...rest } = record;
+        nodes.push(rest);
+      } else if (record.kind === "edge") {
+        const { kind: _kind, ...rest } = record;
+        edges.push(rest);
+      }
+    }
+  } catch (err) {
+    if (err instanceof RangeError && /invalid string length/i.test(err.message)) {
+      throw new Error(
+        `Graph contains a record larger than V8 can hold in a single string (~512 MB). This usually means a node or edge has a multi-hundred-MB content/embedding field. Inspect ${filePath} for the offending record.`,
+        { cause: err }
+      );
+    }
+    throw err;
+  } finally {
+    rl.close();
+    stream.close();
+  }
+  return { nodes, edges };
+}
 async function loadGraph(dirPath) {
   const metaPath = join(dirPath, "metadata.json");
   const graphPath = join(dirPath, "graph.json");
@@ -180,8 +241,8 @@ async function loadGraph(dirPath) {
       expected: CURRENT_SCHEMA_VERSION
     };
   }
-  const graphContent = await readFile(graphPath, "utf-8");
-  return { status: "loaded", graph: JSON.parse(graphContent) };
+  const graph = await streamReadGraph(graphPath);
+  return { status: "loaded", graph };
 }
 
 // src/store/GraphStore.ts
@@ -727,6 +788,84 @@ function groupNodesByImpact(nodes, excludeId) {
 // src/ingest/CodeIngestor.ts
 import * as fs from "fs/promises";
 import * as path from "path";
+import { minimatch } from "minimatch";
+
+// src/ingest/skip-dirs.ts
+var DEFAULT_SKIP_DIRS = /* @__PURE__ */ new Set([
+  // VCS
+  ".git",
+  ".hg",
+  ".svn",
+  // Package managers
+  "node_modules",
+  ".pnpm-store",
+  ".yarn",
+  "vendor",
+  // JS/TS build outputs
+  "dist",
+  "build",
+  "out",
+  "_build",
+  "bin",
+  "obj",
+  "target",
+  "deps",
+  // JS/TS framework / tooling caches
+  ".next",
+  ".nuxt",
+  ".svelte-kit",
+  ".turbo",
+  ".vite",
+  ".cache",
+  ".parcel-cache",
+  ".docusaurus",
+  ".wrangler",
+  ".astro",
+  ".remix",
+  "storybook-static",
+  // Test / coverage / reporter outputs
+  "coverage",
+  ".nyc_output",
+  ".pytest_cache",
+  "playwright-report",
+  "test-results",
+  ".e2e",
+  // Python
+  "__pycache__",
+  ".venv",
+  "venv",
+  ".tox",
+  ".mypy_cache",
+  ".ruff_cache",
+  // JVM
+  ".gradle",
+  ".gradle-home",
+  // IDE / editor
+  ".idea",
+  ".vscode",
+  ".vs",
+  // Harness self
+  ".harness",
+  // AI agent sandboxes (these often contain full repo clones — skipping them
+  // prevents the walker from re-ingesting nested copies of the project)
+  ".claude",
+  ".cursor",
+  ".codex",
+  ".gemini",
+  ".aider",
+  ".agents",
+  ".agentastic",
+  ".playwright-mcp"
+]);
+function resolveSkipDirs(options) {
+  const base = options?.skipDirs ? new Set(options.skipDirs) : new Set(DEFAULT_SKIP_DIRS);
+  if (options?.additionalSkipDirs) {
+    for (const name of options.additionalSkipDirs) base.add(name);
+  }
+  return base;
+}
+
+// src/ingest/CodeIngestor.ts
 var SKIP_METHOD_NAMES = /* @__PURE__ */ new Set(["constructor", "if", "for", "while", "switch"]);
 var FUNCTION_PATTERNS = {
   python: /^def\s+(\w+)\s*\(/,
@@ -796,11 +935,55 @@ function isSupportedSourceFile(name) {
   const ext = name.slice(name.lastIndexOf("."));
   return SUPPORTED_EXTENSIONS.has(ext);
 }
+function isExcluded(relPath, patterns) {
+  if (patterns.length === 0) return false;
+  for (const pattern of patterns) {
+    if (minimatch(relPath, pattern, { dot: true, matchBase: false })) return true;
+    if (!pattern.includes("/") && minimatch(relPath, pattern, { dot: true, matchBase: true })) {
+      return true;
+    }
+  }
+  return false;
+}
+async function readGitignorePatterns(rootDir) {
+  let content;
+  try {
+    content = await fs.readFile(path.join(rootDir, ".gitignore"), "utf-8");
+  } catch {
+    return [];
+  }
+  const patterns = [];
+  for (const rawLine of content.split(/\r?\n/)) {
+    const line = rawLine.trim();
+    if (line === "" || line.startsWith("#")) continue;
+    if (line.startsWith("!")) continue;
+    const rooted = line.startsWith("/");
+    const stripped = rooted ? line.slice(1) : line;
+    const isDirOnly = stripped.endsWith("/");
+    const core = isDirOnly ? stripped.slice(0, -1) : stripped;
+    if (core === "") continue;
+    if (rooted) {
+      patterns.push(isDirOnly ? `${core}/**` : core);
+      if (isDirOnly) patterns.push(`${core}/`);
+    } else {
+      patterns.push(`**/${core}`);
+      patterns.push(`**/${core}/**`);
+      if (isDirOnly) patterns.push(`**/${core}/`);
+    }
+  }
+  return patterns;
+}
 var CodeIngestor = class {
-  constructor(store) {
+  constructor(store, options = {}) {
     this.store = store;
+    this.skipDirs = resolveSkipDirs(options);
+    this.excludePatterns = options.excludePatterns ?? [];
+    this.respectGitignore = options.respectGitignore ?? true;
   }
   store;
+  skipDirs;
+  excludePatterns;
+  respectGitignore;
   async ingest(rootDir) {
     const start = Date.now();
     const errors = [];
@@ -890,19 +1073,55 @@ var CodeIngestor = class {
     }
     files.add(relativePath);
   }
-  async findSourceFiles(dir) {
+  /**
+   * Walk `rootDir` iteratively (BFS via an explicit queue) and return absolute paths
+   * of every supported source file that is not excluded by the configured filters.
+   *
+   * Iterative on purpose: a recursive walker burns one stack frame per nested
+   * directory and crashes with `Maximum call stack size exceeded` on deeply nested
+   * monorepos (issue #274). The queue-based form keeps memory bounded by the
+   * frontier size rather than the deepest path.
+   */
+  async findSourceFiles(rootDir) {
     const results = [];
-    const entries = await fs.readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      if (entry.isDirectory() && entry.name !== "node_modules" && entry.name !== "dist" && entry.name !== "target" && entry.name !== "build" && entry.name !== ".git" && entry.name !== "__pycache__" && entry.name !== ".venv" && entry.name !== ".turbo" && entry.name !== ".harness" && entry.name !== ".next" && entry.name !== ".vscode" && entry.name !== ".idea" && entry.name !== "vendor" && entry.name !== "out" && entry.name !== ".gradle" && entry.name !== ".gradle-home" && entry.name !== "bin" && entry.name !== "obj" && entry.name !== "venv" && entry.name !== "_build" && entry.name !== "deps" && entry.name !== "coverage" && entry.name !== ".nyc_output") {
-        results.push(...await this.findSourceFiles(fullPath));
-      } else if (entry.isFile() && isSupportedSourceFile(entry.name)) {
-        results.push(fullPath);
+    const matchers = await this.buildExcludeMatchers(rootDir);
+    const queue = [rootDir];
+    while (queue.length > 0) {
+      const dir = queue.pop();
+      let entries;
+      try {
+        entries = await fs.readdir(dir, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        const relPath = path.relative(rootDir, fullPath).replaceAll("\\", "/");
+        if (entry.isDirectory()) {
+          if (this.skipDirs.has(entry.name)) continue;
+          if (isExcluded(`${relPath}/`, matchers)) continue;
+          queue.push(fullPath);
+        } else if (entry.isFile()) {
+          if (!isSupportedSourceFile(entry.name)) continue;
+          if (isExcluded(relPath, matchers)) continue;
+          results.push(fullPath);
+        }
       }
     }
     return results;
   }
+  /**
+   * Compose the exclude matchers from `excludePatterns` and (optionally) `.gitignore`.
+   * Returns a list of minimatch-compatible patterns; an empty list means "no excludes".
+   */
+  async buildExcludeMatchers(rootDir) {
+    const patterns = [...this.excludePatterns];
+    if (this.respectGitignore) {
+      const gitignorePatterns = await readGitignorePatterns(rootDir);
+      patterns.push(...gitignorePatterns);
+    }
+    return patterns;
+  }
   extractSymbols(content, fileId, relativePath) {
     const results = [];
     const lines = content.split("\n");
@@ -1867,6 +2086,47 @@ function parseFailureSection(section) {
 // src/ingest/BusinessKnowledgeIngestor.ts
 import * as fs3 from "fs/promises";
 import * as path4 from "path";
+import { z as z2 } from "zod";
+var BUG_TRACK_CATEGORIES = [
+  "build-errors",
+  "test-failures",
+  "runtime-errors",
+  "performance-issues",
+  "database-issues",
+  "security-issues",
+  "ui-bugs",
+  "integration-issues",
+  "logic-errors"
+];
+var KNOWLEDGE_TRACK_CATEGORIES = [
+  "architecture-patterns",
+  "design-patterns",
+  "tooling-decisions",
+  "conventions",
+  "dx",
+  "best-practices"
+];
+var ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
+var SolutionBaseSchema = z2.object({
+  module: z2.string().min(1),
+  tags: z2.array(z2.string()),
+  problem_type: z2.string().min(1),
+  last_updated: z2.string().regex(ISO_DATE, "last_updated must be ISO date YYYY-MM-DD")
+});
+var SolutionDocFrontmatterSchema = z2.discriminatedUnion("track", [
+  SolutionBaseSchema.merge(
+    z2.object({
+      track: z2.literal("bug-track"),
+      category: z2.enum(BUG_TRACK_CATEGORIES)
+    })
+  ),
+  SolutionBaseSchema.merge(
+    z2.object({
+      track: z2.literal("knowledge-track"),
+      category: z2.enum(KNOWLEDGE_TRACK_CATEGORIES)
+    })
+  )
+]);
 var BUSINESS_KNOWLEDGE_TYPES = /* @__PURE__ */ new Set([
   "business_rule",
   "business_process",
@@ -1909,6 +2169,65 @@ var BusinessKnowledgeIngestor = class {
       durationMs: Date.now() - start
     };
   }
+  async ingestSolutions(solutionsDir) {
+    const start = Date.now();
+    const errors = [];
+    let files;
+    try {
+      files = await this.findMarkdownFiles(solutionsDir);
+    } catch {
+      return emptyResult(Date.now() - start);
+    }
+    let nodesAdded = 0;
+    for (const filePath of files) {
+      try {
+        const raw = await fs3.readFile(filePath, "utf-8");
+        const parsed = parseSolutionFrontmatter(raw);
+        if (!parsed) {
+          errors.push(`${filePath}: no frontmatter found`);
+          continue;
+        }
+        const validation = SolutionDocFrontmatterSchema.safeParse(parsed.frontmatter);
+        if (!validation.success) {
+          errors.push(`${filePath}: ${validation.error.message}`);
+          continue;
+        }
+        if (validation.data.track !== "knowledge-track") continue;
+        const relPath = path4.relative(solutionsDir, filePath).replaceAll("\\", "/");
+        const filename = path4.basename(filePath, ".md");
+        const nodeId = `bk:solutions:${validation.data.module}:${filename}`;
+        const titleMatch = parsed.body.match(/^#\s+(.+)$/m);
+        const name = titleMatch ? titleMatch[1].trim() : filename;
+        const node = {
+          id: nodeId,
+          type: "business_concept",
+          name,
+          path: relPath,
+          content: parsed.body.trim(),
+          metadata: {
+            domain: validation.data.module,
+            tags: validation.data.tags,
+            problem_type: validation.data.problem_type,
+            last_updated: validation.data.last_updated,
+            source: "solutions",
+            category: validation.data.category
+          }
+        };
+        this.store.addNode(node);
+        nodesAdded++;
+      } catch (err) {
+        errors.push(`${filePath}: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+    return {
+      nodesAdded,
+      nodesUpdated: 0,
+      edgesAdded: 0,
+      edgesUpdated: 0,
+      errors,
+      durationMs: Date.now() - start
+    };
+  }
   async createNodes(files, knowledgeDir, errors) {
     const entries = [];
     for (const filePath of files) {
@@ -2031,6 +2350,25 @@ function parseFrontmatter(raw) {
     body
   };
 }
+function parseSolutionFrontmatter(raw) {
+  const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
+  if (!match) return null;
+  const yamlBlock = match[1];
+  const body = match[2];
+  const frontmatter = {};
+  for (const line of yamlBlock.split(/\r?\n/)) {
+    const kvMatch = line.match(/^(\w+):\s*(.+)$/);
+    if (!kvMatch) continue;
+    const key = kvMatch[1];
+    const value = kvMatch[2].trim();
+    if (value.startsWith("[") && value.endsWith("]")) {
+      frontmatter[key] = value.slice(1, -1).split(",").map((s) => s.trim());
+    } else {
+      frontmatter[key] = value;
+    }
+  }
+  return { frontmatter, body };
+}
 
 // src/ingest/DecisionIngestor.ts
 import * as fs4 from "fs/promises";
@@ -5417,6 +5755,25 @@ var KnowledgePipelineRunner = class {
         durationMs: 0
       };
     }
+    const solutionsDir = path12.join(options.projectDir, "docs", "solutions");
+    let solutionsResult;
+    try {
+      solutionsResult = await bkIngestor.ingestSolutions(solutionsDir);
+    } catch {
+      solutionsResult = {
+        nodesAdded: 0,
+        nodesUpdated: 0,
+        edgesAdded: 0,
+        edgesUpdated: 0,
+        errors: [],
+        durationMs: 0
+      };
+    }
+    bkResult = {
+      ...bkResult,
+      nodesAdded: bkResult.nodesAdded + solutionsResult.nodesAdded,
+      errors: [...bkResult.errors, ...solutionsResult.errors]
+    };
     const decisionsDir = path12.join(options.projectDir, "docs", "knowledge", "decisions");
     const decisionIngestor = new DecisionIngestor(this.store);
     let decisionResult;
@@ -8645,7 +9002,7 @@ function queryTraceability(store, options) {
 }
 
 // src/constraints/GraphConstraintAdapter.ts
-import { minimatch } from "minimatch";
+import { minimatch as minimatch2 } from "minimatch";
 import { relative as relative5 } from "path";
 var GraphConstraintAdapter = class {
   constructor(store) {
@@ -8697,7 +9054,7 @@ var GraphConstraintAdapter = class {
   resolveLayer(filePath, layers) {
     for (const layer of layers) {
       for (const pattern of layer.patterns) {
-        if (minimatch(filePath, pattern)) {
+        if (minimatch2(filePath, pattern)) {
           return layer;
         }
       }
@@ -9496,6 +9853,7 @@ export {
   D2Parser,
   DEFAULT_BLOCKLIST,
   DEFAULT_PATTERNS,
+  DEFAULT_SKIP_DIRS,
   DecisionIngestor,
   DesignConstraintAdapter,
   DesignIngestor,
@@ -9551,8 +9909,10 @@ export {
   inferDomain,
   linkToCode,
   loadGraph,
+  loadGraphMetadata,
   normalizeIntent,
   project,
   queryTraceability,
+  resolveSkipDirs,
   saveGraph
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@harness-engineering/graph",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "license": "MIT",
   "description": "Knowledge graph for context assembly in Harness Engineering",
   "main": "./dist/index.js",