@sigil-engine/core 0.1.0

package/src/serialize.ts

@@ -0,0 +1,97 @@
+/**
+ * Context document serialization — YAML and JSON.
+ *
+ * YAML is the primary human-readable format.
+ * JSON is the machine-consumption format.
+ *
+ * _meta anchors are serialized as compact flow mappings for readability.
+ */
+
+import { stringify, parse } from "yaml";
+import { readFileSync, writeFileSync } from "fs";
+import type { ContextDocument } from "./schema.js";
+
+// ---------------------------------------------------------------------------
+// Serialize
+// ---------------------------------------------------------------------------
+
+export function serializeYaml(doc: ContextDocument): string {
+  const header = `# context.yaml — generated by sigil extract\n# ${doc._extraction.extracted_at}\n\n`;
+
+  const yamlStr = stringify(doc, {
+    lineWidth: 120,
+    defaultKeyType: "PLAIN",
+    defaultStringType: "QUOTE_DOUBLE",
+    nullStr: "",
+    sortMapEntries: false,
+    flowCollectionPadding: true,
+  });
+
+  // Post-process: compact _meta blocks that are simple (no nested arrays/objects).
+  // _meta blocks with tags arrays are left in standard YAML format.
+  const processed = yamlStr.replace(
+    /^(\s+)_meta:\n((?:\1\s+[^\s].*\n)*)/gm,
+    (match: string, indent: string, body: string) => {
+      // If body contains array items (- "...") or nested objects, skip compaction
+      if (body.includes("\n" + indent + "    - ") || body.includes("\n" + indent + "      ")) {
+        return match; // Leave as-is
+      }
+
+      const lines = body.trim().split("\n").map((l: string) => l.trim()).filter((l: string) => l.length > 0);
+
+      // Only compact if all lines are simple key: value pairs
+      const pairs: string[] = [];
+      for (const line of lines) {
+        const colonIdx = line.indexOf(":");
+        if (colonIdx === -1) return match; // Not a simple pair, abort
+        const key = line.slice(0, colonIdx).trim();
+        let val = line.slice(colonIdx + 1).trim();
+        if (val.startsWith('"') && val.endsWith('"')) {
+          val = val.slice(1, -1);
+        }
+        pairs.push(`${key}: "${val}"`);
+      }
+
+      return `${indent}_meta: { ${pairs.join(", ")} }\n`;
+    },
+  );
+
+  return header + processed;
+}
+
+export function serializeJson(doc: ContextDocument): string {
+  return JSON.stringify(doc, null, 2);
+}
+
+// ---------------------------------------------------------------------------
+// Deserialize
+// ---------------------------------------------------------------------------
+
+export function deserializeYaml(yamlStr: string): ContextDocument {
+  return parse(yamlStr) as ContextDocument;
+}
+
+export function deserializeJson(jsonStr: string): ContextDocument {
+  return JSON.parse(jsonStr) as ContextDocument;
+}
+
+// ---------------------------------------------------------------------------
+// File I/O helpers
+// ---------------------------------------------------------------------------
+
+export function writeContextFile(
+  doc: ContextDocument,
+  outputPath: string,
+  format: "yaml" | "json" = "yaml",
+): void {
+  const content = format === "json" ? serializeJson(doc) : serializeYaml(doc);
+  writeFileSync(outputPath, content, "utf-8");
+}
+
+export function readContextFile(inputPath: string): ContextDocument {
+  const content = readFileSync(inputPath, "utf-8");
+  if (inputPath.endsWith(".json")) {
+    return deserializeJson(content);
+  }
+  return deserializeYaml(content);
+}

package/tests/cache.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect } from "vitest";
+import { getChangedFiles, type FileCache } from "../src/cache.js";
+
+describe("Cache", () => {
+  describe("getChangedFiles", () => {
+    it("identifies added files", () => {
+      const cache: FileCache = {
+        files: {},
+        last_full_extract: "2026-01-01T00:00:00Z",
+      };
+
+      const result = getChangedFiles("/tmp", ["new-file.ts"], cache);
+
+      expect(result.added).toEqual(["new-file.ts"]);
+      expect(result.changed).toEqual([]);
+      expect(result.unchanged).toEqual([]);
+      expect(result.removed).toEqual([]);
+    });
+
+    it("identifies removed files", () => {
+      const cache: FileCache = {
+        files: {
+          "old-file.ts": { hash: "abc123", mtime: 1000 },
+        },
+        last_full_extract: "2026-01-01T00:00:00Z",
+      };
+
+      const result = getChangedFiles("/tmp", [], cache);
+
+      expect(result.removed).toEqual(["old-file.ts"]);
+      expect(result.added).toEqual([]);
+    });
+
+    it("categorizes files not in cache as added", () => {
+      const cache: FileCache = {
+        files: {
+          "existing.ts": { hash: "abc123", mtime: 1000 },
+        },
+        last_full_extract: "2026-01-01T00:00:00Z",
+      };
+
+      const result = getChangedFiles("/tmp", ["existing.ts", "brand-new.ts"], cache);
+
+      expect(result.added).toContain("brand-new.ts");
+    });
+  });
+});

package/tests/diff.test.ts

@@ -0,0 +1,222 @@
+import { describe, it, expect } from "vitest";
+import { createEmptyDocument } from "../src/schema.js";
+import { diffContexts, formatDiff } from "../src/diff.js";
+import type { Entity, Component, Route } from "../src/schema.js";
+
+describe("Diff Engine", () => {
+  function makeDoc(name = "test") {
+    return createEmptyDocument(name);
+  }
+
+  describe("diffContexts", () => {
+    it("returns no changes for identical documents", () => {
+      const doc = makeDoc();
+      const result = diffContexts(doc, doc);
+
+      expect(result.summary.total_changes).toBe(0);
+      expect(result.changes).toEqual([]);
+    });
+
+    it("detects added entities", () => {
+      const oldDoc = makeDoc();
+      const newDoc = makeDoc();
+      newDoc.entities.push({
+        name: "User",
+        _meta: { file: "schema.ts", symbol: "users", type: "entity" },
+        fields: [{ name: "id", type: "uuid", primary: true }],
+        relations: [],
+      });
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.summary.added).toBe(1);
+      expect(result.changes[0].type).toBe("added");
+      expect(result.changes[0].name).toBe("User");
+      expect(result.changes[0].category).toBe("entity");
+    });
+
+    it("detects removed entities", () => {
+      const oldDoc = makeDoc();
+      oldDoc.entities.push({
+        name: "User",
+        _meta: { file: "schema.ts", symbol: "users", type: "entity" },
+        fields: [],
+        relations: [],
+      });
+      const newDoc = makeDoc();
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.summary.removed).toBe(1);
+      expect(result.changes[0].type).toBe("removed");
+      expect(result.changes[0].name).toBe("User");
+    });
+
+    it("detects modified entity fields", () => {
+      const entity: Entity = {
+        name: "User",
+        _meta: { file: "schema.ts", symbol: "users", type: "entity" },
+        fields: [
+          { name: "id", type: "uuid", primary: true },
+          { name: "name", type: "varchar(100)" },
+        ],
+        relations: [],
+      };
+
+      const oldDoc = makeDoc();
+      oldDoc.entities.push(entity);
+
+      const newDoc = makeDoc();
+      newDoc.entities.push({
+        ...entity,
+        fields: [
+          { name: "id", type: "uuid", primary: true },
+          { name: "name", type: "varchar(255)" },  // changed length
+          { name: "email", type: "varchar(255)" },  // new field
+        ],
+      });
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.summary.modified).toBe(1);
+      const change = result.changes[0];
+      expect(change.details).toContain("~ field name: type varchar(100) → varchar(255)");
+      expect(change.details).toContain("+ field: email (varchar(255))");
+    });
+
+    it("detects added components", () => {
+      const oldDoc = makeDoc();
+      const newDoc = makeDoc();
+      newDoc.components.push({
+        name: "Header",
+        _meta: { file: "Header.tsx", symbol: "Header", type: "component" },
+        props: [{ name: "title", type: "string" }],
+        dependencies: [],
+        hooks: [],
+        data_sources: [],
+      });
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.summary.added).toBe(1);
+      expect(result.changes[0].category).toBe("component");
+    });
+
+    it("detects modified component props", () => {
+      const makeComp = (props: Component["props"]): Component => ({
+        name: "Button",
+        _meta: { file: "Button.tsx", symbol: "Button", type: "component" },
+        props,
+        dependencies: [],
+        hooks: [],
+        data_sources: [],
+      });
+
+      const oldDoc = makeDoc();
+      oldDoc.components.push(makeComp([{ name: "label", type: "string" }]));
+
+      const newDoc = makeDoc();
+      newDoc.components.push(makeComp([
+        { name: "label", type: "string" },
+        { name: "variant", type: "'primary' | 'secondary'" },
+      ]));
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.summary.modified).toBe(1);
+      expect(result.changes[0].details).toContain("+ prop: variant: 'primary' | 'secondary'");
+    });
+
+    it("detects added/removed routes", () => {
+      const oldDoc = makeDoc();
+      oldDoc.routes.push({
+        path: "/api/users",
+        _meta: { file: "route.ts", symbol: "route", type: "route" },
+        methods: { GET: { auth: true } },
+      });
+
+      const newDoc = makeDoc();
+      newDoc.routes.push({
+        path: "/api/posts",
+        _meta: { file: "posts/route.ts", symbol: "route", type: "route" },
+        methods: { GET: {}, POST: { body: "JSON" } },
+      });
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.summary.added).toBe(1);
+      expect(result.summary.removed).toBe(1);
+    });
+
+    it("detects stack changes", () => {
+      const oldDoc = makeDoc();
+      oldDoc.stack = { framework: "next.js@14", language: "typescript" };
+
+      const newDoc = makeDoc();
+      newDoc.stack = { framework: "next.js@15", language: "typescript" };
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      expect(result.stack_changes).toHaveLength(1);
+      expect(result.stack_changes[0].name).toBe("framework");
+      expect(result.stack_changes[0].type).toBe("modified");
+    });
+
+    it("detects dependency changes", () => {
+      const oldDoc = makeDoc();
+      oldDoc.dependencies = [
+        { name: "react", version: "^18.0.0" },
+        { name: "lodash", version: "^4.0.0" },
+      ];
+
+      const newDoc = makeDoc();
+      newDoc.dependencies = [
+        { name: "react", version: "^19.0.0" },  // upgraded
+        { name: "zustand", version: "^5.0.0" },  // added
+        // lodash removed
+      ];
+
+      const result = diffContexts(oldDoc, newDoc);
+
+      const depChanges = result.changes.filter((c) => c.category === "dependency");
+      expect(depChanges).toHaveLength(3);
+
+      const added = depChanges.find((c) => c.type === "added");
+      expect(added?.name).toBe("zustand");
+
+      const removed = depChanges.find((c) => c.type === "removed");
+      expect(removed?.name).toBe("lodash");
+
+      const modified = depChanges.find((c) => c.type === "modified");
+      expect(modified?.name).toBe("react");
+    });
+  });
+
+  describe("formatDiff", () => {
+    it("formats empty diff", () => {
+      const result = diffContexts(makeDoc(), makeDoc());
+      const output = formatDiff(result);
+
+      expect(output).toContain("0 changes");
+      expect(output).toContain("No changes detected");
+    });
+
+    it("formats changes with symbols", () => {
+      const oldDoc = makeDoc();
+      const newDoc = makeDoc();
+      newDoc.entities.push({
+        name: "User",
+        _meta: { file: "schema.ts", symbol: "users", type: "entity" },
+        fields: [],
+        relations: [],
+      });
+
+      const result = diffContexts(oldDoc, newDoc);
+      const output = formatDiff(result);
+
+      expect(output).toContain("1 changes");
+      expect(output).toContain("+ User");
+      expect(output).toContain("Entities");
+    });
+  });
+});

package/tests/plugin.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import {
+  registerPlugin,
+  getPlugins,
+  detectPlugins,
+  clearPlugins,
+  type ExtractorPlugin,
+  type ParsedProject,
+  type ExtractResult,
+} from "../src/plugin.js";
+
+describe("Plugin System", () => {
+  beforeEach(() => {
+    clearPlugins();
+  });
+
+  function makeMockPlugin(name: string, detects = true): ExtractorPlugin {
+    return {
+      name,
+      version: "1.0.0",
+      languages: ["test"],
+      detect: () => detects,
+      init: (projectPath: string): ParsedProject => ({
+        projectPath,
+        sourceFiles: [],
+      }),
+      extract: (): ExtractResult => ({
+        entities: [],
+        components: [],
+      }),
+    };
+  }
+
+  describe("registerPlugin", () => {
+    it("registers a plugin", () => {
+      const plugin = makeMockPlugin("test");
+      registerPlugin(plugin);
+
+      expect(getPlugins()).toHaveLength(1);
+      expect(getPlugins()[0].name).toBe("test");
+    });
+
+    it("replaces plugin with same name", () => {
+      const plugin1 = makeMockPlugin("test");
+      const plugin2 = makeMockPlugin("test");
+      plugin2.version = "2.0.0";
+
+      registerPlugin(plugin1);
+      registerPlugin(plugin2);
+
+      expect(getPlugins()).toHaveLength(1);
+      expect(getPlugins()[0].version).toBe("2.0.0");
+    });
+
+    it("registers multiple plugins", () => {
+      registerPlugin(makeMockPlugin("typescript"));
+      registerPlugin(makeMockPlugin("python"));
+      registerPlugin(makeMockPlugin("go"));
+
+      expect(getPlugins()).toHaveLength(3);
+    });
+  });
+
+  describe("detectPlugins", () => {
+    it("returns plugins that detect the project", async () => {
+      registerPlugin(makeMockPlugin("typescript", true));
+      registerPlugin(makeMockPlugin("python", false));
+
+      const detected = await detectPlugins("/some/path");
+
+      expect(detected).toHaveLength(1);
+      expect(detected[0].name).toBe("typescript");
+    });
+
+    it("returns empty array when no plugins detect", async () => {
+      registerPlugin(makeMockPlugin("python", false));
+
+      const detected = await detectPlugins("/some/path");
+      expect(detected).toHaveLength(0);
+    });
+
+    it("supports async detect", async () => {
+      const asyncPlugin: ExtractorPlugin = {
+        ...makeMockPlugin("async-test"),
+        detect: async () => {
+          return true;
+        },
+      };
+
+      registerPlugin(asyncPlugin);
+      const detected = await detectPlugins("/some/path");
+      expect(detected).toHaveLength(1);
+    });
+  });
+
+  describe("clearPlugins", () => {
+    it("removes all plugins", () => {
+      registerPlugin(makeMockPlugin("a"));
+      registerPlugin(makeMockPlugin("b"));
+
+      expect(getPlugins()).toHaveLength(2);
+
+      clearPlugins();
+      expect(getPlugins()).toHaveLength(0);
+    });
+  });
+});

package/tests/schema.test.ts

@@ -0,0 +1,132 @@
+import { describe, it, expect } from "vitest";
+import { createEmptyDocument } from "../src/schema.js";
+import type { ContextDocument, Meta, Entity, Component, Route } from "../src/schema.js";
+
+describe("Schema", () => {
+  describe("createEmptyDocument", () => {
+    it("creates a valid empty document with project name", () => {
+      const doc = createEmptyDocument("test-project");
+
+      expect(doc.version).toBe("1.0");
+      expect(doc.project).toBe("test-project");
+      expect(doc._extraction.extracted_at).toBeTruthy();
+      expect(doc.entities).toEqual([]);
+      expect(doc.components).toEqual([]);
+      expect(doc.routes).toEqual([]);
+      expect(doc.exports).toEqual([]);
+      expect(doc.call_graph).toEqual([]);
+      expect(doc.component_graph).toEqual([]);
+      expect(doc.env_vars).toEqual([]);
+      expect(doc.patterns).toEqual({});
+      expect(doc.conventions).toEqual({});
+      expect(doc.dependencies).toEqual([]);
+      expect(doc.stack).toEqual({});
+    });
+
+    it("sets extraction timestamp to current time", () => {
+      const before = new Date().toISOString();
+      const doc = createEmptyDocument("test");
+      const after = new Date().toISOString();
+
+      expect(doc._extraction.extracted_at >= before).toBe(true);
+      expect(doc._extraction.extracted_at <= after).toBe(true);
+    });
+  });
+
+  describe("Type safety", () => {
+    it("Meta uses generic MetaType categories", () => {
+      const meta: Meta = {
+        file: "src/models/user.ts",
+        symbol: "User",
+        type: "entity",
+        tags: ["orm:drizzle", "db:postgresql"],
+        line: 10,
+        hash: "abc123def456",
+      };
+
+      expect(meta.type).toBe("entity");
+      expect(meta.tags).toContain("orm:drizzle");
+    });
+
+    it("Entity has proper structure", () => {
+      const entity: Entity = {
+        name: "User",
+        _meta: {
+          file: "src/db/schema.ts",
+          symbol: "users",
+          type: "entity",
+          tags: ["orm:drizzle"],
+        },
+        fields: [
+          { name: "id", type: "uuid", primary: true },
+          { name: "email", type: "varchar(255)", unique: true },
+          { name: "name", type: "varchar(100)", nullable: true },
+        ],
+        relations: [
+          { to: "Post", type: "one-to-many", via: "userId" },
+        ],
+      };
+
+      expect(entity.name).toBe("User");
+      expect(entity.fields).toHaveLength(3);
+      expect(entity.fields[0].primary).toBe(true);
+      expect(entity.relations[0].to).toBe("Post");
+    });
+
+    it("Component has proper structure", () => {
+      const comp: Component = {
+        name: "Dashboard",
+        _meta: {
+          file: "src/app/page.tsx",
+          symbol: "Dashboard",
+          type: "component",
+          tags: ["framework:react", "react:client"],
+        },
+        props: [
+          { name: "userId", type: "string", required: true },
+        ],
+        dependencies: ["MetricCard", "ChartPanel"],
+        hooks: ["useState", "useEffect", "useMetrics"],
+        data_sources: ["/api/metrics"],
+        route: "/dashboard",
+      };
+
+      expect(comp._meta.tags).toContain("react:client");
+      expect(comp.hooks).toContain("useMetrics");
+    });
+
+    it("Route has proper structure", () => {
+      const route: Route = {
+        path: "/api/users",
+        _meta: {
+          file: "src/app/api/users/route.ts",
+          symbol: "route",
+          type: "route",
+          tags: ["framework:nextjs"],
+        },
+        methods: {
+          GET: { auth: true, returns: "User[]" },
+          POST: { auth: true, body: "JSON" },
+        },
+      };
+
+      expect(route.methods.GET.auth).toBe(true);
+      expect(route.methods.POST.body).toBe("JSON");
+    });
+
+    it("StackInfo is extensible Record", () => {
+      const stack: ContextDocument["stack"] = {
+        framework: "next.js@15",
+        language: "typescript",
+        orm: "drizzle",
+        database: "postgresql",
+        ui: ["tailwind", "shadcn"],
+        // Any key works — not locked to predefined fields
+        custom_tool: "some-custom-thing",
+      };
+
+      expect(stack.framework).toBe("next.js@15");
+      expect(stack.custom_tool).toBe("some-custom-thing");
+    });
+  });
+});

package/tests/serialize.test.ts

@@ -0,0 +1,92 @@
+import { describe, it, expect } from "vitest";
+import { createEmptyDocument } from "../src/schema.js";
+import {
+  serializeYaml,
+  serializeJson,
+  deserializeYaml,
+  deserializeJson,
+} from "../src/serialize.js";
+
+describe("Serialization", () => {
+  function makeTestDoc() {
+    const doc = createEmptyDocument("test-project");
+    doc.stack = { framework: "next.js@15", language: "typescript" };
+    doc.entities.push({
+      name: "User",
+      _meta: { file: "src/db/schema.ts", symbol: "users", type: "entity", tags: ["orm:drizzle"] },
+      fields: [
+        { name: "id", type: "uuid", primary: true },
+        { name: "email", type: "varchar(255)", unique: true },
+      ],
+      relations: [],
+    });
+    doc.components.push({
+      name: "Dashboard",
+      _meta: { file: "src/app/page.tsx", symbol: "Dashboard", type: "component", tags: ["framework:react"] },
+      props: [{ name: "userId", type: "string" }],
+      dependencies: ["MetricCard"],
+      hooks: ["useState"],
+      data_sources: ["/api/metrics"],
+      route: "/",
+    });
+    doc.patterns = { auth: "next-auth with JWT", styling: "tailwind" };
+    return doc;
+  }
+
+  describe("YAML roundtrip", () => {
+    it("serializes and deserializes without data loss", () => {
+      const doc = makeTestDoc();
+      const yaml = serializeYaml(doc);
+      const restored = deserializeYaml(yaml);
+
+      expect(restored.project).toBe("test-project");
+      expect(restored.version).toBe("1.0");
+      expect(restored.entities).toHaveLength(1);
+      expect(restored.entities[0].name).toBe("User");
+      expect(restored.entities[0].fields).toHaveLength(2);
+      expect(restored.components).toHaveLength(1);
+      expect(restored.components[0].name).toBe("Dashboard");
+      expect(restored.patterns.auth).toBe("next-auth with JWT");
+    });
+
+    it("includes sigil header comment", () => {
+      const doc = makeTestDoc();
+      const yaml = serializeYaml(doc);
+
+      expect(yaml).toContain("# context.yaml — generated by sigil extract");
+    });
+  });
+
+  describe("JSON roundtrip", () => {
+    it("serializes and deserializes without data loss", () => {
+      const doc = makeTestDoc();
+      const json = serializeJson(doc);
+      const restored = deserializeJson(json);
+
+      expect(restored.project).toBe("test-project");
+      expect(restored.entities).toHaveLength(1);
+      expect(restored.components).toHaveLength(1);
+    });
+
+    it("produces valid JSON", () => {
+      const doc = makeTestDoc();
+      const json = serializeJson(doc);
+
+      expect(() => JSON.parse(json)).not.toThrow();
+    });
+  });
+
+  describe("Stack extensibility", () => {
+    it("preserves arbitrary stack keys through YAML roundtrip", () => {
+      const doc = makeTestDoc();
+      doc.stack.custom_field = "custom_value";
+      doc.stack.tools = ["eslint", "prettier"];
+
+      const yaml = serializeYaml(doc);
+      const restored = deserializeYaml(yaml);
+
+      expect(restored.stack.custom_field).toBe("custom_value");
+      expect(restored.stack.tools).toEqual(["eslint", "prettier"]);
+    });
+  });
+});