@entelligentsia/forgecli 1.0.2 → 1.0.3

package/node_modules/@entelligentsia/forge-compress/src/__tests__/compress.test.ts

@@ -0,0 +1,409 @@
+import { describe, it, expect } from "vitest";
+import {
+  compressStoreQuery,
+  compressEntity,
+  compressEntityList,
+  compressMarkdown,
+  compressValidateStore,
+  countTokens,
+  compressIb,
+  compressProgressive,
+  truncateToTokenBudget,
+  lightweightCleanup,
+  verbatimCompact,
+  stripAnsi,
+} from "../index.js";
+
+// ── store-query compressor ──────────────────────────────────
+
+const SAMPLE_QUERY = JSON.stringify(
+  {
+    query: "open tasks in S12",
+    path: "intent-nlp",
+    traversalTrace: [
+      "parsed intent: entity=task, status=open, sprint=S12",
+      "resolved S12 → PROJ-S12",
+      "loaded 8 task records",
+      "filtered by status: 5 matched",
+    ],
+    results: [
+      {
+        id: "PROJ-S12-T01",
+        title: "Add auth middleware",
+        status: "implementing",
+        type: "task",
+        relationships: { sprintId: "PROJ-S12", featureId: "F-AUTH", blockedBy: null },
+        fileRefs: { json: ".forge/store/tasks/PROJ-S12-T01.json", md: ".forge/sprints/S12/tasks/T01/INDEX.md" },
+        storeRef: ".forge/store/tasks/PROJ-S12-T01.json",
+        indexRef: ".forge/sprints/S12/tasks/T01/INDEX.md",
+        excerpt: "# T01: Add auth middleware\n\n## Status: implementing\n\n## Plan\nImplement JWT-based auth middleware...\n\n## Key Changes\n- Added middleware handler\n- Updated route config\n- Added token validation\n\n## Notes\nBlocked by upstream API changes.\nNeed to coordinate with team B.\nAlso requires DB migration.\nSee PR #42 for details.\nReviewed by @alice.",
+      },
+      {
+        id: "PROJ-S12-T02",
+        title: "Fix rate limiter",
+        status: "planned",
+        type: "task",
+        relationships: { sprintId: "PROJ-S12" },
+        fileRefs: { json: ".forge/store/tasks/PROJ-S12-T02.json", md: ".forge/sprints/S12/tasks/T02/INDEX.md" },
+        storeRef: ".forge/store/tasks/PROJ-S12-T02.json",
+        indexRef: ".forge/sprints/S12/tasks/T02/INDEX.md",
+        excerpt: "# T02: Fix rate limiter\n\n## Status: planned\n\nRate limiter is dropping legitimate requests under load.",
+      },
+    ],
+    relatedFileRefs: [".forge/sprints/S12/INDEX.md", ".forge/features/F-AUTH.md"],
+    config: { store: ".forge/store", engineering: ".forge/engineering" },
+    meta: { mode: "intent-nlp", engineVersion: "1.2.0", totalTimeMs: 42 },
+  },
+  null,
+  2,
+);
+
+describe("compressStoreQuery", () => {
+  it("strips envelope fields", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY);
+    const parsed = JSON.parse(result);
+    expect(parsed.traversalTrace).toBeUndefined();
+    expect(parsed.config).toBeUndefined();
+    expect(parsed.meta).toBeUndefined();
+    expect(parsed.relatedFileRefs).toBeUndefined();
+    expect(parsed.results).toHaveLength(2);
+  });
+
+  it("strips fileRefs and storeRef from results", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY);
+    const parsed = JSON.parse(result);
+    expect(parsed.results[0].fileRefs).toBeUndefined();
+    expect(parsed.results[0].storeRef).toBeUndefined();
+    expect(parsed.results[0].indexRef).toBeUndefined();
+  });
+
+  it("strips excerpts by default", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY);
+    const parsed = JSON.parse(result);
+    expect(parsed.results[0].excerpt).toBeUndefined();
+  });
+
+  it("keeps truncated excerpts when requested", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY, {
+      keepExcerpts: true,
+      maxExcerptLines: 3,
+    });
+    const parsed = JSON.parse(result);
+    expect(parsed.results[0].excerpt).toContain("Add auth middleware");
+    expect(parsed.results[0].excerpt).toContain("more lines");
+  });
+
+  it("preserves query and path", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY);
+    const parsed = JSON.parse(result);
+    expect(parsed.query).toBe("open tasks in S12");
+    expect(parsed.path).toBe("intent-nlp");
+  });
+
+  it("limits results count", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY, { maxResults: 1 });
+    const parsed = JSON.parse(result);
+    expect(parsed.results).toHaveLength(1);
+    expect(parsed.truncated).toBe(1);
+  });
+
+  it("strips null relationships", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY);
+    const parsed = JSON.parse(result);
+    expect(parsed.results[0].relationships.blockedBy).toBeUndefined();
+  });
+
+  it("is significantly smaller than input", () => {
+    const result = compressStoreQuery(SAMPLE_QUERY);
+    expect(countTokens(result)).toBeLessThan(countTokens(SAMPLE_QUERY) * 0.5);
+  });
+
+  it("passes through non-JSON input", () => {
+    expect(compressStoreQuery("not json")).toBe("not json");
+  });
+});
+
+// ── entity compressor ───────────────────────────────────────
+
+const SAMPLE_TASK = JSON.stringify(
+  {
+    taskId: "PROJ-S12-T01",
+    sprintId: "PROJ-S12",
+    title: "Add auth middleware",
+    status: "implementing",
+    path: "sprints/S12/tasks/T01",
+    feature_id: "F-AUTH",
+    description: "Implement JWT-based authentication middleware for all API routes.",
+    estimate: "M",
+    dependencies: ["PROJ-S12-T03"],
+    knowledgeUpdates: [{ file: "auth.md", delta: "Added JWT flow" }],
+    planIterations: 2,
+    codeReviewIterations: 1,
+    assignedModel: "claude-opus-4-7",
+    pipeline: "standard",
+    summaries: {
+      plan: {
+        objective: "Design JWT middleware architecture",
+        written_at: "2026-05-20T10:00:00Z",
+        key_changes: ["Created middleware handler", "Updated route config", "Added token validation", "Set up refresh flow"],
+        findings: ["Need to handle expired tokens gracefully"],
+        verdict: "approved",
+        artifact_ref: "PLAN.md",
+      },
+      review_plan: {
+        objective: "Review plan for auth middleware",
+        written_at: "2026-05-20T14:00:00Z",
+        key_changes: ["Approved with minor suggestions"],
+        verdict: "approved",
+      },
+      implementation: {
+        objective: "Implement auth middleware per plan",
+        written_at: "2026-05-21T09:00:00Z",
+        key_changes: ["Implemented JWT parsing", "Added route guards", "Unit tests passing", "Integration tests added", "Error handling complete"],
+        verdict: "approved",
+        artifact_ref: "PROGRESS.md",
+      },
+    },
+  },
+  null,
+  2,
+);
+
+describe("compressEntity", () => {
+  it("keeps essential fields", () => {
+    const result = compressEntity(SAMPLE_TASK);
+    const parsed = JSON.parse(result);
+    expect(parsed.taskId).toBe("PROJ-S12-T01");
+    expect(parsed.title).toBe("Add auth middleware");
+    expect(parsed.status).toBe("implementing");
+    expect(parsed.sprintId).toBe("PROJ-S12");
+  });
+
+  it("drops verbose fields by default", () => {
+    const result = compressEntity(SAMPLE_TASK);
+    const parsed = JSON.parse(result);
+    expect(parsed.description).toBeUndefined();
+    expect(parsed.knowledgeUpdates).toBeUndefined();
+    expect(parsed.assignedModel).toBeUndefined();
+    expect(parsed.path).toBeUndefined();
+    expect(parsed.planIterations).toBeUndefined();
+  });
+
+  it("keeps only latest summary by default", () => {
+    const result = compressEntity(SAMPLE_TASK);
+    const parsed = JSON.parse(result);
+    expect(parsed.summaries).toBeUndefined();
+    expect(parsed.latestPhase).toBe("implementation");
+    expect(parsed.latestSummary.objective).toBe("Implement auth middleware per plan");
+    expect(parsed.latestSummary.verdict).toBe("approved");
+  });
+
+  it("truncates key_changes in summary", () => {
+    const result = compressEntity(SAMPLE_TASK, { maxKeyChanges: 2 });
+    const parsed = JSON.parse(result);
+    expect(parsed.latestSummary.key_changes).toHaveLength(3);
+    expect(parsed.latestSummary.key_changes[2]).toContain("+3 more");
+  });
+
+  it("keeps all summaries when requested", () => {
+    const result = compressEntity(SAMPLE_TASK, { keepSummaries: "all" });
+    const parsed = JSON.parse(result);
+    expect(parsed.summaries).toBeDefined();
+    expect(parsed.summaries.plan).toBeDefined();
+    expect(parsed.summaries.implementation).toBeDefined();
+  });
+
+  it("produces flat format", () => {
+    const result = compressEntity(SAMPLE_TASK, { flatFormat: true });
+    expect(result).toContain("taskId: PROJ-S12-T01");
+    expect(result).toContain("status: implementing");
+    expect(result).not.toContain("{");
+  });
+
+  it("is smaller than input", () => {
+    const result = compressEntity(SAMPLE_TASK);
+    expect(countTokens(result)).toBeLessThan(countTokens(SAMPLE_TASK) * 0.5);
+  });
+});
+
+describe("compressEntityList", () => {
+  it("compresses array of entities", () => {
+    const list = JSON.stringify([JSON.parse(SAMPLE_TASK), JSON.parse(SAMPLE_TASK)], null, 2);
+    const result = compressEntityList(list);
+    const parsed = JSON.parse(result);
+    expect(parsed).toHaveLength(2);
+    expect(parsed[0].taskId).toBe("PROJ-S12-T01");
+    expect(parsed[0].description).toBeUndefined();
+  });
+});
+
+// ── markdown compressor ─────────────────────────────────────
+
+const SAMPLE_MD = `---
+title: Sprint S12 Progress
+date: 2026-05-25
+---
+
+# Sprint S12: Auth & Rate Limiting
+
+## Overview
+This sprint covers the implementation of JWT authentication
+middleware and fixing the rate limiter under high load.
+There are multiple sub-tasks involved.
+
+## Tasks
+
+### T01: Add auth middleware
+- **Status**: implementing
+- **Estimate**: M
+- Started implementation on May 20th
+- Plan was approved after 2 iterations
+- Currently blocked on upstream API
+
+### T02: Fix rate limiter
+- **Status**: planned
+- **Estimate**: S
+- Root cause identified as connection pool exhaustion
+
+## Risks
+- Upstream API team may not deliver on time
+- Rate limiter fix requires load testing infrastructure
+- JWT library has a known CVE that needs patching
+
+## Timeline
+| Milestone | Date | Status |
+|-----------|------|--------|
+| Plan approval | May 20 | Done |
+| Auth impl | May 25 | In progress |
+| Rate limiter | May 28 | Not started |
+`;
+
+describe("compressMarkdown", () => {
+  it("map mode keeps headings and key lines", () => {
+    const result = compressMarkdown(SAMPLE_MD, { mode: "map" });
+    expect(result).toContain("# Sprint S12");
+    expect(result).toContain("### T01");
+    expect(result).toContain("### T02");
+    expect(result).toContain("**Status**");
+    expect(result).toContain("body lines omitted");
+    expect(countTokens(result)).toBeLessThan(countTokens(SAMPLE_MD) * 0.7);
+  });
+
+  it("headings mode extracts structure only", () => {
+    const result = compressMarkdown(SAMPLE_MD, { mode: "headings" });
+    expect(result).toContain("Sprint S12");
+    expect(result).toContain("T01: Add auth middleware");
+    expect(result).toContain("sections");
+    expect(result).not.toContain("implementing");
+  });
+
+  it("truncate mode respects token budget", () => {
+    const result = compressMarkdown(SAMPLE_MD, { mode: "truncate", maxTokens: 50 });
+    expect(countTokens(result)).toBeLessThanOrEqual(60);
+  });
+
+  it("skips frontmatter", () => {
+    const result = compressMarkdown(SAMPLE_MD, { mode: "map" });
+    expect(result).not.toContain("title: Sprint S12 Progress");
+    expect(result).not.toContain("date: 2026");
+  });
+
+  it("handles empty input", () => {
+    expect(compressMarkdown("")).toBe("");
+    expect(compressMarkdown("   ")).toBe("");
+  });
+});
+
+// ── validate-store compressor ───────────────────────────────
+
+describe("compressValidateStore", () => {
+  it("compresses JSON report", () => {
+    const report = JSON.stringify({
+      ok: false,
+      errors: [
+        { entity: "task", id: "PROJ-S12-T01", category: "missing-required", field: "title", message: "title is required" },
+        { entity: "task", id: "PROJ-S12-T02", category: "missing-required", field: "status", message: "status is required" },
+        { entity: "bug", id: "PROJ-B01", category: "orphaned-fk", field: "sprintId", message: "references non-existent sprint" },
+      ],
+      warnings: [
+        { entity: "task", id: "PROJ-S12-T03", category: "stale-path", message: "path does not match filesystem" },
+      ],
+      fixes: [
+        { entity: "task", id: "PROJ-S12-T01", category: "backfill", message: "backfilled path", applied: true },
+      ],
+      summary: { sprints: 2, tasks: 5, bugs: 1, features: 0, errors: 3, warnings: 1, fixes: 1 },
+    });
+    const result = compressValidateStore(report);
+    expect(result).toContain("missing-required (2)");
+    expect(result).toContain("orphaned-fk (1)");
+    expect(result).toContain("1 warnings");
+    expect(result).toContain("1 fixes (1 applied)");
+    expect(countTokens(result)).toBeLessThan(countTokens(report) * 0.6);
+  });
+
+  it("returns ok for valid store", () => {
+    const report = JSON.stringify({
+      ok: true,
+      errors: [],
+      warnings: [],
+      fixes: [],
+      summary: { sprints: 3, tasks: 12, bugs: 2, features: 1, errors: 0, warnings: 0, fixes: 0 },
+    });
+    const result = compressValidateStore(report);
+    expect(result).toContain("3 sprints");
+    expect(result).toContain("12 tasks");
+  });
+
+  it("compresses plain text validation output", () => {
+    const lines = Array.from({ length: 20 }, (_, i) =>
+      `ERROR  PROJ-S12-T${String(i).padStart(2, "0")}: missing required field`
+    ).join("\n");
+    const result = compressValidateStore(lines);
+    expect(result).toContain("20 errors");
+    expect(result).toContain("... +15 more");
+  });
+});
+
+// ── core utilities ──────────────────────────────────────────
+
+describe("core utilities", () => {
+  it("countTokens", () => {
+    expect(countTokens("")).toBe(0);
+    expect(countTokens("hello world")).toBeGreaterThan(0);
+  });
+
+  it("stripAnsi", () => {
+    expect(stripAnsi("\x1b[31mERROR\x1b[0m")).toBe("ERROR");
+    expect(stripAnsi("clean")).toBe("clean");
+  });
+
+  it("truncateToTokenBudget", () => {
+    const long = "word ".repeat(1000);
+    const result = truncateToTokenBudget(long, 50);
+    expect(countTokens(result)).toBeLessThanOrEqual(60);
+  });
+
+  it("compressIb preserves high-entropy lines", () => {
+    let boring = "aaa bbb aaa bbb\n".repeat(30);
+    boring += "unique_identifier_xyz_quartz\n";
+    const result = compressIb(boring, 0.15);
+    expect(result).toContain("unique_identifier_xyz_quartz");
+  });
+
+  it("compressProgressive respects budget", () => {
+    const segs = Array.from({ length: 4 }, (_, i) => `segment ${i} content\n`);
+    const out = compressProgressive(segs, 80);
+    const total = out.reduce((acc, s) => acc + countTokens(s), 0);
+    expect(total).toBeLessThanOrEqual(80);
+  });
+
+  it("lightweightCleanup collapses blanks", () => {
+    const result = lightweightCleanup("a\n\n\n\n\nb");
+    expect(result.match(/\n/g)?.length ?? 0).toBeLessThanOrEqual(3);
+  });
+
+  it("verbatimCompact deduplicates", () => {
+    const result = verbatimCompact("same\n".repeat(20));
+    expect(result).toContain("[20x]");
+  });
+});

package/node_modules/@entelligentsia/forge-compress/src/compressor.ts

@@ -0,0 +1,147 @@
+import { countTokens } from "./tokens.js";
+import {
+  stripAnsi,
+  stripTimestampsAndHashes,
+  normalizeWhitespace,
+  isBoilerplate,
+} from "./strip.js";
+
+export function lightweightCleanup(content: string): string {
+  const lines = content.split("\n");
+  const total = lines.length;
+  const result: string[] = [];
+  let blankCount = 0;
+  let braceRun: string[] = [];
+
+  const flushBraceRun = () => {
+    if (total <= 200 || braceRun.length <= 5) {
+      result.push(...braceRun);
+    } else {
+      result.push(braceRun[0], braceRun[1]);
+      result.push(`[${braceRun.length - 2} brace-only lines collapsed]`);
+    }
+    braceRun = [];
+  };
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      flushBraceRun();
+      blankCount++;
+      if (blankCount <= 1) result.push("");
+      continue;
+    }
+    blankCount = 0;
+    if (/^[}\]);]+$/.test(trimmed)) {
+      braceRun.push(trimmed);
+      continue;
+    }
+    flushBraceRun();
+    result.push(line);
+  }
+  flushBraceRun();
+  return result.join("\n");
+}
+
+export function verbatimCompact(text: string): string {
+  const lines: string[] = [];
+  let blankCount = 0;
+  let prevLine: string | null = null;
+  let repeatCount = 0;
+
+  const flushRepeats = () => {
+    if (repeatCount > 1 && prevLine !== null) {
+      const lastIdx = lines.length - 1;
+      if (lastIdx >= 0) {
+        lines[lastIdx] = `[${repeatCount}x] ${prevLine}`;
+      }
+    }
+    repeatCount = 0;
+    prevLine = null;
+  };
+
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      blankCount++;
+      if (blankCount <= 1) {
+        flushRepeats();
+        lines.push("");
+      }
+      continue;
+    }
+    blankCount = 0;
+    if (isBoilerplate(trimmed)) continue;
+
+    const normalized = normalizeWhitespace(trimmed);
+    const stripped = stripTimestampsAndHashes(normalized);
+
+    if (prevLine !== null && prevLine === stripped) {
+      repeatCount++;
+      continue;
+    }
+    flushRepeats();
+    prevLine = stripped;
+    repeatCount = 1;
+    lines.push(stripped);
+  }
+  flushRepeats();
+  return lines.join("\n");
+}
+
+export function aggressiveCompress(
+  content: string,
+  ext?: string,
+): string {
+  const result: string[] = [];
+  const isPython = ext === "py";
+  const isHtml = ext === "html" || ext === "htm" || ext === "xml" || ext === "svg";
+  const isSql = ext === "sql";
+  const isShell = ext === "sh" || ext === "bash" || ext === "zsh" || ext === "fish";
+  let inBlockComment = false;
+
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    if (inBlockComment) {
+      if (trimmed.includes("*/") || (isHtml && trimmed.includes("-->"))) {
+        inBlockComment = false;
+      }
+      continue;
+    }
+    if (trimmed.startsWith("/*") || (isHtml && trimmed.startsWith("<!--"))) {
+      if (!(trimmed.includes("*/") || trimmed.includes("-->"))) {
+        inBlockComment = true;
+      }
+      continue;
+    }
+    if (trimmed.startsWith("//") && !trimmed.startsWith("///")) continue;
+    if (trimmed.startsWith("*") || trimmed.startsWith("*/")) continue;
+    if (isPython && trimmed.startsWith("#")) continue;
+    if (isSql && trimmed.startsWith("--")) continue;
+    if (isShell && trimmed.startsWith("#") && !trimmed.startsWith("#!")) continue;
+
+    if (/^[}\]);]+$/.test(trimmed)) {
+      const last = result[result.length - 1];
+      if (last && /^[}\]);]+$/.test(last.trim())) {
+        result[result.length - 1] = last + trimmed;
+        continue;
+      }
+    }
+    result.push(trimmed);
+  }
+  return result.join("\n");
+}
+
+export function safeguardRatio(original: string, compressed: string): string {
+  const origTokens = countTokens(original);
+  const compTokens = countTokens(compressed);
+  if (origTokens === 0) return compressed;
+  if (compTokens > origTokens) return original;
+  const ratio = compTokens / origTokens;
+  if (ratio < 0.05 && origTokens < 2000) return original;
+  return compressed;
+}
+
+export { stripAnsi };

package/node_modules/@entelligentsia/forge-compress/src/entropy.ts

@@ -0,0 +1,105 @@
+import { countTokens } from "./tokens.js";
+
+export function shannonEntropy(text: string): number {
+  if (!text) return 0;
+  const freq = new Map<string, number>();
+  for (const ch of text) {
+    freq.set(ch, (freq.get(ch) ?? 0) + 1);
+  }
+  const len = text.length;
+  let entropy = 0;
+  for (const count of freq.values()) {
+    const p = count / len;
+    if (p > 0) entropy -= p * Math.log2(p);
+  }
+  return entropy;
+}
+
+export function normalizedTokenEntropy(text: string): number {
+  if (!text || text.trim().length === 0) return 0;
+  const words = text.trim().split(/\s+/);
+
+  if (words.length <= 1) {
+    const charEntropy = shannonEntropy(text.trim());
+    return Math.min(1, charEntropy / 4.5);
+  }
+
+  const freq = new Map<string, number>();
+  for (const w of words) {
+    freq.set(w, (freq.get(w) ?? 0) + 1);
+  }
+  const n = words.length;
+  const uniqueRatio = freq.size / n;
+  let entropy = 0;
+  for (const count of freq.values()) {
+    const p = count / n;
+    if (p > 0) entropy -= p * Math.log2(p);
+  }
+  const maxEntropy = Math.log2(n);
+  const normalized = maxEntropy > 0 ? entropy / maxEntropy : 0;
+  return Math.min(1, normalized * 0.7 + uniqueRatio * 0.3);
+}
+
+function renderIb(
+  lines: string[],
+  scores: number[],
+  threshold: number,
+): string {
+  const out: string[] = [];
+  let omitRun = 0;
+  for (let i = 0; i < lines.length; i++) {
+    if (scores[i] >= threshold) {
+      if (omitRun > 0) {
+        out.push(`// ... ${omitRun} low-info lines omitted`);
+        omitRun = 0;
+      }
+      out.push(lines[i]);
+    } else {
+      omitRun++;
+    }
+  }
+  if (omitRun > 0) {
+    out.push(`// ... ${omitRun} low-info lines omitted`);
+  }
+  return out.join("\n");
+}
+
+export function compressIb(text: string, targetRatio: number): string {
+  if (!text) return "";
+  const inputTokens = countTokens(text);
+  if (inputTokens === 0) return text;
+
+  const ratio = Math.max(0.02, Math.min(1.0, targetRatio));
+  const lines = text.split("\n");
+  const scores = lines.map((ln) => normalizedTokenEntropy(ln));
+
+  let lo = 0;
+  let hi = 1;
+  let best = renderIb(lines, scores, 0);
+  let bestDiff = Infinity;
+
+  const consider = (thr: number) => {
+    const cand = renderIb(lines, scores, thr);
+    const r = countTokens(cand) / inputTokens;
+    const diff = Math.abs(r - ratio);
+    if (diff < bestDiff) {
+      bestDiff = diff;
+      best = cand;
+    }
+    return r;
+  };
+
+  for (let i = 0; i < 26; i++) {
+    const mid = (lo + hi) / 2;
+    const r = consider(mid);
+    if (r > ratio) {
+      lo = mid;
+    } else {
+      hi = mid;
+    }
+  }
+  for (const thr of [0, 1, lo, hi, (lo + hi) / 2]) {
+    consider(thr);
+  }
+  return best;
+}