@entelligentsia/forgecli 1.0.2 → 1.0.10

package/node_modules/@entelligentsia/forge-compress/src/compressor.ts

@@ -0,0 +1,147 @@
+import { countTokens } from "./tokens.js";
+import {
+  stripAnsi,
+  stripTimestampsAndHashes,
+  normalizeWhitespace,
+  isBoilerplate,
+} from "./strip.js";
+
+export function lightweightCleanup(content: string): string {
+  const lines = content.split("\n");
+  const total = lines.length;
+  const result: string[] = [];
+  let blankCount = 0;
+  let braceRun: string[] = [];
+
+  const flushBraceRun = () => {
+    if (total <= 200 || braceRun.length <= 5) {
+      result.push(...braceRun);
+    } else {
+      result.push(braceRun[0], braceRun[1]);
+      result.push(`[${braceRun.length - 2} brace-only lines collapsed]`);
+    }
+    braceRun = [];
+  };
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      flushBraceRun();
+      blankCount++;
+      if (blankCount <= 1) result.push("");
+      continue;
+    }
+    blankCount = 0;
+    if (/^[}\]);]+$/.test(trimmed)) {
+      braceRun.push(trimmed);
+      continue;
+    }
+    flushBraceRun();
+    result.push(line);
+  }
+  flushBraceRun();
+  return result.join("\n");
+}
+
+export function verbatimCompact(text: string): string {
+  const lines: string[] = [];
+  let blankCount = 0;
+  let prevLine: string | null = null;
+  let repeatCount = 0;
+
+  const flushRepeats = () => {
+    if (repeatCount > 1 && prevLine !== null) {
+      const lastIdx = lines.length - 1;
+      if (lastIdx >= 0) {
+        lines[lastIdx] = `[${repeatCount}x] ${prevLine}`;
+      }
+    }
+    repeatCount = 0;
+    prevLine = null;
+  };
+
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      blankCount++;
+      if (blankCount <= 1) {
+        flushRepeats();
+        lines.push("");
+      }
+      continue;
+    }
+    blankCount = 0;
+    if (isBoilerplate(trimmed)) continue;
+
+    const normalized = normalizeWhitespace(trimmed);
+    const stripped = stripTimestampsAndHashes(normalized);
+
+    if (prevLine !== null && prevLine === stripped) {
+      repeatCount++;
+      continue;
+    }
+    flushRepeats();
+    prevLine = stripped;
+    repeatCount = 1;
+    lines.push(stripped);
+  }
+  flushRepeats();
+  return lines.join("\n");
+}
+
+export function aggressiveCompress(
+  content: string,
+  ext?: string,
+): string {
+  const result: string[] = [];
+  const isPython = ext === "py";
+  const isHtml = ext === "html" || ext === "htm" || ext === "xml" || ext === "svg";
+  const isSql = ext === "sql";
+  const isShell = ext === "sh" || ext === "bash" || ext === "zsh" || ext === "fish";
+  let inBlockComment = false;
+
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    if (inBlockComment) {
+      if (trimmed.includes("*/") || (isHtml && trimmed.includes("-->"))) {
+        inBlockComment = false;
+      }
+      continue;
+    }
+    if (trimmed.startsWith("/*") || (isHtml && trimmed.startsWith("<!--"))) {
+      if (!(trimmed.includes("*/") || trimmed.includes("-->"))) {
+        inBlockComment = true;
+      }
+      continue;
+    }
+    if (trimmed.startsWith("//") && !trimmed.startsWith("///")) continue;
+    if (trimmed.startsWith("*") || trimmed.startsWith("*/")) continue;
+    if (isPython && trimmed.startsWith("#")) continue;
+    if (isSql && trimmed.startsWith("--")) continue;
+    if (isShell && trimmed.startsWith("#") && !trimmed.startsWith("#!")) continue;
+
+    if (/^[}\]);]+$/.test(trimmed)) {
+      const last = result[result.length - 1];
+      if (last && /^[}\]);]+$/.test(last.trim())) {
+        result[result.length - 1] = last + trimmed;
+        continue;
+      }
+    }
+    result.push(trimmed);
+  }
+  return result.join("\n");
+}
+
+export function safeguardRatio(original: string, compressed: string): string {
+  const origTokens = countTokens(original);
+  const compTokens = countTokens(compressed);
+  if (origTokens === 0) return compressed;
+  if (compTokens > origTokens) return original;
+  const ratio = compTokens / origTokens;
+  if (ratio < 0.05 && origTokens < 2000) return original;
+  return compressed;
+}
+
+export { stripAnsi };

package/node_modules/@entelligentsia/forge-compress/src/entropy.ts

@@ -0,0 +1,105 @@
+import { countTokens } from "./tokens.js";
+
+export function shannonEntropy(text: string): number {
+  if (!text) return 0;
+  const freq = new Map<string, number>();
+  for (const ch of text) {
+    freq.set(ch, (freq.get(ch) ?? 0) + 1);
+  }
+  const len = text.length;
+  let entropy = 0;
+  for (const count of freq.values()) {
+    const p = count / len;
+    if (p > 0) entropy -= p * Math.log2(p);
+  }
+  return entropy;
+}
+
+export function normalizedTokenEntropy(text: string): number {
+  if (!text || text.trim().length === 0) return 0;
+  const words = text.trim().split(/\s+/);
+
+  if (words.length <= 1) {
+    const charEntropy = shannonEntropy(text.trim());
+    return Math.min(1, charEntropy / 4.5);
+  }
+
+  const freq = new Map<string, number>();
+  for (const w of words) {
+    freq.set(w, (freq.get(w) ?? 0) + 1);
+  }
+  const n = words.length;
+  const uniqueRatio = freq.size / n;
+  let entropy = 0;
+  for (const count of freq.values()) {
+    const p = count / n;
+    if (p > 0) entropy -= p * Math.log2(p);
+  }
+  const maxEntropy = Math.log2(n);
+  const normalized = maxEntropy > 0 ? entropy / maxEntropy : 0;
+  return Math.min(1, normalized * 0.7 + uniqueRatio * 0.3);
+}
+
+function renderIb(
+  lines: string[],
+  scores: number[],
+  threshold: number,
+): string {
+  const out: string[] = [];
+  let omitRun = 0;
+  for (let i = 0; i < lines.length; i++) {
+    if (scores[i] >= threshold) {
+      if (omitRun > 0) {
+        out.push(`// ... ${omitRun} low-info lines omitted`);
+        omitRun = 0;
+      }
+      out.push(lines[i]);
+    } else {
+      omitRun++;
+    }
+  }
+  if (omitRun > 0) {
+    out.push(`// ... ${omitRun} low-info lines omitted`);
+  }
+  return out.join("\n");
+}
+
+export function compressIb(text: string, targetRatio: number): string {
+  if (!text) return "";
+  const inputTokens = countTokens(text);
+  if (inputTokens === 0) return text;
+
+  const ratio = Math.max(0.02, Math.min(1.0, targetRatio));
+  const lines = text.split("\n");
+  const scores = lines.map((ln) => normalizedTokenEntropy(ln));
+
+  let lo = 0;
+  let hi = 1;
+  let best = renderIb(lines, scores, 0);
+  let bestDiff = Infinity;
+
+  const consider = (thr: number) => {
+    const cand = renderIb(lines, scores, thr);
+    const r = countTokens(cand) / inputTokens;
+    const diff = Math.abs(r - ratio);
+    if (diff < bestDiff) {
+      bestDiff = diff;
+      best = cand;
+    }
+    return r;
+  };
+
+  for (let i = 0; i < 26; i++) {
+    const mid = (lo + hi) / 2;
+    const r = consider(mid);
+    if (r > ratio) {
+      lo = mid;
+    } else {
+      hi = mid;
+    }
+  }
+  for (const thr of [0, 1, lo, hi, (lo + hi) / 2]) {
+    consider(thr);
+  }
+  return best;
+}

package/node_modules/@entelligentsia/forge-compress/src/forge/entity.ts

@@ -0,0 +1,184 @@
+import { countTokens } from "../tokens.js";
+
+interface PhaseSummary {
+  objective?: string;
+  written_at?: string;
+  key_changes?: string[];
+  findings?: string[];
+  verdict?: string;
+  artifact_ref?: string;
+  route?: string;
+  [key: string]: unknown;
+}
+
+const PHASE_ORDER = [
+  "plan",
+  "triage",
+  "review_plan",
+  "implementation",
+  "code_review",
+  "validation",
+  "approve",
+] as const;
+
+export interface CompressEntityOptions {
+  keepSummaries?: boolean | "latest" | "all";
+  maxKeyChanges?: number;
+  flatFormat?: boolean;
+  maxTokens?: number;
+}
+
+export function compressEntity(
+  input: string,
+  opts?: CompressEntityOptions,
+): string {
+  let entity: Record<string, unknown>;
+  try {
+    entity = JSON.parse(input);
+  } catch {
+    return input;
+  }
+
+  const keepSummaries = opts?.keepSummaries ?? "latest";
+  const maxKeyChanges = opts?.maxKeyChanges ?? 3;
+
+  const result: Record<string, unknown> = {};
+
+  const idKey = findIdKey(entity);
+  if (idKey) result[idKey] = entity[idKey];
+  if (entity.title) result.title = entity.title;
+  if (entity.status) result.status = entity.status;
+
+  if (entity.sprintId) result.sprintId = entity.sprintId;
+  if (entity.feature_id) result.feature_id = entity.feature_id;
+  if (entity.severity) result.severity = entity.severity;
+  if (entity.estimate) result.estimate = entity.estimate;
+  if (entity.executionMode) result.executionMode = entity.executionMode;
+  if (entity.pipeline) result.pipeline = entity.pipeline;
+
+  if (Array.isArray(entity.taskIds)) {
+    result.taskIds = entity.taskIds;
+    result.taskCount = entity.taskIds.length;
+  }
+
+  if (Array.isArray(entity.dependencies) && entity.dependencies.length > 0) {
+    result.dependencies = entity.dependencies;
+  }
+
+  const summaries = entity.summaries as Record<string, PhaseSummary> | undefined;
+  if (summaries && typeof summaries === "object") {
+    if (keepSummaries === "all") {
+      result.summaries = compressSummaries(summaries, maxKeyChanges);
+    } else if (keepSummaries === "latest") {
+      const latest = findLatestPhase(summaries);
+      if (latest) {
+        result.latestPhase = latest.phase;
+        result.latestSummary = compressOneSummary(latest.summary, maxKeyChanges);
+      }
+    }
+  }
+
+  if (opts?.flatFormat) {
+    return formatFlat(result);
+  }
+
+  let output = JSON.stringify(result, null, 2);
+  if (opts?.maxTokens && countTokens(output) > opts.maxTokens) {
+    output = JSON.stringify(result);
+  }
+  return output;
+}
+
+export function compressEntityList(
+  input: string,
+  opts?: CompressEntityOptions,
+): string {
+  let entities: Record<string, unknown>[];
+  try {
+    entities = JSON.parse(input);
+  } catch {
+    return input;
+  }
+
+  if (!Array.isArray(entities)) return input;
+
+  const compressed = entities.map((e) =>
+    JSON.parse(compressEntity(JSON.stringify(e), { ...opts, flatFormat: false })),
+  );
+
+  let output = JSON.stringify(compressed, null, 2);
+  if (opts?.maxTokens && countTokens(output) > opts.maxTokens) {
+    output = JSON.stringify(compressed);
+  }
+  return output;
+}
+
+function findIdKey(entity: Record<string, unknown>): string | undefined {
+  for (const key of ["taskId", "sprintId", "bugId", "featureId", "id"]) {
+    if (entity[key]) return key;
+  }
+  return undefined;
+}
+
+function compressSummaries(
+  summaries: Record<string, PhaseSummary>,
+  maxKeyChanges: number,
+): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+  for (const phase of PHASE_ORDER) {
+    if (summaries[phase]) {
+      result[phase] = compressOneSummary(summaries[phase], maxKeyChanges);
+    }
+  }
+  return result;
+}
+
+function compressOneSummary(
+  summary: PhaseSummary,
+  maxKeyChanges: number,
+): Record<string, unknown> {
+  const result: Record<string, unknown> = {};
+  if (summary.objective) result.objective = summary.objective;
+  if (summary.verdict) result.verdict = summary.verdict;
+
+  if (summary.key_changes && summary.key_changes.length > 0) {
+    const shown = summary.key_changes.slice(0, maxKeyChanges);
+    const extra = summary.key_changes.length - maxKeyChanges;
+    result.key_changes = extra > 0 ? [...shown, `+${extra} more`] : shown;
+  }
+
+  if (summary.route) result.route = summary.route;
+  return result;
+}
+
+function findLatestPhase(
+  summaries: Record<string, PhaseSummary>,
+): { phase: string; summary: PhaseSummary } | null {
+  for (let i = PHASE_ORDER.length - 1; i >= 0; i--) {
+    const phase = PHASE_ORDER[i];
+    if (summaries[phase]) {
+      return { phase, summary: summaries[phase] };
+    }
+  }
+  return null;
+}
+
+function formatFlat(obj: Record<string, unknown>, prefix = ""): string {
+  const lines: string[] = [];
+  for (const [key, value] of Object.entries(obj)) {
+    const fullKey = prefix ? `${prefix}.${key}` : key;
+    if (value === null || value === undefined) continue;
+    if (Array.isArray(value)) {
+      if (value.length <= 5 && value.every((v) => typeof v === "string")) {
+        lines.push(`${fullKey}: ${value.join(", ")}`);
+      } else {
+        lines.push(`${fullKey}: [${value.length} items]`);
+      }
+    } else if (typeof value === "object") {
+      lines.push(formatFlat(value as Record<string, unknown>, fullKey));
+    } else {
+      lines.push(`${fullKey}: ${value}`);
+    }
+  }
+  return lines.filter(Boolean).join("\n");
+}

package/node_modules/@entelligentsia/forge-compress/src/forge/index.ts

@@ -0,0 +1,10 @@
+export { compressStoreQuery } from "./query.js";
+export type { CompressQueryOptions } from "./query.js";
+
+export { compressEntity, compressEntityList } from "./entity.js";
+export type { CompressEntityOptions } from "./entity.js";
+
+export { compressMarkdown } from "./markdown.js";
+export type { CompressMarkdownOptions } from "./markdown.js";
+
+export { compressValidateStore } from "./validate.js";

package/node_modules/@entelligentsia/forge-compress/src/forge/markdown.ts

@@ -0,0 +1,122 @@
+import { countTokens, truncateToTokenBudget } from "../tokens.js";
+
+export interface CompressMarkdownOptions {
+  maxTokens?: number;
+  mode?: "map" | "headings" | "truncate";
+}
+
+export function compressMarkdown(
+  input: string,
+  opts?: CompressMarkdownOptions,
+): string {
+  if (!input.trim()) return "";
+
+  const mode = opts?.mode ?? "map";
+  const maxTokens = opts?.maxTokens ?? Infinity;
+
+  switch (mode) {
+    case "headings":
+      return compressHeadings(input, maxTokens);
+    case "truncate":
+      return truncateToTokenBudget(input, maxTokens);
+    case "map":
+    default:
+      return compressMap(input, maxTokens);
+  }
+}
+
+function compressMap(input: string, maxTokens: number): string {
+  const lines = input.split("\n");
+  const picked: string[] = [];
+  let inFrontmatter = false;
+  let frontmatterDone = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmed = line.trim();
+
+    if (i === 0 && trimmed === "---") {
+      inFrontmatter = true;
+      continue;
+    }
+    if (inFrontmatter) {
+      if (trimmed === "---") {
+        inFrontmatter = false;
+        frontmatterDone = true;
+      }
+      continue;
+    }
+
+    if (trimmed.startsWith("#")) {
+      picked.push(line);
+      continue;
+    }
+
+    if (
+      trimmed.startsWith("- **") ||
+      trimmed.startsWith("* **") ||
+      trimmed.startsWith("| ") ||
+      trimmed.startsWith("```")
+    ) {
+      picked.push(line);
+      continue;
+    }
+
+    if (
+      trimmed.startsWith("- [ ]") ||
+      trimmed.startsWith("- [x]") ||
+      trimmed.startsWith("- [X]")
+    ) {
+      picked.push(line);
+      continue;
+    }
+
+    if (i === 0 || (frontmatterDone && picked.length === 0)) {
+      picked.push(line);
+    }
+  }
+
+  if (picked.length === 0) {
+    return truncateToTokenBudget(input, maxTokens);
+  }
+
+  const totalLines = lines.length;
+  const keptLines = picked.length;
+  const omitted = totalLines - keptLines;
+
+  let result = picked.join("\n");
+  if (omitted > 0) {
+    result += `\n\n[${omitted} body lines omitted — ${totalLines} total]`;
+  }
+
+  if (countTokens(result) > maxTokens) {
+    result = truncateToTokenBudget(result, maxTokens);
+  }
+
+  return result;
+}
+
+function compressHeadings(input: string, maxTokens: number): string {
+  const lines = input.split("\n");
+  const headings: string[] = [];
+  let depth = 0;
+
+  for (const line of lines) {
+    const match = line.match(/^(#{1,6})\s+(.+)/);
+    if (match) {
+      const level = match[1].length;
+      const indent = "  ".repeat(level - 1);
+      headings.push(`${indent}${match[2]}`);
+      depth = Math.max(depth, level);
+    }
+  }
+
+  if (headings.length === 0) {
+    return truncateToTokenBudget(input, maxTokens);
+  }
+
+  const result = `${headings.length} sections (depth ${depth}):\n${headings.join("\n")}`;
+  return countTokens(result) > maxTokens
+    ? truncateToTokenBudget(result, maxTokens)
+    : result;
+}

package/node_modules/@entelligentsia/forge-compress/src/forge/query.ts

@@ -0,0 +1,105 @@
+import { countTokens } from "../tokens.js";
+
+interface QueryResultEntry {
+  id: string;
+  title: string;
+  status: string;
+  type: string;
+  relationships?: Record<string, unknown>;
+  fileRefs?: { json: string; md: string };
+  storeRef?: string;
+  indexRef?: string;
+  excerpt?: string | null;
+  [key: string]: unknown;
+}
+
+interface QueryEnvelope {
+  query?: string;
+  path?: string;
+  traversalTrace?: string[];
+  results?: QueryResultEntry[];
+  relatedFileRefs?: string[];
+  count?: number;
+  totalMatched?: number;
+  returned?: number;
+  limit?: number | null;
+  sort?: string | null;
+  config?: Record<string, unknown>;
+  meta?: Record<string, unknown>;
+  [key: string]: unknown;
+}
+
+export interface CompressQueryOptions {
+  keepExcerpts?: boolean;
+  maxExcerptLines?: number;
+  maxResults?: number;
+  maxTokens?: number;
+}
+
+export function compressStoreQuery(
+  input: string,
+  opts?: CompressQueryOptions,
+): string {
+  let envelope: QueryEnvelope;
+  try {
+    envelope = JSON.parse(input);
+  } catch {
+    return input;
+  }
+
+  if (!envelope.results || !Array.isArray(envelope.results)) return input;
+
+  const keepExcerpts = opts?.keepExcerpts ?? false;
+  const maxExcerptLines = opts?.maxExcerptLines ?? 5;
+  const maxResults = opts?.maxResults ?? envelope.results.length;
+
+  const results = envelope.results.slice(0, maxResults).map((r) => {
+    const entry: Record<string, unknown> = {
+      id: r.id,
+      title: r.title,
+      status: r.status,
+      type: r.type,
+    };
+
+    if (r.relationships && Object.keys(r.relationships).length > 0) {
+      const rels: Record<string, unknown> = {};
+      for (const [k, v] of Object.entries(r.relationships)) {
+        if (v !== null && v !== undefined) rels[k] = v;
+      }
+      if (Object.keys(rels).length > 0) entry.relationships = rels;
+    }
+
+    if (keepExcerpts && r.excerpt) {
+      const lines = r.excerpt.split("\n");
+      entry.excerpt =
+        lines.length > maxExcerptLines
+          ? lines.slice(0, maxExcerptLines).join("\n") +
+            `\n... (${lines.length - maxExcerptLines} more lines)`
+          : r.excerpt;
+    }
+
+    return entry;
+  });
+
+  const compact: Record<string, unknown> = { results };
+
+  if (envelope.query) compact.query = envelope.query;
+  if (envelope.path) compact.path = envelope.path;
+  if (typeof envelope.count === "number") compact.count = envelope.count;
+  if (typeof envelope.totalMatched === "number")
+    compact.totalMatched = envelope.totalMatched;
+  if (typeof envelope.returned === "number")
+    compact.returned = envelope.returned;
+
+  if (envelope.results.length > maxResults) {
+    compact.truncated = envelope.results.length - maxResults;
+  }
+
+  let output = JSON.stringify(compact, null, 2);
+
+  if (opts?.maxTokens && countTokens(output) > opts.maxTokens) {
+    output = JSON.stringify(compact);
+  }
+
+  return output;
+}