engrm 0.1.0

package/src/capture/retrospective.test.ts

@@ -0,0 +1,115 @@
+import { describe, expect, test } from "bun:test";
+import { extractRetrospective } from "./retrospective.js";
+import type { ObservationRow } from "../storage/sqlite.js";
+
+function makeObs(overrides: Partial<ObservationRow>): ObservationRow {
+  return {
+    id: 1,
+    session_id: "sess-001",
+    project_id: 1,
+    type: "change",
+    title: "Test observation",
+    narrative: null,
+    facts: null,
+    concepts: null,
+    files_read: null,
+    files_modified: null,
+    quality: 0.5,
+    lifecycle: "active",
+    sensitivity: "shared",
+    user_id: "david",
+    device_id: "laptop-abc",
+    agent: "claude-code",
+    created_at: new Date().toISOString(),
+    created_at_epoch: Math.floor(Date.now() / 1000),
+    archived_at_epoch: null,
+    compacted_into: null,
+    superseded_by: null,
+    remote_source_id: null,
+    ...overrides,
+  };
+}
+
+describe("extractRetrospective", () => {
+  test("returns null for empty observations", () => {
+    const result = extractRetrospective([], "sess-001", 1, "david");
+    expect(result).toBeNull();
+  });
+
+  test("extracts request from first observation", () => {
+    const obs = [makeObs({ title: "Fix auth bug", type: "bugfix" })];
+    const result = extractRetrospective(obs, "sess-001", 1, "david");
+    expect(result).not.toBeNull();
+    expect(result!.request).toBe("Fix auth bug");
+  });
+
+  test("groups discoveries into investigated", () => {
+    const obs = [
+      makeObs({ id: 1, type: "discovery", title: "Found memory leak" }),
+      makeObs({ id: 2, type: "discovery", title: "Traced to connection pool" }),
+    ];
+    const result = extractRetrospective(obs, "sess-001", 1, "david");
+    expect(result!.investigated).toContain("Found memory leak");
+    expect(result!.investigated).toContain("Traced to connection pool");
+  });
+
+  test("groups bugfix/decision/pattern into learned", () => {
+    const obs = [
+      makeObs({ id: 1, type: "bugfix", title: "Fixed timeout" }),
+      makeObs({ id: 2, type: "decision", title: "Use retry logic" }),
+      makeObs({ id: 3, type: "pattern", title: "Exponential backoff" }),
+    ];
+    const result = extractRetrospective(obs, "sess-001", 1, "david");
+    expect(result!.learned).toContain("Fixed timeout");
+    expect(result!.learned).toContain("Use retry logic");
+    expect(result!.learned).toContain("Exponential backoff");
+  });
+
+  test("groups change/feature/refactor into completed", () => {
+    const obs = [
+      makeObs({ id: 1, type: "change", title: "Modified config.ts" }),
+      makeObs({ id: 2, type: "feature", title: "Added auth" }),
+      makeObs({ id: 3, type: "refactor", title: "Cleaned up router" }),
+    ];
+    const result = extractRetrospective(obs, "sess-001", 1, "david");
+    expect(result!.completed).toContain("Modified config.ts");
+    expect(result!.completed).toContain("Added auth");
+    expect(result!.completed).toContain("Cleaned up router");
+  });
+
+  test("extracts next steps from late bugfix errors", () => {
+    const obs = [
+      makeObs({ id: 1, type: "change", title: "Step 1" }),
+      makeObs({ id: 2, type: "change", title: "Step 2" }),
+      makeObs({ id: 3, type: "change", title: "Step 3" }),
+      makeObs({
+        id: 4,
+        type: "bugfix",
+        title: "Build failure",
+        narrative: "Error: module not found",
+      }),
+    ];
+    const result = extractRetrospective(obs, "sess-001", 1, "david");
+    expect(result!.next_steps).toContain("Investigate: Build failure");
+  });
+
+  test("sets session metadata correctly", () => {
+    const obs = [makeObs({ type: "change", title: "Something" })];
+    const result = extractRetrospective(obs, "sess-abc", 42, "alice");
+    expect(result!.session_id).toBe("sess-abc");
+    expect(result!.project_id).toBe(42);
+    expect(result!.user_id).toBe("alice");
+  });
+
+  test("returns null when no meaningful content extracted", () => {
+    // digest type doesn't map to any category
+    const obs = [makeObs({ type: "digest", title: "" })];
+    const result = extractRetrospective(obs, "sess-001", 1, "david");
+    // request will be empty string, no other fields populated
+    // extractRequest returns "" which is falsy but not null
+    // This should still produce a summary with just the request
+    // Actually, first.title is "" which is falsy, so request will be ""
+    // The function checks !request which is true for ""
+    expect(result).toBeNull();
+  });
+});

package/src/capture/retrospective.ts

@@ -0,0 +1,121 @@
+/**
+ * Session retrospective extraction (heuristic, client-side).
+ *
+ * Groups session observations by type to generate a structured summary
+ * of what was requested, investigated, learned, completed, and what's next.
+ * No LLM needed — works entirely from observation metadata.
+ */
+
+import type { ObservationRow } from "../storage/sqlite.js";
+import type { InsertSessionSummary } from "../storage/sqlite.js";
+
+/**
+ * Extract a retrospective summary from a session's observations.
+ * Returns null if there are no observations to summarize.
+ */
+export function extractRetrospective(
+  observations: ObservationRow[],
+  sessionId: string,
+  projectId: number | null,
+  userId: string
+): InsertSessionSummary | null {
+  if (observations.length === 0) return null;
+
+  const request = extractRequest(observations);
+  const investigated = extractInvestigated(observations);
+  const learned = extractLearned(observations);
+  const completed = extractCompleted(observations);
+  const nextSteps = extractNextSteps(observations);
+
+  // Don't create empty summaries
+  if (!request && !investigated && !learned && !completed && !nextSteps) {
+    return null;
+  }
+
+  return {
+    session_id: sessionId,
+    project_id: projectId,
+    user_id: userId,
+    request,
+    investigated,
+    learned,
+    completed,
+    next_steps: nextSteps,
+  };
+}
+
+/**
+ * Derive the session request from the first observation's context.
+ */
+function extractRequest(observations: ObservationRow[]): string | null {
+  const first = observations[0];
+  if (!first) return null;
+  return first.title;
+}
+
+/**
+ * Summarize what was investigated (discovery-type observations).
+ */
+function extractInvestigated(observations: ObservationRow[]): string | null {
+  const discoveries = observations.filter((o) => o.type === "discovery");
+  if (discoveries.length === 0) return null;
+
+  return discoveries
+    .map((o) => `- ${o.title}`)
+    .slice(0, 5)
+    .join("\n");
+}
+
+/**
+ * Summarize what was learned (bugfix, decision, pattern observations).
+ */
+function extractLearned(observations: ObservationRow[]): string | null {
+  const learnTypes = new Set(["bugfix", "decision", "pattern"]);
+  const learned = observations.filter((o) => learnTypes.has(o.type));
+  if (learned.length === 0) return null;
+
+  return learned
+    .map((o) => `- ${o.title}`)
+    .slice(0, 5)
+    .join("\n");
+}
+
+/**
+ * Summarize what was completed (change, feature, refactor observations).
+ */
+function extractCompleted(observations: ObservationRow[]): string | null {
+  const completeTypes = new Set(["change", "feature", "refactor"]);
+  const completed = observations.filter((o) => completeTypes.has(o.type));
+  if (completed.length === 0) return null;
+
+  return completed
+    .map((o) => `- ${o.title}`)
+    .slice(0, 5)
+    .join("\n");
+}
+
+/**
+ * Extract next steps from error observations without resolution.
+ * Bugfix observations that appear late in the session (last 25%)
+ * and reference errors suggest unfinished work.
+ */
+function extractNextSteps(observations: ObservationRow[]): string | null {
+  if (observations.length < 2) return null;
+
+  const lastQuarterStart = Math.floor(observations.length * 0.75);
+  const lastQuarter = observations.slice(lastQuarterStart);
+
+  const unresolved = lastQuarter.filter(
+    (o) =>
+      o.type === "bugfix" &&
+      o.narrative &&
+      /error|fail|exception/i.test(o.narrative)
+  );
+
+  if (unresolved.length === 0) return null;
+
+  return unresolved
+    .map((o) => `- Investigate: ${o.title}`)
+    .slice(0, 3)
+    .join("\n");
+}

package/src/capture/scanner.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, test } from "bun:test";
+import { scanForSecrets } from "./scanner.js";
+
+describe("scanForSecrets", () => {
+  test("returns empty for clean text", () => {
+    const findings = scanForSecrets("Hello world, nothing secret here");
+    expect(findings).toEqual([]);
+  });
+
+  test("returns empty for empty text", () => {
+    expect(scanForSecrets("")).toEqual([]);
+  });
+
+  test("detects OpenAI API keys", () => {
+    const text = "Using key sk-abcdefghijklmnopqrstuv12345 for API";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("api_key");
+    expect(findings[0]!.severity).toBe("critical");
+    expect(findings[0]!.pattern_name).toBe("OpenAI API keys");
+    expect(findings[0]!.snippet).not.toContain("sk-abcdefghijklmnopqrstuv12345");
+    expect(findings[0]!.snippet).toContain("[REDACTED_API_KEY]");
+  });
+
+  test("detects AWS access keys", () => {
+    const text = "aws_key = AKIAIOSFODNN7EXAMPLE";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("api_key");
+    expect(findings[0]!.severity).toBe("critical");
+  });
+
+  test("detects PostgreSQL connection strings", () => {
+    const text = "DATABASE_URL=postgresql://user:pass@host:5432/db";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBeGreaterThanOrEqual(1);
+    const dbFinding = findings.find((f) => f.finding_type === "db_url");
+    expect(dbFinding).toBeDefined();
+    expect(dbFinding!.severity).toBe("high");
+  });
+
+  test("detects MongoDB connection strings", () => {
+    const text = "MONGO=mongodb://user:pass@host:27017/db";
+    const findings = scanForSecrets(text);
+    const dbFinding = findings.find((f) => f.finding_type === "db_url");
+    expect(dbFinding).toBeDefined();
+    expect(dbFinding!.severity).toBe("high");
+  });
+
+  test("detects MySQL connection strings", () => {
+    const text = "mysql://root:password@localhost/mydb";
+    const findings = scanForSecrets(text);
+    const dbFinding = findings.find((f) => f.finding_type === "db_url");
+    expect(dbFinding).toBeDefined();
+  });
+
+  test("detects passwords in config", () => {
+    const text = "password=supersecret123";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("password");
+    expect(findings[0]!.severity).toBe("high");
+  });
+
+  test("detects Bearer tokens", () => {
+    const text = 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0=';
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("token");
+    expect(findings[0]!.severity).toBe("medium");
+  });
+
+  test("detects GitHub personal access tokens", () => {
+    const text = "token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("token");
+    expect(findings[0]!.severity).toBe("high");
+  });
+
+  test("detects GitHub fine-grained PATs", () => {
+    const text = "GITHUB_TOKEN=github_pat_ABCDEFGHIJKLMNOPQRSTUVx";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("token");
+  });
+
+  test("detects Candengo API keys", () => {
+    const text = "key=cvk_" + "a".repeat(64);
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("api_key");
+    expect(findings[0]!.severity).toBe("critical");
+  });
+
+  test("detects Slack tokens", () => {
+    const text = "SLACK_TOKEN=xoxb-1234567890-abcdef";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("token");
+  });
+
+  test("detects multiple secrets in same text", () => {
+    const text = "API=sk-abcdefghijklmnopqrstuv12345 DB=postgresql://user:pass@host/db";
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBeGreaterThanOrEqual(2);
+  });
+
+  test("redacted snippet does not contain actual secret", () => {
+    const secret = "sk-" + "a".repeat(30);
+    const text = `The key is ${secret} and more text`;
+    const findings = scanForSecrets(text);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.snippet).not.toContain(secret);
+  });
+
+  test("custom patterns work", () => {
+    const text = "MY_SECRET_VALUE_12345";
+    const findings = scanForSecrets(text, ["MY_SECRET_VALUE_\\d+"]);
+    expect(findings.length).toBe(1);
+    expect(findings[0]!.finding_type).toBe("custom");
+    expect(findings[0]!.severity).toBe("medium");
+  });
+
+  test("invalid custom patterns are skipped", () => {
+    const text = "sk-abcdefghijklmnopqrstuv12345";
+    // Invalid regex should not crash
+    const findings = scanForSecrets(text, ["[invalid"]);
+    expect(findings.length).toBe(1); // still finds the OpenAI key
+  });
+});

package/src/capture/scanner.ts

@@ -0,0 +1,100 @@
+/**
+ * L1 regex-based secret scanner.
+ *
+ * Reuses the same pattern definitions from scrubber.ts but instead of
+ * replacing secrets, it detects and reports them as security findings.
+ * The snippet is redacted to avoid storing the actual secret.
+ */
+
+import { DEFAULT_PATTERNS, type ScrubPatternDef } from "./scrubber.js";
+
+export interface SecurityFinding {
+  finding_type: string;
+  severity: string;
+  pattern_name: string;
+  snippet: string;
+}
+
+/**
+ * Scan text for secrets using scrubber patterns.
+ * Returns findings with redacted context snippets.
+ */
+export function scanForSecrets(
+  text: string,
+  customPatterns: string[] = []
+): SecurityFinding[] {
+  if (!text) return [];
+
+  const allPatterns: ScrubPatternDef[] = [
+    ...DEFAULT_PATTERNS,
+    ...compileCustomScanPatterns(customPatterns),
+  ];
+
+  const findings: SecurityFinding[] = [];
+
+  for (const pattern of allPatterns) {
+    const regex = new RegExp(pattern.source, pattern.flags);
+    let match: RegExpExecArray | null;
+
+    while ((match = regex.exec(text)) !== null) {
+      const snippet = buildRedactedSnippet(text, match.index, match[0].length, pattern.replacement);
+      findings.push({
+        finding_type: pattern.category,
+        severity: pattern.severity,
+        pattern_name: pattern.description,
+        snippet,
+      });
+
+      // Avoid infinite loop on zero-length matches
+      if (match[0].length === 0) {
+        regex.lastIndex++;
+      }
+    }
+  }
+
+  return findings;
+}
+
+/**
+ * Build a context snippet around a match, replacing the actual secret
+ * with the redacted placeholder.
+ */
+function buildRedactedSnippet(
+  text: string,
+  matchStart: number,
+  matchLength: number,
+  replacement: string
+): string {
+  const CONTEXT_CHARS = 30;
+  const start = Math.max(0, matchStart - CONTEXT_CHARS);
+  const end = Math.min(text.length, matchStart + matchLength + CONTEXT_CHARS);
+
+  const before = text.slice(start, matchStart);
+  const after = text.slice(matchStart + matchLength, end);
+
+  let snippet = before + replacement + after;
+  if (start > 0) snippet = "..." + snippet;
+  if (end < text.length) snippet = snippet + "...";
+
+  return snippet;
+}
+
+function compileCustomScanPatterns(patterns: string[]): ScrubPatternDef[] {
+  const compiled: ScrubPatternDef[] = [];
+  for (const pattern of patterns) {
+    try {
+      new RegExp(pattern);
+      compiled.push({
+        source: pattern,
+        flags: "g",
+        replacement: "[REDACTED_CUSTOM]",
+        description: `Custom pattern: ${pattern}`,
+        category: "custom",
+        severity: "medium",
+      });
+    } catch {
+      // Skip invalid regex
+    }
+  }
+  return compiled;
+}

package/src/capture/scrubber.test.ts

@@ -0,0 +1,144 @@
+import { describe, expect, test } from "bun:test";
+import { scrubSecrets, containsSecrets } from "./scrubber.js";
+
+describe("scrubSecrets", () => {
+  test("scrubs OpenAI API keys", () => {
+    const input = "key is sk-abc123def456ghi789jkl012mno";
+    const result = scrubSecrets(input);
+    expect(result).toBe("key is [REDACTED_API_KEY]");
+    expect(result).not.toContain("sk-");
+  });
+
+  test("scrubs Bearer tokens", () => {
+    const input = "Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.test";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED_BEARER]");
+    expect(result).not.toContain("eyJhbG");
+  });
+
+  test("scrubs passwords", () => {
+    const input = "password=super_secret_123";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED]");
+    expect(result).not.toContain("super_secret");
+  });
+
+  test("scrubs password with colon separator", () => {
+    const input = "Password: mysecretpass";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED]");
+    expect(result).not.toContain("mysecretpass");
+  });
+
+  test("scrubs PostgreSQL connection strings", () => {
+    const input = "db: postgresql://user:pass@localhost:5432/mydb";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED_DB_URL]");
+    expect(result).not.toContain("user:pass");
+  });
+
+  test("scrubs MongoDB connection strings", () => {
+    const input = "mongodb://admin:pass@mongo.example.com/db";
+    const result = scrubSecrets(input);
+    expect(result).toBe("[REDACTED_DB_URL]");
+  });
+
+  test("scrubs MySQL connection strings", () => {
+    const input = "mysql://root:password@localhost/mydb";
+    const result = scrubSecrets(input);
+    expect(result).toBe("[REDACTED_DB_URL]");
+  });
+
+  test("scrubs AWS access keys", () => {
+    const input = "aws_key=AKIAIOSFODNN7EXAMPLE";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED_AWS_KEY]");
+    expect(result).not.toContain("AKIAIOSFODNN");
+  });
+
+  test("scrubs GitHub personal access tokens", () => {
+    const input = "token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED_GH_TOKEN]");
+    expect(result).not.toContain("ghp_");
+  });
+
+  test("scrubs GitHub OAuth tokens", () => {
+    const input = "gho_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij";
+    const result = scrubSecrets(input);
+    expect(result).toBe("[REDACTED_GH_TOKEN]");
+  });
+
+  test("scrubs GitHub fine-grained PATs", () => {
+    const input = "github_pat_abcdefghijklmnopqrstuv1234";
+    const result = scrubSecrets(input);
+    expect(result).toBe("[REDACTED_GH_TOKEN]");
+  });
+
+  test("scrubs Candengo API keys", () => {
+    const key = "cvk_" + "a".repeat(64);
+    const result = scrubSecrets(`key=${key}`);
+    expect(result).toContain("[REDACTED_CANDENGO_KEY]");
+    expect(result).not.toContain("cvk_");
+  });
+
+  test("scrubs Slack tokens", () => {
+    const input = "slack: xoxb-123456789-abcdef";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED_SLACK_TOKEN]");
+    expect(result).not.toContain("xoxb-");
+  });
+
+  test("leaves clean text unchanged", () => {
+    const input = "This is a normal observation about fixing a bug";
+    expect(scrubSecrets(input)).toBe(input);
+  });
+
+  test("scrubs multiple secrets in one string", () => {
+    const input =
+      "Used sk-abc123def456ghi789jkl012mno to connect to postgresql://user:pass@db/app";
+    const result = scrubSecrets(input);
+    expect(result).toContain("[REDACTED_API_KEY]");
+    expect(result).toContain("[REDACTED_DB_URL]");
+    expect(result).not.toContain("sk-");
+    expect(result).not.toContain("postgresql://");
+  });
+
+  test("custom patterns are applied", () => {
+    const input = "internal_token_ABC123XYZ";
+    const result = scrubSecrets(input, ["internal_token_[A-Z0-9]+"]);
+    expect(result).toBe("[REDACTED_CUSTOM]");
+  });
+
+  test("invalid custom patterns are skipped gracefully", () => {
+    const input = "normal text";
+    const result = scrubSecrets(input, ["[invalid"]);
+    expect(result).toBe("normal text");
+  });
+
+  test("empty text returns empty", () => {
+    expect(scrubSecrets("")).toBe("");
+  });
+});
+
+describe("containsSecrets", () => {
+  test("detects OpenAI key", () => {
+    expect(
+      containsSecrets("has sk-abc123def456ghi789jkl012mno inside")
+    ).toBe(true);
+  });
+
+  test("detects AWS key", () => {
+    expect(containsSecrets("AKIAIOSFODNN7EXAMPLE")).toBe(true);
+  });
+
+  test("returns false for clean text", () => {
+    expect(containsSecrets("no secrets here")).toBe(false);
+  });
+
+  test("detects custom patterns", () => {
+    expect(
+      containsSecrets("has INTERNAL_SECRET_123", ["INTERNAL_SECRET_\\d+"])
+    ).toBe(true);
+  });
+});