engrm 0.1.0

package/src/lifecycle/aging.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import { runAgingJob } from "./aging.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+let projectId: number;
+
+const DAY = 86400;
+const NOW = Math.floor(Date.now() / 1000);
+
+function insertObs(
+  overrides: Partial<{
+    lifecycle: string;
+    created_at_epoch: number;
+    title: string;
+  }> = {}
+) {
+  const epoch = overrides.created_at_epoch ?? NOW;
+  db.db
+    .query(
+      `INSERT INTO observations (project_id, type, title, quality, lifecycle, sensitivity, user_id, device_id, agent, created_at, created_at_epoch)
+       VALUES (?, 'discovery', ?, 0.5, ?, 'shared', 'user1', 'dev1', 'claude-code', datetime('now'), ?)`
+    )
+    .run(
+      projectId,
+      overrides.title ?? "Test observation",
+      overrides.lifecycle ?? "active",
+      epoch
+    );
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-aging-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+  const project = db.upsertProject({
+    canonical_id: "github.com/test/repo",
+    name: "repo",
+  });
+  projectId = project.id;
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("runAgingJob", () => {
+  test("transitions active observations older than 30 days to aging", () => {
+    insertObs({ created_at_epoch: NOW - 31 * DAY });
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(1);
+
+    const obs = db.getObservationById(1);
+    expect(obs!.lifecycle).toBe("aging");
+  });
+
+  test("does not transition observations newer than 30 days", () => {
+    insertObs({ created_at_epoch: NOW - 29 * DAY });
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(0);
+
+    const obs = db.getObservationById(1);
+    expect(obs!.lifecycle).toBe("active");
+  });
+
+  test("does not affect pinned observations", () => {
+    insertObs({ lifecycle: "pinned", created_at_epoch: NOW - 60 * DAY });
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(0);
+
+    const obs = db.getObservationById(1);
+    expect(obs!.lifecycle).toBe("pinned");
+  });
+
+  test("does not re-process already aging observations", () => {
+    insertObs({ lifecycle: "aging", created_at_epoch: NOW - 60 * DAY });
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(0);
+  });
+
+  test("does not affect archived observations", () => {
+    insertObs({ lifecycle: "archived", created_at_epoch: NOW - 60 * DAY });
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(0);
+  });
+
+  test("returns accurate count for multiple observations", () => {
+    insertObs({ created_at_epoch: NOW - 31 * DAY, title: "Old 1" });
+    insertObs({ created_at_epoch: NOW - 45 * DAY, title: "Old 2" });
+    insertObs({ created_at_epoch: NOW - 10 * DAY, title: "Recent" });
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(2);
+  });
+
+  test("handles empty database", () => {
+    const result = runAgingJob(db, NOW);
+    expect(result.transitioned).toBe(0);
+  });
+});

package/src/lifecycle/aging.ts

@@ -0,0 +1,36 @@
+/**
+ * Aging job: transition active observations older than 30 days to 'aging'.
+ *
+ * Runs daily (checked on MCP server startup via scheduler).
+ * Aging observations remain in FTS5 and are searchable at 0.7x weight.
+ */
+
+import type { MemDatabase } from "../storage/sqlite.js";
+
+const AGING_THRESHOLD_SECONDS = 30 * 86400; // 30 days
+
+export interface AgingResult {
+  transitioned: number;
+}
+
+/**
+ * Move active observations older than 30 days to aging lifecycle.
+ * Pinned observations are never aged.
+ */
+export function runAgingJob(
+  db: MemDatabase,
+  nowEpoch?: number
+): AgingResult {
+  const now = nowEpoch ?? Math.floor(Date.now() / 1000);
+  const cutoff = now - AGING_THRESHOLD_SECONDS;
+
+  const result = db.db
+    .query(
+      `UPDATE observations SET lifecycle = 'aging'
+       WHERE lifecycle = 'active'
+       AND created_at_epoch < ?`
+    )
+    .run(cutoff);
+
+  return { transitioned: result.changes };
+}

package/src/lifecycle/compaction.test.ts

@@ -0,0 +1,264 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import { runCompactionJob, generateDigest } from "./compaction.js";
+import type { ObservationRow } from "../storage/sqlite.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+let projectId: number;
+
+const DAY = 86400;
+const NOW = Math.floor(Date.now() / 1000);
+
+function insertObs(
+  overrides: Partial<{
+    lifecycle: string;
+    created_at_epoch: number;
+    title: string;
+    session_id: string | null;
+    type: string;
+    facts: string | null;
+    concepts: string | null;
+    narrative: string | null;
+    quality: number;
+  }> = {}
+): number {
+  const epoch = overrides.created_at_epoch ?? NOW - 100 * DAY;
+  // Use db.insertObservation so FTS5 index is properly maintained
+  const obs = db.insertObservation({
+    project_id: projectId,
+    session_id: overrides.session_id ?? "session-1",
+    type: overrides.type ?? "discovery",
+    title: overrides.title ?? "Test observation",
+    narrative: overrides.narrative ?? null,
+    facts: overrides.facts ?? null,
+    concepts: overrides.concepts ?? null,
+    quality: overrides.quality ?? 0.5,
+    lifecycle: overrides.lifecycle ?? "aging",
+    sensitivity: "shared",
+    user_id: "user1",
+    device_id: "dev1",
+    agent: "claude-code",
+  });
+  // Override created_at_epoch to test time-based logic
+  db.db
+    .query("UPDATE observations SET created_at_epoch = ? WHERE id = ?")
+    .run(epoch, obs.id);
+  return obs.id;
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-compaction-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+  const project = db.upsertProject({
+    canonical_id: "github.com/test/repo",
+    name: "repo",
+  });
+  projectId = project.id;
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("runCompactionJob", () => {
+  test("compacts aging observations older than 90 days", () => {
+    insertObs({ title: "Fix auth bug", created_at_epoch: NOW - 100 * DAY });
+    insertObs({ title: "Add logging", created_at_epoch: NOW - 95 * DAY });
+
+    const result = runCompactionJob(db, NOW);
+    expect(result.sessionsCompacted).toBe(1);
+    expect(result.observationsArchived).toBe(2);
+    expect(result.digestsCreated).toBe(1);
+  });
+
+  test("creates digest with correct type and pinned lifecycle", () => {
+    insertObs({ title: "Observation 1" });
+    insertObs({ title: "Observation 2" });
+
+    runCompactionJob(db, NOW);
+
+    // Find the digest
+    const digest = db.db
+      .query<ObservationRow, [string]>(
+        "SELECT * FROM observations WHERE type = ?"
+      )
+      .get("digest");
+
+    expect(digest).not.toBeNull();
+    expect(digest!.lifecycle).toBe("pinned");
+    expect(digest!.type).toBe("digest");
+  });
+
+  test("digest gets max quality from source observations", () => {
+    insertObs({ title: "Low quality", quality: 0.3 });
+    insertObs({ title: "High quality", quality: 0.9 });
+
+    runCompactionJob(db, NOW);
+
+    const digest = db.db
+      .query<ObservationRow, [string]>(
+        "SELECT * FROM observations WHERE type = ?"
+      )
+      .get("digest");
+
+    expect(digest!.quality).toBe(0.9);
+  });
+
+  test("archives source observations and sets compacted_into", () => {
+    const id1 = insertObs({ title: "Obs 1" });
+    const id2 = insertObs({ title: "Obs 2" });
+
+    runCompactionJob(db, NOW);
+
+    const obs1 = db.getObservationById(id1);
+    const obs2 = db.getObservationById(id2);
+    expect(obs1!.lifecycle).toBe("archived");
+    expect(obs2!.lifecycle).toBe("archived");
+    expect(obs1!.compacted_into).toBeGreaterThan(0);
+    expect(obs1!.compacted_into).toBe(obs2!.compacted_into);
+    expect(obs1!.archived_at_epoch).toBe(NOW);
+  });
+
+  test("does not compact pinned observations", () => {
+    insertObs({ lifecycle: "pinned", title: "Pinned obs" });
+    const result = runCompactionJob(db, NOW);
+    expect(result.observationsArchived).toBe(0);
+  });
+
+  test("does not compact observations newer than 90 days", () => {
+    insertObs({ created_at_epoch: NOW - 60 * DAY });
+    const result = runCompactionJob(db, NOW);
+    expect(result.observationsArchived).toBe(0);
+  });
+
+  test("groups by session_id", () => {
+    insertObs({ session_id: "session-A", title: "A1" });
+    insertObs({ session_id: "session-A", title: "A2" });
+    insertObs({ session_id: "session-B", title: "B1" });
+
+    const result = runCompactionJob(db, NOW);
+    expect(result.sessionsCompacted).toBe(2);
+    expect(result.digestsCreated).toBe(2);
+    expect(result.observationsArchived).toBe(3);
+  });
+
+  test("handles observations with null session_id", () => {
+    insertObs({ session_id: null, title: "No session 1" });
+    insertObs({ session_id: null, title: "No session 2" });
+
+    const result = runCompactionJob(db, NOW);
+    expect(result.sessionsCompacted).toBe(1);
+    expect(result.digestsCreated).toBe(1);
+    expect(result.observationsArchived).toBe(2);
+  });
+
+  test("adds digest to sync outbox", () => {
+    insertObs({ title: "Test obs" });
+    runCompactionJob(db, NOW);
+
+    const outbox = db.db
+      .query<{ record_type: string; record_id: number }, []>(
+        "SELECT record_type, record_id FROM sync_outbox WHERE record_type = 'observation'"
+      )
+      .all();
+
+    expect(outbox.length).toBeGreaterThan(0);
+  });
+
+  test("handles empty database", () => {
+    const result = runCompactionJob(db, NOW);
+    expect(result.sessionsCompacted).toBe(0);
+    expect(result.observationsArchived).toBe(0);
+    expect(result.digestsCreated).toBe(0);
+  });
+});
+
+describe("generateDigest", () => {
+  function makeObs(overrides: Partial<ObservationRow> = {}): ObservationRow {
+    return {
+      id: 1,
+      session_id: "s1",
+      project_id: 1,
+      type: overrides.type ?? "discovery",
+      title: overrides.title ?? "Test",
+      narrative: overrides.narrative ?? null,
+      facts: overrides.facts ?? null,
+      concepts: overrides.concepts ?? null,
+      files_read: null,
+      files_modified: null,
+      quality: overrides.quality ?? 0.5,
+      lifecycle: "aging",
+      sensitivity: "shared",
+      user_id: "user1",
+      device_id: "dev1",
+      agent: "claude-code",
+      created_at: "2026-01-01T00:00:00Z",
+      created_at_epoch: NOW,
+      archived_at_epoch: null,
+      compacted_into: null,
+      superseded_by: null,
+      remote_source_id: null,
+    };
+  }
+
+  test("single observation returns its content directly", () => {
+    const obs = makeObs({
+      title: "Found a bug",
+      narrative: "The auth was broken",
+      facts: '["fact1", "fact2"]',
+    });
+    const digest = generateDigest([obs]);
+    expect(digest.title).toBe("Found a bug");
+    expect(digest.narrative).toBe("The auth was broken");
+    expect(digest.facts).toEqual(["fact1", "fact2"]);
+  });
+
+  test("multiple observations are summarised", () => {
+    const obs1 = makeObs({ title: "Fix auth", type: "bugfix" });
+    const obs2 = makeObs({ id: 2, title: "Add tests", type: "feature" });
+    const digest = generateDigest([obs1, obs2]);
+    expect(digest.title).toContain("Fix auth");
+    expect(digest.title).toContain("Add tests");
+    expect(digest.narrative).toContain("[bugfix]");
+    expect(digest.narrative).toContain("[feature]");
+  });
+
+  test("many observations use +N more format", () => {
+    const observations = Array.from({ length: 5 }, (_, i) =>
+      makeObs({ id: i + 1, title: `Obs ${i + 1}` })
+    );
+    const digest = generateDigest(observations);
+    expect(digest.title).toContain("Obs 1");
+    expect(digest.title).toContain("+4 more");
+  });
+
+  test("merges and deduplicates facts", () => {
+    const obs1 = makeObs({ facts: '["fact1", "fact2"]' });
+    const obs2 = makeObs({ id: 2, facts: '["fact2", "fact3"]' });
+    const digest = generateDigest([obs1, obs2]);
+    expect(digest.facts).toContain("fact1");
+    expect(digest.facts).toContain("fact2");
+    expect(digest.facts).toContain("fact3");
+    expect(digest.facts.length).toBe(3); // no duplicates
+  });
+
+  test("unions concepts", () => {
+    const obs1 = makeObs({ concepts: '["auth", "security"]' });
+    const obs2 = makeObs({ id: 2, concepts: '["security", "testing"]' });
+    const digest = generateDigest([obs1, obs2]);
+    expect(digest.concepts).toContain("auth");
+    expect(digest.concepts).toContain("security");
+    expect(digest.concepts).toContain("testing");
+    expect(digest.concepts.length).toBe(3);
+  });
+
+  test("handles empty input", () => {
+    const digest = generateDigest([]);
+    expect(digest.title).toBe("Empty digest");
+  });
+});

package/src/lifecycle/compaction.ts

@@ -0,0 +1,190 @@
+/**
+ * Compaction job: group aging observations >90 days by session,
+ * create digest observations, archive originals.
+ *
+ * Runs weekly (checked on MCP server startup via scheduler).
+ *
+ * Strategy:
+ *   1. Find aging observations older than 90 days
+ *   2. Group by (project_id, session_id)
+ *   3. Generate a digest for each group
+ *   4. Archive source observations (set compacted_into, remove from FTS5)
+ *   5. Add digest to sync outbox
+ */
+
+import type { MemDatabase, ObservationRow } from "../storage/sqlite.js";
+
+const COMPACTION_THRESHOLD_SECONDS = 90 * 86400; // 90 days
+
+export interface CompactionResult {
+  sessionsCompacted: number;
+  observationsArchived: number;
+  digestsCreated: number;
+}
+
+export interface DigestContent {
+  title: string;
+  narrative: string;
+  facts: string[];
+  concepts: string[];
+}
+
+/**
+ * Run the compaction job: group old aging observations, create digests, archive originals.
+ */
+export function runCompactionJob(
+  db: MemDatabase,
+  nowEpoch?: number
+): CompactionResult {
+  const now = nowEpoch ?? Math.floor(Date.now() / 1000);
+  const cutoff = now - COMPACTION_THRESHOLD_SECONDS;
+
+  // Fetch aging observations older than threshold
+  const candidates = db.db
+    .query<ObservationRow, [number]>(
+      `SELECT * FROM observations
+       WHERE lifecycle = 'aging'
+       AND created_at_epoch < ?
+       ORDER BY project_id, session_id, created_at_epoch`
+    )
+    .all(cutoff);
+
+  if (candidates.length === 0) {
+    return { sessionsCompacted: 0, observationsArchived: 0, digestsCreated: 0 };
+  }
+
+  // Group by (project_id, session_id)
+  const groups = new Map<string, ObservationRow[]>();
+  for (const obs of candidates) {
+    const key = `${obs.project_id}:${obs.session_id ?? "__no_session__"}`;
+    const group = groups.get(key);
+    if (group) {
+      group.push(obs);
+    } else {
+      groups.set(key, [obs]);
+    }
+  }
+
+  let sessionsCompacted = 0;
+  let observationsArchived = 0;
+  let digestsCreated = 0;
+
+  for (const [, group] of groups) {
+    const first = group[0]!;
+    const digest = generateDigest(group);
+    const maxQuality = Math.max(...group.map((o) => o.quality));
+
+    // Insert digest observation
+    const digestObs = db.insertObservation({
+      session_id: first.session_id,
+      project_id: first.project_id,
+      type: "digest",
+      title: digest.title,
+      narrative: digest.narrative,
+      facts: digest.facts.length > 0 ? JSON.stringify(digest.facts) : null,
+      concepts:
+        digest.concepts.length > 0 ? JSON.stringify(digest.concepts) : null,
+      quality: maxQuality,
+      lifecycle: "pinned", // digests don't age out
+      sensitivity: first.sensitivity,
+      user_id: first.user_id,
+      device_id: first.device_id,
+      agent: first.agent,
+    });
+
+    // Add digest to sync outbox
+    db.addToOutbox("observation", digestObs.id);
+    digestsCreated++;
+
+    // Archive source observations
+    for (const obs of group) {
+      db.db
+        .query(
+          `UPDATE observations
+           SET lifecycle = 'archived', compacted_into = ?, archived_at_epoch = ?
+           WHERE id = ?`
+        )
+        .run(digestObs.id, now, obs.id);
+
+      // Remove from FTS5
+      db.ftsDelete(obs);
+      observationsArchived++;
+    }
+
+    sessionsCompacted++;
+  }
+
+  return { sessionsCompacted, observationsArchived, digestsCreated };
+}
+
+/**
+ * Generate digest content from a group of observations.
+ * Pure function — no database access.
+ */
+export function generateDigest(observations: ObservationRow[]): DigestContent {
+  if (observations.length === 0) {
+    return { title: "Empty digest", narrative: "", facts: [], concepts: [] };
+  }
+
+  if (observations.length === 1) {
+    const obs = observations[0]!;
+    return {
+      title: obs.title,
+      narrative: obs.narrative ?? "",
+      facts: parseFacts(obs.facts),
+      concepts: parseFacts(obs.concepts),
+    };
+  }
+
+  // Title: summarise the group
+  const first = observations[0]!;
+  const title =
+    observations.length <= 3
+      ? observations.map((o) => o.title).join("; ")
+      : `${first.title} (+${observations.length - 1} more)`;
+
+  // Narrative: bullet points of all titles with their types
+  const bullets = observations.map(
+    (o) => `- [${o.type}] ${o.title}`
+  );
+  const narrative = `Session digest (${observations.length} observations):\n${bullets.join("\n")}`;
+
+  // Facts: merge and deduplicate from all observations
+  const allFacts = new Set<string>();
+  for (const obs of observations) {
+    for (const fact of parseFacts(obs.facts)) {
+      allFacts.add(fact);
+    }
+  }
+
+  // Concepts: union of all concepts
+  const allConcepts = new Set<string>();
+  for (const obs of observations) {
+    for (const concept of parseFacts(obs.concepts)) {
+      allConcepts.add(concept);
+    }
+  }
+
+  return {
+    title,
+    narrative,
+    facts: [...allFacts],
+    concepts: [...allConcepts],
+  };
+}
+
+/**
+ * Parse a JSON array string, returning empty array on failure.
+ */
+function parseFacts(json: string | null): string[] {
+  if (!json) return [];
+  try {
+    const parsed = JSON.parse(json);
+    if (Array.isArray(parsed)) {
+      return parsed.filter((f) => typeof f === "string" && f.length > 0);
+    }
+  } catch {
+    // Not valid JSON
+  }
+  return [];
+}

package/src/lifecycle/purge.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "../storage/sqlite.js";
+import { runPurgeJob } from "./purge.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+let projectId: number;
+
+const DAY = 86400;
+const NOW = Math.floor(Date.now() / 1000);
+
+function insertObs(
+  overrides: Partial<{
+    lifecycle: string;
+    archived_at_epoch: number | null;
+    title: string;
+  }> = {}
+) {
+  db.db
+    .query(
+      `INSERT INTO observations (project_id, type, title, quality, lifecycle, sensitivity, user_id, device_id, agent, created_at, created_at_epoch, archived_at_epoch)
+       VALUES (?, 'discovery', ?, 0.5, ?, 'shared', 'user1', 'dev1', 'claude-code', datetime('now'), ?, ?)`
+    )
+    .run(
+      projectId,
+      overrides.title ?? "Test observation",
+      overrides.lifecycle ?? "archived",
+      NOW - 400 * DAY,
+      overrides.archived_at_epoch ?? null
+    );
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-purge-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+  const project = db.upsertProject({
+    canonical_id: "github.com/test/repo",
+    name: "repo",
+  });
+  projectId = project.id;
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("runPurgeJob", () => {
+  test("deletes archived observations older than 12 months", () => {
+    insertObs({ archived_at_epoch: NOW - 400 * DAY });
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(1);
+    expect(db.getObservationById(1)).toBeNull();
+  });
+
+  test("does not delete archived observations newer than 12 months", () => {
+    insertObs({ archived_at_epoch: NOW - 300 * DAY });
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(0);
+    expect(db.getObservationById(1)).not.toBeNull();
+  });
+
+  test("does not delete pinned observations regardless of age", () => {
+    insertObs({
+      lifecycle: "pinned",
+      archived_at_epoch: NOW - 400 * DAY,
+    });
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(0);
+  });
+
+  test("does not delete active or aging observations", () => {
+    insertObs({ lifecycle: "active", archived_at_epoch: null });
+    insertObs({ lifecycle: "aging", archived_at_epoch: null, title: "Aging obs" });
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(0);
+  });
+
+  test("does not delete archived observations without archived_at_epoch", () => {
+    insertObs({ archived_at_epoch: null });
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(0);
+  });
+
+  test("returns accurate count", () => {
+    insertObs({ archived_at_epoch: NOW - 400 * DAY, title: "Old 1" });
+    insertObs({ archived_at_epoch: NOW - 500 * DAY, title: "Old 2" });
+    insertObs({ archived_at_epoch: NOW - 100 * DAY, title: "Recent" });
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(2);
+  });
+
+  test("handles empty database", () => {
+    const result = runPurgeJob(db, NOW);
+    expect(result.deleted).toBe(0);
+  });
+});