opencode-codex-memory 0.1.2

package/src/paths.ts

@@ -0,0 +1,29 @@
+import path from "path"
+import os from "os"
+
+const MEMORY_DIR_NAME = "memories"
+const MEMORY_DB_NAME = "memory.db"
+
+const OVERRIDE_ENV = "OPENCODE_CODEX_MEMORY_TEST_ROOT"
+
+function dataRoot(): string {
+  const override = process.env[OVERRIDE_ENV]
+  if (override) return override
+  return path.join(os.homedir(), ".local", "share", "opencode")
+}
+
+export function memoryRoot(): string {
+  return path.join(dataRoot(), MEMORY_DIR_NAME)
+}
+
+export function memoryDbPath(): string {
+  return path.join(dataRoot(), MEMORY_DB_NAME)
+}
+
+export function opencodeDbPath(): string {
+  return path.join(dataRoot(), "opencode.db")
+}
+
+export function memorySummaryPath(): string {
+  return path.join(memoryRoot(), "memory_summary.md")
+}

package/src/phase1.ts

@@ -0,0 +1,116 @@
+import { MemoryStore, STAGE1_CONCURRENCY } from "./store.js"
+import { loadTranscript, selectEligibleSessions } from "./capture.js"
+import { redact, isMemoryExcludedFragment } from "./redact.js"
+import { stripCitations } from "./citation.js"
+import { extractViaSubagent } from "./llm.js"
+import { checkRateLimit } from "./ratelimit.js"
+
+export interface Phase1Options {
+  maxAgeDays: number
+  minIdleHours: number
+  // Max rollouts claimed per pass (codex max_rollouts_per_startup / max_claimed).
+  maxClaimed?: number
+  // Retention for stale stage-1 outputs (codex max_unused_days); pruning runs
+  // before the rate gate because it costs no tokens (codex start.rs).
+  maxUnusedDays?: number
+  excludeSession?: string
+  extractModel?: string
+}
+
+export const DEFAULT_PHASE1_OPTIONS: Phase1Options = {
+  maxAgeDays: 10,
+  minIdleHours: 6,
+  maxClaimed: 2,
+  maxUnusedDays: 30,
+}
+
+// codex serializes the full transcript and truncates at 70% of the model
+// context window, falling back to 150k tokens; ~4 chars/token puts the
+// char-estimate equivalent at 600k.
+const TRANSCRIPT_MAX_CHARS = 600_000
+// When truncating, keep the head and the tail: the start carries the user's
+// framing, the end carries the final outcome and feedback.
+const TRANSCRIPT_HEAD_CHARS = 360_000
+const TRANSCRIPT_TAIL_CHARS = 240_000
+
+export async function runPhase1(store: MemoryStore, opts: Phase1Options = DEFAULT_PHASE1_OPTIONS): Promise<void> {
+  store.pruneStage1Outputs(opts.maxUnusedDays ?? 30)
+  const rl = await checkRateLimit("phase1")
+  if (!rl.ok) {
+    console.warn("[opencode-codex-memory] skipping phase1 due to rate limit:", rl.reason)
+    return
+  }
+  const eligible = selectEligibleSessions(store, opts)
+  if (eligible.length === 0) return
+  const claimed = store.claimStage1Jobs(eligible, opts.excludeSession, opts.maxClaimed)
+  if (claimed.length === 0) return
+  const sessionById = new Map(eligible.map((s) => [s.id, s]))
+
+  await runPool(claimed, STAGE1_CONCURRENCY, async (claim) => {
+    const sid = claim.sessionId
+    try {
+      const session = sessionById.get(sid)
+      const sourceUpdatedAt = session?.updated_at ?? Date.now()
+      const transcript = buildTranscript(sid)
+      if (!transcript.trim()) {
+        store.markStage1SucceededNoOutput(sid, claim.ownershipToken, sourceUpdatedAt)
+        return
+      }
+      const result = await extractViaSubagent(sid, transcript, {
+        cwd: session?.directory ?? undefined,
+        model: opts.extractModel,
+      })
+      if (!result) {
+        // Extractor judged the session not worth remembering.
+        store.markStage1SucceededNoOutput(sid, claim.ownershipToken, sourceUpdatedAt)
+        return
+      }
+      store.markStage1Succeeded(sid, claim.ownershipToken, {
+        session_id: sid,
+        source_updated_at: sourceUpdatedAt,
+        raw_memory: redact(result.raw_memory),
+        rollout_summary: redact(result.rollout_summary),
+        // codex redacts the slug too — it becomes a filename.
+        rollout_slug: result.rollout_slug ? redact(result.rollout_slug) : result.rollout_slug,
+        cwd: session?.directory ?? null,
+        generated_at: Date.now(),
+      })
+    } catch (err) {
+      store.markStage1Failed(sid, claim.ownershipToken, (err as Error).message)
+    }
+  })
+}
+
+function buildTranscript(sessionId: string): string {
+  const msgs = loadTranscript(sessionId)
+  if (msgs.length === 0) return ""
+  const lines: string[] = []
+  for (const m of msgs) {
+    if (m.type === "system") continue
+    const role = m.role ?? m.type
+    const text = m.text ?? ""
+    if (!text.trim()) continue
+    // Injected AGENTS.md/<skill> blocks in user content are excluded from
+    // extraction (codex is_memory_excluded_contextual_user_fragment).
+    if (role === "user" && isMemoryExcludedFragment(text)) continue
+    lines.push(`### ${role}\n${redact(stripCitations(text))}`)
+  }
+  const full = lines.join("\n\n")
+  if (full.length <= TRANSCRIPT_MAX_CHARS) return full
+  return (
+    full.slice(0, TRANSCRIPT_HEAD_CHARS) +
+    "\n\n[... transcript truncated ...]\n\n" +
+    full.slice(full.length - TRANSCRIPT_TAIL_CHARS)
+  )
+}
+
+async function runPool<T>(items: T[], concurrency: number, fn: (item: T) => Promise<void>): Promise<void> {
+  let cursor = 0
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (cursor < items.length) {
+      const i = cursor++
+      await fn(items[i])
+    }
+  })
+  await Promise.all(workers)
+}

package/src/phase2.ts

@@ -0,0 +1,101 @@
+import { MemoryStore, PHASE2_COOLDOWN_MS } from "./store.js"
+import { ensureLayout, rebuildRawMemories, writeRolloutSummaries, pruneExtensionResources, writeWorkspaceDiff } from "./workspace.js"
+import { ensureBaseline, captureWorkspaceDiff, resetBaseline, DIFF_ARTIFACT } from "./git-baseline.js"
+import { consolidateViaSubagent } from "./llm.js"
+import { invalidateCache } from "./source.js"
+import { memoryRoot } from "./paths.js"
+import { checkRateLimit } from "./ratelimit.js"
+
+export interface Phase2Options {
+  maxRaw: number
+  maxUnusedDays: number
+  extensionRetentionDays: number
+  consolidationModel?: string
+}
+
+export const DEFAULT_PHASE2_OPTIONS: Phase2Options = {
+  maxRaw: 256,
+  maxUnusedDays: 30,
+  extensionRetentionDays: 7,
+}
+
+let phase2InFlight = false
+
+export async function runPhase2(store: MemoryStore, opts: Phase2Options = DEFAULT_PHASE2_OPTIONS): Promise<{ status: string }> {
+  if (phase2InFlight) return { status: "already_running" }
+  phase2InFlight = true
+  try {
+    const rl = await checkRateLimit("phase2")
+    if (!rl.ok) return { status: "skipped_rate_limit" }
+
+    const claim = store.claimGlobalPhase2Job()
+    if (claim.type !== "claimed") return { status: claim.type }
+
+    try {
+      ensureLayout()
+
+      // Preserves an existing baseline (only initializes a missing one): the
+      // diff below must span last-successful-run -> now so user edits and
+      // ad-hoc notes added since then reach consolidation. Stale stage-1
+      // output pruning happens in phase 1, before the rate gate (codex
+      // start.rs ordering).
+      if (!await ensureBaseline()) {
+        store.markPhase2Failed(claim.ownershipToken, "git baseline failed")
+        return { status: "baseline_failed" }
+      }
+
+      const outputs = store.getPhase2InputSelection(opts.maxRaw, opts.maxUnusedDays)
+      rebuildRawMemories(outputs)
+      writeRolloutSummaries(outputs)
+      pruneExtensionResources(opts.extensionRetentionDays)
+
+      const diff = await captureWorkspaceDiff()
+      if (diff.changes.length === 0) {
+        store.markPhase2Succeeded(claim.ownershipToken, outputs)
+        return { status: "no_workspace_changes" }
+      }
+
+      writeWorkspaceDiff(diff)
+
+      let heartbeatLost = false
+      const heartbeat = setInterval(() => {
+        if (!store.heartbeatPhase2Job(claim.ownershipToken)) {
+          heartbeatLost = true
+        }
+      }, 90_000)
+
+      try {
+        await consolidateViaSubagent(memoryRoot(), DIFF_ARTIFACT, opts.consolidationModel)
+      } finally {
+        clearInterval(heartbeat)
+      }
+
+      // Final synchronous ownership confirmation before the destructive
+      // baseline reset (codex phase2.rs does the same): the periodic flag can
+      // be up to 90s stale, and a stale worker resetting the baseline would
+      // swallow the diff a re-claiming worker is about to consume. The
+      // heartbeat is token+status guarded, so it fails once ownership is lost;
+      // markPhase2Failed is equally guarded and becomes a no-op then.
+      if (heartbeatLost || !store.heartbeatPhase2Job(claim.ownershipToken)) {
+        store.markPhase2Failed(claim.ownershipToken, "ownership lost")
+        return { status: "heartbeat_lost" }
+      }
+
+      if (!await resetBaseline()) {
+        store.markPhase2Failed(claim.ownershipToken, "baseline reset failed")
+        return { status: "baseline_reset_failed" }
+      }
+
+      store.markPhase2Succeeded(claim.ownershipToken, outputs)
+      invalidateCache()
+      return { status: "succeeded" }
+    } catch (err) {
+      store.markPhase2Failed(claim.ownershipToken, (err as Error).message)
+      return { status: "failed" }
+    }
+  } finally {
+    phase2InFlight = false
+  }
+}
+
+export const PHASE2_COOLDOWN = PHASE2_COOLDOWN_MS

package/src/ratelimit.ts

@@ -0,0 +1,26 @@
+export interface RateLimitInfo {
+  ok: boolean
+  reason?: string
+}
+
+let lastPhase1 = 0
+let lastPhase2 = 0
+
+const MIN_PHASE1_INTERVAL_MS = 30_000
+const MIN_PHASE2_INTERVAL_MS = 5 * 60 * 1000
+
+export async function checkRateLimit(kind: "phase1" | "phase2" = "phase1"): Promise<RateLimitInfo> {
+  const now = Date.now()
+  if (kind === "phase1") {
+    if (now - lastPhase1 < MIN_PHASE1_INTERVAL_MS) {
+      return { ok: false, reason: "phase1 rate limit (30s)" }
+    }
+    lastPhase1 = now
+  } else {
+    if (now - lastPhase2 < MIN_PHASE2_INTERVAL_MS) {
+      return { ok: false, reason: "phase2 rate limit (5min)" }
+    }
+    lastPhase2 = now
+  }
+  return { ok: true }
+}

package/src/redact.ts

@@ -0,0 +1,44 @@
+const REDACTIONS: { re: RegExp; replacement: string }[] = [
+  { re: /sk-ant-[A-Za-z0-9_\-]{20,}/g, replacement: "[REDACTED:anthropic-key]" },
+  { re: /sk-[A-Za-z0-9]{20,}/g, replacement: "[REDACTED:openai-key]" },
+  { re: /AKIA[0-9A-Z]{16}/g, replacement: "[REDACTED:aws-key]" },
+  { re: /gh[pousr]_[A-Za-z0-9]{36,}/g, replacement: "[REDACTED:github-token]" },
+  { re: /xox[baprs]-[A-Za-z0-9\-]{10,}/g, replacement: "[REDACTED:slack-token]" },
+  // Case-insensitive with a 16-char floor, matching codex's sanitizer.
+  { re: /bearer\s+[A-Za-z0-9\-\._~+\/=]{16,}/gi, replacement: "Bearer [REDACTED]" },
+  {
+    re: /-----BEGIN [A-Z]+ PRIVATE KEY-----[\s\S]*?-----END [A-Z]+ PRIVATE KEY-----/g,
+    replacement: "[REDACTED:private-key]",
+  },
+  { re: /(password|passwd|pwd|secret|api[_-]?key|token|access[_-]?token)\s*[:=]\s*["']?[^\s"']{4,}["']?/gi, replacement: "$1=[REDACTED]" },
+  { re: /(?:aws_secret_access_key|aws_access_key_id)\s*[:=]\s*["']?[^\s"']{4,}["']?/gi, replacement: "$1=[REDACTED]" },
+]
+
+export function redact(text: string): string {
+  let out = text
+  for (const { re, replacement } of REDACTIONS) {
+    out = out.replace(re, replacement)
+  }
+  return out
+}
+
+function matchesMarkedFragment(text: string, startMarker: string, endMarker: string): boolean {
+  const trimmed = text.trim()
+  return (
+    trimmed.slice(0, startMarker.length).toLowerCase() === startMarker.toLowerCase() &&
+    trimmed.slice(-endMarker.length).toLowerCase() === endMarker.toLowerCase()
+  )
+}
+
+/**
+ * Mirrors codex is_memory_excluded_contextual_user_fragment (phase1.rs):
+ * injected AGENTS.md instruction blocks and <skill> payloads inside user
+ * content are contextual boilerplate, not conversation — they must not be
+ * mined for memories.
+ */
+export function isMemoryExcludedFragment(text: string): boolean {
+  return (
+    matchesMarkedFragment(text, "# AGENTS.md instructions", "</INSTRUCTIONS>") ||
+    matchesMarkedFragment(text, "<skill>", "</skill>")
+  )
+}

package/src/source.ts

@@ -0,0 +1,62 @@
+import fs from "fs"
+import path from "path"
+import { memorySummaryPath, memoryRoot } from "./paths.js"
+import { truncateToTokens } from "./token.js"
+import { fillTemplate } from "./llm.js"
+
+const MEMORY_SUMMARY_TOKEN_LIMIT = 2500
+const READ_PATH_TEMPLATE = "read_path.md"
+
+interface CachedSummary {
+  content: string
+  mtime: number
+}
+
+let cached: CachedSummary | null = null
+
+function readTemplate(): string {
+  const templatePath = path.join(import.meta.dirname, "templates", READ_PATH_TEMPLATE)
+  return fs.readFileSync(templatePath, "utf8")
+}
+
+function readMemorySummary(): string | null {
+  const summaryPath = memorySummaryPath()
+  if (!fs.existsSync(summaryPath)) return null
+
+  const stat = fs.statSync(summaryPath)
+  if (cached && cached.mtime === stat.mtimeMs) {
+    return cached.content
+  }
+
+  const raw = fs.readFileSync(summaryPath, "utf8").trim()
+  if (!raw) return null
+
+  const truncated = truncateToTokens(raw, MEMORY_SUMMARY_TOKEN_LIMIT)
+  cached = {
+    content: truncated,
+    mtime: stat.mtimeMs,
+  }
+  return truncated
+}
+
+export function invalidateCache(): void {
+  cached = null
+}
+
+export function buildMemorySystemPrompt(): string | null {
+  const summary = readMemorySummary()
+  if (!summary) return null
+
+  const template = readTemplate()
+  return fillTemplate(template, {
+    base_path: memoryRoot(),
+    memory_summary: summary,
+  })
+}
+
+export function ensureMemoryLayout(): void {
+  const root = memoryRoot()
+  if (!fs.existsSync(root)) {
+    fs.mkdirSync(root, { recursive: true })
+  }
+}