opencode-codex-memory 0.1.3 → 0.1.5

package/dist/src/paths.d.ts

@@ -0,0 +1,4 @@
+export declare function memoryRoot(): string;
+export declare function memoryDbPath(): string;
+export declare function opencodeDbPath(): string;
+export declare function memorySummaryPath(): string;

package/dist/src/paths.js

@@ -0,0 +1,23 @@
+import path from "path";
+import os from "os";
+const MEMORY_DIR_NAME = "memories";
+const MEMORY_DB_NAME = "memory.db";
+const OVERRIDE_ENV = "OPENCODE_CODEX_MEMORY_TEST_ROOT";
+function dataRoot() {
+    const override = process.env[OVERRIDE_ENV];
+    if (override)
+        return override;
+    return path.join(os.homedir(), ".local", "share", "opencode");
+}
+export function memoryRoot() {
+    return path.join(dataRoot(), MEMORY_DIR_NAME);
+}
+export function memoryDbPath() {
+    return path.join(dataRoot(), MEMORY_DB_NAME);
+}
+export function opencodeDbPath() {
+    return path.join(dataRoot(), "opencode.db");
+}
+export function memorySummaryPath() {
+    return path.join(memoryRoot(), "memory_summary.md");
+}

package/dist/src/phase1.d.ts

@@ -0,0 +1,11 @@
+import { MemoryStore } from "./store.js";
+export interface Phase1Options {
+    maxAgeDays: number;
+    minIdleHours: number;
+    maxClaimed?: number;
+    maxUnusedDays?: number;
+    excludeSession?: string;
+    extractModel?: string;
+}
+export declare const DEFAULT_PHASE1_OPTIONS: Phase1Options;
+export declare function runPhase1(store: MemoryStore, opts?: Phase1Options): Promise<void>;

package/dist/src/phase1.js

@@ -0,0 +1,104 @@
+import { STAGE1_CONCURRENCY } from "./store.js";
+import { loadTranscript, selectEligibleSessions } from "./capture.js";
+import { redact, isMemoryExcludedFragment } from "./redact.js";
+import { stripCitations } from "./citation.js";
+import { extractViaSubagent } from "./llm.js";
+import { checkRateLimit } from "./ratelimit.js";
+export const DEFAULT_PHASE1_OPTIONS = {
+    maxAgeDays: 10,
+    minIdleHours: 6,
+    maxClaimed: 2,
+    maxUnusedDays: 30,
+};
+// codex serializes the full transcript and truncates at 70% of the model
+// context window, falling back to 150k tokens; ~4 chars/token puts the
+// char-estimate equivalent at 600k.
+const TRANSCRIPT_MAX_CHARS = 600_000;
+// When truncating, keep the head and the tail: the start carries the user's
+// framing, the end carries the final outcome and feedback.
+const TRANSCRIPT_HEAD_CHARS = 360_000;
+const TRANSCRIPT_TAIL_CHARS = 240_000;
+export async function runPhase1(store, opts = DEFAULT_PHASE1_OPTIONS) {
+    store.pruneStage1Outputs(opts.maxUnusedDays ?? 30);
+    const rl = await checkRateLimit("phase1");
+    if (!rl.ok) {
+        console.warn("[opencode-codex-memory] skipping phase1 due to rate limit:", rl.reason);
+        return;
+    }
+    const eligible = selectEligibleSessions(store, opts);
+    if (eligible.length === 0)
+        return;
+    const claimed = store.claimStage1Jobs(eligible, opts.excludeSession, opts.maxClaimed);
+    if (claimed.length === 0)
+        return;
+    const sessionById = new Map(eligible.map((s) => [s.id, s]));
+    await runPool(claimed, STAGE1_CONCURRENCY, async (claim) => {
+        const sid = claim.sessionId;
+        try {
+            const session = sessionById.get(sid);
+            const sourceUpdatedAt = session?.updated_at ?? Date.now();
+            const transcript = buildTranscript(sid);
+            if (!transcript.trim()) {
+                store.markStage1SucceededNoOutput(sid, claim.ownershipToken, sourceUpdatedAt);
+                return;
+            }
+            const result = await extractViaSubagent(sid, transcript, {
+                cwd: session?.directory ?? undefined,
+                model: opts.extractModel,
+            });
+            if (!result) {
+                // Extractor judged the session not worth remembering.
+                store.markStage1SucceededNoOutput(sid, claim.ownershipToken, sourceUpdatedAt);
+                return;
+            }
+            store.markStage1Succeeded(sid, claim.ownershipToken, {
+                session_id: sid,
+                source_updated_at: sourceUpdatedAt,
+                raw_memory: redact(result.raw_memory),
+                rollout_summary: redact(result.rollout_summary),
+                // codex redacts the slug too — it becomes a filename.
+                rollout_slug: result.rollout_slug ? redact(result.rollout_slug) : result.rollout_slug,
+                cwd: session?.directory ?? null,
+                generated_at: Date.now(),
+            });
+        }
+        catch (err) {
+            store.markStage1Failed(sid, claim.ownershipToken, err.message);
+        }
+    });
+}
+function buildTranscript(sessionId) {
+    const msgs = loadTranscript(sessionId);
+    if (msgs.length === 0)
+        return "";
+    const lines = [];
+    for (const m of msgs) {
+        if (m.type === "system")
+            continue;
+        const role = m.role ?? m.type;
+        const text = m.text ?? "";
+        if (!text.trim())
+            continue;
+        // Injected AGENTS.md/<skill> blocks in user content are excluded from
+        // extraction (codex is_memory_excluded_contextual_user_fragment).
+        if (role === "user" && isMemoryExcludedFragment(text))
+            continue;
+        lines.push(`### ${role}\n${redact(stripCitations(text))}`);
+    }
+    const full = lines.join("\n\n");
+    if (full.length <= TRANSCRIPT_MAX_CHARS)
+        return full;
+    return (full.slice(0, TRANSCRIPT_HEAD_CHARS) +
+        "\n\n[... transcript truncated ...]\n\n" +
+        full.slice(full.length - TRANSCRIPT_TAIL_CHARS));
+}
+async function runPool(items, concurrency, fn) {
+    let cursor = 0;
+    const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+        while (cursor < items.length) {
+            const i = cursor++;
+            await fn(items[i]);
+        }
+    });
+    await Promise.all(workers);
+}

package/dist/src/phase2.d.ts

@@ -0,0 +1,11 @@
+import { MemoryStore } from "./store.js";
+export interface Phase2Options {
+    maxRaw: number;
+    maxUnusedDays: number;
+    extensionRetentionDays: number;
+    consolidationModel?: string;
+}
+export declare const DEFAULT_PHASE2_OPTIONS: Phase2Options;
+export declare function runPhase2(store: MemoryStore, opts?: Phase2Options): Promise<{
+    status: string;
+}>;

package/dist/src/phase2.js

@@ -0,0 +1,83 @@
+import { ensureLayout, rebuildRawMemories, writeRolloutSummaries, pruneExtensionResources, writeWorkspaceDiff } from "./workspace.js";
+import { ensureBaseline, captureWorkspaceDiff, resetBaseline, DIFF_ARTIFACT } from "./git-baseline.js";
+import { consolidateViaSubagent } from "./llm.js";
+import { invalidateCache } from "./source.js";
+import { memoryRoot } from "./paths.js";
+import { checkRateLimit } from "./ratelimit.js";
+export const DEFAULT_PHASE2_OPTIONS = {
+    maxRaw: 256,
+    maxUnusedDays: 30,
+    extensionRetentionDays: 7,
+};
+let phase2InFlight = false;
+export async function runPhase2(store, opts = DEFAULT_PHASE2_OPTIONS) {
+    if (phase2InFlight)
+        return { status: "already_running" };
+    phase2InFlight = true;
+    try {
+        const rl = await checkRateLimit("phase2");
+        if (!rl.ok)
+            return { status: "skipped_rate_limit" };
+        const claim = store.claimGlobalPhase2Job();
+        if (claim.type !== "claimed")
+            return { status: claim.type };
+        try {
+            ensureLayout();
+            // Preserves an existing baseline (only initializes a missing one): the
+            // diff below must span last-successful-run -> now so user edits and
+            // ad-hoc notes added since then reach consolidation. Stale stage-1
+            // output pruning happens in phase 1, before the rate gate (codex
+            // start.rs ordering).
+            if (!await ensureBaseline()) {
+                store.markPhase2Failed(claim.ownershipToken, "git baseline failed");
+                return { status: "baseline_failed" };
+            }
+            const outputs = store.getPhase2InputSelection(opts.maxRaw, opts.maxUnusedDays);
+            rebuildRawMemories(outputs);
+            writeRolloutSummaries(outputs);
+            pruneExtensionResources(opts.extensionRetentionDays);
+            const diff = await captureWorkspaceDiff();
+            if (diff.changes.length === 0) {
+                store.markPhase2Succeeded(claim.ownershipToken, outputs);
+                return { status: "no_workspace_changes" };
+            }
+            writeWorkspaceDiff(diff);
+            let heartbeatLost = false;
+            const heartbeat = setInterval(() => {
+                if (!store.heartbeatPhase2Job(claim.ownershipToken)) {
+                    heartbeatLost = true;
+                }
+            }, 90_000);
+            try {
+                await consolidateViaSubagent(memoryRoot(), DIFF_ARTIFACT, opts.consolidationModel);
+            }
+            finally {
+                clearInterval(heartbeat);
+            }
+            // Final synchronous ownership confirmation before the destructive
+            // baseline reset (codex phase2.rs does the same): the periodic flag can
+            // be up to 90s stale, and a stale worker resetting the baseline would
+            // swallow the diff a re-claiming worker is about to consume. The
+            // heartbeat is token+status guarded, so it fails once ownership is lost;
+            // markPhase2Failed is equally guarded and becomes a no-op then.
+            if (heartbeatLost || !store.heartbeatPhase2Job(claim.ownershipToken)) {
+                store.markPhase2Failed(claim.ownershipToken, "ownership lost");
+                return { status: "heartbeat_lost" };
+            }
+            if (!await resetBaseline()) {
+                store.markPhase2Failed(claim.ownershipToken, "baseline reset failed");
+                return { status: "baseline_reset_failed" };
+            }
+            store.markPhase2Succeeded(claim.ownershipToken, outputs);
+            invalidateCache();
+            return { status: "succeeded" };
+        }
+        catch (err) {
+            store.markPhase2Failed(claim.ownershipToken, err.message);
+            return { status: "failed" };
+        }
+    }
+    finally {
+        phase2InFlight = false;
+    }
+}

package/dist/src/ratelimit.d.ts

@@ -0,0 +1,5 @@
+export interface RateLimitInfo {
+    ok: boolean;
+    reason?: string;
+}
+export declare function checkRateLimit(kind?: "phase1" | "phase2"): Promise<RateLimitInfo>;

package/dist/src/ratelimit.js

@@ -0,0 +1,20 @@
+let lastPhase1 = 0;
+let lastPhase2 = 0;
+const MIN_PHASE1_INTERVAL_MS = 30_000;
+const MIN_PHASE2_INTERVAL_MS = 5 * 60 * 1000;
+export async function checkRateLimit(kind = "phase1") {
+    const now = Date.now();
+    if (kind === "phase1") {
+        if (now - lastPhase1 < MIN_PHASE1_INTERVAL_MS) {
+            return { ok: false, reason: "phase1 rate limit (30s)" };
+        }
+        lastPhase1 = now;
+    }
+    else {
+        if (now - lastPhase2 < MIN_PHASE2_INTERVAL_MS) {
+            return { ok: false, reason: "phase2 rate limit (5min)" };
+        }
+        lastPhase2 = now;
+    }
+    return { ok: true };
+}

package/dist/src/redact.d.ts

@@ -0,0 +1,8 @@
+export declare function redact(text: string): string;
+/**
+ * Mirrors codex is_memory_excluded_contextual_user_fragment (phase1.rs):
+ * injected AGENTS.md instruction blocks and <skill> payloads inside user
+ * content are contextual boilerplate, not conversation — they must not be
+ * mined for memories.
+ */
+export declare function isMemoryExcludedFragment(text: string): boolean;

package/dist/src/redact.js

@@ -0,0 +1,37 @@
+const REDACTIONS = [
+    { re: /sk-ant-[A-Za-z0-9_\-]{20,}/g, replacement: "[REDACTED:anthropic-key]" },
+    { re: /sk-[A-Za-z0-9]{20,}/g, replacement: "[REDACTED:openai-key]" },
+    { re: /AKIA[0-9A-Z]{16}/g, replacement: "[REDACTED:aws-key]" },
+    { re: /gh[pousr]_[A-Za-z0-9]{36,}/g, replacement: "[REDACTED:github-token]" },
+    { re: /xox[baprs]-[A-Za-z0-9\-]{10,}/g, replacement: "[REDACTED:slack-token]" },
+    // Case-insensitive with a 16-char floor, matching codex's sanitizer.
+    { re: /bearer\s+[A-Za-z0-9\-\._~+\/=]{16,}/gi, replacement: "Bearer [REDACTED]" },
+    {
+        re: /-----BEGIN [A-Z]+ PRIVATE KEY-----[\s\S]*?-----END [A-Z]+ PRIVATE KEY-----/g,
+        replacement: "[REDACTED:private-key]",
+    },
+    { re: /(password|passwd|pwd|secret|api[_-]?key|token|access[_-]?token)\s*[:=]\s*["']?[^\s"']{4,}["']?/gi, replacement: "$1=[REDACTED]" },
+    { re: /(?:aws_secret_access_key|aws_access_key_id)\s*[:=]\s*["']?[^\s"']{4,}["']?/gi, replacement: "$1=[REDACTED]" },
+];
+export function redact(text) {
+    let out = text;
+    for (const { re, replacement } of REDACTIONS) {
+        out = out.replace(re, replacement);
+    }
+    return out;
+}
+function matchesMarkedFragment(text, startMarker, endMarker) {
+    const trimmed = text.trim();
+    return (trimmed.slice(0, startMarker.length).toLowerCase() === startMarker.toLowerCase() &&
+        trimmed.slice(-endMarker.length).toLowerCase() === endMarker.toLowerCase());
+}
+/**
+ * Mirrors codex is_memory_excluded_contextual_user_fragment (phase1.rs):
+ * injected AGENTS.md instruction blocks and <skill> payloads inside user
+ * content are contextual boilerplate, not conversation — they must not be
+ * mined for memories.
+ */
+export function isMemoryExcludedFragment(text) {
+    return (matchesMarkedFragment(text, "# AGENTS.md instructions", "</INSTRUCTIONS>") ||
+        matchesMarkedFragment(text, "<skill>", "</skill>"));
+}

package/dist/src/source.d.ts

@@ -0,0 +1,3 @@
+export declare function invalidateCache(): void;
+export declare function buildMemorySystemPrompt(): string | null;
+export declare function ensureMemoryLayout(): void;

package/dist/src/source.js

@@ -0,0 +1,46 @@
+import fs from "fs";
+import path from "path";
+import { memorySummaryPath, memoryRoot } from "./paths.js";
+import { truncateToTokens } from "./token.js";
+import { fillTemplate } from "./llm.js";
+const MEMORY_SUMMARY_TOKEN_LIMIT = 2500;
+const READ_PATH_TEMPLATE = "read_path.md";
+let cached = null;
+function readTemplate() {
+    const templatePath = path.join(import.meta.dirname, "templates", READ_PATH_TEMPLATE);
+    return fs.readFileSync(templatePath, "utf8");
+}
+function readMemorySummary() {
+    const summaryPath = memorySummaryPath();
+    if (!fs.existsSync(summaryPath))
+        return null;
+    const stat = fs.statSync(summaryPath);
+    if (cached && cached.mtime === stat.mtimeMs) {
+        return cached.content;
+    }
+    const raw = fs.readFileSync(summaryPath, "utf8").trim();
+    if (!raw)
+        return null;
+    const truncated = truncateToTokens(raw, MEMORY_SUMMARY_TOKEN_LIMIT);
+    cached = {
+        content: truncated,
+        mtime: stat.mtimeMs,
+    };
+    return truncated;
+}
+export function invalidateCache() {
+    cached = null;
+}
+export function buildMemorySystemPrompt() {
+    const summary = readMemorySummary();
+    if (!summary)
+        return null;
+    const template = readTemplate();
+    return fillTemplate(template, {
+        base_path: memoryRoot(),
+        memory_summary: summary,
+    });
+}
+export function ensureMemoryLayout() {
+    fs.mkdirSync(memoryRoot(), { recursive: true });
+}

package/dist/src/store.d.ts

@@ -0,0 +1,96 @@
+import type { Database } from "bun:sqlite";
+export declare const DEFAULT_RETRY_REMAINING = 3;
+export declare const STAGE1_LEASE_SECONDS = 3600;
+export declare const PHASE2_LEASE_SECONDS = 3600;
+export declare const STAGE1_RETRY_DELAY_SECONDS = 3600;
+export declare const PHASE2_RETRY_DELAY_SECONDS = 3600;
+export declare const PHASE2_COOLDOWN_MS: number;
+export declare const STAGE1_CONCURRENCY = 8;
+export declare const SCAN_LIMIT = 5000;
+export declare const PRUNE_BATCH_SIZE = 200;
+export type JobKind = "memory_stage1" | "memory_consolidate_global";
+export type JobStatus = "pending" | "running" | "done" | "failed";
+export interface Stage1Output {
+    session_id: string;
+    source_updated_at: number;
+    raw_memory: string;
+    rollout_summary: string;
+    rollout_slug: string | null;
+    cwd?: string | null;
+    generated_at: number;
+    usage_count: number;
+    last_usage: number | null;
+}
+export interface Stage1Claim {
+    sessionId: string;
+    ownershipToken: string;
+}
+export interface ClaimableSession {
+    id: string;
+    updated_at: number;
+}
+export type Phase2ClaimResult = {
+    type: "claimed";
+    workerId: string;
+    ownershipToken: string;
+} | {
+    type: "skipped_cooldown";
+} | {
+    type: "skipped_running";
+} | {
+    type: "skipped_retry_unavailable";
+};
+export declare class MemoryStore {
+    private db;
+    constructor(db?: Database);
+    stage1Outputs(): Stage1Output[];
+    /**
+     * Deletes stale rows; snapshots consumed by the last successful Phase 2 are
+     * protected. Stalest-first, capped per run (codex PRUNE_BATCH_SIZE).
+     */
+    pruneStage1Outputs(maxUnusedDays: number): number;
+    upsertStage1Output(out: Omit<Stage1Output, "usage_count" | "last_usage">): boolean;
+    recordUsage(sessionIds: string[]): void;
+    claimStage1Jobs(sessions: ClaimableSession[], excludeSession?: string, maxClaimed?: number): Stage1Claim[];
+    markStage1Succeeded(sessionId: string, ownershipToken: string, out: Omit<Stage1Output, "usage_count" | "last_usage">): void;
+    /** Extraction succeeded but produced nothing worth keeping: finish the job and drop any stale output. */
+    markStage1SucceededNoOutput(sessionId: string, ownershipToken: string, sourceUpdatedAt: number): void;
+    markStage1Failed(sessionId: string, ownershipToken: string, error: string): void;
+    claimGlobalPhase2Job(): Phase2ClaimResult;
+    heartbeatPhase2Job(ownershipToken: string): boolean;
+    /**
+     * Marks the phase-2 job done and records exactly which stage-1 snapshots the
+     * run consumed (selected_for_phase2), so pruning cannot delete inputs that
+     * still back the consolidated artifacts.
+     */
+    markPhase2Succeeded(ownershipToken: string, selected?: Pick<Stage1Output, "session_id" | "source_updated_at">[]): void;
+    markPhase2Failed(ownershipToken: string, error: string): void;
+    /**
+     * Phase 2 input set, mirroring codex get_phase2_input_selection:
+     * - excludes sessions marked disabled/polluted (their summary files then
+     *   disappear from the workspace and the diff drives forgetting)
+     * - recency: last_usage when the memory has ever been used, otherwise
+     *   source_updated_at
+     * - ranked by usage, then recency
+     */
+    getPhase2InputSelection(maxRaw: number, maxUnusedDays: number): Stage1Output[];
+    /** Mirrors codex delete_thread_memory: remove a deleted session's output + job. */
+    deleteSessionMemory(sessionId: string): void;
+    /**
+     * codex clear_memory_data deletes extracted memories and jobs but explicitly
+     * preserves per-session memory modes: a reset must not re-enable sessions
+     * the user disabled or that were marked polluted.
+     */
+    clearMemoryData(): void;
+    setMemoryMode(sessionId: string, mode: "enabled" | "disabled" | "polluted"): void;
+    /**
+     * Stamp a mode only when the session has no meta row yet — used to mark
+     * sessions seen while generate_memories=false as permanently 'disabled'
+     * (codex stamps memory_mode at thread creation, session.rs), without
+     * overriding an explicit user-set or polluted mode.
+     */
+    stampMemoryModeIfAbsent(sessionId: string, mode: "enabled" | "disabled"): void;
+    getMemoryMode(sessionId: string): "enabled" | "disabled" | "polluted" | null;
+    markPolluted(sessionId: string): void;
+    isPolluted(sessionId: string): boolean;
+}