codemem 0.37.1 → 0.38.0-alpha.2

package/README.md

@@ -24,6 +24,7 @@ codemem --help
 codemem setup --opencode-only
 codemem stats
 codemem search "query"
+codemem distill --limit 10
 codemem serve start
 codemem mcp
 ```

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, REF_BACKFILL_JOB, RawEventSweeper, RefBackfillRunner, SCOPE_BACKFILL_JOB, SESSION_CONTEXT_BACKFILL_JOB, SUMMARY_DEDUP_BACKFILL_JOB, ScopeBackfillRunner, SessionContextBackfillRunner, SummaryDedupBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildRawEventEnvelopeFromCodexHook, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorCreateScopeAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorGrantScopeMembershipAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorListScopeMembershipsAction, coordinatorListScopesAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, coordinatorRevokeScopeMembershipAction, coordinatorUpdateScopeAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, formatHostPort, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingScopeBackfill, hasPendingSessionContextBackfill, hasPendingSummaryDedupBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, listMaintenanceJobs, listPerPeerScopeSyncState, listRetentionScopeIds, loadObserverConfig, loadPublicKey, loadSqliteVec, mdnsEnabled, planReplicationOpsAgePrune, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, retryRawEventFailures, runSyncDaemon, runSyncPass, scanSecretsRetroactive, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
+import { DEDUP_KEY_BACKFILL_JOB, DEFAULT_COORDINATOR_DB_PATH, DedupKeyBackfillRunner, MUTATING_TOOL_NAMES, MemoryStore, ObserverClient, REF_BACKFILL_JOB, RawEventSweeper, RefBackfillRunner, SCOPE_BACKFILL_JOB, SESSION_CONTEXT_BACKFILL_JOB, SUMMARY_DEDUP_BACKFILL_JOB, ScopeBackfillRunner, SessionContextBackfillRunner, SummaryDedupBackfillRunner, SyncRetentionRunner, VERSION, VectorModelMigrationRunner, aiBackfillStructuredContent, applyBootstrapSnapshot, applyDistillRule, backfillMemoryDedupKeys, backfillNarrativeFromBody, backfillTagsText, backfillVectors, buildAuthHeaders, buildBaseUrl, buildDistillReport, buildRawEventEnvelopeFromCodexHook, buildRawEventEnvelopeFromHook, compareMemoryRoleReports, connect, coordinatorCreateGroupAction, coordinatorCreateInviteAction, coordinatorCreateScopeAction, coordinatorDisableDeviceAction, coordinatorEnrollDeviceAction, coordinatorGrantScopeMembershipAction, coordinatorImportInviteAction, coordinatorListBootstrapGrantsAction, coordinatorListDevicesAction, coordinatorListGroupsAction, coordinatorListJoinRequestsAction, coordinatorListScopeMembershipsAction, coordinatorListScopesAction, coordinatorRemoveDeviceAction, coordinatorRenameDeviceAction, coordinatorReviewJoinRequestAction, coordinatorRevokeBootstrapGrantAction, coordinatorRevokeScopeMembershipAction, coordinatorUpdateScopeAction, createBetterSqliteCoordinatorApp, deactivateLowSignalMemories, deactivateLowSignalObservations, dedupNearDuplicateMemories, draftDistillRule, ensureDeviceIdentity, ensureSchemaBootstrapped, exportMemories, extractApplyPatchPaths, fetchAllSnapshotPages, fingerprintPublicKey, flushRawEvents, formatHostPort, getExtractionBenchmarkProfile, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMaintenanceJob, getMemoryArtifactReport, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getRawEventStatus, getSemanticIndexDiagnostics, getSessionExtractionEval, getSessionExtractionEvalScenario, getWorkspaceCodememConfigPath, hasPendingDedupKeyBackfill, hasPendingRefBackfill, hasPendingScopeBackfill, hasPendingSessionContextBackfill, hasPendingSummaryDedupBackfill, hasUnsyncedSharedMemoryChanges, importMemories, initDatabase, isEmbeddingDisabled, judgeDistillReport, listMaintenanceJobs, listPerPeerScopeSyncState, listRetentionScopeIds, loadObserverConfig, loadPublicKey, loadSqliteVec, mdnsEnabled, planReplicationOpsAgePrune, projectMatchesFilter, pruneReplicationOpsUntilCaughtUp, rawEventsGate, readCodememConfigFile, readCodememConfigFileAtPath, readCoordinatorSyncConfig, readImportPayload, renderUnifiedDiff, replayBatchExtraction, replayBatchExtractionWithTierRouting, requestJson, resolveCodememConfigPath, resolveDbPath, resolveHookProject, resolveProject, resolveProjectRoot, retryRawEventFailures, runSyncDaemon, runSyncPass, scanSecretsRetroactive, schema, setPeerProjectFilter, stripJsonComments, stripPrivateObj, stripTrailingCommas, syncPassPreflight, updatePeerAddresses, vacuumDatabase, writeCodememConfigFile } from "@codemem/core";
 import { Command, Option } from "commander";
 import omelette from "omelette";
 import { appendFileSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
@@ -3722,6 +3722,272 @@ scanSecretsCmd.action((opts) => {
 });
 dbCommand.addCommand(scanSecretsCmd);
 //#endregion
+//#region src/commands/distill.ts
+var DistillUsageError = class extends Error {};
+function collectKind(value, previous) {
+	return [...previous, ...value.split(",")].map((kind) => kind.trim()).filter(Boolean);
+}
+function parsePositiveInteger(value, name) {
+	const parsed = Number.parseInt(value, 10);
+	if (!Number.isFinite(parsed) || parsed < 1 || String(parsed) !== value.trim()) throw new DistillUsageError(`${name} must be a positive integer`);
+	return parsed;
+}
+function readContextFile(path, displayPath, scope) {
+	if (!existsSync(path)) return null;
+	const text = readFileSync(path, "utf8");
+	return text.trim() ? {
+		path: displayPath,
+		text,
+		scope
+	} : null;
+}
+function loadDefaultContextDocuments(includeProjectContext, cwd = process.cwd()) {
+	const projectRoot = resolveProjectRoot(cwd) ?? cwd;
+	return [includeProjectContext ? readContextFile(join(projectRoot, "AGENTS.md"), "AGENTS.md", "project") : null, readContextFile(join(homedir(), ".config", "opencode", "AGENTS.md"), "~/.config/opencode/AGENTS.md", "user")].filter((document) => document != null);
+}
+function shouldIncludeProjectContext(opts) {
+	if (opts.allProjects) return false;
+	const currentProject = resolveProject(process.cwd());
+	if (!currentProject) return false;
+	const targetProject = opts.project ? resolveProject(process.cwd(), opts.project) : process.env.CODEMEM_PROJECT?.trim() || currentProject;
+	if (!targetProject) return false;
+	return projectMatchesFilter(targetProject, currentProject);
+}
+function buildFilters(opts) {
+	if (opts.allProjects && opts.project) throw new DistillUsageError("--project cannot be combined with --all-projects");
+	if (opts.allProjects) return void 0;
+	const defaultProject = process.env.CODEMEM_PROJECT?.trim() || null;
+	const project = opts.project ? resolveProject(process.cwd(), opts.project) : defaultProject ?? resolveProject(process.cwd());
+	return project ? { project } : void 0;
+}
+var JUDGE_OVERFETCH_FACTOR = 3;
+var JUDGE_OVERFETCH_MAX_EXTRA = 20;
+function judgeFetchLimit(limit) {
+	return Math.min(limit * JUDGE_OVERFETCH_FACTOR, limit + JUDGE_OVERFETCH_MAX_EXTRA);
+}
+async function runDistill(store, opts, limit) {
+	const minRecurrence = parsePositiveInteger(opts.minRecurrence, "min recurrence");
+	return buildDistillReport(store, {
+		candidate: {
+			includeDocumented: opts.includeDocumented ?? false,
+			maxEvidenceItems: opts.explain ? 10 : 5
+		},
+		contextDocuments: loadDefaultContextDocuments(shouldIncludeProjectContext(opts)),
+		corpus: {
+			filters: buildFilters(opts) ?? null,
+			kinds: opts.kind.length > 0 ? opts.kind : void 0
+		},
+		limit,
+		minRecurrence
+	});
+}
+/**
+* B-side worthiness gate: an LLM judges each deterministic candidate and
+* routine-activity clusters are dropped. Recurrence alone cannot make this
+* call — releases, review passes, and context lookups recur by definition.
+*/
+async function judgeReport(report) {
+	const client = new ObserverClient();
+	return judgeDistillReport(report, async (system, user) => {
+		return (await client.observe(system, user)).raw;
+	});
+}
+function renderDistillReport(report, explain = false) {
+	if (report.candidates.length === 0) return "No distill candidates found.";
+	const lines = [`Distill candidates (${report.candidates.length})`, ""];
+	for (const [index, candidate] of report.candidates.entries()) {
+		lines.push(`${index + 1}. [${candidate.scope}] score=${candidate.score.toFixed(3)} recurrence=${candidate.recurrence} target=${candidate.suggested_target}`);
+		lines.push(`   projects: ${candidate.projects.join(", ") || "(none)"}`);
+		lines.push(`   concepts: ${candidate.concepts.join(", ") || "(none)"}`);
+		lines.push(`   members: ${candidate.member_ids.join(", ")}`);
+		const judge = candidate.judge;
+		if (judge) lines.push(`   judge: ${judge.verdict}${judge.reason ? ` — ${judge.reason}` : ""}`);
+		if (explain) for (const evidence of candidate.evidence) lines.push(`   - ${evidence}`);
+		lines.push("");
+	}
+	return lines.join("\n").trimEnd();
+}
+function resolveCandidateTarget(candidate) {
+	if (candidate.scope === "user") return {
+		path: join(homedir(), ".config", "opencode", "AGENTS.md"),
+		display: "~/.config/opencode/AGENTS.md"
+	};
+	return {
+		path: join(resolveProjectRoot(process.cwd()) ?? process.cwd(), "AGENTS.md"),
+		display: "AGENTS.md"
+	};
+}
+async function draftTopCandidate(report, opts) {
+	const candidate = report.candidates[0];
+	if (!candidate) {
+		if (opts.json) console.log(JSON.stringify({
+			draft: null,
+			reason: "no_candidates"
+		}, null, 2));
+		else {
+			p.intro("codemem distill");
+			p.log.warn("No distill candidates to draft.");
+			p.outro("done");
+		}
+		return;
+	}
+	if (candidate.scope === "project") {
+		const currentProject = resolveProject(process.cwd());
+		if (!(currentProject != null && candidate.projects.every((project) => projectMatchesFilter(project, currentProject)))) {
+			const projects = candidate.projects.join(", ") || "(unknown)";
+			if (opts.json) console.log(JSON.stringify({
+				draft: null,
+				reason: "project_mismatch",
+				representative_id: candidate.representative_id,
+				projects: candidate.projects
+			}, null, 2));
+			else {
+				p.intro("codemem distill draft");
+				p.log.warn(`The top candidate belongs to project "${projects}", not this checkout. Run distill from that repo to draft its AGENTS.md rule.`);
+				p.outro("nothing written");
+			}
+			return;
+		}
+	}
+	const target = resolveCandidateTarget(candidate);
+	const current = existsSync(target.path) ? readFileSync(target.path, "utf8") : "";
+	let rule;
+	let raw;
+	try {
+		const client = new ObserverClient();
+		const drafted = await draftDistillRule(candidate, async (system, user) => {
+			return (await client.observe(system, user)).raw;
+		});
+		rule = drafted.rule;
+		raw = drafted.raw;
+	} catch (error) {
+		const guidance = `drafting needs a configured observer model: ${error instanceof Error ? error.message : String(error)}`;
+		if (opts.json) emitJsonError("draft_failed", guidance, 1);
+		else {
+			p.log.error(guidance);
+			process.exitCode = 1;
+		}
+		return;
+	}
+	if (!rule) {
+		if (opts.json) console.log(JSON.stringify({
+			draft: null,
+			reason: "model_declined",
+			representative_id: candidate.representative_id,
+			raw
+		}, null, 2));
+		else {
+			p.intro("codemem distill draft");
+			p.log.warn("The model declined to draft a rule for the top candidate (too vague).");
+			p.outro("done");
+		}
+		return;
+	}
+	const applied = applyDistillRule(current, rule);
+	const diff = renderUnifiedDiff(target.display, current, applied.text);
+	if (opts.json) {
+		const wrote = Boolean(opts.apply && applied.changed);
+		if (wrote) {
+			mkdirSync(dirname(target.path), { recursive: true });
+			writeFileSync(target.path, applied.text);
+		}
+		console.log(JSON.stringify({ draft: {
+			target: target.display,
+			scope: candidate.scope,
+			representative_id: candidate.representative_id,
+			recurrence: candidate.recurrence,
+			score: candidate.score,
+			rule,
+			diff,
+			already_present: !applied.changed,
+			applied: wrote
+		} }, null, 2));
+		return;
+	}
+	p.intro("codemem distill draft");
+	p.log.info(`Top candidate [${candidate.scope}] recurrence=${candidate.recurrence} → ${target.display}`);
+	p.log.step(`Proposed rule:\n  ${rule}`);
+	if (!applied.changed) {
+		p.log.warn("This rule is already present in the target file.");
+		p.outro("nothing to apply");
+		return;
+	}
+	console.log(diff);
+	if (!opts.apply) {
+		p.outro(`dry run — re-run with --apply to write ${target.display}`);
+		return;
+	}
+	const confirmed = await p.confirm({ message: `Apply this change to ${target.display}?` });
+	if (p.isCancel(confirmed) || !confirmed) {
+		p.outro("aborted — nothing written");
+		return;
+	}
+	mkdirSync(dirname(target.path), { recursive: true });
+	writeFileSync(target.path, applied.text);
+	p.outro(`wrote ${target.display}`);
+}
+async function distillAction(opts) {
+	let store = null;
+	try {
+		store = new MemoryStore(resolveDbPath(resolveDbOpt(opts)));
+		const limit = parsePositiveInteger(opts.limit, "limit");
+		const judging = opts.judge !== false;
+		let result = await runDistill(store, opts, judging ? judgeFetchLimit(limit) : limit);
+		if (judging) {
+			try {
+				result = await judgeReport(result);
+			} catch (error) {
+				const message = error instanceof Error ? error.message : String(error);
+				result = {
+					...result,
+					metadata: {
+						...result.metadata,
+						judged: false,
+						judge_error: message
+					}
+				};
+			}
+			if (result.candidates.length > limit) result = {
+				...result,
+				candidates: result.candidates.slice(0, limit),
+				metadata: {
+					...result.metadata,
+					candidate_count: limit
+				}
+			};
+		}
+		if (opts.draft || opts.apply) {
+			await draftTopCandidate(result, opts);
+			return;
+		}
+		if (opts.json) {
+			console.log(JSON.stringify(result, null, 2));
+			return;
+		}
+		p.intro("codemem distill");
+		console.log(renderDistillReport(result, opts.explain ?? false));
+		if (result.metadata.judged) p.log.info(`judge dropped ${result.metadata.routine_filtered_count ?? 0} routine-activity candidate(s)`);
+		else if (result.metadata.judge_error) p.log.warn(`worthiness judgment skipped (needs a configured observer model): ${result.metadata.judge_error}. Candidates are unfiltered; recurring routine activity may rank high.`);
+		p.outro("review candidates before editing context files");
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		const usageError = error instanceof DistillUsageError;
+		if (opts.json) {
+			emitJsonError(usageError ? "usage_error" : "distill_failed", message, usageError ? 2 : 1);
+			return;
+		}
+		p.log.error(message);
+		process.exitCode = usageError ? 2 : 1;
+	} finally {
+		store?.close();
+	}
+}
+var cmd$3 = new Command("distill").configureHelp(helpStyle).description("Mine recurring memories into reviewable context candidates").option("-p, --project <project>", "project identifier (defaults to git repo root)").option("-A, --all-projects", "mine memories across all projects").option("-k, --kind <kind>", "memory kind to mine (repeat or comma-separate)", collectKind, []).option("-m, --min-recurrence <n>", "minimum member count per candidate", "2").option("-l, --limit <n>", "max candidates", "10").option("-e, --explain", "include evidence snippets in human output").option("--include-documented", "include candidates already represented in context files").option("--no-judge", "skip the observer-model worthiness judgment that drops routine-activity clusters").option("-D, --draft", "draft an AGENTS.md rule for the top candidate and show the diff").option("--apply", "write the drafted rule to the target file (implies --draft; prompts to confirm; with --json applies immediately without prompting)");
+addDbOption(cmd$3);
+addJsonOption(cmd$3);
+cmd$3.action(distillAction);
+var distillCommand = cmd$3;
+//#endregion
 //#region src/commands/embed.ts
 function parseOptionalPositiveInt(value) {
 	if (!value) return void 0;
@@ -4654,6 +4920,51 @@ function createMemoryRoleCompareCommand() {
 	});
 	return cmd;
 }
+function createMemoryArtifactReportCommand() {
+	const cmd = new Command("artifact-report").configureHelp(helpStyle).description("Analyze legacy memory artifacts in a DB snapshot without mutating rows").option("--project <project>", "project identifier (defaults to git repo root)").option("--all-projects", "analyze across all projects").option("--inactive", "include inactive memories");
+	addDbOption(cmd);
+	addJsonOption(cmd);
+	cmd.action((opts) => {
+		try {
+			const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null);
+			const result = getMemoryArtifactReport(resolveDbOpt(opts), {
+				project,
+				allProjects: opts.allProjects === true,
+				includeInactive: opts.inactive === true
+			});
+			if (opts.json) {
+				console.log(JSON.stringify(result, null, 2));
+				return;
+			}
+			p.intro("codemem memory artifact-report");
+			p.log.info([
+				`Memories: ${result.totals.memories}`,
+				`Active: ${result.totals.active}`,
+				`Sessions: ${result.totals.sessions}`
+			].join("\n"));
+			p.log.info("Counts by artifact:");
+			for (const [artifact, count] of Object.entries(result.counts_by_artifact)) p.log.message(`  ${artifact.padEnd(16)} ${String(count)}`);
+			p.log.info("Counts by action:");
+			for (const [action, count] of Object.entries(result.counts_by_action)) p.log.message(`  ${action.padEnd(14)} ${String(count)}`);
+			const topReasons = Object.entries(result.counts_by_reason).sort(([, a], [, b]) => b - a).slice(0, 10);
+			if (topReasons.length > 0) {
+				p.log.info("Top reasons:");
+				for (const [reason, count] of topReasons) p.log.message(`  ${reason.padEnd(36)} ${String(count)}`);
+			}
+			p.log.info(`High-confidence telemetry: ${result.high_confidence_telemetry.total}`);
+			p.outro("done");
+		} catch (error) {
+			const message = error instanceof Error ? error.message : "Artifact report failed";
+			if (opts.json) emitJsonError("artifact_report_failed", message);
+			else {
+				p.log.error(message);
+				process.exitCode = 1;
+			}
+			return;
+		}
+	});
+	return cmd;
+}
 function createMemoryExtractionReportCommand() {
 	const cmd = new Command("extraction-report").configureHelp(helpStyle).description("Score extracted memories for a session against a built-in extraction eval rubric").option("--session-id <id>", "session ID to evaluate").option("--batch-id <id>", "raw-event flush batch ID to evaluate").requiredOption("--scenario <id>", "built-in extraction eval scenario ID").option("--inactive", "include inactive memories");
 	addDbOption(cmd);
@@ -5036,6 +5347,7 @@ memoryCommand.addCommand(createRememberMemoryCommand());
 memoryCommand.addCommand(createInjectMemoryCommand());
 memoryCommand.addCommand(createMemoryRoleReportCommand());
 memoryCommand.addCommand(createMemoryRoleCompareCommand());
+memoryCommand.addCommand(createMemoryArtifactReportCommand());
 memoryCommand.addCommand(createMemoryExtractionReportCommand());
 memoryCommand.addCommand(createMemoryExtractionReplayCommand());
 memoryCommand.addCommand(createMemoryExtractionBenchmarkCommand());
@@ -7408,6 +7720,7 @@ completion.on("command", ({ reply }) => {
 		"config",
 		"coordinator",
 		"db",
+		"distill",
 		"embed",
 		"enqueue-raw-event",
 		"export-memories",
@@ -7489,6 +7802,7 @@ program.addCommand(claudeHookFileContextCommand);
 program.addCommand(codexHookInjectCommand);
 program.addCommand(codexHookIngestCommand);
 program.addCommand(dbCommand);
+program.addCommand(distillCommand);
 program.addCommand(exportMemoriesCommand);
 program.addCommand(importMemoriesCommand);
 program.addCommand(statsCommand);