@makerstufferupper/flypi 0.0.0-stage → 0.1.3

package/src/daemon/update.ts

@@ -0,0 +1,248 @@
+/**
+ * Keeping flypi up to date (docs/architecture.md Phase 7): is a newer flypi published, is it downloaded
+ * already, and when may it be installed?
+ *
+ * `npm run release` is the only thing that publishes, and package.json is the version it bumps, the version
+ * `/healthz` reports and the version `flypi status` prints — so the registry's `latest` is the single answer
+ * to "is there a new package", with no second version file to keep in step.
+ *
+ * The daemon **downloads** the new package (once a day at most, ~0.4 MB, verified against the checksum the
+ * registry published with it) and **installs it only while the daemon is idle** — no page attached and no
+ * turn in flight, held for a stretch, the same idle the memory sweep uses. Installing replaces the files the
+ * daemon is running from, so it is followed by a restart through the CLI; a restart while someone is reading
+ * an answer would be the one thing this must not do.
+ *
+ * **One version at a time.** The version to move to is the next one published above the running one — never
+ * the newest. flypi's own databases (the memory store, the run record) carry a schema, and a schema upgrade
+ * is code that runs when the version that needs it starts: 0.1.0 → 0.3.0 would run 0.3.0's migration against
+ * 0.1.0's schema and skip whatever 0.2.0 would have done to it. So the ladder is walked one published release
+ * per install, each step starting on its own and migrating what it finds, and the stamp is cleared after a
+ * step so the next one is taken from a fresh look rather than from the same answer. When a package has
+ * skipped numbers, the next published release is the step — that is the ladder npm still has.
+ *
+ * A checkout is not an installed package: it has `.git` next to the code, it updates with git, and the
+ * registry has nothing to compare it against. Every entry point here says so rather than checking anyway.
+ */
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve, sep } from "node:path";
+
+/** How long an answer stays good — the same day-long rhythm the memory sweep keeps. */
+export const CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
+/** How long the daemon must have been idle before a downloaded update installs. The job loop passes once a
+ * minute, so this is measured in passes; a lull shorter than this is not a restart. */
+export const IDLE_BEFORE_INSTALL_MS = 10 * 60_000;
+/** The answer, kept next to the pid file so `flypi status` can report it without a network round trip. */
+export const UPDATE_CHECK_FILE = join(".flypi", "update-check.json");
+/** Where a downloaded package waits. One version at a time — `downloadUpdate` clears the rest. */
+export const UPDATE_DIR = join(".flypi", "updates");
+
+export interface UpdateDist {
+	tarball: string;
+	/** `sha512-<base64>`, what npm publishes; `shasum` is the older sha1 hex form it also publishes. */
+	integrity?: string;
+	shasum?: string;
+}
+
+export interface UpdateCheck {
+	checkedAt: string;
+	/** The version installed when the answer was written. `current` in an old stamp is not today's version. */
+	current: string;
+	/** The newest published version — information only; nothing here installs it. */
+	latest: string | null;
+	/** The version to move to: the next one published above `current`. Null when there is none. */
+	next: string | null;
+	error?: string;
+	/** The package for `next` — what gets downloaded and installed. */
+	dist?: UpdateDist;
+	/** The package already downloaded and verified, waiting for an idle moment to install. */
+	download?: { version: string; file: string; at: string };
+}
+
+/** The directory the running code ships in — not `paths.root`, which is the state directory. */
+export function packageDir(): string {
+	return resolve(import.meta.dirname, "..", "..");
+}
+
+/** What the installed package is called, and which version of it this is. */
+export function packageInfo(dir = packageDir()): { name: string; version: string; checkout: boolean } {
+	const pkg = JSON.parse(readFileSync(resolve(dir, "package.json"), "utf8")) as { name?: string; version?: string };
+	// A checkout updates with `git pull`; the registry only knows what `npm publish` put there.
+	return { name: pkg.name ?? "", version: pkg.version ?? "dev", checkout: existsSync(resolve(dir, ".git")) };
+}
+
+/**
+ * Version order, by number: 0.10.0 is newer than 0.9.0, so the parts are not text. A prerelease suffix sorts
+ * as its base version — npm keeps those under the `next` tag, and `published()` filters them out anyway.
+ */
+export function compareVersions(a: string, b: string): number {
+	const parts = (v: string) => (v.split("-")[0] ?? v).split(".").map((p) => Number(p) || 0);
+	const [left, right] = [parts(a), parts(b)];
+	for (let i = 0; i < Math.max(left.length, right.length); i++) {
+		const diff = (left[i] ?? 0) - (right[i] ?? 0);
+		if (diff !== 0) return diff;
+	}
+	return 0;
+}
+
+/** Newest first, by number. */
+export function isNewer(installed: string, latest: string): boolean {
+	return compareVersions(latest, installed) > 0;
+}
+
+/**
+ * The published versions of a package, released ones only: a prerelease is not a rung anyone should be
+ * stepped up through, and npm does not put one in `dist-tags.latest` either.
+ */
+export function published(versions: Record<string, unknown>): string[] {
+	return Object.keys(versions).filter((v) => !v.includes("-")).sort(compareVersions);
+}
+
+/**
+ * The one step above `current` — the whole point of the ladder (see the note at the top of this file). Null
+ * when nothing newer has been published, and when the registry gave no version list to build a ladder from:
+ * guessing a jump is exactly what this rule exists to prevent.
+ */
+export function nextVersion(versions: Record<string, unknown>, current: string): string | null {
+	return published(versions).find((v) => isNewer(current, v)) ?? null;
+}
+
+/**
+ * The step to take, from a stamp written by any install: null when there is nothing newer to move to. A stamp
+ * written by an older install names a step that has since been taken, and `next` is re-checked against the
+ * version running now for that reason.
+ */
+export function updateAvailable(check: UpdateCheck | null, current: string): string | null {
+	return check?.next && isNewer(current, check.next) ? check.next : null;
+}
+
+/**
+ * The version sitting on disk ready to install — only when it is newer than what is running. A stamp left by
+ * an older install still names a tarball that was already applied, and installing it again would be a
+ * restart for nothing.
+ */
+export function stagedUpdate(check: UpdateCheck | null, current: string): string | null {
+	return check?.download?.version && isNewer(current, check.download.version) ? check.download.version : null;
+}
+
+/** Is the answer too old to trust (or absent)? */
+export function dueForCheck(check: UpdateCheck | null, now = Date.now()): boolean {
+	const at = Date.parse(check?.checkedAt ?? "");
+	return Number.isNaN(at) || now - at >= CHECK_INTERVAL_MS;
+}
+
+/**
+ * Ask the registry for the package document — `dist-tags.latest` and every published version with its
+ * `dist` — in one request, because the ladder needs the version list, not just the top of it.
+ *
+ * `fetchImpl` is injectable because a test must not reach the network, and a registry that is down, offline
+ * or has never heard of this package (404 — where this one is until `npm run release` publishes it) is an
+ * answer, not a crash.
+ */
+export async function checkForUpdate(options: { current: string; name: string; fetchImpl?: typeof fetch; timeoutMs?: number }): Promise<UpdateCheck> {
+	const at = new Date().toISOString();
+	const fail = (error: string): UpdateCheck => ({ checkedAt: at, current: options.current, latest: null, next: null, error });
+	const url = `https://registry.npmjs.org/${encodeURIComponent(options.name)}`;
+	try {
+		const response = await (options.fetchImpl ?? fetch)(url, { signal: AbortSignal.timeout(options.timeoutMs ?? 5_000), headers: { accept: "application/json" } });
+		if (!response.ok) return fail(`the registry answered ${response.status} for ${options.name}`);
+		const doc = (await response.json()) as { "dist-tags"?: { latest?: unknown }; versions?: Record<string, { dist?: UpdateDist }> };
+		const versions = doc.versions ?? {};
+		if (Object.keys(versions).length === 0) return fail(`the registry named no published versions of ${options.name} to step through`);
+		const latest = typeof doc["dist-tags"]?.latest === "string" ? doc["dist-tags"].latest : published(versions).at(-1) ?? null;
+		const next = nextVersion(versions, options.current);
+		return {
+			checkedAt: at,
+			current: options.current,
+			latest,
+			next,
+			...(next && versions[next]?.dist?.tarball ? { dist: versions[next].dist } : {}),
+		};
+	} catch (error) {
+		return fail(`the registry could not be reached: ${(error as Error).message}`);
+	}
+}
+
+/**
+ * Fetch the package and keep it for the idle moment. The checksum the registry published with the manifest
+ * is the only proof the bytes are the ones it described, and these bytes become code that runs with the
+ * user's home directory in front of it — so a mismatch is deleted rather than staged, and a manifest with no
+ * checksum to verify against is refused rather than trusted.
+ *
+ * The tarball is flypi's own code (~0.4 MB); npm still resolves its dependencies at install time.
+ */
+export async function downloadUpdate(options: { root: string; version: string; dist: UpdateDist; fetchImpl?: typeof fetch; timeoutMs?: number }): Promise<{ file?: string; error?: string }> {
+	if (!options.dist.integrity?.startsWith("sha512-") && !options.dist.shasum) return { error: "the registry published no checksum to verify the download against" };
+	let bytes: Buffer;
+	try {
+		const response = await (options.fetchImpl ?? fetch)(options.dist.tarball, { signal: AbortSignal.timeout(options.timeoutMs ?? 120_000) });
+		if (!response.ok) return { error: `the registry answered ${response.status} for the package` };
+		bytes = Buffer.from(await response.arrayBuffer());
+	} catch (error) {
+		return { error: `the package could not be downloaded: ${(error as Error).message}` };
+	}
+	if (options.dist.integrity?.startsWith("sha512-")) {
+		const got = createHash("sha512").update(bytes).digest("base64");
+		if (got !== options.dist.integrity.slice("sha512-".length)) return { error: "the downloaded package does not match the registry's sha512" };
+	} else if (createHash("sha1").update(bytes).digest("hex") !== options.dist.shasum) {
+		return { error: "the downloaded package does not match the registry's sha1" };
+	}
+	const dir = resolve(options.root, UPDATE_DIR);
+	mkdirSync(dir, { recursive: true });
+	// One version waiting at a time: the old tarball is code that is no longer going to run.
+	for (const old of readdirSync(dir)) if (!old.includes(options.version)) rmSync(join(dir, old), { force: true });
+	const file = join(dir, basename(options.dist.tarball));
+	writeFileSync(file, bytes);
+	return { file };
+}
+
+export function readUpdateCheck(root: string): UpdateCheck | null {
+	try {
+		return JSON.parse(readFileSync(resolve(root, UPDATE_CHECK_FILE), "utf8")) as UpdateCheck;
+	} catch {
+		return null;
+	}
+}
+
+export function writeUpdateCheck(root: string, check: UpdateCheck): void {
+	// The `.flypi/` directory is created here, not assumed: `flypi update --check` works on a root that has
+	// never run a daemon, where neither the stamp nor the directory exists yet.
+	const file = resolve(root, UPDATE_CHECK_FILE);
+	mkdirSync(dirname(file), { recursive: true });
+	writeFileSync(file, `${JSON.stringify(check, null, 2)}\n`);
+}
+
+/** What installs the downloaded tarball. Pinned to the file that was just verified, so what installs is
+ * what was checked; `--no-audit --no-fund` because this runs in front of a person, not in CI. */
+export function installArgs(file: string): string[] {
+	return ["install", "-g", "--no-audit", "--no-fund", file];
+}
+
+/**
+ * Is the running code the global package `npm install -g` would replace? flypi installed through pnpm, bun,
+ * npx or a second prefix would leave the install landing somewhere else, and the restart would come back on
+ * the old code while the log said it had updated.
+ */
+export function isGlobalInstall(dir = packageDir()): boolean {
+	const out = spawnSync("npm", ["root", "-g"], { encoding: "utf8" });
+	if (out.status !== 0) return false;
+	const root = out.stdout.trim();
+	if (!root) return false;
+	const here = resolve(dir);
+	return here === root || here.startsWith(root + sep);
+}
+
+/** A download is only worth keeping when it is still the step this version should take. */
+export function keepDownload(check: UpdateCheck, previous: UpdateCheck | null): UpdateCheck["download"] {
+	return previous?.download && previous.download.version === check.next ? previous.download : undefined;
+}
+
+/**
+ * Forget the answer after a step has been taken. The next version looks again — a fresh ladder from the
+ * version that is now running, and the next single step — rather than installing the rest of what an older
+ * answer had already picked out.
+ */
+export function clearUpdateCheck(root: string): void {
+	rmSync(resolve(root, UPDATE_CHECK_FILE), { force: true });
+}

package/src/memory/capture.ts

@@ -0,0 +1,109 @@
+/**
+ * Auto-capture v1 (docs/memory.md Phase 4: "heuristic capture on `agent_end` … cheap rules, no extra
+ * model call"). Two kinds of durable fact are recoverable without understanding anything: a sentence the
+ * user wrote *as* an instruction ("remember …", "we never …"), and a command the agent watched fail and
+ * then work on the second try.
+ *
+ * The body of a captured fact stays the user's own words. Rewriting it would put flypi's paraphrase in
+ * front of a model later and lose what was actually said — and the write gate (`NEAR_DUPLICATE`,
+ * `valueSwap` in store.ts) is what stops a capture from becoming a second copy of a fact the model
+ * already stored. Provenance is code's as everywhere else: the user's words are `owner`, what the agent
+ * watched happen is `agent`, and neither is an argument a caller can pass.
+ */
+import type { OriginClass } from "./store.ts";
+
+export interface CapturedFact {
+	body: string;
+	kind: string;
+	origin: OriginClass;
+}
+
+/** What one turn's tools did, in order — the parts of pi's `tool_result` event a rule can use. */
+export interface ToolOutcome {
+	name: string;
+	/** A shell command, when the tool ran one. That is the only outcome worth comparing. */
+	command?: string;
+	isError: boolean;
+}
+
+/** One turn is not a place to write an essay about. */
+const MAX_FACTS = 3;
+/** "not anymore" is a fragment; a fact needs a subject and something said about it. */
+const MIN_WORDS = 4;
+/** The MCP surface caps a body at 8000; a captured sentence never needs near that much. */
+const MAX_CHARS = 500;
+
+/**
+ * The user writing something down on purpose. Global because one prompt can state two facts. It stops at a
+ * sentence end — a period followed by space or the end — so one statement cannot swallow the next one,
+ * while a dotted path inside the fact stays part of it.
+ */
+const STATED = /\b(?:remember|note that|note this|keep in mind|from now on|going forward)\b[:,]?\s*(?:that\s+)?((?:[^.!?]|\.(?!\s|$))+)/gi;
+/** The marker on its own, for the sentence-level test below (a global regex is stateful in `.test`). */
+const STATED_MARK = /\b(?:remember|note that|note this|keep in mind|from now on|going forward)\b/i;
+/** A standing instruction or a correction — the "user corrections" half, in the words people use for it. */
+const STANDING = /\b(never|always|no longer|not anymore|anymore|instead of|prefer|only use|do not|don't|does not|doesn't|must not|use instead)\b/i;
+// shortcut: a word list, so it reads "I never said anything about it" as an instruction and misses a
+// correction phrased unusually. If that noise shows up in the store, the upgrade is the NLI judge already
+// sitting behind the write gate (src/memory/judge.ts) judging these too, not a longer list here.
+
+/**
+ * What this turn is worth remembering, in the order it was said. Deliberately conservative: a capture
+ * that writes junk costs a row and a retrieval slot forever, while a fact it misses is one the model can
+ * still be asked to store — so every rule here has to be something a sentence states outright.
+ */
+export function captureFromTurn(prompt: string, tools: ToolOutcome[] = []): CapturedFact[] {
+	const facts: CapturedFact[] = [];
+	const add = (fact: CapturedFact | undefined) => {
+		if (fact && !facts.some((existing) => existing.body === fact.body)) facts.push(fact);
+	};
+
+	for (const match of prompt.matchAll(STATED)) add(factOf(match[1], "fact", "owner"));
+	for (const sentence of sentences(prompt)) {
+		// Already handled as a stated fact: capturing it twice, once with the "remember" prefix and once
+		// without, is two rows for one instruction.
+		if (STATED_MARK.test(sentence)) continue;
+		if (STANDING.test(sentence)) add(factOf(sentence, "preference", "owner"));
+	}
+
+	// Failed-then-fixed: the second command working after the first failed is a fact the agent earned, so it
+	// is `agent`-origin even though nobody typed it. Matched on the attempt — the same arguments, whatever
+	// runs them — because `npm run build` then `pnpm run build` is one fix, while `ls` then `cat` are two
+	// unrelated commands and calling the second one a fix would invent a fact.
+	const commands = tools.flatMap((tool) => (tool.command ? [{ command: tool.command, isError: tool.isError }] : []));
+	for (const failed of commands.filter((tool) => tool.isError)) {
+		const fix = commands.find((tool) => !tool.isError && tool.command !== failed.command && sameAttempt(failed.command, tool.command));
+		if (fix) add(factOf(`\`${failed.command}\` failed; \`${fix.command}\` worked`, "fact", "agent"));
+	}
+	return facts.slice(0, MAX_FACTS);
+}
+
+/** Sentence boundaries, kept simple: a captured fact must not run across two instructions. */
+function sentences(prompt: string): string[] {
+	return prompt
+		.split(/(?<=[.!?])\s+|\n+/)
+		.map((sentence) => sentence.trim())
+		.filter(Boolean);
+}
+
+function factOf(text: string | undefined, kind: string, origin: OriginClass): CapturedFact | undefined {
+	const raw = (text ?? "").replace(/\s+/g, " ").trim().slice(0, MAX_CHARS);
+	// A question is someone asking, not someone telling: "don't we use pnpm?" states no preference.
+	if (!raw || raw.endsWith("?") || wordCount(raw) < MIN_WORDS) return undefined;
+	// One shape for every rule: some patterns match a sentence with its period, some stop before it, and a
+	// store where "use pnpm" and "use pnpm." are two different strings invites the reader to dedupe them.
+	const body = raw.replace(/[.!?]+$/, "").trim();
+	return { body, kind, origin };
+}
+
+const wordCount = (text: string): number => (text.match(/[\p{L}\p{N}]+/gu) ?? []).length;
+/** A command without its runner or its flags: what makes two of them the same attempt. */
+const shape = (command: string): string[] => command.trim().split(/\s+/).slice(1).filter((token) => !token.startsWith("-"));
+/**
+ * Two commands are the same attempt when the only difference is what runs them. Both empty is *not* a match:
+ * `false` then `true` is two commands with nothing in common, and calling that a fix would invent a fact.
+ */
+const sameAttempt = (failed: string, fix: string): boolean => {
+	const tried = new Set(shape(failed));
+	return shape(fix).some((token) => tried.has(token));
+};

package/src/memory/config.ts

@@ -0,0 +1,90 @@
+/**
+ * `memory` in `.flypi/agent/settings.json` (docs/memory.md Phase 3). Defaults are chosen to work on
+ * a machine with nothing installed, which is why `provider: "none"` is an escape hatch you can take
+ * and not a failure state: search then runs FTS-only and says so.
+ *
+ * `consolidation` and `wiki` are Phase 6 and absent here on purpose.
+ */
+import { readFileSync } from "node:fs";
+import type { FlypiPaths } from "../agent/paths.ts";
+
+/**
+ * CHOSEN (2026-10-11, docs/decisions.md): EmbeddingGemma 2 — text, images and audio in one vector
+ * space, 8,192-token window. The `onnx-community` mirror, not `google/`: the upstream repo ships
+ * safetensors only and could not be loaded here. Non-English is covered (100+ languages).
+ */
+export const EMBED_MODEL = "onnx-community/embeddinggemma-2-ONNX";
+
+/**
+ * CHOSEN (2026-10-11, docs/decisions.md): the contradiction judge is `nli-deberta-v3-xsmall` — an NLI
+ * cross-encoder, not a chat model. Chosen on measurement (`scripts/benchmark-nli.ts`): the smallest
+ * checkpoint that read every contradiction in the pair set correctly, 87 MB at q8, ~6 ms a pair.
+ */
+export const JUDGE_MODEL = "Xenova/nli-deberta-v3-xsmall";
+
+export interface EmbeddingsConfig {
+	provider: "transformers" | "none";
+	model: string;
+	/** Part of the index identity key: two quantizations are two vector spaces (risk 7). */
+	quantized: boolean;
+	cacheDir: string;
+}
+
+/**
+ * The contradiction judge (docs/memory.md Phase 4). On by default: switched off, the store can only
+ * judge what tokens can judge, and the measured cost is 87 MB of weights and ~6 ms per pair. It shares
+ * the cache directory with the embeddings because it shares the runtime that reads them.
+ */
+export interface JudgeConfig {
+	provider: "transformers" | "none";
+	model: string;
+	quantized: boolean;
+	cacheDir: string;
+}
+
+export interface MemoryConfig {
+	embeddings: EmbeddingsConfig;
+	judge: JudgeConfig;
+	/**
+	 * The review fork (docs/memory.md Phase 4.8): how many delivered answers between forks, `0` = off. It is
+	 * a config key with an off switch for the same reason the judge is — a fork is a whole model run over the
+	 * conversation, ~30K tokens on a route that does not serve it from cache, with no human in the loop.
+	 */
+	review: { interval: number };
+}
+
+export function memoryConfig(paths: FlypiPaths): MemoryConfig {
+	let settings: unknown;
+	try {
+		settings = JSON.parse(readFileSync(paths.settingsPath, "utf8"));
+	} catch {
+		settings = {}; // no settings file yet: the defaults have to work
+	}
+	return parseMemoryConfig(settings, paths);
+}
+
+export function parseMemoryConfig(settings: unknown, paths: FlypiPaths): MemoryConfig {
+	const memory = (settings as { memory?: Record<string, unknown> } | undefined)?.memory;
+	const raw = memory?.embeddings as Record<string, unknown> | undefined;
+	const judge = memory?.judge as Record<string, unknown> | undefined;
+	const provider = (value: unknown): "transformers" | "none" => (value === "none" || value === false ? "none" : "transformers");
+	const model = (value: unknown, fallback: string) => (typeof value === "string" && value.length > 0 ? value : fallback);
+	// A number of turns, so a non-number is the default rather than a fork every turn: `true` in the file must
+	// not become `1`, and a negative one must not become "fire on every turn" through a modulo.
+	const interval = (value: unknown, fallback: number) => (typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : fallback);
+	return {
+		embeddings: {
+			provider: provider(raw?.provider),
+			model: model(raw?.model, EMBED_MODEL),
+			quantized: raw?.quantized !== false,
+			cacheDir: model(raw?.cacheDir, paths.modelsDir),
+		},
+		judge: {
+			provider: provider(judge?.provider),
+			model: model(judge?.model, JUDGE_MODEL),
+			quantized: judge?.quantized !== false,
+			cacheDir: model(judge?.cacheDir, paths.modelsDir),
+		},
+		review: { interval: interval(memory?.reviewInterval, 10) },
+	};
+}

package/src/memory/consolidate.ts

@@ -0,0 +1,218 @@
+/**
+ * The consolidation sweep (docs/memory.md Phase 6) — the part no per-turn pass can do.
+ *
+ * Why a sweep exists at all, when the write gate refuses near-copies and `search()` already collapses
+ * same-fact rows at read time: read-time collapsing **hides** a duplicate, it does not remove it. Both rows
+ * stay `active`, both keep taking a slot in the top-5 fetch every later write is judged against, and they
+ * **split `use_count`/`cited_count` between them** — which is exactly what the promotion gates below read,
+ * so a fact cited across three turns but split over two copies may never reach ≥ 3 on either. And nothing
+ * else in the system ever looks at the store as a whole: the write gate sees five neighbours at insert time
+ * and then never again.
+ *
+ * The LLM "reflect" phase this phase used to name is gone — Phase 4.8's review fork does that judging while
+ * the context is warm, instead of re-reading conversations later. What is left here is deterministic, which
+ * is also what makes it safe to run unattended: **cluster, gate, then write — and only the last step
+ * writes**, so a sweep interrupted halfway has changed nothing. hermes is the precedent for the shape: its
+ * curator runs the deterministic transitions always and leaves the model-driven half off by default
+ * (`agent/curator.py`, `DEFAULT_CONSOLIDATE = False`).
+ *
+ * The safety story is one line and it lives in the store, not here: a sweep may supersede and it may not
+ * delete (`UNATTENDED_ACTORS` in src/memory/store.ts).
+ */
+import { cosine } from "./embed.ts";
+import {
+	NEAR_DUPLICATE,
+	sameFact,
+	valueSwap,
+	type MemoryRow,
+	type MemoryStore,
+	type OriginClass,
+} from "./store.ts";
+
+/** The queue kind the daemon registers a handler for (src/daemon/index.ts). */
+export const CONSOLIDATE_JOB = "memory.consolidate";
+
+/**
+ * The trigger, on flypi's conditions rather than a 3am cron (docs/memory.md Phase 6): the store is idle
+ * **and** (this many rows have arrived since the last sweep, **or** the last one is older than this). A
+ * desktop app is off at 3am some nights, so a cron would simply not run.
+ */
+export const MIN_NEW_ROWS = 25;
+export const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * The promotion gates: memory gets promoted because it kept being useful, not because it was written
+ * confidently. `citations` counts turns whose answer actually cited the row, `queries` how many different
+ * questions led to it (one question asked five times is one query), and `score` is the citation rate over
+ * the turns it was retrieved for — a row that is always fetched and never used has earned nothing.
+ */
+export const GATE_CITATIONS = 3;
+export const GATE_QUERIES = 3;
+export const GATE_SCORE = 0.75;
+
+export interface TurnEvidence {
+	userText: string;
+	retrieved: number[];
+	cited: number[];
+}
+
+export interface MergePlan {
+	survivor: number;
+	merged: number[];
+}
+
+export interface SweepPlan {
+	merges: MergePlan[];
+	promotions: number[];
+}
+
+/** What one pass did, in the shape `job_runs.detail` prints and the tests read. */
+export interface SweepReport extends SweepPlan {
+	rowsBefore: number;
+	rowsAfter: number;
+}
+
+/**
+ * Authority order. A merge retires rows, so it must never retire the more authoritative copy: an `owner`
+ * row (the user's own words) and an `agent` row that says the same thing collapse to the `owner` one, or
+ * the sweep would quietly downgrade a fact the user stated into one the agent inferred.
+ */
+const AUTHORITY: Record<OriginClass, number> = { owner: 3, agent: 2, untrusted: 1, system: 0 };
+
+/** Which copy of a duplicated fact survives: authority first, then what earned its place, then oldest. */
+function compareSurvivor(a: MemoryRow, b: MemoryRow): number {
+	return (
+		AUTHORITY[b.origin_class] - AUTHORITY[a.origin_class] ||
+		b.cited_count - a.cited_count ||
+		b.use_count - a.use_count ||
+		a.id - b.id
+	);
+}
+
+/**
+ * Are these two rows the same fact? Two signals, both deterministic:
+ *
+ * - the vectors say so (`NEAR_DUPLICATE`, the same threshold the write gate uses) — but only inside one
+ *   index, which is why `activeVectors` hands back the current model's vectors and nothing else. This is
+ *   also the pass that catches what a re-embed makes: a new vector space is a different set of near pairs.
+ * - token containment says so (`sameFact`, the rule the read path already uses to hide a duplicate from
+ *   the model). The sweep is where it stops being merely hidden.
+ *
+ * Two shapes are corrections, not copies, and merging one would retire whichever side the survivor rule
+ * happened to prefer: a pair whose difference is one changed value (`valueSwap`), and a pair where one side
+ * is negated and the other is not. The second is the one the vectors are worst at — measured, "we do not use
+ * pnpm" against "we use pnpm" sits at 0.9835, above the duplicate threshold — so the guard runs before both
+ * signals. Telling a contradiction from a copy in general is the NLI judge's job at the write gate
+ * (src/memory/judge.ts); this is the part that needs no model.
+ */
+const NEGATION = /\b(?:not|never|no|none|without|don'?t|doesn'?t|didn'?t|isn'?t|aren'?t|won'?t|cannot|can'?t)\b/;
+const polarityMismatch = (a: string, b: string): boolean => NEGATION.test(a) !== NEGATION.test(b);
+
+function sameStoredFact(a: MemoryRow, b: MemoryRow, vectors: Map<number, Float32Array>): boolean {
+	if (valueSwap(a.body, b.body) !== undefined || polarityMismatch(a.body, b.body)) return false;
+	const left = vectors.get(a.id);
+	const right = vectors.get(b.id);
+	if (left && right && cosine(left, right) >= NEAR_DUPLICATE) return true;
+	return sameFact(a.body, b.body);
+}
+
+/**
+ * Connected components over the same-fact relation, in id order. Union-find rather than one anchor per
+ * cluster because the relation is not transitive in one hop: a reworded chain A~B~C has to land in one
+ * cluster even when A and C share no tokens, or the copies at the ends of it stay active forever.
+ * shortcut: every pair is compared, like `vectorSearch`'s full scan — fine to a few tens of thousands of
+ * rows, and the same place a vector index would plug in.
+ */
+export function clusterRows(rows: MemoryRow[], vectors: Map<number, Float32Array>): number[][] {
+	const parent = new Map<number, number>(rows.map((row) => [row.id, row.id]));
+	const find = (id: number): number => {
+		const above = parent.get(id)!;
+		if (above === id) return id;
+		const root = find(above);
+		parent.set(id, root);
+		return root;
+	};
+	for (let i = 0; i < rows.length; i += 1) {
+		for (let j = i + 1; j < rows.length; j += 1) {
+			if (sameStoredFact(rows[i]!, rows[j]!, vectors)) parent.set(find(rows[i]!.id), find(rows[j]!.id));
+		}
+	}
+	const clusters = new Map<number, number[]>();
+	for (const row of rows) {
+		const root = find(row.id);
+		clusters.set(root, [...(clusters.get(root) ?? []), row.id]);
+	}
+	return [...clusters.values()];
+}
+
+/**
+ * What the gates read for one row — over the turns that cited **it or any row it absorbed**. That is the
+ * whole point of carrying citations through a merge: two copies one criterion short of a gate each must not
+ * stay permanently short, and `turns.cited_json` is an immutable record of what a turn actually wrote, so
+ * the citations are followed at read time rather than rewritten into history.
+ */
+export function promotionEvidence(turns: TurnEvidence[], ids: number[]): { citations: number; queries: number; retrieved: number; score: number } {
+	const wanted = new Set(ids);
+	let citations = 0;
+	let retrieved = 0;
+	const queries = new Set<string>();
+	for (const turn of turns) {
+		if (turn.retrieved.some((id) => wanted.has(id))) retrieved += 1;
+		if (turn.cited.some((id) => wanted.has(id))) {
+			citations += 1;
+			queries.add(turn.userText);
+		}
+	}
+	return { citations, queries: queries.size, retrieved, score: retrieved === 0 ? 0 : citations / retrieved };
+}
+
+export function passesGates(evidence: { citations: number; queries: number; score: number }): boolean {
+	return evidence.citations >= GATE_CITATIONS && evidence.queries >= GATE_QUERIES && evidence.score >= GATE_SCORE;
+}
+
+/**
+ * The plan, with nothing written: clusters of two or more collapse to one row, and the rows that survive a
+ * merge are gated on evidence that includes the citations of the rows they absorbed.
+ */
+export function planSweep(rows: MemoryRow[], vectors: Map<number, Float32Array>, turns: TurnEvidence[]): SweepPlan {
+	const byId = new Map(rows.map((row) => [row.id, row]));
+	const merges: MergePlan[] = [];
+	for (const cluster of clusterRows(rows, vectors)) {
+		if (cluster.length < 2) continue;
+		const survivor = cluster.map((id) => byId.get(id)!).reduce((best, row) => (compareSurvivor(row, best) < 0 ? row : best)).id;
+		merges.push({ survivor, merged: cluster.filter((id) => id !== survivor) });
+	}
+	const absorbed = new Map<number, number[]>(merges.map((merge) => [merge.survivor, merge.merged]));
+	const retired = new Set(merges.flatMap((merge) => merge.merged));
+	const promotions = rows
+		.map((row) => row.id)
+		.filter((id) => !retired.has(id))
+		.filter((id) => passesGates(promotionEvidence(turns, [id, ...(absorbed.get(id) ?? [])])));
+	return { merges, promotions };
+}
+
+/**
+ * Is a sweep due? Idle first — a sweep reorganises the store a conversation is being answered from, so it
+ * waits for the same quiet the re-embed job waits for. Then either enough new rows to be worth a pass, or a
+ * day since the last one. A store that has never been swept is not due until it has the rows: an empty store
+ * has nothing to consolidate, and a daemon that swept on start-up would queue work for every fresh checkout.
+ */
+export function sweepDue(state: { newRows: number; lastSweepAt: string | null; at: string; idle: boolean }): boolean {
+	if (!state.idle) return false;
+	if (state.newRows >= MIN_NEW_ROWS) return true;
+	if (state.lastSweepAt === null) return false;
+	return Date.parse(state.at) - Date.parse(state.lastSweepAt) >= SWEEP_INTERVAL_MS;
+}
+
+/**
+ * One pass, end to end: read, plan, then apply. The apply half is four store calls and nothing else, which
+ * is what makes "a sweep can be interrupted without half-applying" true rather than asserted.
+ */
+export function consolidate(store: MemoryStore, options: { actor?: string } = {}): SweepReport {
+	const actor = options.actor ?? "sweep";
+	const rowsBefore = store.memoryRowCount();
+	const plan = planSweep(store.activeMemories(), store.activeVectors(), store.citationEvidence());
+	for (const merge of plan.merges) store.applyMerge(merge, actor);
+	for (const id of plan.promotions) store.applyPromotion(id, actor);
+	return { ...plan, rowsBefore, rowsAfter: store.memoryRowCount() };
+}