@makerstufferupper/flypi 0.0.0-stage → 0.1.3

package/src/memory/wiki.ts

@@ -0,0 +1,140 @@
+/**
+ * The wiki (docs/memory.md Phase 6): a **render-only projection** of the `active` rows.
+ *
+ * It exists so a human can read what the store has learned, and so duplication is *visible* — two near
+ * copies of a fact sit next to each other on one page in a way they never are in a search result. It is not
+ * a second source of truth and never becomes one: recall keeps reading `memories`, nothing reads these
+ * tables back as a fact, and the whole thing is rebuildable from scratch at any time. There is no editing,
+ * no managed blocks and no human-note region — a view has nothing editable, and that is a decision, not a
+ * deferral.
+ *
+ * A page is *a query plus a template*: one page per `kind` (the `topic_key`), sections clustered by the
+ * row's first tag, and the provenance spine (`#id` citations) emitted by **code**. The model never decides
+ * which claims back a section, which is what makes the spine byte-identical for the same active set — and
+ * in this build the model writes no wiki prose at all, so the whole render is deterministic and costs
+ * nothing. `wiki_pages.model` is null for that reason, and is where a future prose pass would name itself.
+ *
+ * It runs as a job after a consolidation sweep (src/daemon/index.ts), not on every write.
+ */
+import type { DatabaseSync } from "node:sqlite";
+import type { MemoryRow, MemoryStore } from "./store.ts";
+
+export const WIKI_JOB = "wiki.render";
+
+export interface WikiSection {
+	heading: string;
+	body: string;
+	claim_ids: number[];
+}
+
+export interface WikiPage {
+	slug: string;
+	title: string;
+	topic_key: string;
+	body: string;
+	claim_ids: number[];
+	model: string | null;
+	generated_at: string;
+	sections: WikiSection[];
+}
+
+/** `kind` is free-form text, so the page key is the kind made into a slug. Two kinds that slug the same
+ *  way are one page — which is also what makes the primary key impossible to collide. */
+const slugOf = (kind: string): string => kind.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "untitled";
+
+/** The heading a claim files under: its first tag, or `general` for an untagged row. */
+const headingOf = (row: MemoryRow): string => row.tags[0] ?? "general";
+
+/** One claim, cited by its row id: the spine a reader can follow back to the memory itself. */
+const claim = (row: MemoryRow): string => `[#${row.id}] ${row.body}`;
+
+/**
+ * Rebuild every page from the active rows. Drop-then-insert inside one transaction, so a render that throws
+ * halfway leaves the previous pages rather than half a wiki.
+ */
+export function renderWiki(store: MemoryStore, db: DatabaseSync): { pages: number; sections: number } {
+	const generatedAt = new Date().toISOString();
+	const byTopic = new Map<string, { title: string; rows: MemoryRow[] }>();
+	for (const row of store.activeMemories()) {
+		const slug = slugOf(row.kind);
+		const page = byTopic.get(slug);
+		// The title is the kind as it was actually written, taken from the oldest row on the page: `kind` is
+		// free-form, and "Deploy targets" and "deploy targets" are one page whose title should not depend on
+		// which row happened to be inserted first.
+		if (!page) byTopic.set(slug, { title: row.kind, rows: [row] });
+		else page.rows.push(row);
+	}
+
+	db.exec("BEGIN");
+	try {
+		db.prepare("DELETE FROM wiki_sections").run();
+		db.prepare("DELETE FROM wiki_pages").run();
+		const pages = db.prepare("INSERT INTO wiki_pages (slug, title, topic_key, body, claim_ids, model, generated_at) VALUES (?, ?, ?, ?, ?, ?, ?)");
+		const sections = db.prepare("INSERT INTO wiki_sections (slug, heading, body, claim_ids, generated_at) VALUES (?, ?, ?, ?, ?)");
+		let pageRows = 0;
+		let sectionRows = 0;
+		for (const [slug, page] of [...byTopic.entries()].sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
+			const byHeading = new Map<string, MemoryRow[]>();
+			for (const row of page.rows) {
+				const heading = headingOf(row);
+				byHeading.set(heading, [...(byHeading.get(heading) ?? []), row]);
+			}
+			const rendered = [...byHeading.entries()]
+				.sort((a, b) => (a[0] < b[0] ? -1 : 1))
+				.map(([heading, rows]) => {
+					const ordered = rows.sort((a, b) => a.id - b.id);
+					return { heading, body: ordered.map(claim).join("\n"), claim_ids: ordered.map((row) => row.id) };
+				});
+			const body = [`# ${page.title}`, "", ...rendered.flatMap((section) => [`## ${section.heading}`, "", section.body, ""])].join("\n").trimEnd();
+			const claimIds = rendered.flatMap((section) => section.claim_ids);
+			pages.run(slug, page.title, page.rows[0]!.kind, body, JSON.stringify(claimIds), null, generatedAt);
+			for (const section of rendered) {
+				sections.run(slug, section.heading, section.body, JSON.stringify(section.claim_ids), generatedAt);
+				sectionRows += 1;
+			}
+			pageRows += 1;
+		}
+		db.exec("COMMIT");
+		return { pages: pageRows, sections: sectionRows };
+	} catch (error) {
+		db.exec("ROLLBACK");
+		throw error;
+	}
+}
+
+/** What the memory browser shows: every page with its sections, spine included. */
+export function readWiki(db: DatabaseSync): WikiPage[] {
+	const pages = db.prepare("SELECT slug, title, topic_key, body, claim_ids, model, generated_at FROM wiki_pages ORDER BY slug").all() as unknown as {
+		slug: string;
+		title: string;
+		topic_key: string;
+		body: string;
+		claim_ids: string;
+		model: string | null;
+		generated_at: string;
+	}[];
+	const sections = db.prepare("SELECT slug, heading, body, claim_ids, generated_at FROM wiki_sections ORDER BY slug, heading, id").all() as unknown as {
+		slug: string;
+		heading: string;
+		body: string;
+		claim_ids: string;
+		generated_at: string;
+	}[];
+	const bySlug = new Map<string, WikiSection[]>();
+	for (const section of sections) {
+		bySlug.set(section.slug, [
+			...(bySlug.get(section.slug) ?? []),
+			{ heading: section.heading, body: section.body, claim_ids: JSON.parse(section.claim_ids) as number[] },
+		]);
+	}
+	return pages.map((page) => ({
+		slug: page.slug,
+		title: page.title,
+		topic_key: page.topic_key,
+		body: page.body,
+		claim_ids: JSON.parse(page.claim_ids) as number[],
+		model: page.model,
+		generated_at: page.generated_at,
+		sections: bySlug.get(page.slug) ?? [],
+	}));
+}

package/src/secrets/cli.ts

@@ -0,0 +1,248 @@
+/**
+ * `flypi secret …` (docs/secrets.md Phase 5.5).
+ *
+ * Two jobs, and they are not the same one:
+ *
+ * - `print NAME` is the command a config file references — `"apiKey": "!flypi secret print NAME"` in
+ *   auth.json, or an `env`/`headers` value in mcp.json (pi runs a leading `!` command and uses its stdout,
+ *   docs/models.md). That is why this exists as a process rather than only as a UI form: it has to answer in
+ *   any shell, with nothing in the environment, and print the value and nothing else.
+ * - `list`, `get`, `put`, `rotate`, `delete`, `import-ssh` are the ones a person types.
+ *
+ * Reads open the memory database read-only, so a second process can answer while the daemon holds the write
+ * lock (risk 15). Writes are short transactions: the plan's rule is that the CLI is never a second long-lived
+ * writer, and a put that returns in one statement does not become one.
+ */
+import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, join } from "node:path";
+import type { DatabaseSync } from "node:sqlite";
+import type { FlypiPaths } from "../agent/paths.ts";
+import { flypiPaths } from "../agent/paths.ts";
+import { openMemoryDb, openMemoryDbReadOnly } from "../memory/db.ts";
+import { secretFilePath } from "./config.ts";
+import { closeSecretSource } from "./redact.ts";
+import { keyfileProvider, type SecretKind } from "./crypto.ts";
+import { SecretStore, type SecretRow } from "./vault.ts";
+
+const USAGE = `usage: flypi secret list | get NAME | print NAME | put NAME [value] [--kind K] [--meta k=v]
+       flypi secret rotate NAME [value] | delete NAME | undelete NAME | path NAME
+       flypi secret import-ssh ~/.ssh/id_ed25519.pub | key | rotate-key
+a value left out of put/rotate is read from stdin, which keeps it out of the process list`;
+
+const READS = ["list", "get", "print", "key"];
+const WRITES = ["put", "rotate", "delete", "undelete", "path", "import-ssh", "rotate-key"];
+
+export interface SecretCliOptions {
+	/** Defaults to the checkout the command was run from, like every other flypi entrypoint. */
+	root?: string;
+	out?: (line: string) => void;
+	err?: (line: string) => void;
+	/** Only tests pass these; a real run opens the databases itself. */
+	db?: DatabaseSync;
+	stdin?: string;
+}
+
+/** Returns the exit code, and never calls `process.exit`, so a test can assert on it. */
+export function runSecretCli(argv: string[], options: SecretCliOptions = {}): number {
+	const paths = flypiPaths(options.root ?? process.cwd());
+	const say = options.out ?? ((line: string) => console.log(line));
+	const fail = options.err ?? ((line: string) => console.error(line));
+	const args = argv;
+	const command = args[0] ?? "list";
+	const reading = READS.includes(command);
+	if (!reading && !WRITES.includes(command)) {
+		fail(USAGE);
+		return 2;
+	}
+
+	let db: DatabaseSync;
+	try {
+		db = options.db ?? (reading ? openMemoryDbReadOnly(paths.memoryDbPath) : openMemoryDb(paths.memoryDbPath));
+	} catch (cause) {
+		fail(`flypi secret: ${(cause as Error).message}`);
+		return 1;
+	}
+	// A read command opened the database read-only, so it must know that: `print` still answers, it just
+	// cannot record the use while another process owns the writer.
+	const store = new SecretStore(db, keyfileProvider(paths.secretKeyPath), reading && options.db === undefined);
+	try {
+		const run = reading ? (a: string[]) => read(a, store, paths, say, fail) : (a: string[]) => write(a, store, paths, say, fail, options.stdin);
+		return run(args);
+	} catch (cause) {
+		// One place, so a bad name or a key the provider does not hold is a sentence, not a stack trace.
+		fail(`flypi secret: ${(cause as Error).message}`);
+		return 1;
+	} finally {
+		if (!options.db) db.close();
+		// A CLI run is short-lived, but the redaction source is process-wide: leave it where we found it.
+		closeSecretSource();
+	}
+}
+
+function read(args: string[], store: SecretStore, paths: FlypiPaths, say: (line: string) => void, fail: (line: string) => void): number {
+	const [command, ...rest] = args;
+	if (command === "print") {
+		const name = rest[0];
+		if (!name) {
+			fail("flypi secret print NAME");
+			return 2;
+		}
+		const value = store.value(name, { actor: "cli", action: "revealed" });
+		if (value === undefined) {
+			fail(`no secret named "${name}"`);
+			return 1;
+		}
+		// The value and nothing else, because a config file's API key is this process's stdout.
+		say(value);
+		return 0;
+	}
+	if (command === "get") {
+		const name = rest[0];
+		if (!name) {
+			fail("flypi secret get NAME");
+			return 2;
+		}
+		const row = store.row(name);
+		if (!row) {
+			fail(`no secret named "${name}"`);
+			return 1;
+		}
+		say(`name        ${row.name}`);
+		say(`kind        ${row.kind}`);
+		say(`mask        ${row.mask}`);
+		say(`key         v${row.key_version}`);
+		say(`created     ${row.created_at}${row.updated_at ? ` (updated ${row.updated_at})` : ""}`);
+		say(`used        ${row.use_count} time(s)${row.last_used_at ? `, last ${row.last_used_at}` : ""}`);
+		if (row.status === "deleted") say(`status      deleted (flypi secret undelete ${row.name} brings it back)`);
+		for (const [key, value] of Object.entries(row.meta)) say(`meta        ${key}=${value}`);
+		for (const use of store.uses(row.name)) say(`history     ${use.created_at} ${use.actor} ${use.action}${use.reason ? ` (${use.reason})` : ""}`);
+		return 0;
+	}
+	if (command === "key") {
+		const drift = store.keyDrift();
+		say(`key provider ${drift.keyProvider}, current v${drift.current}`);
+		say(`rows         ${drift.rows}, ${drift.stale} sealed under an older key`);
+		if (drift.stale > 0) say("             `flypi secret rotate-key` re-seals them");
+		return 0;
+	}
+	const rows = store.list(rest[0] ?? "", true);
+	if (rows.length === 0) {
+		say("flypi secret: the vault is empty. `flypi secret put NAME` stores one, or the web UI's Secrets tab.");
+		return 0;
+	}
+	say(`${"name".padEnd(22)} ${"kind".padEnd(16)} ${"mask".padEnd(28)} used  status`);
+	for (const row of rows) say(listLine(row));
+	return 0;
+}
+
+function write(args: string[], store: SecretStore, paths: FlypiPaths, say: (line: string) => void, fail: (line: string) => void, stdin?: string): number {
+	const [command, name, ...rest] = args;
+	if (!name && command !== "import-ssh" && command !== "rotate-key") {
+		fail(`flypi secret ${command} NAME`);
+		return 2;
+	}
+	const kind = flag(args, "--kind") as SecretKind | undefined;
+	if (command === "put" || command === "rotate") {
+		const value = valueFrom(rest, stdin, fail);
+		if (value === undefined) return 1;
+		if (command === "put") {
+			const result = store.put({ name: name!, value, kind, meta: metaFrom(args), actor: "cli" });
+			if (result.status === "exists") say(`flypi secret: already stored as "${result.name}"`);
+			else if (result.status === "name taken") say(`flypi secret: "${name}" is taken — \`flypi secret rotate ${name}\` replaces its value`);
+			else say(`flypi secret: stored ${name} (#${result.id})`);
+			return result.status === "stored" ? 0 : 1;
+		}
+		const result = store.rotate({ name: name!, value, kind, actor: "cli" });
+		say(result.status === "rotated" ? `flypi secret: rotated ${name}` : `flypi secret: no secret named "${name}"`);
+		return result.status === "rotated" ? 0 : 1;
+	}
+	if (command === "delete" || command === "undelete") {
+		const result = command === "delete" ? store.remove(name!, { actor: "cli" }) : store.restore(name!, { actor: "cli" });
+		const done = result.status === "deleted" || result.status === "restored";
+		say(done ? `flypi secret: ${result.status} ${name}${command === "delete" ? ` (\`flypi secret undelete ${name}\` brings it back)` : ""}` : `flypi secret: no secret named "${name}"`);
+		return done ? 0 : 1;
+	}
+	if (command === "path") {
+		// A path, for the tools that only take one: ssh, gpg. The plaintext goes in a 0400 file under
+		// .flypi/secrets/ — never the environment, which every child process would inherit (risk 13).
+		const value = store.value(name!, { actor: "cli", action: "used" });
+		if (value === undefined) {
+			fail(`no secret named "${name}"`);
+			return 1;
+		}
+		mkdirSync(paths.secretsDir, { recursive: true });
+		const file = secretFilePath(paths, name!);
+		writeFileSync(file, value, { mode: 0o400 });
+		chmodSync(file, 0o400);
+		say(file);
+		return 0;
+	}
+	if (command === "import-ssh") {
+		// The paths come after the subcommand, so the first one landed in `name` above.
+		const files = name ? [name, ...rest] : [];
+		if (files.length === 0) {
+			fail("flypi secret import-ssh ~/.ssh/id_ed25519.pub [more files…] — a public key is safe to store; a private one is your call");
+			return 2;
+		}
+		let stored = 0;
+		for (const target of files) {
+			const file = target.startsWith("~") ? join(homedir(), target.slice(1)) : target;
+			let text: string;
+			try {
+				text = readFileSync(file, "utf8");
+			} catch {
+				fail(`cannot read ${file}`);
+				continue;
+			}
+			// Named after the file, because that is the name a user would write in a config file, and the kind
+			// follows from the extension: `.pub` is public, anything else is a private key.
+			const base = basename(file);
+			const keyKind: SecretKind = base.endsWith(".pub") ? "ssh_public_key" : "ssh_private_key";
+			const result = store.put({ name: `ssh_${base}`, value: text.trim(), kind: keyKind, meta: { source: file }, actor: "cli" });
+			if (result.status === "stored") stored += 1;
+			say(result.status === "stored" ? `flypi secret: stored ssh_${base} (#${result.id}) as ${keyKind}` : `flypi secret: already stored as "${result.name}"`);
+		}
+		return stored > 0 ? 0 : 1;
+	}
+	const rotated = store.rotateKey("cli");
+	const reencrypted = store.reencrypt();
+	say(`flypi secret: new key v${rotated.version}; re-sealed ${reencrypted.rows} row(s)`);
+	return 0;
+}
+
+/** stdin when no value was typed on the command line — argv is visible in `ps`, a credential should not be. */
+function valueFrom(rest: string[], stdin: string | undefined, fail: (line: string) => void): string | undefined {
+	// Only a leading argument that is not a flag is the value. `put NAME --kind password` used to store the
+	// literal "--kind". A value that starts with a dash belongs on stdin, as the usage says.
+	const first = rest[0];
+	if (first !== undefined && !first.startsWith("-")) return first;
+	const piped = stdin?.trim();
+	if (piped) return piped;
+	fail('no value. Pass one as an argument, or pipe it: printf %s "$TOKEN" | flypi secret put NAME');
+	return undefined;
+}
+
+function metaFrom(args: string[]): Record<string, string> {
+	const meta: Record<string, string> = {};
+	for (const pair of valuesOf(args, "--meta")) {
+		const at = pair.indexOf("=");
+		if (at > 0) meta[pair.slice(0, at)] = pair.slice(at + 1);
+	}
+	return meta;
+}
+
+const flag = (args: string[], name: string): string | undefined => valuesOf(args, name)[0];
+
+function valuesOf(args: string[], name: string): string[] {
+	const out: string[] = [];
+	for (let i = 0; i < args.length; i += 1) {
+		const value = args[i + 1];
+		if (args[i] === name && value !== undefined) out.push(value);
+	}
+	return out;
+}
+
+function listLine(row: SecretRow): string {
+	return `${row.name.padEnd(22)} ${row.kind.padEnd(16)} ${row.mask.padEnd(28)} ${String(row.use_count).padEnd(5)} ${row.status}`;
+}

package/src/secrets/config.ts

@@ -0,0 +1,133 @@
+/**
+ * `secrets` in `.flypi/agent/settings.json` (docs/secrets.md Phase 5.5), and the reference syntax the
+ * config files use to point at a vault entry.
+ *
+ * Only the options this code acts on are here. `redact` is not one of them: redaction is the point of the
+ * feature, so it is always on. `shellEnv` is not either, for the opposite reason — a vault value never
+ * enters a process environment, because the environment is inherited by every child and lands in crash
+ * reports (risk 13). What is left is on/off, which key provider to use, and whether `secret_get` asks
+ * first.
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { FlypiPaths } from "../agent/paths.ts";
+import { keyfileProvider, type KeyProvider } from "./crypto.ts";
+
+export interface SecretsConfig {
+	enabled: boolean;
+	/** The seam, not a menu: `keyfile` is the only provider implemented, and a name it does not know is an error the user should see. */
+	keyProvider: string;
+	keyfile: string;
+	/** `secret_get` is held until the user says yes — the permission gate's mechanism, the vault's decision. */
+	askBeforeUse: boolean;
+}
+
+export function secretsConfig(paths: FlypiPaths): SecretsConfig {
+	let settings: unknown;
+	try {
+		settings = JSON.parse(readFileSync(paths.settingsPath, "utf8"));
+	} catch {
+		settings = {}; // no settings file yet: the defaults have to work
+	}
+	return parseSecretsConfig(settings, paths);
+}
+
+export function parseSecretsConfig(settings: unknown, paths: FlypiPaths): SecretsConfig {
+	const raw = (settings as { secrets?: Record<string, unknown> } | undefined)?.secrets;
+	return {
+		enabled: raw?.enabled !== false,
+		keyProvider: typeof raw?.keyProvider === "string" && raw.keyProvider.length > 0 ? raw.keyProvider : "keyfile",
+		keyfile: typeof raw?.keyfile === "string" && raw.keyfile.length > 0 ? raw.keyfile : paths.secretKeyPath,
+		askBeforeUse: raw?.askBeforeUse !== false,
+	};
+}
+
+/**
+ * The provider the settings name. "keyfile" is the only one, and saying so beats inventing a plugin point
+ * Phase 5.5 does not need: another provider (a macOS keychain, say) would appear here and in `keyProvider`
+ * above together, and nothing else.
+ */
+export function keyProviderFor(config: SecretsConfig): KeyProvider {
+	if (config.keyProvider !== "keyfile") {
+		throw new Error(`secrets.keyProvider is "${config.keyProvider}" and no such provider exists — the only one is "keyfile"`);
+	}
+	return keyfileProvider(config.keyfile);
+}
+
+/**
+ * pi's own reference syntax, used here rather than invented: a leading `!` in an `apiKey`, `headers` or `env`
+ * value is a command whose stdout becomes the value (docs/models.md, docs/mcp.md). flypi's half of that is a
+ * command that prints a vault value, so a config file can hold a reference and still be portable, shareable
+ * and free of secrets. Anything that ends in `secret print NAME` counts, because that is how the command looks
+ * both as `!flypi secret print NAME` and as `!node /path/to/flypi/scripts/flypi.mjs secret print NAME` —
+ * there is no `flypi` on PATH until Phase 7 packages one.
+ */
+export const SECRET_REF = /!\S.*\ssecret\s+print\s+([A-Za-z0-9][A-Za-z0-9._-]*)\s*$/;
+
+/** The vault name a config value points at, if it points at one. */
+export function secretRefName(value: unknown): string | undefined {
+	if (typeof value !== "string") return undefined;
+	return value.match(SECRET_REF)?.[1];
+}
+
+/** One config entry that points at a vault entry. This is the "used by" half of the UI (docs/secrets.md). */
+export interface SecretReference {
+	file: string;
+	/** Where in that file, so a user can fix the reference rather than guess: `mcpServers.github.headers.Authorization`. */
+	path: string;
+	name: string;
+}
+
+/**
+ * The references in the three files pi reads for credentials. Read defensively: a file that does not parse is
+ * reported by the settings panel already, and a references panel that throws would be worse than one that
+ * shows less.
+ */
+export function secretReferences(paths: FlypiPaths): SecretReference[] {
+	const found: SecretReference[] = [];
+	for (const [file, label] of [
+		[paths.authPath, "auth.json"],
+		[paths.mcpPath, "mcp.json"],
+		[paths.modelsPath, "models.json"],
+	] as const) {
+		let raw: unknown;
+		try {
+			raw = JSON.parse(readFileSync(file, "utf8"));
+		} catch {
+			continue;
+		}
+		walk(raw, label, (path, value) => {
+			const name = secretRefName(value);
+			if (name === undefined) return;
+			found.push({ file: label, path, name });
+		});
+	}
+	return found;
+}
+
+function walk(value: unknown, path: string, visit: (path: string, value: unknown) => void): void {
+	if (typeof value === "string") {
+		visit(path, value);
+		return;
+	}
+	if (Array.isArray(value)) {
+		value.forEach((item, index) => walk(item, `${path}[${index}]`, visit));
+		return;
+	}
+	if (typeof value === "object" && value !== null) {
+		for (const [key, item] of Object.entries(value)) walk(item, `${path}.${key}`, visit);
+	}
+}
+
+/**
+ * A vault name, and the name `secret_file` will use as a file. One rule for both, because a name that is not
+ * a safe file name cannot be written for ssh to read, and a name with a space in it cannot be written in a
+ * config file's `!flypi secret print NAME`.
+ */
+export const SECRET_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+/** Where `secret_file` puts a plaintext a command needs a path to. `mode` is the caller's, 0400. */
+export function secretFilePath(paths: FlypiPaths, name: string): string {
+	if (!SECRET_NAME.test(name)) throw new Error(`"${name}" is not a usable secret name: letters, digits, dot, dash and underscore`);
+	return join(paths.secretsDir, name);
+}