@danypops/pi-packed 0.1.0 → 0.1.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danypops/pi-packed",
-  "version": "0.1.0",
+  "version": "0.1.7",
   "description": "Package service for the Pi agent: search/info/install/updates + /packages TUI, backed by a long-running Bun service",
   "type": "module",
   "keywords": [

package/src/cache.ts

@@ -1,7 +1,9 @@
 /** TTLCache — the smart-proxy concern, nothing more. */
+import { CACHE_TTL_MS } from "./constants.ts";
+
 export class TTLCache {
 	private m = new Map<string, { body: string; expires: number }>();
-	constructor(private ttlMs = 5 * 60_000) {}
+	constructor(private ttlMs = CACHE_TTL_MS) {}
 
 	get(key: string): string | undefined {
 		const e = this.m.get(key);

package/src/catalog.ts

@@ -1,44 +1,45 @@
 /**
- * catalog.ts — event producer #2: bulk snapshot of every pi package.
- * npm's search API has no ETag/conditional requests (CDN max-age only),
- * so freshness is TTL-based: resync on serve start + interval.
+ * catalog.ts — the sync pipeline (apt update / pkg update analog).
+ * Paginates the upstream registry into the local SQLite mirror and records
+ * sync metadata (the Release/repomd checksum role) in sync_meta.
  */
-import { writeFile, readFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
 import type { Pkg, Registry } from "./ports.ts";
+import { PI_PACKAGE_KEYWORD } from "./constants.ts";
+import { openDb, replaceAll, getSyncMeta, dbPath } from "./db.ts";
+import { createLogger } from "./log.ts";
 
-export interface CatalogSnapshot {
-	fetchedAt: string;
-	packages: Pkg[];
-}
-
-function catalogPath(dir: string): string {
-	return join(dir, "catalog.json");
-}
+const log = createLogger("catalog");
+import type { SyncMeta } from "./db.ts";
 
-export async function saveCatalog(dir: string, snap: CatalogSnapshot): Promise<void> {
-	await mkdir(dir, { recursive: true });
-	await writeFile(catalogPath(dir), JSON.stringify(snap), { mode: 0o600 });
+export interface CatalogStatus {
+	stale: boolean;
+	meta?: SyncMeta;
 }
 
-export async function loadCatalog(dir: string): Promise<CatalogSnapshot | undefined> {
+export function catalogStatus(dir: string, ttlMs: number): CatalogStatus {
+	const db = openDb(dbPath(dir));
 	try {
-		return JSON.parse(await readFile(catalogPath(dir), "utf8")) as CatalogSnapshot;
-	} catch {
-		return undefined;
+		const meta = getSyncMeta(db);
+		if (!meta) return { stale: true };
+		return { stale: Date.now() - Date.parse(meta.fetchedAt) > ttlMs, meta };
+	} finally {
+		db.close();
 	}
 }
 
-export function catalogStale(snap: CatalogSnapshot | undefined, ttlMs: number): boolean {
-	if (!snap?.fetchedAt) return true;
-	return Date.now() - Date.parse(snap.fetchedAt) > ttlMs;
-}
-
-/** Full sync: paginate the whole keyword universe into a lean snapshot. */
-export async function syncCatalog(reg: Registry, dir: string, query = "keywords:pi-package"): Promise<number> {
+/** Full mirror sync: upstream pages → atomic SQLite replace. */
+export async function syncCatalog(reg: Registry, dir: string, query: string = PI_PACKAGE_KEYWORD): Promise<number> {
+	const t0 = Date.now();
+	log.info("mirror sync started", { query });
 	const packages = await reg.searchAll(query);
-	await saveCatalog(dir, { fetchedAt: new Date().toISOString(), packages });
-	return packages.length;
+	const db = openDb(dbPath(dir));
+	try {
+		replaceAll(db, packages, "npm:" + query);
+		log.info("mirror sync complete", { packages: packages.length, ms: Date.now() - t0 });
+		return packages.length;
+	} finally {
+		db.close();
+	}
 }
 
 export function startCatalogSync(
@@ -49,9 +50,9 @@ export function startCatalogSync(
 ): () => void {
 	async function sync(): Promise<void> {
 		try {
-			if (catalogStale(await loadCatalog(dir), ttlMs)) {
+			if (catalogStatus(dir, ttlMs).stale) {
 				const n = await syncCatalog(reg, dir);
-				console.error(`[packed] catalog synced: ${n} packages`);
+				log.info("scheduled sync complete", { packages: n });
 			}
 		} catch (e) {
 			onError?.(e);

package/src/cli.ts

@@ -7,20 +7,25 @@
 import { buildSearchQuery, clampLimit } from "./ports.ts";
 import type { Installer, Registry } from "./ports.ts";
 import { readInstalledPackages } from "./installed.ts";
-import { checkUpdates, loadUpdates } from "./watcher.ts";
-import { loadCatalog } from "./catalog.ts";
+import { checkUpdates } from "./watcher.ts";
+import { syncCatalog } from "./catalog.ts";
+import { openDb, searchLocal, catalogList, getSyncMeta, latestVersion, dbPath } from "./db.ts";
 import { NAME_RE } from "./install.ts";
+import {
+	VERSION, SEARCH_DEFAULT_LIMIT, SEARCH_MAX_LIMIT, NPM_REGISTRY_BASE, SEARCH_PAGE_SIZE, MIRROR_PAGE_DELAY_MS,
+} from "./constants.ts";
 
 const SOURCE_RE = /^(npm:[A-Za-z0-9@._/-]+|git:[A-Za-z0-9@:._/-]+|https:\/\/[A-Za-z0-9@:._/?=&%~-]+)$/;
 
 const USAGE = `packed — package service for the Pi agent
 
 usage:
-  packed search <query> [--limit N] [--json]   search pi packages on npm
+  packed search <query> [--offline] [--json]   search npm (or the local mirror with --offline)
   packed info <name> [--json]                  package details
-  packed updates [--cached] [--json]           available updates (cached = daemon snapshot)
+  packed updates [--json]                      updates per the local mirror
+  packed mirror [--json]                       sync upstream into the local SQLite index
   packed installed [--json]                    installed pi packages
-  packed catalog [--json]                      full pi-package catalog snapshot
+  packed catalog [--json]                      local package index (apt-cache stats)
   packed install <source>                      pi install npm:|git:|https://…
   packed remove <name>                         remove by bare npm name
   packed serve                                 run the long-running daemon
@@ -46,17 +51,19 @@ interface Flags {
 	json: boolean;
 	limit: number;
 	cached: boolean;
+	offline: boolean;
 }
 
 function parseFlags(rest: string[]): { flags: Flags; pos: string[] } {
-	const flags: Flags = { json: false, limit: 10, cached: false };
+	const flags: Flags = { json: false, limit: SEARCH_DEFAULT_LIMIT, cached: false, offline: false };
 	const pos: string[] = [];
 	for (let i = 0; i < rest.length; i++) {
 		const a = rest[i]!;
 		if (a === "--json") flags.json = true;
 		else if (a === "--cached") flags.cached = true;
-		else if (a === "--limit" && i + 1 < rest.length) flags.limit = Number(rest[++i]) || 10;
-		else if (a.startsWith("--limit=")) flags.limit = Number(a.slice(8)) || 10;
+		else if (a === "--offline") flags.offline = true;
+		else if (a === "--limit" && i + 1 < rest.length) flags.limit = Number(rest[++i]) || SEARCH_DEFAULT_LIMIT;
+		else if (a.startsWith("--limit=")) flags.limit = Number(a.slice(8)) || SEARCH_DEFAULT_LIMIT;
 		else pos.push(a);
 	}
 	return { flags, pos };
@@ -70,11 +77,26 @@ const usageErr = (out: string): CliResult => ({ code: 2, out });
 
 const commands: Record<string, { usage: string; run: Command }> = {
 	search: {
-		usage: "packed search <query> [--limit N] [--json]",
+		usage: "packed search <query> [--offline] [--limit N] [--json]",
 		async run(_rest, d, flags, pos) {
 			const q = pos[0];
 			if (!q) return usageErr(`usage: ${commands["search"]!.usage}\n`);
-			const { results, total } = await d.reg.search(buildSearchQuery(q), clampLimit(flags.limit, 10, 50));
+			const limit = clampLimit(flags.limit, SEARCH_DEFAULT_LIMIT, SEARCH_MAX_LIMIT);
+			// --offline: query the SQLite mirror only (apt-cache search analog)
+			if (flags.offline) {
+				const db = openDb(dbPath(d.stateDir));
+				try {
+					const results = searchLocal(db, q, limit);
+					if (flags.json) return ok(JSON.stringify({ query: q, total: results.length, results, offline: true }) + "\n");
+					if (results.length === 0) return ok(`no mirrored packages match "${q}" (run: packed mirror)\n`);
+					let out = `${results.length} mirrored package(s):\n\n`;
+					for (const p of results) out += `  ${p.name}@${p.version}\n    ${p.description ?? ""}\n`;
+					return ok(out);
+				} finally {
+					db.close();
+				}
+			}
+			const { results, total } = await d.reg.search(buildSearchQuery(q), limit);
 			if (flags.json) return ok(JSON.stringify({ query: q, total, results }) + "\n");
 			if (results.length === 0) return ok(`no pi packages found for "${q}"\n`);
 			let out = `${total} package(s) (showing ${results.length}):\n\n`;
@@ -83,6 +105,15 @@ const commands: Record<string, { usage: string; run: Command }> = {
 		},
 	},
 
+	mirror: {
+		usage: "packed mirror [--json]  (sync the upstream registry into the local SQLite index — the apt update analog)",
+		async run(_rest, d, flags) {
+			const n = await syncCatalog(d.reg, d.stateDir);
+			if (flags.json) return ok(JSON.stringify({ synced: n }) + "\n");
+			return ok(`mirrored ${n} packages into the local index\n`);
+		},
+	},
+
 	info: {
 		usage: "packed info <name> [--json]",
 		async run(_rest, d, flags, pos) {
@@ -98,18 +129,19 @@ const commands: Record<string, { usage: string; run: Command }> = {
 	},
 
 	updates: {
-		usage: "packed updates [--cached] [--json]",
+		usage: "packed updates [--json]  (from the local mirror — run `packed mirror` first)",
 		async run(_rest, d, flags) {
+			const db = openDb(dbPath(d.stateDir));
 			let updates;
-			if (flags.cached) {
-				updates = (await loadUpdates(d.stateDir))?.updates ?? [];
-			} else {
-				updates = await checkUpdates(d.reg, readInstalledPackages(d.piHome));
+			try {
+				updates = checkUpdates((name) => latestVersion(db, name), readInstalledPackages(d.piHome));
+			} finally {
+				db.close();
 			}
 			if (flags.json) {
 				return ok(JSON.stringify({ checkedAt: new Date().toISOString(), updates }) + "\n");
 			}
-			if (updates.length === 0) return ok("all pi packages up to date\n");
+			if (updates.length === 0) return ok("all pi packages up to date (per the local mirror)\n");
 			let out = `${updates.length} update(s) available:\n\n`;
 			for (const u of updates) out += `  ${u.name}  ${u.installed} → ${u.latest}\n`;
 			return ok(out + "\nrun: pi update --extensions\n");
@@ -128,12 +160,21 @@ const commands: Record<string, { usage: string; run: Command }> = {
 	catalog: {
 		usage: "packed catalog [--json]",
 		async run(_rest, d, flags) {
-			const snap = await loadCatalog(d.stateDir);
-			const packages = snap?.packages ?? [];
-			if (flags.json) return ok(JSON.stringify(snap ?? { packages: [] }) + "\n");
-			let out = `${packages.length} packages in catalog (fetched ${snap?.fetchedAt ?? "never"})\n\n`;
-			for (const p of packages.slice(0, 50)) out += `  ${p.name}@${p.version}\n`;
-			return ok(out);
+			const db = openDb(dbPath(d.stateDir));
+			try {
+				const meta = getSyncMeta(db);
+				const packages = catalogList(db);
+				if (flags.json) {
+					return ok(JSON.stringify({ fetchedAt: meta?.fetchedAt, sha256: meta?.sha256, packages }) + "\n");
+				}
+				let out = `${packages.length} packages in the local index`;
+				if (meta) out += ` (synced ${meta.fetchedAt}, sha256:${meta.sha256.slice(0, 12)}…)`;
+				out += "\n\n";
+				for (const p of packages.slice(0, 50)) out += `  ${p.name}@${p.version}\n`;
+				return ok(out);
+			} finally {
+				db.close();
+			}
 		},
 	},
 
@@ -175,7 +216,7 @@ const commands: Record<string, { usage: string; run: Command }> = {
 	version: {
 		usage: "packed version",
 		async run() {
-			return ok("0.1.0\n");
+			return ok(VERSION + "\n");
 		},
 	},
 };
@@ -226,7 +267,11 @@ if (import.meta.main) {
 		const { resolveRegistry } = await import("./client.ts");
 		const { ExecInstaller } = await import("./install.ts");
 		const dir = stateDir();
-		const reg = await resolveRegistry(dir, "https://registry.npmjs.org");
+		// mirror talks to UPSTREAM, not the daemon cache — apt update semantics.
+		const reg =
+			args[0] === "mirror"
+				? new (await import("./registry.ts")).HttpRegistry(NPM_REGISTRY_BASE, SEARCH_PAGE_SIZE, MIRROR_PAGE_DELAY_MS)
+				: await resolveRegistry(dir, NPM_REGISTRY_BASE);
 		const { code, out } = await cliRun(args, {
 			reg,
 			inst: new ExecInstaller(),

package/src/client.ts

@@ -7,6 +7,7 @@ import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import type { PkgInfo, Registry, SearchPage } from "./ports.ts";
 import { HttpRegistry } from "./registry.ts";
+import { DAEMON_HOST, PROBE_TIMEOUT_MS, REGISTRY_FETCH_TIMEOUT_MS, PORT_FILE, TOKEN_FILE } from "./constants.ts";
 
 export class DaemonRegistry implements Registry {
 	constructor(
@@ -17,7 +18,7 @@ export class DaemonRegistry implements Registry {
 	private async get<T>(path: string): Promise<T> {
 		const res = await fetch(`${this.base}${path}`, {
 			headers: { authorization: `Bearer ${this.token}` },
-			signal: AbortSignal.timeout(15_000),
+			signal: AbortSignal.timeout(REGISTRY_FETCH_TIMEOUT_MS),
 		});
 		if (!res.ok) throw new Error(`daemon HTTP ${res.status}`);
 		return (await res.json()) as T;
@@ -55,16 +56,16 @@ export async function probe(dir: string): Promise<DaemonHandle | undefined> {
 	let port: string;
 	let token: string;
 	try {
-		port = readFileSync(join(dir, "port"), "utf8").trim();
-		token = readFileSync(join(dir, "token"), "utf8").trim();
+		port = readFileSync(join(dir, PORT_FILE), "utf8").trim();
+		token = readFileSync(join(dir, TOKEN_FILE), "utf8").trim();
 	} catch {
 		return undefined;
 	}
-	const base = `http://127.0.0.1:${port}`;
+	const base = `http://${DAEMON_HOST}:${port}`;
 	try {
 		const res = await fetch(`${base}/health`, {
 			headers: { authorization: `Bearer ${token}` },
-			signal: AbortSignal.timeout(800),
+			signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
 		});
 		if (res.ok) return { base, token };
 	} catch {

package/src/constants.ts

@@ -0,0 +1,50 @@
+/**
+ * constants.ts — every magic value in one place, named.
+ * (lexicon: replace-magic-number-with-symbolic-constant)
+ */
+
+// --- Upstream ---
+export const NPM_REGISTRY_BASE = "https://registry.npmjs.org";
+
+// --- Search / pagination ---
+export const SEARCH_DEFAULT_LIMIT = 10;
+export const SEARCH_MAX_LIMIT = 50;
+export const SEARCH_PAGE_SIZE = 250; // npm registry max page size
+export const PI_PACKAGE_KEYWORD = "keywords:pi-package";
+
+// --- Upstream etiquette (429s) ---
+export const RETRY_MAX_ATTEMPTS = 6;
+export const RETRY_BASE_DELAY_MS = 2_000; // 2+4+8+16+32s spans npm's ~60s search window
+export const PAGE_DELAY_MS = 100; // politeness pause between catalog pages
+export const MIRROR_PAGE_DELAY_MS = 400; // manual full-sync: extra polite (burst limits)
+
+// --- Cache / fetch ---
+export const CACHE_TTL_MS = 5 * 60_000;
+export const PROBE_TIMEOUT_MS = 800;
+export const REGISTRY_FETCH_TIMEOUT_MS = 15_000;
+
+// --- Daemon ---
+export const DAEMON_HOST = "127.0.0.1";
+export const WATCH_INTERVAL_DEFAULT_MS = 30 * 60_000; // updates diff cadence
+export const CATALOG_INTERVAL_DEFAULT_MS = 6 * 3_600_000; // full mirror TTL
+export const IDLE_BUDGET_DEFAULT_MS = 10 * 60_000; // on-demand self-exit
+export const WATCHDOG_TICK_MS = 15_000;
+
+// --- Identity ---
+export const VERSION = "0.1.0";
+
+// --- State-dir file names ---
+export const TOKEN_FILE = "token";
+export const PORT_FILE = "port";
+export const UPDATES_FILE = "updates.json";
+export const DB_FILE = "packed.db";
+export const SETTINGS_FILE = "settings.json";
+
+// --- Environment knobs ---
+export const ENV = {
+	HOME: "PI_PACKED_HOME",
+	PI_HOME: "PI_PACKED_PI_HOME",
+	WATCH_SECS: "PI_PACKED_WATCH_SECS",
+	CATALOG_SECS: "PI_PACKED_CATALOG_SECS",
+	IDLE_SECS: "PI_PACKED_IDLE_SECS",
+} as const;

package/src/daemon.ts

@@ -9,6 +9,11 @@ import { ExecInstaller } from "./install.ts";
 import { loadOrCreateToken, writePort, idleExpired, envMs, stateDir } from "./state.ts";
 import { startWatcher } from "./watcher.ts";
 import { startCatalogSync } from "./catalog.ts";
+import { openDb, latestVersion, dbPath } from "./db.ts";
+import {
+	DAEMON_HOST, ENV, WATCH_INTERVAL_DEFAULT_MS, CATALOG_INTERVAL_DEFAULT_MS,
+	IDLE_BUDGET_DEFAULT_MS, WATCHDOG_TICK_MS,
+} from "./constants.ts";
 import { readInstalledPackages, defaultPiHome } from "./installed.ts";
 
 export function serveMain(): void {
@@ -20,7 +25,7 @@ export function serveMain(): void {
 	let lastActive = Date.now();
 	const server = Bun.serve({
 		port: 0,
-		hostname: "127.0.0.1",
+		hostname: DAEMON_HOST,
 		fetch: (req) => {
 			lastActive = Date.now();
 			return app.fetch(req);
@@ -29,14 +34,16 @@ export function serveMain(): void {
 	if (!server.port) throw new Error("failed to bind listener");
 	writePort(dir, server.port);
 
-	const stopWatcher = startWatcher(reg, dir, () => readInstalledPackages(defaultPiHome()), {
-		intervalMs: envMs("PI_PACKED_WATCH_SECS", 30 * 60_000),
+	// Event producer #1: drift detection from the local mirror (default 30min).
+	const watcherDb = openDb(dbPath(dir));
+	const stopWatcher = startWatcher((name) => latestVersion(watcherDb, name), dir, () => readInstalledPackages(defaultPiHome()), {
+		intervalMs: envMs(ENV.WATCH_SECS, WATCH_INTERVAL_DEFAULT_MS),
 	});
-	const stopCatalog = startCatalogSync(reg, dir, envMs("PI_PACKED_CATALOG_SECS", 6 * 3_600_000));
+	const stopCatalog = startCatalogSync(reg, dir, envMs(ENV.CATALOG_SECS, CATALOG_INTERVAL_DEFAULT_MS));
 
 	// Idle watchdog: for on-demand spawns. PI_PACKED_IDLE_SECS=0 disables it
 	// (systemd or another supervisor owns the lifecycle then).
-	const idleBudget = envMs("PI_PACKED_IDLE_SECS", 10 * 60_000);
+	const idleBudget = envMs(ENV.IDLE_SECS, IDLE_BUDGET_DEFAULT_MS);
 	let watchdog: ReturnType<typeof setInterval> | undefined;
 	if (idleBudget > 0) {
 		watchdog = setInterval(() => {
@@ -44,13 +51,14 @@ export function serveMain(): void {
 				console.error(`[packed] idle for ${idleBudget / 1000}s, exiting`);
 				shutdown();
 			}
-		}, 15_000);
+		}, WATCHDOG_TICK_MS);
 	}
 
 	function shutdown(): void {
 		if (watchdog) clearInterval(watchdog);
 		stopWatcher();
 		stopCatalog();
+		watcherDb.close();
 		server.stop(true);
 		process.exit(0);
 	}
@@ -58,6 +66,6 @@ export function serveMain(): void {
 	process.on("SIGINT", shutdown);
 
 	console.error(
-		`[packed] listening on 127.0.0.1:${server.port} (state ${dir}, watch ${envMs("PI_PACKED_WATCH_SECS", 30 * 60_000) / 1000}s, idle ${idleBudget / 1000}s)`,
+		`[packed] listening on ${DAEMON_HOST}:${server.port} (state ${dir}, watch ${envMs(ENV.WATCH_SECS, WATCH_INTERVAL_DEFAULT_MS) / 1000}s, idle ${idleBudget / 1000}s)`,
 	);
 }

package/src/db.ts

@@ -0,0 +1,124 @@
+/**
+ * db.ts — the local registry index (SQLite), the pkgng/YUM model.
+ * Master-index role: sync_meta (source, time, count, payload checksum —
+ * the APT Release / repomd.xml analog). Payload: packages + FTS5 mirror.
+ */
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import type { Pkg } from "./ports.ts";
+import { DB_FILE } from "./constants.ts";
+
+export interface SyncMeta {
+	source: string;
+	fetchedAt: string;
+	packageCount: number;
+	sha256: string;
+}
+
+export function dbPath(dir: string): string {
+	return join(dir, DB_FILE);
+}
+
+export function openDb(path: string): Database {
+	if (path !== ":memory:") mkdirSync(join(path, ".."), { recursive: true });
+	const db = new Database(path, { create: true });
+	db.exec("PRAGMA journal_mode = WAL");
+	db.exec(`CREATE TABLE IF NOT EXISTS packages (
+		name TEXT PRIMARY KEY,
+		version TEXT NOT NULL DEFAULT '',
+		description TEXT,
+		date TEXT
+	)`);
+	db.exec(`CREATE TABLE IF NOT EXISTS sync_meta (
+		id INTEGER PRIMARY KEY CHECK (id = 1),
+		source TEXT NOT NULL,
+		fetched_at TEXT NOT NULL,
+		package_count INTEGER NOT NULL,
+		sha256 TEXT NOT NULL
+	)`);
+	// FTS5 mirror; rebuilt wholesale on each sync.
+	db.exec(
+		`CREATE VIRTUAL TABLE IF NOT EXISTS packages_fts USING fts5(name, description)`,
+	);
+	return db;
+}
+
+/** Canonical checksum over the payload — the Release-file analog: cheap
+ * change detection between syncs and integrity for the local mirror. */
+function payloadHash(pkgs: Pkg[]): string {
+	const h = createHash("sha256");
+	for (const p of pkgs) h.update(`${p.name}@${p.version}\n`);
+	return h.digest("hex");
+}
+
+/** Atomic full-catalog replace (the mirror write side of `apt update`). */
+export function replaceAll(db: Database, pkgs: Pkg[], source: string): SyncMeta {
+	const meta: SyncMeta = {
+		source,
+		fetchedAt: new Date().toISOString(),
+		packageCount: pkgs.length,
+		sha256: payloadHash(pkgs),
+	};
+	// INSERT OR REPLACE: npm pagination is unstable — rankings shift between
+	// pages and a package can appear twice in one sync (rowid reuse is fine).
+	const insert = db.prepare("INSERT OR REPLACE INTO packages (name, version, description, date) VALUES (?, ?, ?, ?)");
+	const insertFts = db.prepare("INSERT INTO packages_fts (rowid, name, description) VALUES (?, ?, ?)");
+	db.transaction(() => {
+		db.exec("DELETE FROM packages");
+		db.exec("DELETE FROM packages_fts");
+		for (const p of pkgs) {
+			const { lastInsertRowid } = insert.run(p.name, p.version, p.description ?? null, p.date ?? null);
+			insertFts.run(Number(lastInsertRowid), p.name, p.description ?? "");
+		}
+		db.prepare(
+			"INSERT INTO sync_meta (id, source, fetched_at, package_count, sha256) VALUES (1, ?, ?, ?, ?) " +
+				"ON CONFLICT(id) DO UPDATE SET source=excluded.source, fetched_at=excluded.fetched_at, " +
+				"package_count=excluded.package_count, sha256=excluded.sha256",
+		).run(meta.source, meta.fetchedAt, meta.packageCount, meta.sha256);
+	})();
+	return meta;
+}
+
+export function getSyncMeta(db: Database): SyncMeta | undefined {
+	return db
+		.query("SELECT source, fetched_at AS fetchedAt, package_count AS packageCount, sha256 FROM sync_meta WHERE id = 1")
+		.get() as SyncMeta | undefined;
+}
+
+export function catalogList(db: Database, limit = 0, offset = 0): Pkg[] {
+	const sql =
+		"SELECT name, version, description, date FROM packages ORDER BY name" +
+		(limit > 0 ? ` LIMIT ${Math.floor(limit)} OFFSET ${Math.floor(offset)}` : "");
+	return db.query(sql).all() as Pkg[];
+}
+
+/** FTS5 over the mirror (the `apt-cache search` analog). Sanitizes user
+ * input into AND-joined quoted terms; falls back to LIKE for hostile input. */
+export function searchLocal(db: Database, q: string, limit = 50): Pkg[] {
+	const terms = q.trim().split(/\s+/).filter(Boolean);
+	if (terms.length === 0) return [];
+	try {
+		const match = terms.map((t) => `"${t.replaceAll('"', '""')}"*`).join(" ");
+		return db
+			.query(
+				`SELECT p.name, p.version, p.description, p.date
+				 FROM packages_fts f JOIN packages p ON p.rowid = f.rowid
+				 WHERE packages_fts MATCH ? ORDER BY rank LIMIT ?`,
+			)
+			.all(match, limit) as Pkg[];
+	} catch {
+		// FTS syntax hostility → substring fallback
+		const like = `%${q}%`;
+		return db
+			.query("SELECT name, version, description, date FROM packages WHERE name LIKE ? OR description LIKE ? ORDER BY name LIMIT ?")
+			.all(like, like, limit) as Pkg[];
+	}
+}
+
+/** Latest mirrored version of one package (watcher's lookup). */
+export function latestVersion(db: Database, name: string): string | undefined {
+	const row = db.query("SELECT version FROM packages WHERE name = ?").get(name) as { version: string } | null;
+	return row?.version;
+}

package/src/installed.ts

@@ -3,6 +3,7 @@
 import { readFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+import { ENV, SETTINGS_FILE } from "./constants.ts";
 import type { InstalledPkg } from "./ports.ts";
 
 export function splitNpmSource(spec: string): [name: string, version: string] {
@@ -32,7 +33,7 @@ function nodeModulesVersion(piHome: string, name: string): string | undefined {
 export function readInstalledPackages(piHome: string): InstalledPkg[] {
 	let settings: { packages?: unknown[] };
 	try {
-		settings = JSON.parse(readFileSync(join(piHome, "settings.json"), "utf8"));
+		settings = JSON.parse(readFileSync(join(piHome, SETTINGS_FILE), "utf8"));
 	} catch {
 		return [];
 	}
@@ -51,6 +52,7 @@ export function readInstalledPackages(piHome: string): InstalledPkg[] {
 }
 
 export function defaultPiHome(): string {
-	if (process.env["PI_PACKED_PI_HOME"]) return process.env["PI_PACKED_PI_HOME"];
+	const envHome = process.env[ENV.PI_HOME];
+	if (envHome) return envHome;
 	return join(homedir(), ".pi", "agent");
 }

package/src/log.ts

@@ -0,0 +1,42 @@
+/**
+ * log.ts — structured logging for the service. JSON lines to stderr
+ * (stdout belongs to CLI output; the TUI owns the terminal UI).
+ * Level via PI_PACKED_LOG_LEVEL (debug|info|warn|error, default info).
+ * Sinks are injectable so tests can capture without a terminal.
+ */
+
+export type LogLevel = "debug" | "info" | "warn" | "error";
+export type LogSink = (line: string) => void;
+
+const ORDER: Record<LogLevel, number> = { debug: 10, info: 20, warn: 30, error: 40 };
+const LEVELS = new Set(Object.keys(ORDER));
+
+export interface LogFields {
+	[key: string]: unknown;
+}
+
+export interface Logger {
+	debug(msg: string, fields?: LogFields): void;
+	info(msg: string, fields?: LogFields): void;
+	warn(msg: string, fields?: LogFields): void;
+	error(msg: string, fields?: LogFields): void;
+}
+
+function envLevel(): LogLevel {
+	const raw = process.env["PI_PACKED_LOG_LEVEL"] ?? "info";
+	return (LEVELS.has(raw) ? raw : "info") as LogLevel;
+}
+
+export function createLogger(module: string, sink: LogSink = (l) => console.error(l), minLevel?: LogLevel): Logger {
+	const threshold = ORDER[minLevel ?? envLevel()];
+	function emit(level: LogLevel, msg: string, fields?: LogFields): void {
+		if (ORDER[level] < threshold) return;
+		sink(JSON.stringify({ ts: new Date().toISOString(), level, module, msg, ...fields }));
+	}
+	return {
+		debug: (m, f) => emit("debug", m, f),
+		info: (m, f) => emit("info", m, f),
+		warn: (m, f) => emit("warn", m, f),
+		error: (m, f) => emit("error", m, f),
+	};
+}

package/src/ports.ts

@@ -62,10 +62,12 @@ export interface UpdatesSnapshot {
 }
 
 /** Scope every query to pi packages unless the caller already qualified it. */
+import { PI_PACKAGE_KEYWORD } from "./constants.ts";
+
 export function buildSearchQuery(q: string): string {
 	const t = q.trim();
 	if (t.includes("keywords:")) return t;
-	return t === "" ? "keywords:pi-package" : `keywords:pi-package ${t}`;
+	return t === "" ? PI_PACKAGE_KEYWORD : `${PI_PACKAGE_KEYWORD} ${t}`;
 }
 
 export function clampLimit(v: number, def: number, max: number): number {

package/src/registry.ts

@@ -3,11 +3,50 @@
  * Lean mapping = Facade over npm's verbose package documents.
  */
 import type { Pkg, PkgInfo, Registry, SearchPage } from "./ports.ts";
+import {
+	NPM_REGISTRY_BASE, SEARCH_PAGE_SIZE, RETRY_MAX_ATTEMPTS, RETRY_BASE_DELAY_MS, PAGE_DELAY_MS,
+} from "./constants.ts";
+import { createLogger } from "./log.ts";
+
+const log = createLogger("registry");
+
+/** Upstream etiquette: honor Retry-After on 429, exponential backoff
+ * otherwise, give up after RETRY_MAX_ATTEMPTS. */
+async function fetchWithRetry(url: string, init: RequestInit | undefined, baseDelayMs: number): Promise<Response> {
+	let lastErr: Error | undefined;
+	for (let attempt = 1; attempt <= RETRY_MAX_ATTEMPTS; attempt++) {
+		const t0 = Date.now();
+		try {
+			const res = await fetch(url, init);
+			const ms = Date.now() - t0;
+			if (res.status === 429 && attempt < RETRY_MAX_ATTEMPTS) {
+				const ra = res.headers.get("retry-after");
+				const delayMs =
+					ra !== null && Number(ra) > 0
+						? Number(ra) * 1000 // authoritative only when positive
+						: baseDelayMs * 2 ** (attempt - 1); // npm sends 0: use exponential
+				log.warn("429 rate-limited, backing off", { attempt, delayMs, ms, url: url.slice(0, 120) });
+				await Bun.sleep(delayMs);
+				continue;
+			}
+			log.debug("fetch", { status: res.status, ms, attempt, url: url.slice(0, 120) });
+			return res;
+		} catch (e) {
+			lastErr = e instanceof Error ? e : new Error(String(e));
+			log.warn("fetch error, retrying", { attempt, error: lastErr.message, url: url.slice(0, 120) });
+			if (attempt < RETRY_MAX_ATTEMPTS) await Bun.sleep(baseDelayMs * 2 ** (attempt - 1));
+		}
+	}
+	log.error("retry budget exhausted", { attempts: RETRY_MAX_ATTEMPTS, url: url.slice(0, 120) });
+	throw lastErr ?? new Error("retry budget exhausted");
+}
 
 export class HttpRegistry implements Registry {
 	constructor(
-		private base = "https://registry.npmjs.org",
-		private pageSize = 250,
+		private base = NPM_REGISTRY_BASE,
+		private pageSize = SEARCH_PAGE_SIZE,
+		private pageDelayMs = PAGE_DELAY_MS,
+		private retryBaseDelayMs = RETRY_BASE_DELAY_MS,
 	) {}
 
 	async search(query: string, limit: number): Promise<SearchPage> {
@@ -16,7 +55,7 @@ export class HttpRegistry implements Registry {
 
 	async searchPage(query: string, from: number, size: number): Promise<SearchPage> {
 		const params = new URLSearchParams({ text: query, size: String(size), from: String(from) });
-		const res = await fetch(`${this.base}/-/v1/search?${params}`);
+		const res = await fetchWithRetry(`${this.base}/-/v1/search?${params}`, undefined, this.retryBaseDelayMs);
 		if (!res.ok) throw new Error(`npm search: HTTP ${res.status}`);
 		const doc = (await res.json()) as {
 			total?: number;
@@ -32,20 +71,28 @@ export class HttpRegistry implements Registry {
 	}
 
 	async searchAll(query: string): Promise<Pkg[]> {
-		const out: Pkg[] = [];
+		// Map by name: npm's ranking shifts mid-pagination and a package can
+		// appear on two pages — first occurrence wins.
+		const byName = new Map<string, Pkg>();
 		let from = 0;
 		for (;;) {
+			if (from > 0 && this.pageDelayMs > 0) await Bun.sleep(this.pageDelayMs);
 			const { results, total } = await this.searchPage(query, from, this.pageSize);
-			out.push(...results);
+			if (results.length === 0 || from >= total) break;
+			for (const p of results) {
+				if (!byName.has(p.name)) byName.set(p.name, p);
+			}
 			from += results.length;
-			if (results.length === 0 || from >= total) return out;
 		}
+		return [...byName.values()];
 	}
 
 	async info(name: string): Promise<PkgInfo> {
-		const res = await fetch(`${this.base}/${encodeURIComponent(name).replace("%2F", "/")}`, {
-			headers: { accept: "application/vnd.npm.install-v1+json" },
-		});
+		const res = await fetchWithRetry(
+			`${this.base}/${encodeURIComponent(name).replace("%2F", "/")}`,
+			{ headers: { accept: "application/vnd.npm.install-v1+json" } },
+			this.retryBaseDelayMs,
+		);
 		if (!res.ok) throw new Error(`npm info ${name}: HTTP ${res.status}`);
 		const doc = (await res.json()) as {
 			name?: string;

package/src/service.ts

@@ -7,7 +7,11 @@ import { buildSearchQuery, clampLimit } from "./ports.ts";
 import type { Installer, Registry } from "./ports.ts";
 import { TTLCache } from "./cache.ts";
 import { loadUpdates } from "./watcher.ts";
-import { loadCatalog } from "./catalog.ts";
+import { openDb, searchLocal, catalogList, getSyncMeta, dbPath } from "./db.ts";
+import { VERSION, SEARCH_DEFAULT_LIMIT, SEARCH_MAX_LIMIT } from "./constants.ts";
+import { createLogger } from "./log.ts";
+
+const log = createLogger("service");
 
 export interface Deps {
 	reg: Registry;
@@ -35,12 +39,22 @@ export function createApp(deps: Deps): { fetch: (req: Request) => Promise<Respon
 		const path = url.pathname;
 
 		if (path === "/health" && req.method === "GET") {
-			return json({ ok: true, version: "0.1.0" });
+			return json({ ok: true, version: VERSION });
 		}
 
 		if (path === "/search" && req.method === "GET") {
 			const q = url.searchParams.get("q") ?? "";
-			const limit = clampLimit(Number(url.searchParams.get("limit")), 10, 50);
+			const limit = clampLimit(Number(url.searchParams.get("limit")), SEARCH_DEFAULT_LIMIT, SEARCH_MAX_LIMIT);
+			// offline=1: serve from the SQLite mirror (apt-cache search analog)
+			if (url.searchParams.get("offline") === "1") {
+				const db = openDb(dbPath(deps.stateDir));
+				try {
+					const results = searchLocal(db, q, limit);
+					return json({ query: q, total: results.length, results, offline: true });
+				} finally {
+					db.close();
+				}
+			}
 			try {
 				const { results, total } = await deps.reg.search(buildSearchQuery(q), limit);
 				return json({ query: q, total, results });
@@ -84,8 +98,13 @@ export function createApp(deps: Deps): { fetch: (req: Request) => Promise<Respon
 		}
 
 		if (path === "/catalog" && req.method === "GET") {
-			const snap = await loadCatalog(deps.stateDir);
-			return json(snap ?? { packages: [] });
+			const db = openDb(dbPath(deps.stateDir));
+			try {
+				const meta = getSyncMeta(db);
+				return json({ fetchedAt: meta?.fetchedAt, sha256: meta?.sha256, packages: catalogList(db) });
+			} finally {
+				db.close();
+			}
 		}
 
 		return err(404, "not found");
@@ -93,6 +112,7 @@ export function createApp(deps: Deps): { fetch: (req: Request) => Promise<Respon
 
 	return {
 		async fetch(req: Request): Promise<Response> {
+			const t0 = Date.now();
 			if (req.headers.get("authorization") !== `Bearer ${deps.token}`) {
 				return err(401, "missing or invalid bearer token");
 			}
@@ -100,13 +120,17 @@ export function createApp(deps: Deps): { fetch: (req: Request) => Promise<Respon
 			if (req.method === "GET" && !["/health", "/updates", "/catalog"].includes(new URL(req.url).pathname)) {
 				const hit = cache.get(req.url);
 				if (hit) {
+					log.debug("request", { path: new URL(req.url).pathname, cache: "hit", ms: Date.now() - t0 });
 					return new Response(hit, { headers: { "content-type": "application/json", "x-cache": "hit" } });
 				}
 				const res = await route(req);
 				if (res.status === 200) cache.set(req.url, await res.clone().text());
+				log.debug("request", { path: new URL(req.url).pathname, status: res.status, cache: "miss", ms: Date.now() - t0 });
 				return res;
 			}
-			return route(req);
+			const res = await route(req);
+			log.debug("request", { path: new URL(req.url).pathname, status: res.status, ms: Date.now() - t0 });
+			return res;
 		},
 	};
 }

package/src/state.ts

@@ -2,10 +2,12 @@
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import { join } from "node:path";
+import { ENV, TOKEN_FILE, PORT_FILE } from "./constants.ts";
 import { randomBytes } from "node:crypto";
 
 export function stateDir(): string {
-	if (process.env["PI_PACKED_HOME"]) return process.env["PI_PACKED_HOME"];
+	const envHome = process.env[ENV.HOME];
+	if (envHome) return envHome;
 	try {
 		return join(homedir(), ".cache", "pi-packed");
 	} catch {
@@ -14,7 +16,7 @@ export function stateDir(): string {
 }
 
 export function loadOrCreateToken(dir: string): string {
-	const path = join(dir, "token");
+	const path = join(dir, TOKEN_FILE);
 	try {
 		const tok = readFileSync(path, "utf8").trim();
 		if (tok) return tok;
@@ -29,7 +31,7 @@ export function loadOrCreateToken(dir: string): string {
 
 export function writePort(dir: string, port: number): void {
 	mkdirSync(dir, { recursive: true });
-	writeFileSync(join(dir, "port"), String(port) + "\n", { mode: 0o600 });
+	writeFileSync(join(dir, PORT_FILE), String(port) + "\n", { mode: 0o600 });
 }
 
 export function idleExpired(lastActiveMs: number, nowMs: number, budgetMs: number): boolean {

package/src/watcher.ts

@@ -5,10 +5,14 @@
  */
 import { writeFile, readFile, mkdir } from "node:fs/promises";
 import { join } from "node:path";
-import type { InstalledPkg, Registry, UpdateEntry, UpdatesSnapshot } from "./ports.ts";
+import { UPDATES_FILE } from "./constants.ts";
+import { createLogger } from "./log.ts";
+
+const log = createLogger("watcher");
+import type { InstalledPkg, UpdateEntry, UpdatesSnapshot } from "./ports.ts";
 
 function updatesPath(dir: string): string {
-	return join(dir, "updates.json");
+	return join(dir, UPDATES_FILE);
 }
 
 export async function saveUpdates(dir: string, snap: UpdatesSnapshot): Promise<void> {
@@ -24,20 +28,18 @@ export async function loadUpdates(dir: string): Promise<UpdatesSnapshot | undefi
 	}
 }
 
-/** Pure diff: drift = latest ≠ what we have (registry latest is authoritative). */
-export async function checkUpdates(reg: Registry, installed: InstalledPkg[]): Promise<UpdateEntry[]> {
+/** Pure diff against the local mirror (apt list --upgradable semantics):
+ * drift = mirrored latest ≠ what we have. The mirror is refreshed by
+ * catalogSync; updates are computed offline, exactly like APT. */
+export function checkUpdates(latestOf: (name: string) => string | undefined, installed: InstalledPkg[]): UpdateEntry[] {
 	const now = new Date().toISOString();
 	const updates: UpdateEntry[] = [];
 	for (const p of installed) {
 		const have = p.installed || p.pinned;
 		if (!have) continue;
-		try {
-			const info = await reg.info(p.name);
-			if (info.version && info.version !== have) {
-				updates.push({ name: p.name, installed: have, latest: info.version, detectedAt: now });
-			}
-		} catch (e) {
-			console.error(`updates: ${p.name}: ${e instanceof Error ? e.message : e}`);
+		const latest = latestOf(p.name);
+		if (latest && latest !== have) {
+			updates.push({ name: p.name, installed: have, latest, detectedAt: now });
 		}
 	}
 	return updates;
@@ -50,16 +52,16 @@ export interface WatcherOptions {
 
 /** Producer loop: immediate check, then on a timer. Returns a stop function. */
 export function startWatcher(
-	reg: Registry,
+	latestOf: (name: string) => string | undefined,
 	stateDir: string,
 	readInstalled: () => InstalledPkg[],
 	opts: WatcherOptions,
 ): () => void {
 	async function check(): Promise<void> {
 		try {
-			const updates = await checkUpdates(reg, readInstalled());
+			const updates = checkUpdates(latestOf, readInstalled());
 			await saveUpdates(stateDir, { checkedAt: new Date().toISOString(), updates });
-			if (updates.length > 0) console.error(`[packed] updates available: ${updates.length}`);
+			log.info("updates check", { updates: updates.length });
 		} catch (e) {
 			opts.onError?.(e);
 		}