@broberg/lens-daemon 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,15 @@
-# Temporary Holding Version
+# @broberg/lens-daemon
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The Lens module of the cardmem daemon as a package (lens F001.5). TypeScript source, Bun runtime.
+
+```ts
+import { ensureLensSchema, createLensRouter, createLensMcpRouter, LENS_DAEMON_VERSION } from '@broberg/lens-daemon';
+
+ensureLensSchema(db);                                  // inside getDb()'s schema transaction
+app.route('/lens', createLensRouter(lensDeps));        // every /lens/* route
+app.route('/mcp', createLensMcpRouter(lensDeps));
+```
+
+`lensDeps.resolveSandboxedPath` must be the daemon's own sandbox check. The package holds no copy of it, and every path call refuses until it is set.
+
+Source: github.com/broberg-ai/lens `packages/daemon`.

package/package.json

@@ -1,6 +1,55 @@
 {
   "name": "@broberg/lens-daemon",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0",
+  "description": "The Lens module of the cardmem daemon (flows, critic, coverage, capture sessions, the /lens/* routes and the cardmem-lens MCP) as a package. The daemon mounts createLensRouter(deps) + createLensMcpRouter(deps). Bun runtime.",
+  "type": "module",
+  "license": "MIT",
+  "dependencies": {
+    "@broberg/ai-sdk": "0.56.0",
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "@mozilla/readability": "^0.6.0",
+    "hono": "^4.6.14",
+    "pixelmatch": "^6.0.0",
+    "playwright": "1.61.1",
+    "pngjs": "^7.0.0",
+    "sharp": "^0.33.5",
+    "turndown": "^7.2.4",
+    "zod": "^3.23.8",
+    "@broberg/lens-engine": "0.10.2",
+    "@broberg/lens-svg": "0.1.0"
+  },
+  "devDependencies": {
+    "@happy-dom/global-registrator": "^20.10.6",
+    "@types/bun": "^1.1.14",
+    "@types/pngjs": "^6.0.5",
+    "@types/turndown": "^5.0.6",
+    "typescript": "^5.6.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/broberg-ai/lens",
+    "directory": "packages/daemon"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/*.selftest.ts",
+    "!src/test-support",
+    "!src/fixtures",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "bun": ">=1.1"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "bun test"
+  }
 }

package/src/artifact-backfill.ts

@@ -0,0 +1,253 @@
+// F276.4 — put the whole history in the cloud. Christian, 2026-08-25, asked without
+// hedging: "jeg vil have ALT fra Lens historik i skyen."
+//
+// MEASURED before building, so the job is scoped to what exists rather than to a guess:
+// 6588 files, 982 MB, median 88 KB, largest 7.1 MB, 6585 PNG + 3 webm. Not one file
+// exceeds the 25 MB upload ceiling, so nothing is refused on size — which makes this a
+// throughput problem, not a selection problem.
+//
+// IT WALKS THE DISK, NOT THE DATABASE, and that is the load-bearing choice. The tables
+// know which artifacts they BELIEVE exist; the disk knows which ones are actually there.
+// Those differ in both directions — a row can point at bytes that were deleted, and bytes
+// can sit in the directory with no row left pointing at them. Walking the rows would
+// silently skip the second kind and report a complete backfill.
+
+import { readdir, stat } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
+import type { LensDeps } from './service.ts';
+import { enqueueArtifact, uploadStats } from './artifact-uploads.ts';
+
+export interface SkippedDir {
+  /** The directory as it exists on disk. */
+  dir: string;
+  /** Why it is not going to the cloud. A skip must be able to say its reason out
+   *  loud; a skip with no reason is indistinguishable from a lookup that failed. */
+  reason: string;
+  /** How many files it holds, so the size of what is NOT uploaded is visible. */
+  files: number;
+}
+
+export interface BackfillResult {
+  /** Files found on disk. */
+  scanned: number;
+  /** Newly queued by this pass. */
+  enqueued: number;
+  /** Already known to the queue — the re-run case, and the proof it is idempotent. */
+  already_queued: number;
+  /** Directories held back ON PURPOSE, each with its reason and file count. Separate
+   *  from `unattributable` because they are different facts: one is a decision, the
+   *  other is a defect, and collapsing them is exactly how 967 files went missing. */
+  skipped: SkippedDir[];
+  /** Files whose path does not yield a project slug. Reported, never skipped silently:
+   *  an unexplained gap between `scanned` and `enqueued` reads as a bug in the count. */
+  unattributable: string[];
+  /** The queue AFTER this pass. */
+  queue: ReturnType<typeof uploadStats>;
+  /** Set when the local storage backend is not a directory we can walk. */
+  note: string | null;
+}
+
+/**
+ * Directories whose name is NOT the project slug cardmem knows it by.
+ *
+ * MEASURED, not guessed (2026-08-25): seven of the 32 directories on disk matched no
+ * project. Three were renames the artifact tree never heard about, and their 600 files
+ * would have failed five times each and then been abandoned — silently, because the
+ * only thing a caller sees is a `failed` count with no names in it.
+ *
+ * Christian confirmed each mapping himself; `torrent-search-api` is not a guess from
+ * the repo name, it is his answer: "Hedder moovyy nu."
+ */
+export const DIR_ALIASES: Record<string, string> = {
+  xrt81: 'xrt81-com',
+  'broberg-ai': 'broberg-ai-site',
+  'torrent-search-api': 'moovyy',
+};
+
+/**
+ * Directories that are deliberately NOT uploaded, each with the reason it is held back.
+ *
+ * These are held back BY DECISION. Today they also happen to fail the project lookup,
+ * which means they are currently safe by accident — and an accident is not a policy.
+ * Naming them here makes the hold-back survive someone later fixing the lookup.
+ *
+ * `capture-sessions` is Christian's call, given 2026-08-25 when asked directly whether
+ * 329 screenshots of him signing in to App Store Connect and Google Play should go to
+ * the cloud: "Nej." No credentials are visible in them (all 329 verified as PNG), and
+ * they stay on the machine anyway.
+ */
+export const SKIP_DIRS: Record<string, string> = {
+  'capture-sessions': "owner decision 2026-08-25: login recordings stay on the Mac",
+  _external: 'screenshots of third-party sites — not this fleet\'s project history',
+  'lens-statepath-test': 'one-off engine test run, not project history',
+  'lens-mint-test': 'one-off engine test run, not project history',
+};
+
+export type Attribution =
+  | { kind: 'project'; slug: string }
+  | { kind: 'skipped'; dir: string; reason: string }
+  | { kind: 'unattributable' };
+
+/**
+ * Artifact keys are `<dir>/<kind>/…`. The first segment is USUALLY the project slug —
+ * but "usually" is the whole bug, so this returns which of three things happened rather
+ * than a slug-or-null that flattens two of them into one.
+ */
+export function attribute(relPath: string): Attribution {
+  const first = relPath.split(sep)[0];
+  if (!first || first === relPath) return { kind: 'unattributable' };
+  const reason = SKIP_DIRS[first];
+  if (reason) return { kind: 'skipped', dir: first, reason };
+  return { kind: 'project', slug: DIR_ALIASES[first] ?? first };
+}
+
+async function walk(dir: string, out: string[]): Promise<void> {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const e of entries) {
+    const p = join(dir, e.name);
+    if (e.isDirectory()) await walk(p, out);
+    else if (e.isFile()) out.push(p);
+  }
+}
+
+export async function backfillFromDisk(deps: LensDeps & { artifactRoot?: string }): Promise<BackfillResult> {
+  const root = deps.artifactRoot ?? defaultRoot();
+  if (deps.storage.backend !== 'local') {
+    // Not a failure — tigris artifacts are already durable — but it must SAY so rather
+    // than return zeros, which would read identically to "there was nothing to do".
+    return {
+      scanned: 0,
+      enqueued: 0,
+      already_queued: 0,
+      skipped: [],
+      unattributable: [],
+      queue: uploadStats(deps.db),
+      note: `storage backend is '${deps.storage.backend}', not a local directory — nothing to walk`,
+    };
+  }
+
+  const files: string[] = [];
+  await walk(root, files);
+
+  let enqueued = 0;
+  let already = 0;
+  const unattributable: string[] = [];
+  const skipped = new Map<string, SkippedDir>();
+
+  for (const abs of files) {
+    const rel = relative(root, abs);
+    const a = attribute(rel);
+    if (a.kind === 'unattributable') {
+      unattributable.push(rel);
+      continue;
+    }
+    if (a.kind === 'skipped') {
+      const seen = skipped.get(a.dir);
+      if (seen) seen.files++;
+      else skipped.set(a.dir, { dir: a.dir, reason: a.reason, files: 1 });
+      continue;
+    }
+    // POSIX-style key, matching how storage.put wrote it.
+    const key = rel.split(sep).join('/');
+    const before = countKey(deps.db, key);
+    enqueueArtifact(deps.db, a.slug, key);
+    if (before) already++;
+    else enqueued++;
+  }
+
+  // A row already in the queue under a dead directory name keeps failing forever
+  // unless it is repointed — the alias fixes NEW rows, this fixes the ones the old
+  // mapping already wrote. Also clears their failed state so the drain retries them.
+  repointAliased(deps.db);
+  applySkipPolicy(deps.db);
+
+  return {
+    scanned: files.length,
+    enqueued,
+    already_queued: already,
+    skipped: [...skipped.values()].sort((x, y) => y.files - x.files),
+    unattributable,
+    queue: uploadStats(deps.db),
+    note: null,
+  };
+}
+
+/**
+ * Move queue rows written under a pre-rename directory onto the current slug, and
+ * un-fail them.
+ *
+ * Without this the alias only helps files enqueued from now on: the 27 rows already
+ * sitting at `failed` with "project xrt81 is not known to cardmem" would stay failed,
+ * and a re-run would report `already_queued` for them — a count that reads like
+ * success and means the opposite.
+ */
+export function repointAliased(db: LensDeps['db']): number {
+  let moved = 0;
+  for (const [from, to] of Object.entries(DIR_ALIASES)) {
+    const r = db
+      .prepare(
+        `UPDATE lens_artifact_uploads
+            SET project_slug = ?, status = 'pending', attempts = 0, last_error = NULL
+          WHERE project_slug = ? AND status != 'done'`,
+      )
+      .run(to, from);
+    moved += Number(r.changes ?? 0);
+  }
+  return moved;
+}
+
+/**
+ * Park the rows belonging to a held-back directory in their OWN state.
+ *
+ * 6588 rows were already queued before the skip list existed, 365 of them from
+ * directories the owner has since said should stay on the Mac. Left as `pending` they
+ * would be retried five times each and land in `failed` — where they would be
+ * indistinguishable from a genuine upload failure, and would keep the queue looking
+ * broken forever. `skipped` says what actually happened, and carries the reason.
+ */
+export function applySkipPolicy(db: LensDeps['db']): number {
+  let parked = 0;
+  for (const [dir, reason] of Object.entries(SKIP_DIRS)) {
+    const r = db
+      .prepare(
+        `UPDATE lens_artifact_uploads
+            SET status = 'skipped', last_error = ?
+          WHERE status != 'done' AND status != 'skipped' AND artifact_key LIKE ?`,
+      )
+      .run(reason, `${dir}/%`);
+    parked += Number(r.changes ?? 0);
+  }
+  return parked;
+}
+
+function countKey(db: LensDeps['db'], key: string): boolean {
+  const r = db.query('SELECT 1 AS n FROM lens_artifact_uploads WHERE artifact_key = ?').get(key);
+  return Boolean(r);
+}
+
+function defaultRoot(): string {
+  return join(
+    process.env.HOME ?? '',
+    'Library/Application Support/cardmem/lens-artifacts',
+  );
+}
+
+/** Total bytes on disk, so a human can see the size of the job before starting it. */
+export async function backfillSize(root?: string): Promise<{ files: number; bytes: number }> {
+  const files: string[] = [];
+  await walk(root ?? defaultRoot(), files);
+  let bytes = 0;
+  for (const f of files) {
+    try {
+      bytes += (await stat(f)).size;
+    } catch {
+      /* counted in `files`, not in `bytes` — a file that vanished mid-walk */
+    }
+  }
+  return { files: files.length, bytes };
+}