@broberg/lens-daemon 0.0.0-stage → 0.1.5

package/src/artifact-uploads.ts

@@ -0,0 +1,397 @@
+// F276.1 — every Lens artifact reaches the cloud, so the owner can open it on a phone.
+//
+// THE DEFECT THIS CLOSES, in Christian's words (2026-08-25): "Alle Lens billeder bør da
+// placeres i Cloud så jeg kan tilgå dem fra alle steder og maskiner eller kan jeg jo ikke
+// se dem på min iPhone PWA." He was right, and measuring it found something larger — the
+// whole Lens surface reads from 127.0.0.1, so on a phone it does not merely show broken
+// images, it shows "daemon offline". Lens had never worked on his phone at all.
+//
+// A QUEUE, NOT AN INLINE UPLOAD. Three reasons, each of which has a failure attached:
+//   · a capture must not get slower, or people stop running them;
+//   · an upload failure must not fail the RUN — the local copy is still valid evidence,
+//     and losing a run because object storage hiccuped is the worse outcome;
+//   · the backfill of 6588 historic files (F276.4) is this same operation at scale, so it
+//     uses this machinery rather than a second implementation that drifts from it.
+//
+// WHAT IS DELIBERATELY NOT HERE: deletion. The local copy stays. It is the fast path on
+// the Mac and the fallback when a cloud copy is missing, and removing a working path
+// before its replacement is proven is the naked cutover the harness contract forbids.
+
+import type { Database } from 'bun:sqlite';
+import type { LensStorage } from './storage.ts';
+import type { CloudClientLike } from './service.ts';
+
+/**
+ * The queue's schema, owned by the module that reads and writes it.
+ *
+ * It lives here rather than inline in state.ts so a test can build a real queue table
+ * WITHOUT calling getDb — which is a module-level singleton that ignores its path
+ * argument after the first call. A second test file that touched it handed the next
+ * file someone else's database, and the assertion that failed named a value that test
+ * had never written. One definition, two callers, no copy to drift.
+ */
+export const UPLOAD_QUEUE_DDL = `
+  CREATE TABLE IF NOT EXISTS lens_artifact_uploads (
+    artifact_key TEXT PRIMARY KEY,
+    project_slug TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'pending',
+    cloud_artifact_id TEXT,
+    attempts INTEGER NOT NULL DEFAULT 0,
+    last_error TEXT,
+    enqueued_at INTEGER NOT NULL,
+    uploaded_at INTEGER,
+    claimed_at INTEGER
+  );
+  CREATE INDEX IF NOT EXISTS lens_artifact_uploads_status_idx
+    ON lens_artifact_uploads (status, enqueued_at);
+`;
+
+/** Create the queue table if it is not there. Idempotent. */
+export function ensureUploadQueue(db: Database): void {
+  db.exec(UPLOAD_QUEUE_DDL);
+  // Additive for daemons whose table predates F276.5. Look first rather than
+  // swallow the ALTER's error: a catch-all would also hide a locked or
+  // read-only database, and the drain would then fail later on a column it
+  // believes exists.
+  const cols = db.query('PRAGMA table_info(lens_artifact_uploads)').all() as { name: string }[];
+  if (!cols.some((c) => c.name === 'claimed_at')) {
+    db.exec('ALTER TABLE lens_artifact_uploads ADD COLUMN claimed_at INTEGER');
+  }
+}
+
+/** A claim older than this is treated as abandoned and returns to the queue. Long
+ *  enough that a genuinely slow 7 MB upload is never stolen mid-flight; short enough
+ *  that a daemon killed mid-batch does not strand its rows for a human to notice. */
+export const CLAIM_TTL_MS = 10 * 60 * 1000;
+
+export interface UploadRow {
+  artifact_key: string;
+  project_slug: string;
+  status: 'pending' | 'uploading' | 'done' | 'failed' | 'skipped';
+  cloud_artifact_id: string | null;
+  attempts: number;
+  last_error: string | null;
+  enqueued_at: number;
+  uploaded_at: number | null;
+}
+
+/** Give up after this many tries. A key that fails five times is broken in a way
+ *  retrying will not fix (bytes gone, file corrupt), and an infinite retry would
+ *  hide it behind a queue that never drains. */
+export const MAX_ATTEMPTS = 5;
+
+/**
+ * Queue an artifact for upload. Idempotent BY THE SCHEMA — artifact_key is the
+ * primary key, so enqueueing the same key twice does nothing.
+ *
+ * That is not a micro-optimisation: F276.4 re-runs the backfill over 6588 files and
+ * must not create 6588 duplicates on the second pass. Expressing it in the table
+ * rather than in a guard means no caller can forget it.
+ */
+/**
+ * Directories that are deliberately NOT uploaded, each with the reason it is held back.
+ *
+ * Applied where a file is QUEUED (enqueueArtifact), not only during a backfill —
+ * otherwise a live flow under `_external` would be held back only by the project
+ * lookup failing, which is the accident this list exists to replace (F006.4).
+ *
+ * These are held back BY DECISION. Today they also happen to fail the project lookup,
+ * which means they are currently safe by accident — and an accident is not a policy.
+ * Naming them here makes the hold-back survive someone later fixing the lookup.
+ *
+ * `capture-sessions` is Christian's call, given 2026-08-25 when asked directly whether
+ * 329 screenshots of him signing in to App Store Connect and Google Play should go to
+ * the cloud: "Nej." No credentials are visible in them (all 329 verified as PNG), and
+ * they stay on the machine anyway.
+ */
+export const SKIP_DIRS: Record<string, string> = {
+  'capture-sessions': "owner decision 2026-08-25: login recordings stay on the Mac",
+  _external: 'screenshots of third-party sites — not this fleet\'s project history',
+  'lens-statepath-test': 'one-off engine test run, not project history',
+  'lens-mint-test': 'one-off engine test run, not project history',
+};
+
+
+export function enqueueArtifact(db: Database, projectSlug: string, key: string | null): void {
+  if (!key) return;
+  const skipReason = SKIP_DIRS[key.split('/')[0] ?? ''];
+  db.prepare(
+    `INSERT OR IGNORE INTO lens_artifact_uploads (artifact_key, project_slug, status, enqueued_at, last_error)
+     VALUES (?, ?, ?, ?, ?)`,
+  ).run(key, projectSlug, skipReason ? 'skipped' : 'pending', Date.now(), skipReason ?? null);
+}
+
+/** Queue several at once — the flow path has one per step. */
+export function enqueueArtifacts(db: Database, projectSlug: string, keys: (string | null)[]): void {
+  for (const k of keys) enqueueArtifact(db, projectSlug, k);
+}
+
+export interface DrainResult {
+  /** Uploaded successfully in this pass. */
+  uploaded: number;
+  /** Failed this pass but will be retried. */
+  retryable: number;
+  /** Given up on (attempts exhausted, or the bytes are gone). */
+  abandoned: number;
+  /** Still waiting AFTER this pass — so a caller can tell "done" from "batch full". */
+  remaining: number;
+  /** Why nothing happened, when nothing happened. Stated rather than implied by a
+   *  row of zeros, because "queue empty" and "not signed in" look identical in
+   *  counts and mean completely different things. */
+  idle_reason: string | null;
+}
+
+/**
+ * Upload one batch. Returns counts AND, when nothing moved, the reason.
+ *
+ * The `idle_reason` field exists because of the failure this whole epic is about: a
+ * result of all zeros could mean the queue is empty, or that we are signed out, or
+ * that storage is unprovisioned. Three different situations, one indistinguishable
+ * report — so the reason is stated.
+ */
+export async function drainUploads(
+  db: Database,
+  storage: LensStorage,
+  cloud: CloudClientLike | null,
+  opts: {
+    batch?: number;
+    projectId?: (slug: string) => Promise<string | null>;
+    /** Injectable clock, so the stale-claim path is testable without waiting 10 minutes. */
+    now?: () => number;
+  } = {},
+): Promise<DrainResult> {
+  const empty = (reason: string | null): DrainResult => ({
+    uploaded: 0,
+    retryable: 0,
+    abandoned: 0,
+    remaining: pendingCount(db),
+    idle_reason: reason,
+  });
+
+  if (!cloud) return empty('not signed in to cardmem — run `cardmem login`; nothing was uploaded');
+
+  const batch = Math.max(1, Math.min(opts.batch ?? 25, 200));
+  const now = opts.now?.() ?? Date.now();
+  releaseStaleClaims(db, now);
+  const rows = claimBatch(db, batch, now);
+
+  if (rows.length === 0) {
+    // "Nothing to do" and "another drain has everything claimed" both produce zero rows
+    // and mean opposite things — the first says stop, the second says come back. Say
+    // which one it is rather than letting a row of zeros stand for both.
+    const inFlight = db
+      .query("SELECT COUNT(*) AS n FROM lens_artifact_uploads WHERE status = 'uploading'")
+      .get() as { n: number } | undefined;
+    return empty(
+      Number(inFlight?.n ?? 0) > 0
+        ? `${inFlight!.n} artifact(s) are claimed by another drain in flight — nothing to take right now`
+        : null,
+    );
+  }
+
+  let uploaded = 0;
+  let retryable = 0;
+  let abandoned = 0;
+  const projectIds = new Map<string, string | null>();
+
+  for (const row of rows) {
+    try {
+      // Resolve the cloud project id once per slug per pass. Guessing one would put a
+      // project's screenshots into a different project's store.
+      if (!projectIds.has(row.project_slug)) {
+        projectIds.set(row.project_slug, await resolveProjectId(cloud, row.project_slug, opts.projectId));
+      }
+      const projectId = projectIds.get(row.project_slug);
+      if (!projectId) throw new Error(`project ${row.project_slug} is not known to cardmem`);
+
+      const bytes = await storage.get(row.artifact_key);
+      if (!bytes) {
+        // The row claims an artifact and the storage does not have it. Retrying cannot
+        // conjure bytes, so this is ABANDONED immediately rather than burning five
+        // attempts — and it is recorded as a distinct outcome, because a missing file
+        // is a finding, not a transient error.
+        markFailed(db, row.artifact_key, 'bytes are GONE from local storage', MAX_ATTEMPTS);
+        abandoned++;
+        continue;
+      }
+
+      const placed = await cloud.callTool<{ artifact_id: string }>('cardmem_place_artifact', {
+        project_id: projectId,
+        name: artifactName(row.artifact_key),
+        mime: row.artifact_key.endsWith('.webm') ? 'video/webm' : 'image/png',
+        base64: bytes.toString('base64'),
+        source: 'lens',
+      });
+
+      db.prepare(
+        `UPDATE lens_artifact_uploads
+         SET status = 'done', cloud_artifact_id = ?, uploaded_at = ?, last_error = NULL,
+             attempts = attempts + 1
+         WHERE artifact_key = ?`,
+      ).run(placed.artifact_id, Date.now(), row.artifact_key);
+      uploaded++;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      const attempts = row.attempts + 1;
+      markFailed(db, row.artifact_key, msg, attempts);
+      if (attempts >= MAX_ATTEMPTS) abandoned++;
+      else retryable++;
+    }
+  }
+
+  return { uploaded, retryable, abandoned, remaining: pendingCount(db), idle_reason: null };
+}
+
+/**
+ * Take a batch of rows OUT of the pending pool in the same statement that reads them.
+ *
+ * THE BUG THIS CLOSES, measured 2026-08-25: the old code selected `status='pending'`,
+ * uploaded (hundreds of ms of network), and only then wrote `done`. For that whole
+ * window the row still read `pending` to anyone else — and two drains exist by design,
+ * the daemon's 30 s background tick and the HTTP route. On project xrt81-com that put
+ * 16 of 27 files in the cloud twice, ids ~65 ms apart, while the local queue showed a
+ * flawless 27. A count taken on the side that does the writing cannot see a duplicate
+ * it wrote twice.
+ *
+ * `artifact_key` being the PRIMARY KEY makes ENQUEUEING idempotent, which is true and
+ * is an acceptance criterion on F276.4. It says nothing about DRAINING. Idempotent
+ * insert and exactly-once processing are different properties.
+ */
+function claimBatch(db: Database, batch: number, now: number): UploadRow[] {
+  return db
+    .prepare(
+      `UPDATE lens_artifact_uploads
+          SET status = 'uploading', claimed_at = ?
+        WHERE artifact_key IN (
+          SELECT artifact_key FROM lens_artifact_uploads
+           WHERE status = 'pending' AND attempts < ?
+           ORDER BY enqueued_at ASC LIMIT ?
+        )
+        RETURNING *`,
+    )
+    .all(now, MAX_ATTEMPTS, batch) as UploadRow[];
+}
+
+/**
+ * Return rows whose claim went stale to the queue.
+ *
+ * Without this, the cure for double-upload becomes a cause of never-upload: a daemon
+ * killed mid-batch would leave its rows in `uploading` forever, and they would be
+ * invisible to `pending` and to `failed` alike — the silent gap this epic is about,
+ * reintroduced by its own fix.
+ */
+export function releaseStaleClaims(db: Database, now: number = Date.now()): number {
+  const r = db
+    .prepare(
+      `UPDATE lens_artifact_uploads
+          SET status = 'pending', claimed_at = NULL
+        WHERE status = 'uploading' AND (claimed_at IS NULL OR claimed_at < ?)`,
+    )
+    .run(now - CLAIM_TTL_MS);
+  return Number(r.changes ?? 0);
+}
+
+function markFailed(db: Database, key: string, error: string, attempts: number): void {
+  db.prepare(
+    `UPDATE lens_artifact_uploads
+     SET attempts = ?, last_error = ?, status = ?
+     WHERE artifact_key = ?`,
+  ).run(attempts, error.slice(0, 500), attempts >= MAX_ATTEMPTS ? 'failed' : 'pending', key);
+}
+
+/** Rows that are not finished — pending AND in flight elsewhere. A row another drain
+ *  is currently uploading is still not done, and excluding it would make `remaining: 0`
+ *  mean "nothing left" while an upload is still running. */
+function pendingCount(db: Database): number {
+  const r = db
+    .query(
+      `SELECT COUNT(*) AS n FROM lens_artifact_uploads
+        WHERE (status = 'pending' AND attempts < ?) OR status = 'uploading'`,
+    )
+    .get(MAX_ATTEMPTS) as { n: number } | undefined;
+  return Number(r?.n ?? 0);
+}
+
+async function resolveProjectId(
+  cloud: CloudClientLike,
+  slug: string,
+  override?: (slug: string) => Promise<string | null>,
+): Promise<string | null> {
+  if (override) return override(slug);
+  const res = await cloud.callTool<{ projects?: Array<{ id: string; slug: string }> }>(
+    'cardmem_list_projects',
+    {},
+  );
+  return res?.projects?.find((p) => p.slug === slug)?.id ?? null;
+}
+
+/** A readable filename from an artifact key, so the Artifacts surface is browsable
+ *  rather than a wall of hashes. `cardmem/flows/<run>/step-3.png` → `lens-flows-<run>-step-3.png`. */
+export function artifactName(key: string): string {
+  const cleaned = key.replace(/^[^/]+\//, '').replace(/\//g, '-');
+  return `lens-${cleaned}`;
+}
+
+/** What the queue holds, for the surface and for a human asking "is it done yet?". */
+export function uploadStats(db: Database): {
+  pending: number;
+  done: number;
+  failed: number;
+  /** Claimed by a drain and in flight right now (F276.5). Its own bucket: in-flight and
+   *  not-yet-started are different facts, and sharing one value of `status` between them
+   *  is what let two drains upload the same file. */
+  uploading: number;
+  /** Held back on purpose (F276.4). Counted SEPARATELY from failed: one is a decision
+   *  the owner made, the other is something broken, and a queue that shows them as one
+   *  number cannot be used to tell whether the backfill is finished. */
+  skipped: number;
+  total: number;
+  /** pending + uploading + done + failed + skipped. If this ever differs from `total`,
+   *  a row is in a state nothing counts — the silent gap this epic exists to close. */
+  accounted: number;
+} {
+  const row = db
+    .query(
+      `SELECT
+         SUM(CASE WHEN status = 'pending' AND attempts < ? THEN 1 ELSE 0 END) AS pending,
+         SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) AS done,
+         SUM(CASE WHEN status = 'failed' OR (status = 'pending' AND attempts >= ?) THEN 1 ELSE 0 END) AS failed,
+         SUM(CASE WHEN status = 'uploading' THEN 1 ELSE 0 END) AS uploading,
+         SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped,
+         COUNT(*) AS total
+       FROM lens_artifact_uploads`,
+    )
+    .get(MAX_ATTEMPTS, MAX_ATTEMPTS) as Record<string, number> | undefined;
+  const pending = Number(row?.pending ?? 0);
+  const done = Number(row?.done ?? 0);
+  const failed = Number(row?.failed ?? 0);
+  const skipped = Number(row?.skipped ?? 0);
+  const uploading = Number(row?.uploading ?? 0);
+  return {
+    pending,
+    done,
+    failed,
+    uploading,
+    skipped,
+    total: Number(row?.total ?? 0),
+    accounted: pending + uploading + done + failed + skipped,
+  };
+}
+
+/** Map artifact keys → their cloud artifact id, for the read path. Only 'done' rows,
+ *  so a queued-but-not-yet-uploaded key resolves to nothing rather than to a url that
+ *  404s — a link in the durable form pointing at nothing is worse than no link. */
+export function cloudIdsFor(db: Database, keys: string[]): Map<string, string> {
+  const out = new Map<string, string>();
+  if (keys.length === 0) return out;
+  const placeholders = keys.map(() => '?').join(',');
+  const rows = db
+    .query(
+      `SELECT artifact_key, cloud_artifact_id FROM lens_artifact_uploads
+       WHERE status = 'done' AND cloud_artifact_id IS NOT NULL
+         AND artifact_key IN (${placeholders})`,
+    )
+    .all(...keys) as Array<{ artifact_key: string; cloud_artifact_id: string }>;
+  for (const r of rows) out.set(r.artifact_key, r.cloud_artifact_id);
+  return out;
+}