@openparachute/vault 0.5.1 → 0.5.2-rc.2

package/src/transcription-worker.test.ts

@@ -56,20 +56,60 @@ function makeWorker(opts: {
   fetchImpl: typeof fetch;
   retention?: "keep" | "until_transcribed" | "never";
   maxAttempts?: number;
+  // Override the store the worker sees (race-injection tests wrap `store`).
+  store?: Store;
+  logger?: { error: (...args: unknown[]) => void; info?: (...args: unknown[]) => void };
 }) {
+  const workerStore = opts.store ?? (store as unknown as Store);
   return startTranscriptionWorker({
     vaultList: () => ["default"],
-    getStore: () => store as unknown as Store,
+    getStore: () => workerStore,
     scribeUrl: "http://scribe.test",
     resolveAssetsDir: () => assetsRoot,
     getAudioRetention: () => opts.retention ?? "keep",
     pollIntervalMs: 10_000_000, // never auto-fire; tests drive ticks manually
     maxAttempts: opts.maxAttempts ?? 3,
     fetchImpl: opts.fetchImpl,
-    logger: silentLogger,
+    logger: opts.logger ?? silentLogger,
   });
 }
 
+/**
+ * Wrap a store so the first `interfereTimes` `updateNote` calls carrying an
+ * `if_updated_at` precondition fire `userEdit()` (a concurrent user write that
+ * bumps `updated_at`) just before delegating — forcing the precondition stale
+ * exactly that many times. Non-OC writes (and the last-resort no-precondition
+ * write) pass straight through. Drives the vault#435 worker race tests.
+ *
+ * NOTE: duplicated in src/vault.test.ts (routes-layer race tests) — keep in sync.
+ */
+function withRace(
+  base: Store,
+  interfereTimes: number,
+  userEdit: () => Promise<void>,
+): Store {
+  let fired = 0;
+  return new Proxy(base, {
+    get(target, prop, receiver) {
+      if (prop === "updateNote") {
+        return async (id: string, updates: any) => {
+          if (updates?.if_updated_at !== undefined && fired < interfereTimes) {
+            fired++;
+            // bun:sqlite stamps `updated_at` at ms granularity. Sleep so the
+            // concurrent user write lands at a strictly-greater timestamp than
+            // the precondition the worker captured — making the conflict
+            // deterministic rather than racing inside the same millisecond.
+            await new Promise((r) => setTimeout(r, 5));
+            await userEdit();
+          }
+          return (target as any).updateNote(id, updates);
+        };
+      }
+      return Reflect.get(target, prop, receiver);
+    },
+  }) as Store;
+}
+
 describe("transcription worker", () => {
   test("happy path: replaces _Transcript pending._ and clears stub marker", async () => {
     const note = await store.createNote(
@@ -305,8 +345,10 @@ describe("transcription worker", () => {
   });
 
   test("FIFO: oldest pending is processed first", async () => {
-    await store.createNote("s", { id: "f1", metadata: { transcribe_stub: true } });
-    await store.createNote("s", { id: "f2", metadata: { transcribe_stub: true } });
+    // Seed the placeholder body so the transcript patches in place (the real
+    // capture shape); ordering is what this test verifies.
+    await store.createNote("_Transcript pending._", { id: "f1", metadata: { transcribe_stub: true } });
+    await store.createNote("_Transcript pending._", { id: "f2", metadata: { transcribe_stub: true } });
     seedAudio("memos/first.webm");
     seedAudio("memos/second.webm");
     await store.addAttachment("f1", "memos/first.webm", "audio/webm", {
@@ -513,6 +555,173 @@ describe("transcription worker", () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// Optimistic concurrency on the worker's note writes (vault#435).
+//
+// The worker patches the memo note in two read-modify-write cycles — the
+// success path (replace the placeholder/marker with the transcript) and the
+// failure path (`applyFailureMarker`). Without an `if_updated_at` precondition
+// a user edit landing between read and write is silently clobbered (the
+// static-write/stale-read class of vault#208). These tests inject a concurrent
+// user PATCH right before the worker's OC write to assert:
+//   (a) the user's edit is NOT clobbered,
+//   (b) the worker RE-APPLIES the transcript/marker correctly on fresh content,
+//   (c) the double-conflict policy (worker = resilient): last-resort apply when
+//       still safe, safe-skip otherwise — never crash the sweep.
+// ---------------------------------------------------------------------------
+describe("transcription worker — optimistic concurrency (vault#435)", () => {
+  test("success patch: single race → re-applies transcript onto the user's fresh content", async () => {
+    // Placeholder body so the transcript would normally surgical-replace in
+    // place. The user appends a line mid-flight; the worker's first OC write
+    // conflicts, it re-reads, and the surgical replace lands on the FRESH
+    // content — preserving the user's append.
+    await store.createNote(
+      "# memo\n\n_Transcript pending._\n",
+      { id: "oc-succ-1", metadata: { transcribe_stub: true } },
+    );
+    seedAudio("memos/ocs1.webm");
+    await store.addAttachment("oc-succ-1", "memos/ocs1.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    const raceStore = withRace(store as unknown as Store, 1, async () => {
+      await store.updateNote("oc-succ-1", { append: "\n\nUSER EDIT" });
+    });
+    const worker = makeWorker({
+      store: raceStore,
+      fetchImpl: mkFetchMock([{ text: "the transcript" }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("oc-succ-1");
+    // (b) transcript replaced the placeholder in place; (a) user edit survives.
+    expect(note!.content).toBe("# memo\n\nthe transcript\n\n\nUSER EDIT");
+    expect((note!.metadata as any)?.transcribe_stub).toBeUndefined();
+    const [att] = await store.getAttachments("oc-succ-1");
+    expect(att.metadata?.transcribe_status).toBe("done");
+    expect(att.metadata?.transcript).toBe("the transcript");
+  });
+
+  test("failure marker: single race → re-applies marker onto the user's fresh content", async () => {
+    // maxAttempts=1 so a single scribe failure is terminal → applyFailureMarker.
+    await store.createNote(
+      "# memo\n\n_Transcript pending._\n",
+      { id: "oc-fail-1", metadata: { transcribe_stub: true } },
+    );
+    seedAudio("memos/ocf1.webm");
+    await store.addAttachment("oc-fail-1", "memos/ocf1.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    const raceStore = withRace(store as unknown as Store, 1, async () => {
+      await store.updateNote("oc-fail-1", { append: "\n\nUSER EDIT" });
+    });
+    const worker = makeWorker({
+      store: raceStore,
+      maxAttempts: 1,
+      fetchImpl: mkFetchMock([{ error: "scribe down", status: 500 }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("oc-fail-1");
+    // (b) marker replaced the placeholder in place; (a) user edit survives.
+    expect(note!.content).toBe("# memo\n\n_Transcription unavailable._\n\n\nUSER EDIT");
+    expect((note!.metadata as any)?.transcribe_stub).toBeUndefined();
+    const [att] = await store.getAttachments("oc-fail-1");
+    expect(att.metadata?.transcribe_status).toBe("failed");
+  });
+
+  test("double conflict + still safe → last-resort apply (append path stays applicable)", async () => {
+    // No placeholder in the body → the transform takes the APPEND branch, which
+    // is always safe against fresh content. Interfere on BOTH the first write
+    // and the retry write → the worker falls back to the precondition-less
+    // last-resort apply (safe because the stub survives + append is safe).
+    await store.createNote(
+      "user prose with no marker",
+      { id: "oc-dbl-safe", metadata: { transcribe_stub: true } },
+    );
+    seedAudio("memos/ocd1.webm");
+    await store.addAttachment("oc-dbl-safe", "memos/ocd1.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    let edits = 0;
+    const raceStore = withRace(store as unknown as Store, 2, async () => {
+      edits++;
+      await store.updateNote("oc-dbl-safe", { append: ` e${edits}` });
+    });
+    const worker = makeWorker({
+      store: raceStore,
+      fetchImpl: mkFetchMock([{ text: "TRANSCRIPT" }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("oc-dbl-safe");
+    // (a) both user appends survive; (c) transcript still appended (last-resort).
+    expect(note!.content).toBe("user prose with no marker e1 e2\n\nTRANSCRIPT");
+    expect((note!.metadata as any)?.transcribe_stub).toBeUndefined();
+  });
+
+  test("double conflict + unsafe (user cleared the stub) → safe-skip, note untouched", async () => {
+    // The racing user edit clears `transcribe_stub` → the last-resort safety
+    // gate refuses to blind-write. The worker skips the note write entirely
+    // and the user's content is left exactly as they left it. The attachment
+    // transcript is still recorded (we never throw work away).
+    await store.createNote(
+      "# memo\n\n_Transcript pending._\n",
+      { id: "oc-dbl-skip", metadata: { transcribe_stub: true } },
+    );
+    seedAudio("memos/ocd2.webm");
+    await store.addAttachment("oc-dbl-skip", "memos/ocd2.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    let edit = 0;
+    const raceStore = withRace(store as unknown as Store, 2, async () => {
+      edit++;
+      // Second interference clears the stub (user opts out mid-flight).
+      if (edit >= 2) {
+        await store.updateNote("oc-dbl-skip", {
+          content: "I took this note over",
+          metadata: {},
+        });
+      } else {
+        await store.updateNote("oc-dbl-skip", { append: " edit1" });
+      }
+    });
+    const worker = makeWorker({
+      store: raceStore,
+      fetchImpl: mkFetchMock([{ text: "WOULD CLOBBER" }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("oc-dbl-skip");
+    // (c) safe-skip: the worker did NOT overwrite the user's takeover content.
+    expect(note!.content).toBe("I took this note over");
+    expect(note!.content).not.toContain("WOULD CLOBBER");
+    // Transcript still durable on the attachment row.
+    const [att] = await store.getAttachments("oc-dbl-skip");
+    expect(att.metadata?.transcribe_status).toBe("done");
+    expect(att.metadata?.transcript).toBe("WOULD CLOBBER");
+  });
+});
+
 describe("transcription worker — auth + context", () => {
   test("attaches multipart context part when getContextPredicates returns entries", async () => {
     await store.createNote("stub", { id: "ctx1", metadata: { transcribe_stub: true } });
@@ -702,7 +911,8 @@ describe("transcription worker — hook-driven", () => {
   });
 
   test("attachment:created event triggers a cycle before the sweep fires", async () => {
-    await hookedStore.createNote("stub", { id: "h1", metadata: { transcribe_stub: true } });
+    // Placeholder body so the transcript patches in place (real capture shape).
+    await hookedStore.createNote("_Transcript pending._", { id: "h1", metadata: { transcribe_stub: true } });
     seedAudio("memos/h1.webm");
 
     let callCount = 0;
@@ -1117,3 +1327,259 @@ describe("transcription worker — auto-origin (vault#353)", () => {
     }
   });
 });
+
+// ---------------------------------------------------------------------------
+// finding F — never destroy content on the legacy in-body memo path. These
+// fixtures carry the `![[<audio>]]` embed (the REAL production capture shape
+// from recorder.ts memoNoteContent), which the pre-existing failure tests all
+// omitted — which is why the data-loss branch went uncaught.
+// ---------------------------------------------------------------------------
+
+describe("transcription worker — legacy in-body memo content safety (finding F)", () => {
+  // Canonical capture body: header + _Recorded_ + placeholder + ![[embed]].
+  const captureBody = (audio: string) =>
+    `# 🎙️ Voice memo\n\n_Recorded sometime._\n\n_Transcript pending._\n\n![[${audio}]]\n`;
+
+  test("failure with placeholder present → marker replaces placeholder, embed intact", async () => {
+    await store.createNote(captureBody("memos/f1.webm"), {
+      id: "f-marker-1",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f1.webm");
+    await store.addAttachment("f-marker-1", "memos/f1.webm", "audio/webm", {
+      transcribe_status: "pending",
+      transcribe_attempts: 2,
+    });
+
+    const worker = makeWorker({
+      fetchImpl: mkFetchMock([{ error: "scribe down", status: 500 }]),
+      maxAttempts: 3,
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("f-marker-1");
+    expect(note!.content).toBe(
+      "# 🎙️ Voice memo\n\n_Recorded sometime._\n\n_Transcription unavailable._\n\n![[memos/f1.webm]]\n",
+    );
+    // Embed survives — the whole point.
+    expect(note!.content).toContain("![[memos/f1.webm]]");
+    expect((note!.metadata as any)?.transcribe_stub).toBeUndefined();
+  });
+
+  test("failure with placeholder ABSENT (user-edited body + embed) → marker APPENDED, content preserved", async () => {
+    // The data-loss regression: user edited the note while pending, removing
+    // the _Transcript pending._ placeholder but keeping their text + the
+    // embed. The old code full-replaced the body with the bare marker here,
+    // destroying both. Now we append.
+    const editedBody = "# 🎙️ My trip notes\n\nWent to the coast today.\n\n![[memos/f2.webm]]\n";
+    await store.createNote(editedBody, {
+      id: "f-marker-2",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f2.webm");
+    await store.addAttachment("f-marker-2", "memos/f2.webm", "audio/webm", {
+      transcribe_status: "pending",
+      transcribe_attempts: 2,
+    });
+
+    const worker = makeWorker({
+      fetchImpl: mkFetchMock([{ error: "scribe down", status: 500 }]),
+      maxAttempts: 3,
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("f-marker-2");
+    // User text + embed preserved; marker appended.
+    expect(note!.content).toBe(`${editedBody}\n\n_Transcription unavailable._`);
+    expect(note!.content).toContain("Went to the coast today.");
+    expect(note!.content).toContain("![[memos/f2.webm]]");
+    expect((note!.metadata as any)?.transcribe_stub).toBeUndefined();
+  });
+
+  test("double terminal failure → exactly one marker (idempotent, no stacking)", async () => {
+    // First failure writes the marker + clears the stub. Re-arm the stub and
+    // fail again — the marker must NOT stack. (Re-arming models a retry that
+    // fails again.)
+    await store.createNote(captureBody("memos/f3.webm"), {
+      id: "f-marker-3",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f3.webm");
+    await store.addAttachment("f-marker-3", "memos/f3.webm", "audio/webm", {
+      transcribe_status: "pending",
+      transcribe_attempts: 2,
+    });
+
+    const worker1 = makeWorker({
+      fetchImpl: mkFetchMock([{ error: "down", status: 500 }]),
+      maxAttempts: 3,
+    });
+    try { await worker1.tick(); } finally { await worker1.stop(); }
+
+    // Re-arm stub + reset attachment to pending (what a retry does), then fail again.
+    const note1 = await store.getNote("f-marker-3");
+    await store.updateNote("f-marker-3", {
+      metadata: { ...((note1!.metadata as any) ?? {}), transcribe_stub: true },
+      skipUpdatedAt: true,
+    });
+    const [att] = await store.getAttachments("f-marker-3");
+    await store.setAttachmentMetadata(att.id, {
+      ...(att.metadata ?? {}),
+      transcribe_status: "pending",
+      transcribe_attempts: 2,
+    });
+
+    const worker2 = makeWorker({
+      fetchImpl: mkFetchMock([{ error: "down again", status: 500 }]),
+      maxAttempts: 3,
+    });
+    try { await worker2.tick(); } finally { await worker2.stop(); }
+
+    const note = await store.getNote("f-marker-3");
+    const occurrences = note!.content.split("_Transcription unavailable._").length - 1;
+    expect(occurrences).toBe(1);
+    expect(note!.content).toContain("![[memos/f3.webm]]");
+  });
+
+  test("success with placeholder absent (stub set) → transcript appended, content preserved", async () => {
+    // Mirror of the failure-append case, on the success path. User edited the
+    // note (cleared the placeholder) but kept the stub. The old code
+    // full-replaced the body with the bare transcript; now we append.
+    const editedBody = "# 🎙️ My trip notes\n\nWent to the coast today.\n\n![[memos/f4.webm]]\n";
+    await store.createNote(editedBody, {
+      id: "f-success-1",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f4.webm");
+    await store.addAttachment("f-success-1", "memos/f4.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    const worker = makeWorker({
+      fetchImpl: mkFetchMock([{ text: "the spoken transcript" }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("f-success-1");
+    expect(note!.content).toBe(`${editedBody}\n\nthe spoken transcript`);
+    expect(note!.content).toContain("Went to the coast today.");
+    expect(note!.content).toContain("![[memos/f4.webm]]");
+    expect((note!.metadata as any)?.transcribe_stub).toBeUndefined();
+  });
+
+  test("first-try success with placeholder present + embed → transcript replaces placeholder in place", async () => {
+    // Pins the canonical first-try-success shape the retry round-trip must match.
+    await store.createNote(captureBody("memos/f5.webm"), {
+      id: "f-success-2",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f5.webm");
+    await store.addAttachment("f-success-2", "memos/f5.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    const worker = makeWorker({
+      fetchImpl: mkFetchMock([{ text: "the spoken words" }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("f-success-2");
+    expect(note!.content).toBe(
+      "# 🎙️ Voice memo\n\n_Recorded sometime._\n\nthe spoken words\n\n![[memos/f5.webm]]\n",
+    );
+  });
+
+  test("transcript containing `$&` round-trips byte-identical (no String.replace injection) — first-try success", async () => {
+    // Regression: String.replace with a STRING replacement treats `$&` etc. as
+    // special. A transcript with `$&` must land verbatim in the body, not as
+    // the matched marker text.
+    const dollarTranscript = "it matched $& exactly, and $1 too, plus $` and $'";
+    await store.createNote(captureBody("memos/f6.webm"), {
+      id: "f-dollar-1",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f6.webm");
+    await store.addAttachment("f-dollar-1", "memos/f6.webm", "audio/webm", {
+      transcribe_status: "pending",
+    });
+
+    const worker = makeWorker({
+      fetchImpl: mkFetchMock([{ text: dollarTranscript }]),
+    });
+    try {
+      await worker.tick();
+    } finally {
+      await worker.stop();
+    }
+
+    const note = await store.getNote("f-dollar-1");
+    expect(note!.content).toBe(
+      `# 🎙️ Voice memo\n\n_Recorded sometime._\n\n${dollarTranscript}\n\n![[memos/f6.webm]]\n`,
+    );
+    // Belt-and-suspenders: the literal `$&` must be present verbatim.
+    expect(note!.content).toContain("it matched $& exactly");
+  });
+
+  test("transcript containing `$&` round-trips byte-identical on the RETRY path (marker → transcript)", async () => {
+    // Same injection guard, but exercised on the retry surgical-replace: the
+    // transcript replaces `_Transcription unavailable._` in place.
+    const dollarTranscript = "retry text with $& and $0 and $$ literal";
+    await store.createNote(captureBody("memos/f7.webm"), {
+      id: "f-dollar-2",
+      metadata: { transcribe_stub: true },
+    });
+    seedAudio("memos/f7.webm");
+    await store.addAttachment("f-dollar-2", "memos/f7.webm", "audio/webm", {
+      transcribe_status: "pending",
+      transcribe_attempts: 2, // one more failure → terminal at maxAttempts=3
+    });
+
+    // Phase 1: terminal failure → marker replaces placeholder.
+    const worker1 = makeWorker({
+      fetchImpl: mkFetchMock([{ error: "down", status: 500 }]),
+      maxAttempts: 3,
+    });
+    try { await worker1.tick(); } finally { await worker1.stop(); }
+    const failed = await store.getNote("f-dollar-2");
+    expect(failed!.content).toContain("_Transcription unavailable._");
+
+    // Re-arm stub + reset attachment (what the retry route does).
+    await store.updateNote("f-dollar-2", {
+      metadata: { ...((failed!.metadata as any) ?? {}), transcribe_stub: true },
+      skipUpdatedAt: true,
+    });
+    const [att] = await store.getAttachments("f-dollar-2");
+    await store.setAttachmentMetadata(att.id, {
+      ...(att.metadata ?? {}),
+      transcribe_status: "pending",
+    });
+
+    // Phase 2: success with a `$&`-bearing transcript → replaces the marker.
+    const worker2 = makeWorker({
+      fetchImpl: mkFetchMock([{ text: dollarTranscript }]),
+    });
+    try { await worker2.tick(); } finally { await worker2.stop(); }
+
+    const note = await store.getNote("f-dollar-2");
+    expect(note!.content).toBe(
+      `# 🎙️ Voice memo\n\n_Recorded sometime._\n\n${dollarTranscript}\n\n![[memos/f7.webm]]\n`,
+    );
+    expect(note!.content).toContain("retry text with $& and $0 and $$ literal");
+  });
+});