@openparachute/vault 0.4.7-rc.1 → 0.4.8-rc.4

package/src/server.ts

@@ -24,12 +24,17 @@ import { defaultHookRegistry } from "../core/src/hooks.ts";
 import { registerTriggers } from "./triggers.ts";
 import { route } from "./routing.ts";
 import { startTranscriptionWorker, registerTranscriptionHook, type TranscriptionWorker } from "./transcription-worker.ts";
+import { setTranscriptionWorker } from "./transcription-registry.ts";
 import { assetsDir } from "./routes.ts";
-import { resolveScribeAuthToken } from "./scribe-env.ts";
+import { resolveScribeAuthToken, ensureScribeBearer } from "./scribe-env.ts";
+import { getCachedScribeUrl } from "./scribe-discovery.ts";
+import { readEnvFile, setEnvVar } from "./config.ts";
 import { resolveBindHostname } from "./bind.ts";
 import { MirrorManager } from "./mirror-manager.ts";
 import { setMirrorManager } from "./mirror-registry.ts";
 import { buildMirrorDeps, resolveMirrorVaultName } from "./mirror-deps.ts";
+import { selfRegister } from "./self-register.ts";
+import pkg from "../package.json" with { type: "json" };
 
 // Register webhook triggers from global config. Replaces the old hardcoded
 // tts-hook and transcription-hook with config-driven webhooks.
@@ -47,8 +52,9 @@ function registerConfiguredTriggers(): void {
   // both will process the same attachments. The trigger's `missing_metadata`
   // guard keeps it idempotent once the worker marks `transcript` on the
   // attachment, but the noise is worth flagging.
-  if (process.env.SCRIBE_URL) {
-    const scribeHost = safeHost(process.env.SCRIBE_URL);
+  const probedScribeUrl = getCachedScribeUrl();
+  if (probedScribeUrl) {
+    const scribeHost = safeHost(probedScribeUrl);
     for (const t of config.triggers) {
       if (t.action.send !== "attachment") continue;
       if (scribeHost && safeHost(t.action.webhook) === scribeHost) {
@@ -74,17 +80,35 @@ loadEnvFile();
 registerConfiguredTriggers();
 
 /**
- * Start the transcription worker if SCRIBE_URL is configured. The worker
- * polls every vault for attachments with `metadata.transcribe_status = "pending"`
- * and sends the audio to scribe. Absent SCRIBE_URL, the worker stays off
- * — `{transcribe: true}` uploads still enqueue, they just wait.
+ * Start the transcription worker if scribe is discoverable. Scribe URL
+ * resolution order (per `scribe-discovery.ts`): `SCRIBE_URL` env var, then
+ * `~/.parachute/services.json` `parachute-scribe` entry, then nothing.
+ *
+ * Bearer generation (vault#353): if neither `SCRIBE_AUTH_TOKEN` nor the
+ * legacy `SCRIBE_TOKEN` is set, generate a fresh 32-byte base64url bearer
+ * and persist it to vault's `.env` so subsequent restarts use the same
+ * value. Idempotent — calls after the first see the existing token. The
+ * operator (or hub install) is expected to mirror this bearer to scribe's
+ * `SCRIBE_AUTH_TOKEN`; without that mirror, scribe will reject the
+ * Authorization header on a 401 and transcription fails with a friendly
+ * error captured on the transcript note.
  */
+const scribeUrl = getCachedScribeUrl();
 let transcriptionWorker: TranscriptionWorker | null = null;
-if (process.env.SCRIBE_URL) {
+if (scribeUrl) {
+  // Generate + persist the shared bearer on first boot. Subsequent boots
+  // pick up the existing value and don't rotate. Loading the .env back
+  // into process.env happens above (`loadEnvFile()`); we re-load here to
+  // pick up the just-written value without restart.
+  const { created, token } = ensureScribeBearer(readEnvFile, setEnvVar);
+  if (created) {
+    process.env.SCRIBE_AUTH_TOKEN = token;
+    console.log("[transcribe] generated SCRIBE_AUTH_TOKEN (32 bytes, base64url) — mirror this value into scribe's config");
+  }
   transcriptionWorker = startTranscriptionWorker({
     vaultList: () => listVaults(),
     getStore: (name) => getVaultStore(name),
-    scribeUrl: process.env.SCRIBE_URL,
+    scribeUrl,
     scribeToken: resolveScribeAuthToken(),
     resolveAssetsDir: (vault) => assetsDir(vault),
     getAudioRetention: (vault) => readVaultConfig(vault)?.audio_retention ?? "keep",
@@ -97,9 +121,12 @@ if (process.env.SCRIBE_URL) {
     transcriptionWorker,
     (store) => getVaultNameForStore(store as never),
   );
-  console.log(`[transcribe] worker started → ${process.env.SCRIBE_URL}`);
+  // Expose the worker to the REST retry endpoint so retries kick immediately
+  // instead of waiting for the sweep. Idempotent on second boot.
+  setTranscriptionWorker(transcriptionWorker);
+  console.log(`[transcribe] worker started → ${scribeUrl}`);
 } else {
-  console.log("[transcribe] worker disabled (set SCRIBE_URL to enable)");
+  console.log("[transcribe] worker disabled (no scribe in services.json and SCRIBE_URL unset)");
 }
 
 if (process.env.VAULT_AUTH_TOKEN?.trim()) {
@@ -151,6 +178,14 @@ if (listVaults().length === 0) {
   }
 }
 
+// vault#266 — self-register manifest + installDir into services.json so
+// hub's discovery / admin SPA can find vault without a `parachute install
+// vault` round-trip. Idempotent (re-runs produce the same row when nothing
+// changed); never throws (boot must not fail on bookkeeping). The merge-
+// preserving `upsertService` ensures any hub-stamped fields on the row
+// survive — see self-register.ts header for the v0.6 vs v0.7 design note.
+selfRegister({ version: pkg.version });
+
 // Migrate tag schemas from vault.yaml → DB for each vault.
 // Only inserts schemas that don't already exist in the DB (safe across restarts).
 for (const vaultName of listVaults()) {

package/src/transcript-note.test.ts

@@ -0,0 +1,171 @@
+/**
+ * Tests for vault transcript-note materialization (vault#353).
+ *
+ * Covers two surfaces:
+ *   - `buildTranscriptNote` — pure, frontmatter shape per design Q3.
+ *   - `upsertTranscriptNote` — DB-backed, create + retry-update flow.
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from "bun:test";
+import { Database } from "bun:sqlite";
+import { mkdirSync, rmSync } from "fs";
+import { join } from "path";
+import { tmpdir } from "os";
+import { BunStore } from "./vault-store.ts";
+import { buildTranscriptNote, transcriptPathFor, upsertTranscriptNote } from "./transcript-note.ts";
+
+let db: Database;
+let store: BunStore;
+let tmpDir: string;
+
+beforeEach(() => {
+  tmpDir = join(tmpdir(), `transcript-note-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  mkdirSync(tmpDir, { recursive: true });
+  db = new Database(join(tmpDir, "test.db"));
+  store = new BunStore(db);
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("transcriptPathFor", () => {
+  test("appends `.transcript` to the audio path", () => {
+    expect(transcriptPathFor("inbox/2026-05-21.m4a")).toBe("inbox/2026-05-21.m4a.transcript");
+  });
+
+  test("handles nested paths", () => {
+    expect(transcriptPathFor("a/b/c/voice.webm")).toBe("a/b/c/voice.webm.transcript");
+  });
+});
+
+describe("buildTranscriptNote — success shape", () => {
+  test("frontmatter includes transcript_of, status, attachment id, duration, and provider", () => {
+    const result = buildTranscriptNote({
+      attachmentPath: "inbox/voice.webm",
+      attachmentId: "att-1",
+      attachmentNoteId: "note-1",
+      status: "complete",
+      text: "hello world",
+      provider: "groq",
+      durationMs: 1234,
+      createdAt: new Date("2026-05-21T09:13:42Z"),
+    });
+    expect(result.path).toBe("inbox/voice.webm.transcript");
+    expect(result.content).toBe("hello world");
+    expect(result.tags).toEqual(["transcript", "capture"]);
+    expect(result.metadata.transcript_of).toBe("inbox/voice.webm");
+    expect(result.metadata.transcript_attachment_id).toBe("att-1");
+    expect(result.metadata.transcript_status).toBe("complete");
+    expect(result.metadata.transcript_provider).toBe("groq");
+    expect(result.metadata.transcript_duration_ms).toBe(1234);
+    expect(result.metadata.title).toBe("Transcript of voice.webm");
+    expect(result.createdAt).toBe("2026-05-21T09:13:42.000Z");
+    expect(result.metadata.transcript_error).toBeUndefined();
+  });
+
+  test("provider/duration omitted when not supplied", () => {
+    const result = buildTranscriptNote({
+      attachmentPath: "voice.m4a",
+      attachmentId: "att-x",
+      attachmentNoteId: "note-x",
+      status: "complete",
+      text: "no provider info",
+    });
+    expect(result.metadata.transcript_provider).toBeUndefined();
+    expect(result.metadata.transcript_duration_ms).toBeUndefined();
+  });
+});
+
+describe("buildTranscriptNote — failure shape", () => {
+  test("body is empty + transcript_error captured", () => {
+    const result = buildTranscriptNote({
+      attachmentPath: "inbox/voice.webm",
+      attachmentId: "att-1",
+      attachmentNoteId: "note-1",
+      status: "failed",
+      error: "no transcription provider configured",
+    });
+    expect(result.content).toBe("");
+    expect(result.metadata.transcript_status).toBe("failed");
+    expect(result.metadata.transcript_error).toBe("no transcription provider configured");
+  });
+
+  test("falls back to 'unknown error' when no error string is supplied", () => {
+    const result = buildTranscriptNote({
+      attachmentPath: "voice.m4a",
+      attachmentId: "att-1",
+      attachmentNoteId: "note-1",
+      status: "failed",
+    });
+    expect(result.metadata.transcript_error).toBe("unknown error");
+  });
+});
+
+describe("upsertTranscriptNote", () => {
+  test("creates a new note + link on first call", async () => {
+    const audioOwner = await store.createNote("# Voice memo\n", { id: "owner" });
+    await store.addAttachment(audioOwner.id, "memos/a.webm", "audio/webm");
+
+    const note = await upsertTranscriptNote(store, {
+      attachmentPath: "memos/a.webm",
+      attachmentId: "att-1",
+      attachmentNoteId: audioOwner.id,
+      status: "complete",
+      text: "spoken words",
+      provider: "groq",
+      durationMs: 999,
+    });
+    expect(note.content).toBe("spoken words");
+    expect(note.path).toBe("memos/a.webm.transcript");
+
+    const fetched = await store.getNoteByPath("memos/a.webm.transcript");
+    expect(fetched?.id).toBe(note.id);
+    expect((fetched?.metadata as any)?.transcript_status).toBe("complete");
+    expect(fetched?.tags).toContain("transcript");
+    expect(fetched?.tags).toContain("capture");
+  });
+
+  test("overwrites existing transcript note in place on retry (id preserved)", async () => {
+    const owner = await store.createNote("# Voice memo\n", { id: "owner-2" });
+    await store.addAttachment(owner.id, "memos/b.webm", "audio/webm");
+
+    const first = await upsertTranscriptNote(store, {
+      attachmentPath: "memos/b.webm",
+      attachmentId: "att-2",
+      attachmentNoteId: owner.id,
+      status: "failed",
+      error: "no transcription provider configured",
+    });
+    expect(first.content).toBe("");
+
+    const retried = await upsertTranscriptNote(store, {
+      attachmentPath: "memos/b.webm",
+      attachmentId: "att-2",
+      attachmentNoteId: owner.id,
+      status: "complete",
+      text: "transcript that finally landed",
+      durationMs: 500,
+    });
+    expect(retried.id).toBe(first.id);
+    expect(retried.content).toBe("transcript that finally landed");
+    expect((retried.metadata as any)?.transcript_status).toBe("complete");
+    expect((retried.metadata as any)?.transcript_error).toBeUndefined();
+  });
+
+  test("attempting to upsert when the attachmentNoteId is missing still creates the note (link skipped)", async () => {
+    // No owner note created — the link create will throw or silently skip;
+    // either way the transcript note must still land so the failure has
+    // a visible record. (Defensive: a deleted-attachment race.)
+    const note = await upsertTranscriptNote(store, {
+      attachmentPath: "missing/a.webm",
+      attachmentId: "att-x",
+      attachmentNoteId: "nonexistent-note",
+      status: "failed",
+      error: "audio file not found",
+    });
+    expect(note.path).toBe("missing/a.webm.transcript");
+    expect((note.metadata as any)?.transcript_status).toBe("failed");
+  });
+});

package/src/transcript-note.ts

@@ -0,0 +1,189 @@
+/**
+ * Transcript-note materialization for the vault↔scribe auto-transcribe path
+ * (vault#353, design 2026-05-21 Part 2, design question 3).
+ *
+ * When the transcription worker resolves an audio attachment, it asks this
+ * module to create (or update) a sibling `<attachment-path>.transcript.md`
+ * note. The note's frontmatter links back to the original audio attachment
+ * via `transcript_of`, lets vault graph queries surface the relation, and
+ * captures success / failure state in a single uniform shape.
+ *
+ * Design Q3's exact shape:
+ *
+ *   ---
+ *   title: Transcript of <attachment-name>
+ *   tags: [transcript, capture]
+ *   created_at: <iso>
+ *   transcript_of: <attachment-path>
+ *   transcript_status: complete | failed
+ *   transcript_provider: <name or unset>
+ *   transcript_duration_ms: <wall-clock ms>
+ *   transcript_error: <error-message — failed only>
+ *   ---
+ *
+ *   <transcript text — empty body when failed>
+ *
+ * The `transcript_status` values are uniform: `complete` for success and
+ * `failed` for any terminal failure (provider missing, scribe 5xx, timeout,
+ * etc.). The cause is in `transcript_error`. This matches the design's
+ * "same failure substrate" stance — no first-boot-specific branch, just
+ * different error strings.
+ *
+ * ## Retry semantics
+ *
+ * When the retry endpoint re-runs transcription on an already-failed
+ * transcript note, the same note is updated in place — `transcript_of`,
+ * `path`, and `id` all stay constant. Callers identify the transcript by
+ * note path (`<attachment-path>.transcript.md`) and call `upsertTranscriptNote`
+ * to overwrite. The transcript-of relation is materialized as both a
+ * frontmatter scalar AND a vault link (via `links.add` on create) so graph
+ * queries can find it without parsing frontmatter.
+ */
+
+import type { Store, Note } from "../core/src/types.ts";
+
+export type TranscriptStatus = "complete" | "failed";
+
+export interface TranscriptNoteInput {
+  /**
+   * Path of the source audio attachment (relative to the vault assets dir).
+   * Used to derive the transcript note's path and the `transcript_of`
+   * frontmatter scalar.
+   */
+  attachmentPath: string;
+  /**
+   * Attachment row id of the source audio. Stamped onto the transcript note
+   * frontmatter (`transcript_attachment_id`) so the retry endpoint can find
+   * the original audio in one DB lookup without walking links or paths.
+   */
+  attachmentId: string;
+  /**
+   * The note that owns the audio attachment. The transcript-of relation
+   * (vault link) is established to this note so graph queries can find the
+   * transcript from either side.
+   */
+  attachmentNoteId: string;
+  /** Outcome — `complete` (transcript text available) or `failed`. */
+  status: TranscriptStatus;
+  /**
+   * Transcript body. Required when `status === "complete"`; ignored otherwise.
+   * Failure transcript notes have an empty body — the error string is in
+   * frontmatter, not in the note body, so failures don't read like content.
+   */
+  text?: string;
+  /** Error cause for `status === "failed"`. Required for failed status. */
+  error?: string;
+  /** Scribe-reported provider (e.g. "groq", "whisper"). Optional. */
+  provider?: string;
+  /** Wall-clock duration of the scribe call. Optional; 0 if unknown. */
+  durationMs?: number;
+  /** Created-at override (defaults to `new Date()`). */
+  createdAt?: Date;
+}
+
+/**
+ * Compute the canonical transcript-note path for an audio attachment.
+ *
+ * Example: `inbox/Voice 2026-05-21 09-13.m4a` → `inbox/Voice 2026-05-21 09-13.m4a.transcript`
+ *
+ * The `.md` extension is implicit in the vault path normalization (paths are
+ * stored without `.md` per vault convention; on export to portable markdown,
+ * `.md` is appended).
+ */
+export function transcriptPathFor(attachmentPath: string): string {
+  return `${attachmentPath}.transcript`;
+}
+
+/**
+ * Build the body+metadata for a transcript note from the worker's input.
+ * Pure — no Store calls — so unit tests can assert shape without a DB.
+ * Exposed for tests and for any future caller that wants to materialize a
+ * transcript note independently of the worker.
+ */
+export function buildTranscriptNote(input: TranscriptNoteInput): {
+  path: string;
+  content: string;
+  metadata: Record<string, unknown>;
+  tags: string[];
+  createdAt: string;
+} {
+  const created = (input.createdAt ?? new Date()).toISOString();
+  const filename = input.attachmentPath.split("/").pop() ?? input.attachmentPath;
+  const metadata: Record<string, unknown> = {
+    title: `Transcript of ${filename}`,
+    transcript_of: input.attachmentPath,
+    transcript_attachment_id: input.attachmentId,
+    transcript_status: input.status,
+  };
+  if (input.provider) metadata.transcript_provider = input.provider;
+  if (typeof input.durationMs === "number") {
+    metadata.transcript_duration_ms = input.durationMs;
+  }
+  if (input.status === "failed") {
+    metadata.transcript_error = input.error ?? "unknown error";
+  }
+  const body = input.status === "complete" ? (input.text ?? "") : "";
+  return {
+    path: transcriptPathFor(input.attachmentPath),
+    content: body,
+    metadata,
+    tags: ["transcript", "capture"],
+    createdAt: created,
+  };
+}
+
+/**
+ * Create or update the transcript note at `<attachmentPath>.transcript.md`.
+ *
+ * - If no note exists at that path, creates one with the standard shape,
+ *   establishes a `transcript_of` link to the attachment-owning note, and
+ *   returns the new note.
+ * - If a transcript note already exists (retry path), overwrites its content
+ *   + metadata in place, preserves the existing note id, and returns the
+ *   updated note. Links are NOT re-created — the existing relation survives.
+ *
+ * Errors from the Store (path collision with a non-transcript note, etc.)
+ * bubble up; the worker catches them and logs.
+ */
+export async function upsertTranscriptNote(
+  store: Store,
+  input: TranscriptNoteInput,
+): Promise<Note> {
+  const built = buildTranscriptNote(input);
+  const existing = await store.getNoteByPath(built.path);
+  if (existing) {
+    await store.updateNote(existing.id, {
+      content: built.content,
+      metadata: built.metadata,
+      // Preserve the original created_at; the retry doesn't reset it.
+      skipUpdatedAt: false,
+    });
+    // Re-apply tags via tagNote — updateNote() doesn't accept a tags field
+    // (the engine treats tag mutation as a separate op). tagNote upserts
+    // existing tags (no duplicates), so it's safe to call even when tags
+    // are unchanged from the prior write.
+    await store.tagNote(existing.id, built.tags);
+    const updated = await store.getNote(existing.id);
+    return updated ?? existing;
+  }
+  const created = await store.createNote(built.content, {
+    path: built.path,
+    tags: built.tags,
+    metadata: built.metadata,
+    created_at: built.createdAt,
+  });
+  // Materialize the transcript-of relation as a typed link too, so graph
+  // queries (find-path, neighborhood) surface the transcript without
+  // walking frontmatter. The relation is named "transcript_of" so it
+  // matches the frontmatter scalar; if the target note has been deleted
+  // by the time we get here (race during retry), skip silently — the
+  // frontmatter scalar still carries the relation for query-by-path.
+  try {
+    await store.createLink(created.id, input.attachmentNoteId, "transcript_of");
+  } catch {
+    // Target may be missing or the link constraint may reject — failure
+    // here doesn't invalidate the transcript itself, which is still
+    // queryable via the `transcript_of` frontmatter scalar + path.
+  }
+  return created;
+}

package/src/transcription-registry.ts

@@ -0,0 +1,22 @@
+/**
+ * Process-singleton holder for the active TranscriptionWorker (vault#353).
+ *
+ * Mirrors `mirror-registry.ts`: `server.ts` constructs the worker on boot
+ * (when scribe is discoverable), and the REST retry endpoint
+ * (`/api/notes/:id/retry-transcription`) picks it up here to call `kick()`
+ * for an event-driven re-run. Absent the worker (no scribe), the retry
+ * endpoint still flips the attachment back to `pending` so the sweep would
+ * pick it up — but it'll just sit there until scribe shows up.
+ */
+
+import type { TranscriptionWorker } from "./transcription-worker.ts";
+
+let activeWorker: TranscriptionWorker | null = null;
+
+export function setTranscriptionWorker(worker: TranscriptionWorker | null): void {
+  activeWorker = worker;
+}
+
+export function getTranscriptionWorker(): TranscriptionWorker | null {
+  return activeWorker;
+}