@openparachute/vault 0.7.2 → 0.7.3-rc.10

package/src/embedding-worker.ts

@@ -0,0 +1,226 @@
+/**
+ * Embed-on-write worker (semantic search MVP — EXPERIMENTAL).
+ *
+ * Event-driven happy path + a sweep safety net — structurally the same
+ * shape as `transcription-worker.ts`, simplified (no retry/backoff state
+ * machine; a failed embed just leaves the chunk stale for the next edit
+ * or sweep pass to retry):
+ *
+ *   - **Event path (hot):** a note create/update dispatches the
+ *     `HookRegistry`'s "created"/"updated" event. `registerEmbeddingHook`
+ *     calls `worker.kick(vault, note)`, which chunks the note
+ *     (`chunkNoteContent`), diffs against existing `note_vectors` rows
+ *     (`planStaleness`), prunes anything obsolete (the note shrank, or the
+ *     model changed), and embeds ONLY the stale chunks. A no-op edit
+ *     (identical content, same model) makes ZERO provider calls — the
+ *     freshness gate short-circuits before the network/CPU work.
+ *   - **Sweep path (backfill + safety net):** every `sweepIntervalMs`,
+ *     walks each vault's notes lacking a fresh vector for the active
+ *     model (`getNotesPendingEmbedding`) and embeds them. This is what
+ *     actually drains the v27 migration's implicit backfill (the
+ *     migration itself does no embedding — see `schema.ts`'s
+ *     `migrateToV27` doc comment) and catches anything a dropped hook
+ *     dispatch or a mid-backfill restart left behind.
+ *
+ * Concurrency: the event path rides `HookRegistry`'s own
+ * `HOOK_CONCURRENCY` cap for free (the handler IS the capped unit of
+ * work). The sweep processes its candidate list SEQUENTIALLY — simple,
+ * safe, and matches the real per-note embed latency the P0 eval measured
+ * (roughly 1–5 notes/sec depending on model); a bulk backfill of a few
+ * thousand notes finishes in minutes, not hours, and never spawns
+ * unbounded parallel provider calls.
+ */
+
+import type { Store, Note } from "../core/src/types.ts";
+import type { HookRegistry, NoteHookPayload } from "../core/src/hooks.ts";
+import type { EmbeddingProvider } from "../core/src/embedding/provider.ts";
+import { chunkNoteContent } from "../core/src/embedding/chunker.ts";
+import { planStaleness, contentHash } from "../core/src/embedding/staleness.ts";
+import { getNoteVectorRows, deleteObsoleteVectorRows, upsertNoteVector, getNotesPendingEmbedding } from "../core/src/embedding/vectors.ts";
+import { normalize } from "../core/src/embedding/vector-codec.ts";
+
+/** Mirrors transcription-worker's safety-net cadence. */
+const DEFAULT_SWEEP_MS = 30_000;
+
+export interface EmbeddingWorkerOpts {
+  /**
+   * `undefined` when `EMBEDDINGS_ENABLED=false` (the off switch — see
+   * `src/embedding/select.ts`). The worker is still constructed and
+   * started either way (simpler wiring in `server.ts`, always one
+   * lifecycle to manage) — `embedNote`/`sweepOnce` just no-op when there's
+   * no provider to invoke, which is the entire "drain no-ops" contract.
+   */
+  provider: EmbeddingProvider | undefined;
+  /** Every vault name currently open (mirrors the transcription worker's `vaultList`). */
+  vaultList: () => string[];
+  /** Resolve (or open) a vault's `Store` by name. */
+  getStore: (name: string) => Store;
+  /** Sweep cadence (ms). Default 30s. */
+  sweepIntervalMs?: number;
+  logger?: { error: (...args: unknown[]) => void; log?: (...args: unknown[]) => void };
+}
+
+export class EmbeddingWorker {
+  private readonly provider: EmbeddingProvider | undefined;
+  private readonly vaultList: () => string[];
+  private readonly getStore: (name: string) => Store;
+  private readonly sweepIntervalMs: number;
+  private readonly logger: { error: (...args: unknown[]) => void; log?: (...args: unknown[]) => void };
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private sweeping = false;
+
+  constructor(opts: EmbeddingWorkerOpts) {
+    this.provider = opts.provider;
+    this.vaultList = opts.vaultList;
+    this.getStore = opts.getStore;
+    this.sweepIntervalMs = opts.sweepIntervalMs ?? DEFAULT_SWEEP_MS;
+    this.logger = opts.logger ?? console;
+  }
+
+  /**
+   * (Re-)embed one note's stale chunks. Cheap-first: chunking + the
+   * staleness diff are pure CPU, no provider call; obsolete-row pruning is
+   * a plain DELETE. Only when at least one chunk is genuinely stale does
+   * this reach the provider — and if the provider reports itself
+   * unavailable, this returns without erroring (the chunk stays stale;
+   * the next edit or sweep pass retries once a provider is configured).
+   *
+   * No provider at all (`EMBEDDINGS_ENABLED=false`) → no-op, immediately —
+   * the drain contract's off switch (see `EmbeddingWorkerOpts.provider`).
+   *
+   * Empty-content chunks (a blank note, or a note whose content is only
+   * whitespace) are filtered out of the staleness plan before it reaches
+   * the provider — an empty chunk is never meaningfully embeddable, and
+   * without this filter a blank note would re-enter `getNotesPendingEmbedding`
+   * every sweep forever (nothing to write makes it "pending" again next
+   * pass), hammering a provider that may reject empty input outright.
+   */
+  async embedNote(store: Store, note: Note): Promise<void> {
+    if (!this.provider) return; // off switch — see EmbeddingWorkerOpts.provider
+    if (typeof note.content !== "string") return;
+    const db = store.db;
+    const chunks = chunkNoteContent(note.content).filter((c) => c.text.trim().length > 0);
+    const existing = getNoteVectorRows(db, note.id);
+    const plan = planStaleness(existing, chunks, this.provider.model);
+
+    // Cheap sync write — prunes ghost chunks from a shrunk note or a model
+    // change BEFORE any embed call, so a concurrent semantic search never
+    // ranks against stale content. See core/src/embedding/staleness.ts.
+    // An empty note's chunks were just filtered to [] above, so this also
+    // prunes any leftover rows from before the note went empty.
+    deleteObsoleteVectorRows(db, note.id, plan.obsoleteIxs);
+
+    if (plan.stale.length === 0) return; // no-op edit (or nothing embeddable) — zero provider calls
+
+    const avail = await this.provider.available();
+    if (!avail.ok) return; // not configured/ready — leave stale, retried later
+
+    try {
+      const { vectors } = await this.provider.embed({ texts: plan.stale.map((c) => c.text) });
+      for (let i = 0; i < plan.stale.length; i++) {
+        const chunk = plan.stale[i]!;
+        const vector = vectors[i];
+        if (!vector) continue; // defensive: provider returned fewer vectors than requested
+        upsertNoteVector(db, note.id, chunk, normalize(vector), this.provider.model, contentHash(chunk.text));
+      }
+    } catch (err) {
+      this.logger.error(`[embedding] embed failed for note ${note.id}:`, err);
+      // Leave stale — idempotent: the next edit (or the sweep) re-derives
+      // the same plan and retries.
+    }
+  }
+
+  /** Event-path entry point — called from the registered `onNote` hook. */
+  async kick(vaultName: string, note: Note): Promise<void> {
+    const store = this.getStore(vaultName);
+    await this.embedNote(store, note);
+  }
+
+  /**
+   * One backfill/safety-net pass across every open vault. Returns counts
+   * for caller logging. Re-entrancy guarded (`sweeping`) — a slow pass
+   * (a big vault, a slow provider) never overlaps with the next timer tick.
+   * No provider at all → no-op immediately (same off switch as `embedNote`).
+   *
+   * L1 (reviewer): no per-note lock between this sweep and a concurrent
+   * `embedNote` triggered by the event-path hook for the SAME note — by
+   * design, matching `HookRegistry`'s own documented "no per-note
+   * serialization, last write wins" contract. Worst case is a transient
+   * window where a stored vector reflects a slightly-stale version; it
+   * self-heals on the NEXT edit (or the next sweep pass), since
+   * `planStaleness` re-diffs against the note's CURRENT content_hash every
+   * time, never trusting a prior run's result.
+   */
+  async sweepOnce(): Promise<{ processed: number; vaults: number }> {
+    if (!this.provider) return { processed: 0, vaults: 0 }; // off switch
+    if (this.sweeping) return { processed: 0, vaults: 0 };
+    this.sweeping = true;
+    try {
+      let processed = 0;
+      const vaults = this.vaultList();
+      for (const vaultName of vaults) {
+        const store = this.getStore(vaultName);
+        const pending = getNotesPendingEmbedding(store.db, this.provider.model);
+        for (const row of pending) {
+          const note = await store.getNote(row.id);
+          if (!note) continue; // deleted between the scan and now
+          await this.embedNote(store, note);
+          processed++;
+        }
+      }
+      return { processed, vaults: vaults.length };
+    } finally {
+      this.sweeping = false;
+    }
+  }
+
+  /** Start the recurring sweep. No-op if already started. */
+  start(): void {
+    if (this.timer) return;
+    this.timer = setInterval(() => {
+      void this.sweepOnce().catch((err) => this.logger.error("[embedding] sweep failed:", err));
+    }, this.sweepIntervalMs);
+    this.timer.unref?.();
+  }
+
+  /** Stop the recurring sweep. */
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+  }
+}
+
+/**
+ * Register the embed-on-write hook. No `when` predicate (unlike
+ * `registerTranscriptionHook`'s `transcribe_status === "pending"` check)
+ * — a `NoteHook` predicate only receives the note payload, not the store,
+ * so it has no way to cheaply consult `note_vectors`. Instead every
+ * create/update dispatches the handler, and `embedNote`'s OWN
+ * content-hash diff (checked first, before any provider call) is what
+ * makes a no-op edit cost nothing — the same end result, just checked one
+ * layer in. `HookRegistry`'s concurrency cap still bounds how many of
+ * these run at once.
+ */
+export function registerEmbeddingHook(
+  registry: HookRegistry,
+  worker: EmbeddingWorker,
+  resolveVault: (store: Store) => string | undefined,
+  logger: { error: (...args: unknown[]) => void } = console,
+): () => void {
+  return registry.onNote({
+    name: "embedding-on-write",
+    event: ["created", "updated"],
+    handler: async (payload: NoteHookPayload, store: Store) => {
+      const vault = resolveVault(store);
+      if (!vault) {
+        logger.error(
+          `[embedding] could not resolve vault for note ${(payload as Note).id}; sweep will pick it up`,
+        );
+        return;
+      }
+      await worker.kick(vault, payload as Note);
+    },
+  });
+}

package/src/mcp-http.ts

@@ -109,7 +109,7 @@ export async function handleScopedMcp(
   const instruction = await getServerInstruction(vaultName, auth);
   return handleMcp(
     req,
-    () => generateScopedMcpTools(vaultName, auth, callerBearer ?? null),
+    () => generateScopedMcpTools(vaultName, auth, callerBearer ?? null, req),
     `parachute-vault/${vaultName}`,
     vaultName,
     auth,
@@ -229,6 +229,8 @@ export async function handleMcp(
         tag?: string;
         cycle?: unknown;
         referencing_tags?: unknown;
+        /** Attachment-tickets design (§2c "errors as JIT docs") — a short, imperative next-step, distinct from the older free-form `hint`. */
+        how_to?: string;
       };
       // Honest-queries validation errors (vault#550) — `limit`/`offset`/date
       // values that are structurally invalid rather than merely "no
@@ -399,6 +401,16 @@ export async function handleMcp(
           error_type: e.error_type,
           field: e.field,
           hint: e.hint,
+          // Forwarded when present (undefined keys are fine — JSON-RPC
+          // `data` just omits them) so a `structuredError()` call site that
+          // stamps extra fields — `limit`/`got`/`extension` (size/type
+          // refusals) or `how_to` (attachment-tickets' JIT-docs field, §2c)
+          // — isn't silently truncated down to error_type/field/hint by
+          // this backstop.
+          ...(e.limit !== undefined ? { limit: e.limit } : {}),
+          ...(e.got !== undefined ? { got: e.got } : {}),
+          ...(e.extension !== undefined ? { extension: e.extension } : {}),
+          ...(e.how_to !== undefined ? { how_to: e.how_to } : {}),
         });
       }
       return {

package/src/mcp-tools.ts

@@ -43,6 +43,8 @@ import {
 import { chooseHubOrigin, mintHubJwt, revokeHubJwt } from "./mcp-install.ts";
 import { looksLikeJwt } from "./hub-jwt.ts";
 import { readGlobalConfig, DEFAULT_PORT } from "./config.ts";
+import { getBaseUrl } from "./oauth-discovery.ts";
+import { getSharedAttachmentTicketProvider } from "./attachment-tickets.ts";
 
 /**
  * Filter a vault projection to entries an in-scope tag contributes to.
@@ -110,6 +112,10 @@ export async function getServerInstruction(
     description: config?.description ?? null,
     projection,
     coordinates: resolveVaultCoordinates(),
+    // Bun always wires an in-process AttachmentTicketProvider (see
+    // `generateScopedMcpTools` below) — the connect-time brief can
+    // unconditionally teach the ticket tools on this door.
+    attachments: { ticketsEnabled: true },
   });
 }
 
@@ -143,6 +149,16 @@ export function generateScopedMcpTools(
   vaultName: string,
   auth?: AuthResult,
   callerBearer?: string | null,
+  /**
+   * The incoming MCP request, when available (omitted by the many
+   * test-only call sites that construct tools without a live request).
+   * Threaded through ONLY so the attachment-ticket tools can resolve a
+   * request-accurate `urlBase` (honoring `X-Forwarded-Host`/proto, same as
+   * `getBaseUrl`) — falls back to `resolveVaultCoordinates()`'s configured/
+   * expose-state origin when omitted, so ticket tools are still present
+   * (just less precisely origined) in that case.
+   */
+  req?: Request,
 ): McpToolDef[] {
   const store = getVaultStore(vaultName);
 
@@ -225,20 +241,39 @@ export function generateScopedMcpTools(
     ? (info) => logStrictBypass(info)
     : undefined;
 
-  const tools = generateMcpTools(
-    store,
-    expandVisibility || nearTraversable || ifExistsVisible || aggregateVisibility || writeContext || strictBypass
-      ? {
-          ...(expandVisibility ? { expandVisibility } : {}),
-          ...(nearTraversable ? { nearTraversable } : {}),
-          ...(ifExistsVisible ? { ifExistsVisible } : {}),
-          ...(aggregateVisibility ? { aggregateVisibility } : {}),
-          ...(writeContext ? { writeContext } : {}),
-          ...(strictBypass ? { strictBypass } : {}),
-          ...(onStrictBypass ? { onStrictBypass } : {}),
-        }
-      : undefined,
-  );
+  // Attachment tickets (Wave 1): bun always wires the in-process provider
+  // (see src/attachment-tickets.ts) — unlike the tag-scope predicates
+  // above, this isn't conditional on `scoped` because the tools should be
+  // listed for every session, scoped or not. Tag-scope confidentiality is
+  // still enforced (`noteVisible`, awaited inline inside the ticket tools'
+  // own async execute — see its doc comment in generateMcpTools for why
+  // this doesn't need the shared allowedHolder machinery the OTHER
+  // predicates above do).
+  const ticketNoteVisible = scoped
+    ? async (note: Note) => {
+        const allowed = await expandTokenTagScope(store, rawTags);
+        return noteWithinTagScope(note, allowed, rawTags);
+      }
+    : undefined;
+  const ticketUrlBase = req
+    ? `${getBaseUrl(req).replace(/\/$/, "")}/vault/${vaultName}`
+    : `${resolveVaultCoordinates().hubOrigin.replace(/\/$/, "")}/vault/${vaultName}`;
+
+  const tools = generateMcpTools(store, {
+    ...(expandVisibility ? { expandVisibility } : {}),
+    ...(nearTraversable ? { nearTraversable } : {}),
+    ...(ifExistsVisible ? { ifExistsVisible } : {}),
+    ...(aggregateVisibility ? { aggregateVisibility } : {}),
+    ...(writeContext ? { writeContext } : {}),
+    ...(strictBypass ? { strictBypass } : {}),
+    ...(onStrictBypass ? { onStrictBypass } : {}),
+    attachmentTickets: {
+      provider: getSharedAttachmentTicketProvider(),
+      vaultName,
+      urlBase: ticketUrlBase,
+      ...(ticketNoteVisible ? { noteVisible: ticketNoteVisible } : {}),
+    },
+  });
 
   overrideVaultInfo(tools, vaultName, auth);
   applyTagDependencyGuards(tools, vaultName);

package/src/onboarding-seed.test.ts

@@ -23,14 +23,17 @@ import { BunStore } from "./vault-store.ts";
 import { seedOnboardingNotes } from "./onboarding-seed.ts";
 import {
   applySeedPack,
+  ARCHIVED_TAG,
   CAPTURE_ANYTHING_PATH,
   CONNECT_AI_PATH,
   GETTING_STARTED_PATH,
   NOTES_REQUIRED_TAGS,
+  STARTER_ONTOLOGY_PACK,
   SURFACE_STARTER_PACK,
   SURFACE_STARTER_PATH,
   TAGS_GRAPH_PATH,
   WELCOME_PATH,
+  welcomePack,
   YOURS_TO_KEEP_PATH,
 } from "../core/src/seed-packs.ts";
 import {
@@ -290,6 +293,67 @@ describe("applySeedPack — surface-starter via add-pack", () => {
   });
 });
 
+describe("applySeedPack — tag description preservation on re-apply (Aaron-ratified 2026-07-17)", () => {
+  test("fresh apply: a brand-new tag gets the pack's description, reported as touched not preserved", async () => {
+    const result = await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    expect(result.tags).toContain(ARCHIVED_TAG.name);
+    expect(result.preservedTagDescriptions).toEqual([]);
+
+    const record = await store.getTagRecord(ARCHIVED_TAG.name);
+    expect(record!.description).toBe(ARCHIVED_TAG.description);
+  });
+
+  test("re-apply unmodified: description is still the pack's text — upserted, not preserved", async () => {
+    await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    const rerun = await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    expect(rerun.preservedTagDescriptions).toEqual([]);
+
+    const record = await store.getTagRecord(ARCHIVED_TAG.name);
+    expect(record!.description).toBe(ARCHIVED_TAG.description);
+  });
+
+  test("re-apply after a user edit: the hand-tuned description survives and is reported preserved", async () => {
+    await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    const edited = "My own house rules for archiving — nothing like the default.";
+    await store.upsertTagRecord(ARCHIVED_TAG.name, { description: edited });
+
+    const rerun = await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    expect(rerun.preservedTagDescriptions).toContain(ARCHIVED_TAG.name);
+    // Still listed in `tags` — the tag itself was touched (fields/parent_names
+    // upsert unconditionally); only the description write was skipped.
+    expect(rerun.tags).toContain(ARCHIVED_TAG.name);
+
+    const record = await store.getTagRecord(ARCHIVED_TAG.name);
+    expect(record!.description).toBe(edited);
+  });
+
+  test("capture byte-identity anchor (NOTES_REQUIRED_TAGS) holds under either apply order", async () => {
+    // welcome, then starter-ontology.
+    await applySeedPack(store, welcomePack());
+    expect((await store.getTagRecord("capture"))!.description).toBe(
+      NOTES_REQUIRED_TAGS[0]!.description,
+    );
+    const afterOntology = await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    expect(afterOntology.preservedTagDescriptions).not.toContain("capture");
+    expect((await store.getTagRecord("capture"))!.description).toBe(
+      NOTES_REQUIRED_TAGS[0]!.description,
+    );
+  });
+
+  test("capture byte-identity anchor holds in the reverse order too", async () => {
+    // starter-ontology, then welcome.
+    await applySeedPack(store, STARTER_ONTOLOGY_PACK);
+    expect((await store.getTagRecord("capture"))!.description).toBe(
+      NOTES_REQUIRED_TAGS[0]!.description,
+    );
+    const afterWelcome = await applySeedPack(store, welcomePack());
+    expect(afterWelcome.preservedTagDescriptions).not.toContain("capture");
+    expect((await store.getTagRecord("capture"))!.description).toBe(
+      NOTES_REQUIRED_TAGS[0]!.description,
+    );
+  });
+});
+
 describe("vault-info / projection pointer (A2)", () => {
   test("projection carries getting_started when the note exists", async () => {
     // Absent before seeding → no pointer.