@openparachute/vault 0.7.3-rc.9 → 0.7.4-rc.1

package/src/embeddings-routes.ts

@@ -0,0 +1,178 @@
+/**
+ * HTTP surface for the semantic-search (embeddings) opt-in toggle.
+ *
+ *   GET  /vault/<name>/.parachute/embeddings — read the persisted setting +
+ *                                              the running/effective/env state
+ *   PUT  /vault/<name>/.parachute/embeddings — flip the persisted setting
+ *
+ * The 0.7.3 fast-follow: 0.7.3 shipped the persisted `embeddings_enabled`
+ * config.yaml setting + `resolveEmbeddingsEnabled` (env override → persisted
+ * → off), but the ONLY way to flip it was hand-editing config.yaml or setting
+ * an env var. This module gives the vault admin SPA a real toggle over that
+ * setting.
+ *
+ * URL note: same reasoning as `mirror-routes.ts` — the admin SPA's static
+ * bundle owns `/vault/<name>/admin/*` (vault#252), so the API surface lives
+ * under the `.parachute/` module-protocol namespace (sibling to
+ * `.parachute/config`, `.parachute/mirror`, `.parachute/usage`).
+ *
+ * Auth: `vault:admin`, gated upstream in `routing.ts` (this module is the
+ * after-auth handler). `embeddings_enabled` is host-global config that affects
+ * every vault on the server, but it's reached through the same per-vault admin
+ * surface every other settings page uses; the UI copy names the host-wide
+ * scope.
+ *
+ * **Activation is restart-to-apply, not hot.** The embedding provider is
+ * resolved ONCE at boot (`getSharedEmbeddingProvider`) and captured into every
+ * open store (`Store.embeddingProvider` + `embeddingDisabledReason`, baked at
+ * construction) and into the embedding worker (`EmbeddingWorker.provider`, a
+ * private readonly field). Re-resolving the provider mid-run would mean
+ * mutating readonly fields across the worker AND every already-open store AND
+ * resetting a deliberately-once module memo — not clean. So this endpoint
+ * PERSISTS the setting and reports `restart_required` when the persisted
+ * change hasn't taken effect in the running process yet. Hot-reconfigure is a
+ * possible follow-up.
+ *
+ * **The `EMBEDDINGS_ENABLED` env var still wins.** It's the low-level override
+ * (`true`/`1` on, `false`/`0` off, else defer). When it's forcing a value the
+ * snapshot carries `env_override` so the UI can flag that the persisted toggle
+ * is advisory until the env var is removed — the toggle never lies about
+ * what's actually in force.
+ */
+
+import { readGlobalConfig, writeGlobalConfig } from "./config.ts";
+import { embeddingsEnabledEnvOverride, resolveEmbeddingsEnabled } from "./embedding/select.ts";
+import { isSharedEmbeddingProviderActive } from "./vault-store.ts";
+
+/**
+ * Approximate size of the bundled ONNX model (`bge-small-en-v1.5`, q8) that
+ * lazy-downloads on the FIRST embed after enabling. Surfaced so the UI can
+ * warn before the operator flips the switch. (The npm dependency ships ~270MB
+ * of runtime, but the model weights fetched on first use are ~34MB.)
+ */
+export const EMBEDDING_MODEL_DOWNLOAD_MB = 34;
+
+const CORS = { "Access-Control-Allow-Origin": "*" } as const;
+
+/**
+ * The wire shape the admin SPA reads/writes. Both GET and PUT return this so
+ * the SPA reuses one decoder.
+ */
+export interface EmbeddingsSettingsSnapshot {
+  /**
+   * The persisted `embeddings_enabled` in config.yaml — the value the toggle
+   * reflects. `false` when unset (semantic search is opt-in, default off).
+   */
+  enabled: boolean;
+  /**
+   * The `EMBEDDINGS_ENABLED` env override, tri-state: `true` (forced on),
+   * `false` (forced off), or `null` (no env var — defers to `enabled`). When
+   * non-null the env var wins over the persisted setting.
+   */
+  env_override: boolean | null;
+  /**
+   * Convenience flag: `true` exactly when `env_override` is non-null. The UI
+   * shows an advisory banner ("an env var is forcing this") when set.
+   */
+  env_forced: boolean;
+  /**
+   * What a fresh boot would resolve: env override, else persisted, else off.
+   * This is the state the running process will land in on next restart.
+   */
+  effective: boolean;
+  /**
+   * Whether semantic search is LIVE in the running process right now (the
+   * boot-resolved provider is present). Distinct from `effective`: a flip of
+   * the persisted setting changes `effective` immediately but not `active`.
+   */
+  active: boolean;
+  /**
+   * `true` when `active !== effective` — the operator's intended state is
+   * persisted but the running process hasn't picked it up. Restart to apply.
+   */
+  restart_required: boolean;
+  /** First-enable model-download size hint (MB) for the UI copy. */
+  model_download_mb: number;
+}
+
+/**
+ * Build the snapshot from the persisted setting + env + running state. Pure
+ * over its inputs (env + persisted + active) so it's unit-testable without a
+ * live server; the `active` reading is injected (defaults to the real
+ * process-shared provider state).
+ */
+export function buildEmbeddingsSnapshot(opts?: {
+  env?: NodeJS.ProcessEnv;
+  persistedEnabled?: boolean;
+  active?: boolean;
+}): EmbeddingsSettingsSnapshot {
+  const env = opts?.env ?? process.env;
+  const persisted = opts?.persistedEnabled ?? false;
+  const active = opts?.active ?? isSharedEmbeddingProviderActive();
+  const override = embeddingsEnabledEnvOverride(env);
+  const effective = resolveEmbeddingsEnabled(env, persisted);
+  return {
+    enabled: persisted,
+    env_override: override ?? null,
+    env_forced: override !== undefined,
+    effective,
+    active,
+    restart_required: active !== effective,
+    model_download_mb: EMBEDDING_MODEL_DOWNLOAD_MB,
+  };
+}
+
+/**
+ * `GET /vault/<name>/.parachute/embeddings` — return the current settings
+ * snapshot. Always 200 (auth enforced upstream). `activeOverride` is a test
+ * seam; production passes nothing and the running provider state is read.
+ */
+export function handleEmbeddingsGet(activeOverride?: boolean): Response {
+  const persisted = readGlobalConfig().embeddings_enabled;
+  const snapshot = buildEmbeddingsSnapshot({ persistedEnabled: persisted, active: activeOverride });
+  return Response.json(snapshot, { headers: CORS });
+}
+
+/**
+ * `PUT /vault/<name>/.parachute/embeddings` — persist a new
+ * `embeddings_enabled` value. Body: `{ "enabled": boolean }`.
+ *
+ * Reuses the config write path (`writeGlobalConfig` — never hand-rolls YAML),
+ * so the serialize logic that already knows how to persist `embeddings_enabled`
+ * is the single source of truth. Persists even when the env var is forcing a
+ * value (the operator's persisted intent is recorded; the env override still
+ * wins for `effective`/`active`, and the snapshot says so). Returns the same
+ * shape as GET, reflecting the new persisted value + the unchanged running
+ * state (so `restart_required` is honest right after the write).
+ */
+export async function handleEmbeddingsPut(req: Request, activeOverride?: boolean): Promise<Response> {
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch (err) {
+    return Response.json(
+      { error: "Invalid JSON body", message: (err as Error).message ?? String(err) },
+      { status: 400 },
+    );
+  }
+  if (typeof body !== "object" || body === null || Array.isArray(body)) {
+    return Response.json(
+      { error: "Invalid body", field: "enabled", message: "Expected a JSON object { enabled: boolean }." },
+      { status: 400 },
+    );
+  }
+  const enabled = (body as { enabled?: unknown }).enabled;
+  if (typeof enabled !== "boolean") {
+    return Response.json(
+      { error: "Invalid body", field: "enabled", message: "`enabled` must be a boolean." },
+      { status: 400 },
+    );
+  }
+
+  const config = readGlobalConfig();
+  config.embeddings_enabled = enabled;
+  writeGlobalConfig(config);
+
+  const snapshot = buildEmbeddingsSnapshot({ persistedEnabled: enabled, active: activeOverride });
+  return Response.json(snapshot, { headers: CORS });
+}

package/src/live-frame-parity.test.ts

@@ -98,6 +98,18 @@ describe("buildSnapshotFrames — chunking + done flag", () => {
     for (const f of frames) expect(new TextEncoder().encode(f).byteLength).toBeLessThan(1_000_000);
     expect(frames.flatMap((f) => JSON.parse(f).notes).length).toBe(8);
   });
+
+  it("is shape-agnostic — frames a lean NoteIndex entry verbatim", () => {
+    // A lean subscription hands `toNoteIndex`-projected entries; the framer
+    // serializes them byte-for-byte, no content field re-added.
+    const lean = { id: "x", byteSize: 3, preview: "abc", displayTitle: "abc", tags: ["chat"], metadata: {} };
+    const frames = buildSnapshotFrames([lean as any]);
+    expect(frames.length).toBe(1);
+    const f = JSON.parse(frames[0]!);
+    expect(f.done).toBe(true);
+    expect(f.notes).toEqual([lean]);
+    expect(f.notes[0].content).toBeUndefined();
+  });
 });
 
 describe("parseClientMessage", () => {
@@ -171,6 +183,15 @@ describe("validateWsSubscribeQuery — same rejects as the SSE route (byte-ident
     const v = validateWsSubscribeQuery(new URL("http://x/vault/v/api/subscribe?tag=chat&path_prefix=meetings/"));
     expect("queryOpts" in v).toBe(true);
   });
+  it("resolves include_content — default TRUE (full), `false`/`0` → lean, `true`/`1` → full", () => {
+    const at = (q: string) =>
+      validateWsSubscribeQuery(new URL(`http://x/vault/v/api/subscribe?tag=chat${q}`)) as { includeContent: boolean };
+    expect(at("").includeContent).toBe(true); // absent → full (byte-unchanged default)
+    expect(at("&include_content=false").includeContent).toBe(false); // opt into lean
+    expect(at("&include_content=0").includeContent).toBe(false);
+    expect(at("&include_content=true").includeContent).toBe(true);
+    expect(at("&include_content=1").includeContent).toBe(true);
+  });
   it("shares the SSE route's queryOpts-level guard (cursor/has_links/date filters)", () => {
     // The belt-and-suspenders layer both doors + the SSE route route through:
     // a date filter isn't expressible as a flat URL param (removed 0.6.4), but

package/src/mcp-http.ts

@@ -180,9 +180,15 @@ export async function handleMcp(
     }
     try {
       const result = await tool.execute((args ?? {}) as Record<string, unknown>);
-      return {
-        content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }],
-      };
+      // The one wrapper change (attachments-for-agents design, Wave 2):
+      // `resultContent`, when a tool defines it, decides the MCP content
+      // blocks instead of the universal single-text-block default —
+      // `read-attachment`'s image branch is the only current user (needs a
+      // REAL {type:"image"} block alongside the row-JSON text block).
+      const content = tool.resultContent
+        ? tool.resultContent(result)
+        : [{ type: "text" as const, text: JSON.stringify(result, null, 2) }];
+      return { content };
     } catch (err) {
       // vault#555 fix 6 — never re-wrap an already-formed McpError. Passing
       // the SAME instance straight through is strictly correct (no
@@ -231,6 +237,12 @@ export async function handleMcp(
         referencing_tags?: unknown;
         /** Attachment-tickets design (§2c "errors as JIT docs") — a short, imperative next-step, distinct from the older free-form `hint`. */
         how_to?: string;
+        /** `read-attachment` (Wave 2) refusals — `image_too_large` / `unsupported_attachment_type` carry the attachment's actual byte size. */
+        size?: number;
+        /** `read-attachment` `image_too_large` — the 4 MiB cap it exceeded. */
+        max_bytes?: number;
+        /** `read-attachment` `unsupported_attachment_type` — the mime type that couldn't be read. */
+        mime_type?: string;
       };
       // Honest-queries validation errors (vault#550) — `limit`/`offset`/date
       // values that are structurally invalid rather than merely "no
@@ -411,6 +423,11 @@ export async function handleMcp(
           ...(e.got !== undefined ? { got: e.got } : {}),
           ...(e.extension !== undefined ? { extension: e.extension } : {}),
           ...(e.how_to !== undefined ? { how_to: e.how_to } : {}),
+          // read-attachment (Wave 2) refusal fields — same forward-when-present
+          // discipline as the ticket fields just above.
+          ...(e.size !== undefined ? { size: e.size } : {}),
+          ...(e.max_bytes !== undefined ? { max_bytes: e.max_bytes } : {}),
+          ...(e.mime_type !== undefined ? { mime_type: e.mime_type } : {}),
         });
       }
       return {

package/src/mcp-tools.ts

@@ -45,6 +45,7 @@ import { looksLikeJwt } from "./hub-jwt.ts";
 import { readGlobalConfig, DEFAULT_PORT } from "./config.ts";
 import { getBaseUrl } from "./oauth-discovery.ts";
 import { getSharedAttachmentTicketProvider } from "./attachment-tickets.ts";
+import { createFsAttachmentBytesProvider } from "./attachment-bytes.ts";
 
 /**
  * Filter a vault projection to entries an in-scope tag contributes to.
@@ -112,10 +113,12 @@ export async function getServerInstruction(
     description: config?.description ?? null,
     projection,
     coordinates: resolveVaultCoordinates(),
-    // Bun always wires an in-process AttachmentTicketProvider (see
+    // Bun always wires an in-process AttachmentTicketProvider AND a fresh
+    // fs-backed AttachmentBytesProvider per session (see
     // `generateScopedMcpTools` below) — the connect-time brief can
-    // unconditionally teach the ticket tools on this door.
-    attachments: { ticketsEnabled: true },
+    // unconditionally teach both the ticket tools and read-attachment on
+    // this door.
+    attachments: { ticketsEnabled: true, readEnabled: true },
   });
 }
 
@@ -273,6 +276,15 @@ export function generateScopedMcpTools(
       urlBase: ticketUrlBase,
       ...(ticketNoteVisible ? { noteVisible: ticketNoteVisible } : {}),
     },
+    // Attachment bytes (Wave 2 model lane): bun always wires a fresh fs
+    // provider per session — cheap (stateless), unlike the ticket
+    // provider's process-wide shared Map. Same `ticketNoteVisible`
+    // tag-scope predicate as the ticket seam above (identical contract:
+    // "is the owning note in scope").
+    attachmentBytes: {
+      provider: createFsAttachmentBytesProvider(vaultName),
+      ...(ticketNoteVisible ? { noteVisible: ticketNoteVisible } : {}),
+    },
   });
 
   overrideVaultInfo(tools, vaultName, auth);

package/src/oauth-discovery.ts

@@ -27,6 +27,7 @@
  */
 
 import { getHubOrigin } from "./hub-jwt.ts";
+import { SCOPE_READ, SCOPE_WRITE } from "./scopes.ts";
 
 /**
  * OAuth scopes vault publishes through discovery, RESOURCE-NARROWED to the
@@ -87,6 +88,36 @@ export function handleProtectedResource(req: Request, vaultName: string): Respon
   });
 }
 
+/**
+ * Protected Resource Metadata (RFC 9728) for the CANONICAL ROOT `/mcp`
+ * endpoint (U1). Same document shape as `handleProtectedResource`, with two
+ * deliberate differences that follow from the root being vault-AGNOSTIC (the
+ * vault is derived from the token, not the URL):
+ *
+ *   - `resource` is the origin-root `<base>/mcp`, not a `/vault/<name>/mcp`.
+ *   - `scopes_supported` advertises the UN-NARROWED forms `vault:read` /
+ *     `vault:write` (there's no vault name to narrow to here). A spec-following
+ *     client reads these and requests the broad forms; the hub's consent picker
+ *     narrows the grant to a chosen vault at authorization time (that path
+ *     already exists hub-side), minting a token stamped with a
+ *     `vault:<name>:<verb>` scope + `aud=vault.<name>` — exactly what the root
+ *     endpoint derives the target vault from. `admin` is intentionally omitted:
+ *     the interactive connect flow grants read/write; admin is an operator
+ *     concern, not something the consent picker offers.
+ *
+ * Served at the RFC 9728 §3.1 path-insertion location for the root resource:
+ * `/.well-known/oauth-protected-resource/mcp`.
+ */
+export function handleRootProtectedResource(req: Request): Response {
+  const base = getBaseUrl(req);
+  return Response.json({
+    resource: `${base}/mcp`,
+    authorization_servers: [getHubOrigin()],
+    scopes_supported: [SCOPE_READ, SCOPE_WRITE],
+    bearer_methods_supported: ["header"],
+  });
+}
+
 /**
  * OAuth 2.0 Authorization Server Metadata (RFC 8414).
  *