opencode-episodic-memory 0.1.3 → 0.3.0

package/src/embed-sidecar.mjs

@@ -0,0 +1,110 @@
+// Persistent Node-side embedding server. stdout is reserved for NDJSON protocol
+// messages; all diagnostics (including dependency chatter) go to stderr.
+const originalConsole = globalThis.console;
+globalThis.console = {
+  ...originalConsole,
+  log: (...args) => originalConsole.error(...args),
+  info: (...args) => originalConsole.error(...args),
+  debug: (...args) => originalConsole.error(...args),
+  warn: (...args) => originalConsole.error(...args),
+};
+
+// Keep this fallback synchronized with DEFAULT_MODEL in embed.ts; embed.test.ts
+// guards against accidental drift between the Bun host and Node sidecar.
+const model = process.env.EPISODIC_EMBED_MODEL ?? "Snowflake/snowflake-arctic-embed-m-v1.5";
+const MAX_REQUEST_TEXTS = 64;
+const batchSize = positiveIntegerEnv("EPISODIC_EMBED_BATCH_SIZE", 32, MAX_REQUEST_TEXTS);
+let embedder;
+let queue = Promise.resolve();
+
+function positiveIntegerEnv(name, defaultValue, maximum) {
+  const value = process.env[name];
+  if (value === undefined) return defaultValue;
+  if (!/^[1-9]\d*$/.test(value)) throw new Error(`Invalid ${name} ${JSON.stringify(value)}; expected an integer from 1 to ${maximum}.`);
+  const parsed = Number(value);
+  if (!Number.isSafeInteger(parsed) || parsed > maximum) {
+    throw new Error(`Invalid ${name} ${JSON.stringify(value)}; expected an integer from 1 to ${maximum}.`);
+  }
+  return parsed;
+}
+
+function send(response) {
+  process.stdout.write(`${JSON.stringify(response)}\n`);
+}
+
+function requestError(id, error) {
+  send({ id, error: error instanceof Error ? error.message : String(error) });
+}
+
+function validRequest(value) {
+  return value && typeof value === "object" && Number.isSafeInteger(value.id)
+    && Array.isArray(value.texts) && value.texts.length <= MAX_REQUEST_TEXTS
+    && value.texts.every((text) => typeof text === "string");
+}
+
+async function embed(texts) {
+  const vectors = [];
+  for (let offset = 0; offset < texts.length; offset += batchSize) {
+    const batch = texts.slice(offset, offset + batchSize);
+    const output = await embedder(batch, { pooling: "cls", normalize: true });
+    const dimensions = output.dims.at(-1);
+    if (!Number.isSafeInteger(dimensions) || dimensions <= 0) throw new Error("model returned invalid embedding dimensions");
+    const data = output.data;
+    if (data.length !== batch.length * dimensions) throw new Error("model returned an invalid embedding batch");
+    for (let index = 0; index < batch.length; index++) {
+      vectors.push(Array.from(data.slice(index * dimensions, (index + 1) * dimensions)));
+    }
+  }
+  return vectors;
+}
+
+async function initialize() {
+  try {
+    const { pipeline } = await import("@huggingface/transformers");
+    embedder = await pipeline("feature-extraction", model, { dtype: "q8" });
+    send({ ready: true });
+  } catch (error) {
+    send({ ready: false, error: error instanceof Error ? error.message : String(error) });
+    process.exitCode = 1;
+    process.stdin.destroy();
+    throw error;
+  }
+}
+
+const initialization = initialize();
+let remainder = "";
+process.stdin.setEncoding("utf8");
+process.stdin.on("data", (chunk) => {
+  remainder += chunk;
+  let newline;
+  while ((newline = remainder.indexOf("\n")) >= 0) {
+    const line = remainder.slice(0, newline);
+    remainder = remainder.slice(newline + 1);
+    if (!line) continue;
+    let request;
+    try {
+      request = JSON.parse(line);
+    } catch {
+      requestError(null, "request must be valid JSON");
+      continue;
+    }
+    if (!validRequest(request)) {
+      requestError(request && typeof request === "object" && "id" in request ? request.id : null, `request must have an integer id and at most ${MAX_REQUEST_TEXTS} string texts`);
+      continue;
+    }
+    queue = queue.then(async () => {
+      try {
+        await initialization;
+        send({ id: request.id, vectors: await embed(request.texts) });
+      } catch (error) {
+        requestError(request.id, error);
+      }
+    });
+  }
+});
+process.stdin.on("error", () => process.exit());
+process.stdin.on("end", () => process.exit());
+process.on("SIGTERM", () => process.exit());
+process.on("SIGINT", () => process.exit());
+
+await initialization.catch(() => {});

package/src/embed.ts

@@ -1,14 +1,7 @@
-// Local, offline embeddings via Transformers.js. CLS-pooled + L2-normalized,
-// so cosine similarity is a plain dot product.
-//
-// Model: Snowflake/snowflake-arctic-embed-m-v1.5 (q8) — 768 dims, Apache-2.0,
-// official ONNX export in the model repo. Chosen over Xenova/bge-small-en-v1.5
-// by empirical eval on our real corpus (2026-07-22, see
-// docs/embedding-model-eval.md): equal top-1, better top-3, and far better
-// score separation (negatives max ~0.33 vs bge's ~0.66), so minScore
-// thresholding is meaningful. Asymmetric retriever: queries get a task
-// prefix, documents go through unmodified.
-import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
+// Local, offline embeddings. The default backend lives in a system-Node
+// sidecar so importing the OpenCode plugin never loads ML native addons into
+// its embedded Bun process.
+import { fileURLToPath } from "node:url";
 
 export const DEFAULT_MODEL = "Snowflake/snowflake-arctic-embed-m-v1.5";
 
@@ -20,34 +13,295 @@ export const QUERY_PREFIX = "Represent this sentence for searching relevant pass
 // degrade embeddings (and this model's window is 512 tokens anyway).
 export const MAX_CHARS = 2000;
 
-let cached: Promise<FeatureExtractionPipeline> | null = null;
+const DEFAULT_BATCH_SIZE = 32;
+const MAX_BATCH_SIZE = 64;
+const DEFAULT_READY_TIMEOUT_MS = 10 * 60 * 1000;
+const DEFAULT_REQUEST_TIMEOUT_MS = 2 * 60 * 1000;
+const MAX_TIMEOUT_MS = 2_147_483_647;
 
-export function getEmbedder(): Promise<FeatureExtractionPipeline> {
-  if (!cached) {
-    // transformers.js declares pipeline<"feature-extraction"> as its task-metadata
-    // record, not the FeatureExtractionPipeline instance it returns at runtime, so
-    // this cast restores the documented return type (matches HuggingFace's examples).
-    cached = pipeline("feature-extraction", process.env.EPISODIC_EMBED_MODEL ?? DEFAULT_MODEL, {
-      dtype: "q8",
-    }) as Promise<FeatureExtractionPipeline>;
-    // A rejected promise (e.g. failed model download) would poison the cache
-    // for the lifetime of the process; reset so the next call retries.
-    cached.catch(() => { if (cached) cached = null; });
+export type EmbedMode = "sidecar" | "inline";
+
+type PendingRequest = {
+  resolve: (vectors: Float32Array[]) => void;
+  reject: (error: Error) => void;
+  count: number;
+};
+
+type Sidecar = {
+  process: Bun.Subprocess<"pipe", "pipe", "pipe">;
+  pending: Map<number, PendingRequest>;
+  ready: Promise<void>;
+  resolveReady: () => void;
+  rejectReady: (error: Error) => void;
+  stderr: string;
+  stdout: string;
+  dimensions: number | null;
+};
+
+class SidecarUnavailableError extends Error {}
+
+let sidecar: Sidecar | null = null;
+let nextRequestId = 1;
+
+export function getEmbedMode(): EmbedMode {
+  const mode = process.env.EPISODIC_EMBED_MODE ?? "sidecar";
+  if (mode === "sidecar" || mode === "inline") return mode;
+  throw new Error(`Invalid EPISODIC_EMBED_MODE ${JSON.stringify(mode)}; expected "sidecar" or "inline".`);
+}
+
+function tail(value: string, addition: string): string {
+  return (value + addition).slice(-8_192);
+}
+
+function positiveIntegerEnv(name: string, defaultValue: number, maximum: number): number {
+  const value = process.env[name];
+  if (value === undefined) return defaultValue;
+  if (!/^[1-9]\d*$/.test(value)) throw new Error(`Invalid ${name} ${JSON.stringify(value)}; expected an integer from 1 to ${maximum}.`);
+  const parsed = Number(value);
+  if (!Number.isSafeInteger(parsed) || parsed > maximum) {
+    throw new Error(`Invalid ${name} ${JSON.stringify(value)}; expected an integer from 1 to ${maximum}.`);
+  }
+  return parsed;
+}
+
+function sidecarError(message: string, child: Sidecar): Error {
+  const details = child.stderr.trim();
+  return new SidecarUnavailableError(details ? `${message}: ${details}` : message);
+}
+
+function rejectAll(child: Sidecar, error: Error): void {
+  for (const { reject } of child.pending.values()) reject(error);
+  child.pending.clear();
+  child.rejectReady(error);
+}
+
+function sidecarGone(child: Sidecar, error: Error): void {
+  if (sidecar === child) sidecar = null;
+  rejectAll(child, error);
+  child.process.kill();
+}
+
+function protocolFailure(child: Sidecar, message: string): void {
+  const error = new Error(`Embedding sidecar protocol error: ${message}`);
+  sidecarGone(child, error);
+}
+
+function vectorsFromResponse(value: unknown, expectedCount: number, child: Sidecar): Float32Array[] {
+  if (!Array.isArray(value) || value.length !== expectedCount) {
+    throw new Error(`expected ${expectedCount} vectors, got ${Array.isArray(value) ? value.length : "a non-array"}`);
+  }
+
+  const vectors = value.map((vector) => {
+    if (!Array.isArray(vector) || vector.length === 0 || !vector.every((n) => typeof n === "number" && Number.isFinite(n))) {
+      throw new Error("vectors must be non-empty arrays of finite numbers");
+    }
+    if (child.dimensions !== null && vector.length !== child.dimensions) {
+      throw new Error(`expected ${child.dimensions} dimensions, got ${vector.length}`);
+    }
+    return new Float32Array(vector);
+  });
+
+  const dimensions = vectors[0]?.length;
+  if (vectors.some((vector) => vector.length !== dimensions)) throw new Error("vectors have inconsistent dimensions");
+  child.dimensions ??= dimensions ?? null;
+  return vectors;
+}
+
+function handleLine(child: Sidecar, line: string): void {
+  let response: unknown;
+  try {
+    response = JSON.parse(line);
+  } catch {
+    protocolFailure(child, "stdout contained invalid JSON");
+    return;
+  }
+  if (!response || typeof response !== "object" || Array.isArray(response)) {
+    protocolFailure(child, "response must be an object");
+    return;
+  }
+
+  const record = response as Record<string, unknown>;
+  if ("ready" in record) {
+    if (record.ready === true) child.resolveReady();
+    else {
+      sidecarGone(child, new SidecarUnavailableError(`Embedding sidecar failed to start: ${typeof record.error === "string" ? record.error : "unknown error"}`));
+    }
+    return;
+  }
+  if (typeof record.id !== "number" || !Number.isSafeInteger(record.id)) {
+    protocolFailure(child, "response has no valid request id");
+    return;
+  }
+  const pending = child.pending.get(record.id);
+  if (!pending) {
+    protocolFailure(child, `response has unknown request id ${record.id}`);
+    return;
+  }
+  child.pending.delete(record.id);
+  if (typeof record.error === "string") {
+    pending.reject(new Error(`Embedding sidecar request failed: ${record.error}`));
+    return;
+  }
+  try {
+    pending.resolve(vectorsFromResponse(record.vectors, pending.count, child));
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    pending.reject(new Error(`Embedding sidecar protocol error: ${message}`));
+    protocolFailure(child, message);
+  }
+}
+
+async function drainStdout(child: Sidecar): Promise<void> {
+  const reader = child.process.stdout.getReader();
+  const decoder = new TextDecoder();
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      child.stdout += decoder.decode(value, { stream: true });
+      let newline: number;
+      while ((newline = child.stdout.indexOf("\n")) >= 0) {
+        const line = child.stdout.slice(0, newline);
+        child.stdout = child.stdout.slice(newline + 1);
+        if (line) handleLine(child, line);
+      }
+    }
+    if (child.stdout.trim()) protocolFailure(child, "stdout ended with an incomplete response");
+  } catch (error) {
+    if (sidecar === child) sidecarGone(child, sidecarError(`Could not read embedding sidecar output (${String(error)})`, child));
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+async function drainStderr(child: Sidecar): Promise<void> {
+  const reader = child.process.stderr.getReader();
+  const decoder = new TextDecoder();
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      child.stderr = tail(child.stderr, decoder.decode(value, { stream: true }));
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+
+function startSidecar(): Sidecar {
+  if (sidecar) return sidecar;
+  const nodeBinary = process.env.EPISODIC_NODE_BINARY ?? "node";
+  const sidecarPath = fileURLToPath(new URL("./embed-sidecar.mjs", import.meta.url));
+  let resolveReady!: () => void;
+  let rejectReady!: (error: Error) => void;
+  const ready = new Promise<void>((resolve, reject) => {
+    resolveReady = resolve;
+    rejectReady = reject;
+  });
+  // The rejected ready promise is also observed by each request; suppress a
+  // transient unhandled-rejection warning while the first request is starting.
+  ready.catch(() => {});
+
+  let childProcess: Bun.Subprocess<"pipe", "pipe", "pipe">;
+  try {
+    childProcess = Bun.spawn([nodeBinary, sidecarPath], {
+      env: process.env,
+      stdin: "pipe",
+      stdout: "pipe",
+      stderr: "pipe",
+      detached: true,
+    });
+  } catch (error) {
+    throw new Error(`Could not start embedding sidecar using ${JSON.stringify(nodeBinary)}. Install Node 20+ or set EPISODIC_NODE_BINARY: ${String(error)}`);
+  }
+  childProcess.unref();
+  const child: Sidecar = { process: childProcess, pending: new Map(), ready, resolveReady, rejectReady, stderr: "", stdout: "", dimensions: null };
+  sidecar = child;
+  void drainStdout(child);
+  void drainStderr(child);
+  void childProcess.exited.then(() => {
+    if (sidecar === child) {
+      sidecar = null;
+      rejectAll(child, sidecarError("Embedding sidecar exited unexpectedly", child));
+    }
+  });
+  return child;
+}
+
+function awaitReady(child: Sidecar, timeoutMs: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      const error = new SidecarUnavailableError(`Embedding sidecar did not become ready within ${timeoutMs}ms`);
+      sidecarGone(child, error);
+      reject(error);
+    }, timeoutMs);
+    child.ready.then(
+      () => {
+        clearTimeout(timeout);
+        resolve();
+      },
+      (error) => {
+        clearTimeout(timeout);
+        reject(error);
+      },
+    );
+  });
+}
+
+async function requestSidecar(texts: string[], readyTimeoutMs: number, requestTimeoutMs: number, retried = false): Promise<Float32Array[]> {
+  let child: Sidecar;
+  try {
+    child = startSidecar();
+    await awaitReady(child, readyTimeoutMs);
+    const id = nextRequestId++;
+    return await new Promise<Float32Array[]>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        sidecarGone(child, new SidecarUnavailableError(`Embedding sidecar request timed out after ${requestTimeoutMs}ms`));
+      }, requestTimeoutMs);
+      const resolveRequest = (vectors: Float32Array[]) => {
+        clearTimeout(timeout);
+        resolve(vectors);
+      };
+      const rejectRequest = (error: Error) => {
+        clearTimeout(timeout);
+        reject(error);
+      };
+      child.pending.set(id, { resolve: resolveRequest, reject: rejectRequest, count: texts.length });
+      if (sidecar !== child) {
+        child.pending.delete(id);
+        rejectRequest(new SidecarUnavailableError("Embedding sidecar became unavailable before the request was sent"));
+        return;
+      }
+      try {
+        child.process.stdin.write(`${JSON.stringify({ id, texts })}\n`);
+      } catch (error) {
+        child.pending.delete(id);
+        const unavailable = new SidecarUnavailableError(`Could not write to embedding sidecar: ${String(error)}`);
+        sidecarGone(child, unavailable);
+        rejectRequest(unavailable);
+      }
+    });
+  } catch (error) {
+    if (!retried && error instanceof SidecarUnavailableError) return requestSidecar(texts, readyTimeoutMs, requestTimeoutMs, true);
+    throw error;
   }
-  return cached;
 }
 
 async function embedRaw(texts: string[]): Promise<Float32Array[]> {
   if (texts.length === 0) return [];
-  const e = await getEmbedder();
-  const out = await e(texts.map((t) => t.slice(0, MAX_CHARS)), { pooling: "cls", normalize: true });
-  const dims: number = out.dims[out.dims.length - 1];
-  // out.data is DataArray (a union incl. bigint typed arrays); a feature-extraction
-  // tensor with normalize:true is a Float32Array at runtime, so this cast is safe.
-  const flat = new Float32Array(out.data as Float32Array);
+  const prepared = texts.map((text) => text.slice(0, MAX_CHARS));
+  if (getEmbedMode() === "inline") {
+    // Unsafe under affected OpenCode/Bun versions: this intentionally loads
+    // Transformers.js only when the caller explicitly opts in.
+    const { embedInline } = await import("./embed-inline.ts");
+    return embedInline(prepared);
+  }
+  const batchSize = positiveIntegerEnv("EPISODIC_EMBED_BATCH_SIZE", DEFAULT_BATCH_SIZE, MAX_BATCH_SIZE);
+  const readyTimeoutMs = positiveIntegerEnv("EPISODIC_EMBED_READY_TIMEOUT_MS", DEFAULT_READY_TIMEOUT_MS, MAX_TIMEOUT_MS);
+  const requestTimeoutMs = positiveIntegerEnv("EPISODIC_EMBED_REQUEST_TIMEOUT_MS", DEFAULT_REQUEST_TIMEOUT_MS, MAX_TIMEOUT_MS);
   const vectors: Float32Array[] = [];
-  for (let i = 0; i < texts.length; i++) {
-    vectors.push(flat.subarray(i * dims, (i + 1) * dims));
+  for (let index = 0; index < prepared.length; index += batchSize) {
+    vectors.push(...await requestSidecar(prepared.slice(index, index + batchSize), readyTimeoutMs, requestTimeoutMs));
   }
   return vectors;
 }
@@ -57,6 +311,5 @@ export const embed = embedRaw;
 
 /** Embed a search query. Prepends the retrieval prefix. */
 export function embedQuery(query: string): Promise<Float32Array[]> {
-  const q = query.startsWith(QUERY_PREFIX) ? query : QUERY_PREFIX + query;
-  return embedRaw([q]);
+  return embedRaw([query.startsWith(QUERY_PREFIX) ? query : QUERY_PREFIX + query]);
 }

package/src/format.ts

@@ -3,6 +3,12 @@
 // here so the two front-ends can't drift apart.
 import type { SourceMessage } from "./reader";
 import type { SearchHit } from "./store";
+import { Buffer } from "node:buffer";
+
+const MAX_CONTEXT_BODY_BYTES = 600;
+const MAX_CONTEXT_TOOLS_BYTES = 200;
+const MAX_CONTEXT_FIELD_BYTES = 100;
+const MAX_CONTEXT_SESSION_FIELD_BYTES = 300;
 
 // Discriminated result so callers handle the parse error explicitly (no cast to
 // strip the error arm off a union). `ms` is undefined when no date was given.
@@ -55,6 +61,46 @@ export function renderTranscript(
   return lines.join("\n");
 }
 
+// Render a bounded live-source window around a search-hit anchor. The helper in
+// reader.ts has already applied the privacy gate and validated the anchor.
+export function renderTranscriptContext(
+  meta: { title: string; time_created: number; directory: string; id: string },
+  context: { messages: SourceMessage[]; anchorIndex: number; sliceStart: number; total: number }
+): string {
+  const lines = [
+    `# ${truncateContext(meta.title, MAX_CONTEXT_SESSION_FIELD_BYTES)}`,
+    `${fmtDate(meta.time_created)} — ${truncateContext(meta.directory, MAX_CONTEXT_SESSION_FIELD_BYTES)} — ${truncateContext(meta.id, MAX_CONTEXT_FIELD_BYTES)}`,
+    `Context around message ${context.anchorIndex + 1}/${context.total}`,
+    "",
+  ];
+  for (const [offset, message] of context.messages.entries()) {
+    const position = context.sliceStart + offset;
+    const text = message.parts.filter((part) => part.type === "text" && part.text).map((part) => part.text).join("\n");
+    const tools = message.parts.filter((part) => part.type === "tool" && part.tool).map((part) => part.tool);
+    lines.push(`## ${truncateContext(message.role, MAX_CONTEXT_FIELD_BYTES)} — ${truncateContext(message.id, MAX_CONTEXT_FIELD_BYTES)} — ${position + 1}/${context.total}${position === context.anchorIndex ? " (anchor)" : ""}`);
+    if (text) lines.push(truncateContext(text, MAX_CONTEXT_BODY_BYTES));
+    if (tools.length) lines.push(truncateContext(`*(tools: ${tools.join(", ")})*`, MAX_CONTEXT_TOOLS_BYTES));
+    if (message.contextPartsOmitted) lines.push(`*(${message.contextPartsOmitted} parts omitted from bounded context)*`);
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+function truncateContext(value: string, byteLimit: number): string {
+  if (Buffer.byteLength(value, "utf8") <= byteLimit) return value;
+  const suffix = "... [truncated]";
+  const contentBudget = byteLimit - Buffer.byteLength(suffix, "utf8");
+  let bytes = 0;
+  let result = "";
+  for (const codePoint of value) {
+    const codePointBytes = Buffer.byteLength(codePoint, "utf8");
+    if (bytes + codePointBytes > contentBudget) break;
+    result += codePoint;
+    bytes += codePointBytes;
+  }
+  return result + suffix;
+}
+
 // One search hit as a markdown block. snippetLength defaults to 400 (plugin
 // tool output); the CLI passes 220 to keep terminal output brief. scoreLabel
 // names the score field: "score" for vector (cosine ~0.4–0.7) and BM25, "rrf"
@@ -62,7 +108,9 @@ export function renderTranscript(
 // AGENTS.md) so the number isn't misread against the cosine thresholds.
 export function formatHit(h: SearchHit, snippetLength = 400, scoreLabel = "score"): string {
   const snippet = h.text.replace(/\s+/g, " ").slice(0, snippetLength);
-  return `## ${fmtDate(h.time_created)} — ${h.title}\nsession: ${h.session_id}  ${scoreLabel}: ${h.score.toFixed(3)}\n${h.directory}\n> ${snippet}`;
+  const anchor = h.anchor_message_id ?? "unavailable (refresh/reindex required)";
+  const source = h.source_id ? `source: ${h.source_id}\n` : "";
+  return `## ${fmtDate(h.time_created)} — ${h.title}\n${source}session: ${h.session_id}  ${scoreLabel}: ${h.score.toFixed(3)}\nanchor: ${anchor}\n${h.directory}\n> ${snippet}`;
 }
 
 export function formatHits(hits: SearchHit[], snippetLength = 400, scoreLabel = "score"): string {

package/src/indexer.ts

@@ -1,10 +1,10 @@
 // Incremental, idempotent indexer. Watermark = session.time_updated; a session
 // is re-embedded only when the source changed since we last indexed it.
 import type { Database } from "bun:sqlite";
-import { getTranscriptChecked, listSessions, type SourceSession } from "./reader";
+import { getTranscriptChecked, listSessions, transcriptHasMarker, type SourceSession } from "./reader";
 import { parseTranscript, exchangeText } from "./parser";
 import { embed } from "./embed";
-import { getIndexedSession, replaceSessionChunks } from "./store";
+import type { IndexStore } from "./store";
 
 export interface SyncResult {
   scanned: number;
@@ -17,20 +17,33 @@ export interface SyncResult {
 
 export async function syncSession(
   source: Database,
-  index: Database,
+  index: IndexStore,
   s: SourceSession,
   force = false
 ): Promise<"indexed" | "fresh" | "excluded" | "empty"> {
-  const prior = getIndexedSession(index, s.id);
+  const removeIfRemoteExcluded = async (): Promise<boolean> => {
+    if (!index.remote || !transcriptHasMarker(source, s.id)) return false;
+    await index.removeSession(s.id);
+    return true;
+  };
+  // Remote freshness must never preserve metadata for a newly excluded session.
+  // Local mode intentionally keeps its established cheap freshness-first path.
+  const checked = index.remote ? getTranscriptChecked(source, s.id) : undefined;
+  if (checked?.excluded) {
+    await index.removeSession(s.id);
+    return "excluded";
+  }
+  const prior = await index.getIndexedSession(s.id);
+  if (await removeIfRemoteExcluded()) return "excluded";
   if (!force && prior && prior.source_time_updated >= s.time_updated) return "fresh";
 
   // Authoritative opt-out gate lives inside getTranscriptChecked (raw-blob
   // scan before any read); parseTranscript's own parsed-text check is a
   // harmless redundant fast path for the non-excluded branch.
-  const checked = getTranscriptChecked(source, s.id);
-  const { exchanges, excluded } = checked.excluded
+  const transcript = checked ?? getTranscriptChecked(source, s.id);
+  const { exchanges, excluded } = transcript.excluded
     ? { exchanges: [], excluded: true }
-    : parseTranscript(checked.messages);
+    : parseTranscript(transcript.messages);
   const meta = {
     id: s.id, project_id: s.project_id, parent_id: s.parent_id,
     title: s.title, directory: s.directory,
@@ -38,28 +51,37 @@ export async function syncSession(
   };
 
   if (excluded) {
-    replaceSessionChunks(index, meta, [], "excluded");
+    // A remote index is an opt-in upload boundary: unlike the local index's
+    // useful excluded tombstone, it must retain no metadata for marked chats.
+    if (index.remote) await index.removeSession(s.id);
+    else await index.replaceSessionChunks(meta, [], "excluded");
     return "excluded";
   }
   if (exchanges.length === 0) {
-    replaceSessionChunks(index, meta, [], "empty");
+    if (await removeIfRemoteExcluded()) return "excluded";
+    await index.replaceSessionChunks(meta, [], "empty");
     return "empty";
   }
 
   const date = new Date(s.time_created).toISOString().slice(0, 10);
   const texts = exchanges.map((e) => exchangeText(s.title, date, e));
   const vectors = await embed(texts);
-  replaceSessionChunks(
-    index,
+  // Embedding can take long enough for the source conversation to change. Run
+  // the cheap authoritative raw-marker check again immediately before a remote
+  // upload so a marker added during embedding never exports the prepared data.
+  if (await removeIfRemoteExcluded()) return "excluded";
+  await index.replaceSessionChunks(
     meta,
-    exchanges.map((e, i) => ({ seq: i, time_created: e.time, text: texts[i], embedding: vectors[i] }))
+    exchanges.map((e, i) => ({
+      seq: i, time_created: e.time, text: texts[i], embedding: vectors[i], anchor_message_id: e.anchorMessageId,
+    }))
   );
   return "indexed";
 }
 
 export async function syncAll(
   source: Database,
-  index: Database,
+  index: IndexStore,
   opts: { force?: boolean; onProgress?: (done: number, total: number, title: string) => void } = {}
 ): Promise<SyncResult> {
   const sessions = listSessions(source);
@@ -76,7 +98,7 @@ export async function syncAll(
 
   // Prune index rows whose session no longer exists in the source DB;
   // otherwise their stale (possibly wrong-dims) chunks linger forever.
-  result.pruned = pruneOrphans(source, index, sessions);
+  result.pruned = await pruneOrphans(source, index, sessions);
 
   return result;
 }
@@ -85,15 +107,6 @@ export async function syncAll(
 // source DB. Extracted so the plugin's full-reindex path can call it without
 // re-running the whole sync. Pass already-fetched sessions to avoid a redundant
 // query in syncAll; omitted, it re-reads the source.
-export function pruneOrphans(source: Database, index: Database, knownSource?: SourceSession[]): number {
-  const sourceIds = new Set((knownSource ?? listSessions(source)).map((s) => s.id));
-  const indexedIds = index.prepare<{ id: string }, []>("SELECT id FROM sessions").all();
-  let pruned = 0;
-  for (const { id } of indexedIds) {
-    if (sourceIds.has(id)) continue;
-    index.run("DELETE FROM chunks WHERE session_id = ?", [id]);
-    index.run("DELETE FROM sessions WHERE id = ?", [id]);
-    pruned++;
-  }
-  return pruned;
+export async function pruneOrphans(source: Database, index: IndexStore, knownSource?: SourceSession[]): Promise<number> {
+  return index.pruneOrphans((knownSource ?? listSessions(source)).map((s) => s.id));
 }

package/src/parser.ts

@@ -23,6 +23,7 @@ export function hasExcludeMarker(messages: SourceMessage[]): boolean {
 }
 
 export interface Exchange {
+  anchorMessageId: string;
   user: string;
   assistant: string;
   tools: string[];
@@ -60,7 +61,7 @@ export function parseTranscript(messages: SourceMessage[]): {
     if (m.role === "user") {
       const text = textOf(m.parts);
       if (!text) continue; // e.g. pure tool-result turns
-      current = { user: text, assistant: "", tools: [], time: m.timeCreated };
+      current = { anchorMessageId: m.id, user: text, assistant: "", tools: [], time: m.timeCreated };
       exchanges.push(current);
     } else if (m.role === "assistant" && current) {
       const text = textOf(m.parts);
@@ -73,7 +74,7 @@ export function parseTranscript(messages: SourceMessage[]): {
   return { exchanges: exchanges.filter((e) => e.user || e.assistant), excluded: false };
 }
 
-// Text stored per chunk (also displayed by episodic_read). Capped at 4000 chars
+// Text stored per chunk (also displayed by episodic_read_session). Capped at 4000 chars
 // to keep storage sane; the embedding step (embed.ts) further truncates to 2000
 // chars where retrieval quality peaks. The head of an exchange carries the
 // most signal.