@3sln/trove 0.0.16 → 0.0.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@3sln/trove",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "type": "module",
   "description": "Trove — a self-hostable, plugin-extensible Google Drive. Semantic search, pluggable storage (S3 / filesystem / NAS), and a VS Code-style contribution system with sandboxed plugins.",
   "repository": {

package/packages/core/src/index.js

@@ -36,7 +36,7 @@ export { EmbeddingProvider, LocalHashEmbedding, HttpEmbedding } from './search/e
 export { SqliteVectorStore, SqliteKeywordStore, SEARCH_DB_KEY } from './search/sqliteStores.js';
 
 export { IndexerRegistry, textIndexer, chunkText } from './indexers/registry.js';
-export { PluginService, PackageStore, StoragePackageStore, PluginInstallStore, SqlitePluginInstallStore, MemoryPluginInstallStore, parsePluginPackage, capabilityList, ALL_CAPABILITIES, IndexerRuntime, InProcessIndexerRuntime, PluginIndexers, matchFromSelector } from './plugins/index.js';
+export { PluginService, PackageStore, StoragePackageStore, PluginInstallStore, SqlitePluginInstallStore, MemoryPluginInstallStore, parsePluginPackage, capabilityList, ALL_CAPABILITIES, IndexerRuntime, InProcessIndexerRuntime, WorkerLoaderIndexerRuntime, PluginIndexers, matchFromSelector } from './plugins/index.js';
 export { UploadManager, KvSessionStore, DEFAULT_PART_SIZE } from './uploads.js';
 // Encryption at rest: the bucket holds ciphertext, the drive holds the key. Protects
 // against the STORAGE host (a leaked bucket credential, a storage vendor who is not the

package/packages/core/src/indexers/registry.js

@@ -64,9 +64,20 @@ export class IndexerRegistry {
   unregister(id) {
     this.indexers.delete(id);
   }
+  /**
+   * The redacted view — what `/api/indexers` answers with. Deliberately not the indexer
+   * itself: an indexer holds a `match` and an `index` that runs plugin code, and neither
+   * belongs in a response.
+   */
   list() {
     return [...this.indexers.values()].map((i) => ({ id: i.id, displayName: i.displayName || i.id }));
   }
+
+  /**
+   * The LIVE indexer, for a caller that means to run it — `backfillIndexer` after an
+   * install, which needs the real `match` and `index` that `list()` strips.
+   */
+  get(id) { return this.indexers.get(id) || null; }
   matching(node) {
     return [...this.indexers.values()].filter((i) => {
       try {

package/packages/core/src/plugins/index.js

@@ -15,6 +15,7 @@ export { parsePluginPackage, capabilityList, ALL_CAPABILITIES, serverIndexers, d
 export * from './identity.js';
 export { CONTRIBUTION_TYPES, contributionsOfType } from './contributions.js';
 export { IndexerRuntime, InProcessIndexerRuntime } from './runtime.js';
+export { WorkerLoaderIndexerRuntime } from './workerLoaderRuntime.js';
 export { clampContribution, DEFAULT_CAPS } from '../indexers/contribution.js';
 export { PluginIndexers, matchFromSelector } from './indexers.js';
 
@@ -79,6 +80,16 @@ export class PluginService {
     if (pkg.indexers.length && !this.indexers) {
       throw TroveError.unsupported('Server indexers are disabled on this deployment');
     }
+    // ...or when it HAS one that cannot run here — the in-process runner on workerd,
+    // which cannot import a `data:` URL. That case DEGRADES rather than refusing: a
+    // plugin is more than its indexer, and refusing the install would take the viewers,
+    // commands and everything else away over one part the deployment cannot host.
+    //
+    // What it must not do is what it used to: run anyway and fail once per file forever,
+    // with nothing said. So the reason is recorded ON the record — `indexersSkipped` —
+    // and travels to the client, which is what turns "my search is empty" into a sentence
+    // someone can act on. Asked once per install, not per node.
+    const probe = pkg.indexers.length ? (await this.indexers?.probe?.() ?? { ok: true }) : { ok: true };
 
     const version = pkg.manifest.version || '0';
     // CONTENT-addressed. The ref used to be `<account>/<pluginId>/<version>.zip`, written
@@ -112,21 +123,48 @@ export class PluginService {
     if (prev?.packageRef && prev.packageRef !== ref && !(await this.#refIsShared(prev.packageRef, account))) {
       await this.packages.delete(prev.packageRef).catch(() => {});
     }
-    // Register + backfill any server indexers this package ships.
-    if (this.indexers && pkg.indexers.length) {
-      try { await this.indexers.activate(record); }
+    // Register + backfill any server indexers this package ships — unless the probe
+    // just said this deployment cannot run them, in which case registering would only
+    // queue up a failure per file.
+    if (this.indexers && pkg.indexers.length && probe.ok) {
+      // REGISTER ONLY. The backfill used to run here, inline and awaited inside the
+      // install request, which is the same mistake `completeUpload` made at a larger
+      // scale: re-reading every matching file in a drive is not work a request can
+      // finish, and on Workers the isolate goes before it gets far. The caller schedules
+      // it instead — `beginBackfill`, which lands in the same Durable Object that owns
+      // scans and reindexes, and reports through the same task record.
+      try { await this.indexers.activate(record, { backfill: false }); }
       catch (err) { console.error(`activating indexers for ${record.pluginId} failed:`, err.message); }
+    } else if (!probe.ok) {
+      console.warn(`server indexers for ${record.pluginId} are not running: ${probe.reason}`);
     }
-    return this.#publicRecord(record);
+    return this.#annotate(this.#publicRecord(record));
+  }
+
+  /**
+   * Why this record's indexers are not running, or null when they are.
+   *
+   * DERIVED, not stored. The reason is a property of the DEPLOYMENT, not of the install:
+   * a drive that gains a Worker Loader binding should stop reporting "skipped" the moment
+   * it restarts, without anyone rewriting rows — and a drive that loses one should start.
+   * Storing it at install would freeze an answer that is only true until the next deploy.
+   *
+   * The probe caches its own result, so this costs nothing after the first call.
+   */
+  async #annotate(record) {
+    if (!record?.indexers?.length) return record;
+    const probe = await this.indexers?.probe?.() ?? { ok: true };
+    return probe.ok ? record : { ...record, indexersSkipped: probe.reason };
   }
 
   /** An account's installed plugins (secrets stripped). */
   async list(principal) {
-    return (await this.installs.list(accountOf(principal))).map((r) => this.#publicRecord(r));
+    const rows = (await this.installs.list(accountOf(principal))).map((r) => this.#publicRecord(r));
+    return Promise.all(rows.map((r) => this.#annotate(r)));
   }
   async get(principal, pluginId) {
     const r = await this.installs.get(accountOf(principal), pluginId);
-    return r ? this.#publicRecord(r) : null;
+    return r ? this.#annotate(this.#publicRecord(r)) : null;
   }
 
   /** Download the package blob (for a device to sync + enable). */

package/packages/core/src/plugins/indexers.js

@@ -34,6 +34,14 @@ export class PluginIndexers {
    */
   async activate(record, { backfill = true } = {}) {
     const specs = record.indexers || [];
+    // The probe first, so a deployment that cannot run this plugin's indexers says so
+    // once, here, instead of failing per file — and so the saying of it is undone the
+    // moment the deployment can. A record with no indexers has nothing to diagnose.
+    if (specs.length) {
+      const probe = await this.probe();
+      await this.#diagnose(record, probe);
+      if (!probe.ok) return 0;
+    }
     // Retire anything this plugin used to declare and no longer does. Both this and
     // deactivate() iterated only the record in hand, so an indexer dropped by an
     // upgrade stayed registered — running code the user upgraded away from on every
@@ -66,6 +74,50 @@ export class PluginIndexers {
     return specs.length;
   }
 
+  /**
+   * Can indexers actually run on this deployment? Passed straight through to the
+   * runtime — PluginIndexers is the coordinator, not the thing that knows.
+   */
+  async probe() { return this.runtime?.probe?.() ?? { ok: true }; }
+
+  /**
+   * Say — or stop saying — that this plugin's indexers are not running.
+   *
+   * A standing DIAGNOSTIC rather than a field nobody draws, because the symptom is an
+   * absence: a drive whose plugin indexers never ran looks exactly like a drive with
+   * nothing to index. The issues list is where the drive already admits to things it
+   * cannot do, and it carries a retry, so the answer to "why is my search empty" arrives
+   * in the same place as every other one.
+   *
+   * Raised and cleared at ACTIVATION, which happens on install and again at every boot —
+   * so a deployment that gains a Worker Loader binding drops the diagnostic on its next
+   * start, and one that loses it picks the diagnostic back up, with nobody rewriting a
+   * stored answer.
+   */
+  async #diagnose(record, probe) {
+    const issues = this.vfs?.issues;
+    if (!issues) return;
+    try {
+      if (probe.ok) {
+        await issues.clear('plugin-indexers', record.pluginId);
+        return;
+      }
+      await issues.raise({
+        kind: 'plugin-indexers',
+        subject: record.pluginId,
+        severity: 'warning',
+        title: `“${record.pluginId}” cannot index on this deployment — what it would add to search is missing`,
+        detail: probe.reason,
+        remedy: 'Bind a Worker Loader (`[[worker_loaders]]` in wrangler.toml) so plugin indexers can run in their own isolate.',
+        // Retrying re-activates, which re-probes: the honest way to ask again after the
+        // binding has been added, rather than a button that only clears the message.
+        retry: { op: 'reactivate-indexers', pluginId: record.pluginId },
+      });
+    } catch (err) {
+      console.error('could not record the plugin-indexer diagnostic:', err.message);
+    }
+  }
+
   /** Unregister + purge every indexer a record declared. */
   async deactivate(record) {
     for (const spec of record.indexers || []) {

package/packages/core/src/plugins/runtime.js

@@ -33,6 +33,27 @@ export class IndexerRuntime {
    */
   async run(spec, node, ctx) { throw TroveError.unsupported('IndexerRuntime.run'); }
   async close() {}
+
+  /**
+   * Can this runtime actually execute an indexer HERE, on this deployment?
+   *
+   * Asked once, at install, because the alternative is what shipped: a runtime that
+   * installs happily and then fails on every single file, silently, for the life of the
+   * deployment. The in-process runner loads code by importing a `data:` URL — which
+   * Node and Bun allow and **workerd does not** — so on Cloudflare every plugin indexer
+   * failed with `No such module "data:text/javascript;base64,…"`, once per node, with
+   * nothing to see at install time and nothing in the UI to explain the empty index.
+   *
+   * The design doc's provider matrix already called for this (§7, the `CF plain / none`
+   * row: *"install-scope check refuses server-indexer plugins on this deployment, with a
+   * clear message"*). It could not fire, because it keyed on the runtime being ABSENT
+   * and the broken runtime is present — it just cannot run.
+   *
+   * Default true: a runtime that does not implement a probe is one that works.
+   *
+   * @returns {Promise<{ok: true} | {ok: false, reason: string}>}
+   */
+  async probe() { return { ok: true }; }
 }
 
 /**
@@ -59,10 +80,7 @@ export class InProcessIndexerRuntime extends IndexerRuntime {
     if (!p) {
       const code = spec.files?.[spec.entry];
       if (!code) return Promise.reject(TroveError.invalid(`Indexer "${spec.id}" is missing its entry "${spec.entry}"`));
-      // base64 data URL — percent-encoded JS confuses some runtimes' data: loader
-      // (they fall back to a text module), whereas base64 imports reliably.
-      const url = 'data:text/javascript;base64,' + bytesToBase64(code);
-      p = import(/* @vite-ignore */ url).then((mod) => {
+      p = importModule(code).then((mod) => {
         const fn = mod.default || mod.index;
         if (typeof fn !== 'function') throw TroveError.invalid(`Indexer "${spec.id}" has no default/index export`);
         return fn;
@@ -72,6 +90,33 @@ export class InProcessIndexerRuntime extends IndexerRuntime {
     return p;
   }
 
+  /**
+   * Try the loader itself, once, on a module that does nothing.
+   *
+   * The probe imports a real `data:` URL rather than sniffing for a runtime name.
+   * Feature-detection over branding: workerd is the case that prompted this, but the
+   * question is "does dynamic import of a data: URL work here", and only doing it
+   * answers that. Cached — including the failure, which is a property of the runtime
+   * and will not change while the process lives.
+   */
+  async probe() {
+    // A REALISTICALLY SIZED module, not `export default 1`. The tiny version answered a
+    // question nobody was asking: Bun imports a small data: URL happily and refuses one
+    // over ~1.5 KB with ENAMETOOLONG, so a 12-byte probe passed and every real indexer —
+    // the audiobook one is 34 KB — failed per file. That is the same silent shape this
+    // probe exists to prevent, so it now loads something the size of a real entry.
+    const padding = 'x'.repeat(PROBE_BYTES);
+    const code = new TextEncoder().encode(`export default 1; //${padding}`);
+    this._probe ||= importModule(code)
+      .then(() => ({ ok: true }))
+      .catch((err) => ({
+        ok: false,
+        reason: `this deployment's JavaScript runtime cannot load plugin code dynamically (${err?.message || err}). `
+          + 'Server indexers need an isolate runtime — on Cloudflare, a Worker Loader binding.',
+      }));
+    return this._probe;
+  }
+
   async run(spec, node, ctx) {
     const fn = await this.#load(spec);
     const result = await withTimeout(
@@ -83,6 +128,58 @@ export class InProcessIndexerRuntime extends IndexerRuntime {
   }
 }
 
+/**
+ * How big a module the probe pretends to load. Larger than any plausible bundled indexer
+ * entry (the audiobook one is 34 KB), because the failure being detected is a SIZE limit
+ * and a probe under it proves nothing.
+ */
+export const PROBE_BYTES = 64 * 1024;
+
+/**
+ * Import a module from bytes, on whichever runtime this is.
+ *
+ * The two schemes are exactly complementary, which is why both are here:
+ *
+ *   data:   Node takes it at any size (2.6 MB tested). Bun refuses over ~1.5 KB with
+ *           `NameTooLong` — it resolves the URL as a path, so ENAMETOOLONG.
+ *   blob:   Bun takes it at any size. Node's ESM loader supports exactly `file:`, `data:`
+ *           and `node:`, so it throws ERR_UNSUPPORTED_ESM_URL_SCHEME.
+ *
+ * data: first because it needs no cleanup and the same bytes give the same URL, so the
+ * engine's module cache makes a re-import free. An object URL is a live registry entry
+ * that leaks until revoked, which is the only reason it is the fallback rather than the
+ * default.
+ *
+ * Neither works on workerd. That is what `WorkerLoaderIndexerRuntime` is for, and what
+ * `probe()` reports when it is missing.
+ */
+async function importModule(code) {
+  // base64 rather than percent-encoded: some runtimes' data: loader treats
+  // percent-encoded JS as a text module instead of code.
+  try {
+    return await import(/* @vite-ignore */ 'data:text/javascript;base64,' + bytesToBase64(code));
+  } catch (dataErr) {
+    if (typeof URL.createObjectURL !== 'function' || typeof Blob !== 'function') throw dataErr;
+    let url;
+    try {
+      url = URL.createObjectURL(new Blob([code], { type: 'text/javascript' }));
+    } catch {
+      throw dataErr; // report the data: failure — it is the one that describes the runtime
+    }
+    try {
+      return await import(/* @vite-ignore */ url);
+    } catch {
+      throw dataErr;
+    } finally {
+      // Safe here and only here: an ESM module is fully instantiated by the time the
+      // import resolves, and `#load` caches the promise so this URL is never resolved
+      // again. Without it every indexer would pin its own source in memory for the life
+      // of the process.
+      URL.revokeObjectURL?.(url);
+    }
+  }
+}
+
 function bytesToBase64(bytes) {
   let bin = '';
   const CHUNK = 0x8000;

package/packages/core/src/plugins/workerLoaderRuntime.js

@@ -0,0 +1,160 @@
+// WorkerLoaderIndexerRuntime — the Cloudflare arm of the provider matrix.
+//
+// This is step 4 of docs/design/server-plugins-and-indexers.md §11, and the reason the
+// in-process runner is not the answer on Workers: it loads indexer code by importing a
+// `data:` URL, which workerd refuses outright. Every plugin indexer on a Worker
+// deployment failed with `No such module "data:text/javascript;base64,…"`, once per
+// file, silently. See `InProcessIndexerRuntime.probe()`.
+//
+// A Worker Loader is the fix the doc already chose over Workers-for-Platforms dispatch
+// namespaces: no per-install upload step, so the install flow stays identical across
+// runtimes — every deployment stores the same package blob, and only execution differs.
+// The parent Worker never evaluates plugin code at all; the loader builds a genuinely
+// separate isolate for it.
+//
+// THE BYTES PROBLEM, and why it is solved with a URL rather than RPC. An indexer reads
+// its file adaptively — the audiobook one walks MP4 boxes, so which range it wants next
+// depends on what the last range said. That is a conversation, and a conversation across
+// an isolate boundary is either RPC plumbing on every deployment or one thing the
+// sandbox can already do: fetch. So the host mints a time-limited presigned read URL
+// (§8, `ctx.presignRead()`) and the shim implements `readRange` as a ranged GET against
+// it. The sandbox reaches exactly one object, for a few minutes, and the host holds no
+// open channel it has to police.
+//
+// What the sandbox is allowed to do is therefore precisely: fetch that URL. `env` carries
+// no bindings — no storage, no database, no secrets beyond the plugin's own config — and
+// the entrypoint is called once with one node.
+
+import { TroveError } from '../errors.js';
+import { withTimeout } from '../retry.js';
+import { clampContribution, DEFAULT_CAPS } from '../indexers/contribution.js';
+import { IndexerRuntime } from './runtime.js';
+
+/**
+ * The module that runs INSIDE the sandbox, wrapping the plugin's own entry.
+ *
+ * It exists because the plugin's `index(node, ctx)` contract is a function call and a
+ * dynamic Worker's contract is `fetch(request)`. This is the adapter between them, and
+ * it is the only host-authored code in there.
+ *
+ * `readRange` is deliberately the same shape the in-process context offers, so an
+ * indexer cannot tell which runtime it is on — that is what makes the audiobook indexer
+ * work unchanged on both.
+ */
+export const SHIM = `
+import * as entry from './entry.js';
+
+const fn = entry.default || entry.index;
+
+export default {
+  async fetch(request) {
+    const { node, url, maxBytes, config, secrets } = await request.json();
+    if (typeof fn !== 'function') {
+      return Response.json({ error: 'no default/index export' }, { status: 500 });
+    }
+    // One ranged GET per call, against the presigned URL and nothing else. \`end\` is
+    // exclusive here (as it is host-side) and inclusive in an HTTP Range header — the
+    // off-by-one that would silently drop the last byte of every read.
+    const readRange = async (start = 0, end = node.size) => {
+      const from = Math.max(0, Math.min(start, node.size));
+      const to = Math.min(end, from + maxBytes, node.size);
+      if (to <= from) return new Uint8Array(0);
+      // Concatenation rather than a template literal, deliberately: this whole module is
+      // itself a template literal, and a nested one has to be escaped through two levels.
+      // The first version got that wrong and every sandbox died with "Failed to start
+      // Worker" — a syntax error in generated code, which is invisible at the call site.
+      const res = await fetch(url, { headers: { Range: 'bytes=' + from + '-' + (to - 1) } });
+      if (!res.ok && res.status !== 206) throw new Error('range read failed: ' + res.status);
+      return new Uint8Array(await res.arrayBuffer());
+    };
+    const ctx = {
+      readRange,
+      readBytes: () => readRange(0, node.size),
+      readText: async () => new TextDecoder().decode(await readRange(0, node.size)),
+      maxBytes, config, secrets,
+    };
+    try {
+      return Response.json({ contribution: (await fn(node, ctx)) ?? null });
+    } catch (err) {
+      return Response.json({ error: err?.message || String(err) }, { status: 500 });
+    }
+  },
+};
+`;
+
+export class WorkerLoaderIndexerRuntime extends IndexerRuntime {
+  /**
+   * @param {object} opts
+   * @param {{get: Function}} opts.loader the `worker_loaders` binding (env.LOADER)
+   * @param {string} [opts.compatibilityDate] what the sandbox is compiled against
+   * @param {number} [opts.timeoutMs]
+   * @param {object} [opts.caps]
+   */
+  constructor({ loader, compatibilityDate = '2025-01-01', timeoutMs = 10_000, caps = DEFAULT_CAPS } = {}) {
+    super();
+    if (!loader?.get) throw TroveError.invalid('WorkerLoaderIndexerRuntime needs a worker_loaders binding');
+    this.loader = loader;
+    this.compatibilityDate = compatibilityDate;
+    this.timeoutMs = timeoutMs;
+    this.caps = { ...DEFAULT_CAPS, ...caps };
+  }
+
+  /**
+   * The binding either exists or it does not, and if it does it works — unlike the
+   * in-process runner, there is no capability here that a runtime might withhold.
+   * Constructing this class already refused a missing binding.
+   */
+  async probe() { return { ok: true }; }
+
+  async run(spec, node, ctx) {
+    // A presigned URL is the whole mechanism, so its absence is a hard error rather
+    // than a degraded path. It fails LOUDLY and per-install-shaped: a backend that
+    // cannot presign cannot host server indexers on Workers, and saying that is more
+    // use than an empty contribution.
+    if (typeof ctx.presignRead !== 'function') {
+      throw TroveError.unsupported(
+        `Indexer "${spec.id}" needs a presigned read URL to run in a sandbox, and this storage backend cannot mint one`,
+      );
+    }
+    const url = await ctx.presignRead();
+
+    const code = spec.files?.[spec.entry];
+    if (!code) throw TroveError.invalid(`Indexer "${spec.id}" is missing its entry "${spec.entry}"`);
+
+    // Keyed by cacheKey, which embeds the package DIGEST — so a reinstall at the same
+    // version gets a different isolate rather than the old code kept warm. The loader
+    // may reuse an isolate for a repeated id; that is exactly what we want within a
+    // digest and exactly what we must not have across one.
+    const stub = this.loader.get(spec.cacheKey || spec.id, async () => ({
+      compatibilityDate: this.compatibilityDate,
+      mainModule: 'shim.js',
+      modules: {
+        'shim.js': SHIM,
+        'entry.js': new TextDecoder().decode(code),
+      },
+      // No bindings. The sandbox gets its instructions in the request body and its bytes
+      // from one URL; there is nothing else for it to reach.
+      env: {},
+    }));
+
+    const res = await withTimeout(
+      stub.getEntrypoint().fetch('https://indexer.invalid/run', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          node: { id: node.id, name: node.name, contentType: node.contentType, size: node.size },
+          url,
+          maxBytes: ctx.maxBytes ?? 2 * 1024 * 1024,
+          config: ctx.config || {},
+          secrets: ctx.secrets || {},
+        }),
+      }),
+      this.timeoutMs,
+      `Indexer "${spec.id}" timed out after ${this.timeoutMs}ms`,
+    );
+
+    const body = await res.json().catch(() => ({ error: 'indexer returned a non-JSON response' }));
+    if (!res.ok || body.error) throw TroveError.invalid(`Indexer "${spec.id}" failed: ${body.error || res.status}`);
+    return clampContribution(body.contribution, this.caps);
+  }
+}

package/packages/core/src/sidecar/document.js

@@ -22,7 +22,7 @@
 export const SIDECAR_VERSION = 1;
 
 export function emptyDoc(nodeId) {
-  return { v: SIDECAR_VERSION, nodeId, clock: 0, tags: {}, comments: {}, subscribers: {} };
+  return { v: SIDECAR_VERSION, nodeId, clock: 0, tags: {}, comments: {}, subscribers: {}, data: {} };
 }
 
 // A stamp orders and tie-breaks a write. Higher clock wins; equal clock → higher
@@ -101,6 +101,53 @@ export function removeTag(doc, name, { actor, at } = {}) {
   if (newer(s, cur)) doc.tags[name] = { present: false, value: cur?.value, ...s };
 }
 
+/**
+ * A plugin's own key/value data for this item.
+ *
+ * SCOPED, and the scope is not the plugin's to choose: the caller passes the plugin id it
+ * has already authenticated, exactly as `/api/index/:indexerId` does for contributions.
+ * Two plugins writing `position` to the same book are writing two different things, and a
+ * flat namespace would make one silently overwrite the other.
+ *
+ * Same LWW register as a tag, for the same reason: a listener finishes a chapter on their
+ * phone and opens the book on a laptop that has been offline for a day. Both wrote. The
+ * later write wins, and `newer` breaks a tie the same total, deterministic way it does
+ * everywhere else in this document — so both devices converge on the same answer without
+ * either having to be authoritative.
+ */
+export function setData(doc, scope, key, value, { actor, at } = {}) {
+  if (!scope || !key) return null;
+  doc.data ||= {};
+  doc.data[scope] ||= {};
+  const s = stamp(doc, actor, at);
+  const cur = doc.data[scope][key];
+  if (newer(s, cur)) doc.data[scope][key] = { present: true, value, ...s };
+  return doc.data[scope][key];
+}
+
+/**
+ * Forget one key. A TOMBSTONE rather than a delete, because a delete that removed the
+ * entry would be silently undone by any replica that still had the old value — the
+ * absence has to be a fact with a stamp, or it cannot win a merge.
+ */
+export function removeData(doc, scope, key, { actor, at } = {}) {
+  if (!scope || !key) return;
+  doc.data ||= {};
+  doc.data[scope] ||= {};
+  const s = stamp(doc, actor, at);
+  const cur = doc.data[scope][key];
+  if (newer(s, cur)) doc.data[scope][key] = { present: false, value: cur?.value, ...s };
+}
+
+/** What a plugin sees of its own scope: present keys, values only. */
+export function dataOf(doc, scope) {
+  const out = {};
+  for (const [key, cell] of Object.entries(doc?.data?.[scope] || {})) {
+    if (cell?.present) out[key] = cell.value;
+  }
+  return out;
+}
+
 export function subscribe(doc, userId, { muted = false, actor, at } = {}) {
   if (!userId) return;
   const s = stamp(doc, actor ?? userId, at);
@@ -119,7 +166,7 @@ export function unsubscribe(doc, userId, { actor, at } = {}) {
 export function mergeDoc(a, b) {
   if (!a) return structuredCloneSafe(b);
   if (!b) return structuredCloneSafe(a);
-  const out = { v: SIDECAR_VERSION, nodeId: a.nodeId || b.nodeId, clock: Math.max(a.clock || 0, b.clock || 0), tags: {}, comments: {}, subscribers: {} };
+  const out = { v: SIDECAR_VERSION, nodeId: a.nodeId || b.nodeId, clock: Math.max(a.clock || 0, b.clock || 0), tags: {}, comments: {}, subscribers: {}, data: {} };
 
   for (const key of union(a.tags, b.tags)) out.tags[key] = pick(a.tags[key], b.tags[key]);
   // Documents written before the register was removed still carry one, and a merge that
@@ -131,6 +178,17 @@ export function mergeDoc(a, b) {
   }
   for (const key of union(a.subscribers, b.subscribers)) out.subscribers[key] = pick(a.subscribers[key], b.subscribers[key]);
 
+  // Data is a map of maps — scope, then key — so it merges a level deeper than the rest.
+  // Merged per KEY rather than per scope: two devices that each wrote a different key in
+  // the same scope must end up with both, and picking a whole scope would drop one.
+  out.data = {};
+  for (const scope of union(a.data, b.data)) {
+    out.data[scope] = {};
+    for (const key of union(a.data?.[scope], b.data?.[scope])) {
+      out.data[scope][key] = pick(a.data?.[scope]?.[key], b.data?.[scope]?.[key]);
+    }
+  }
+
   for (const id of union(a.comments, b.comments)) {
     out.comments[id] = mergeComment(a.comments[id], b.comments[id]);
   }

package/packages/core/src/sidecar/index.js

@@ -8,7 +8,7 @@ import { SidecarStore } from './store.js';
 import { SidecarManager } from './manager.js';
 import {
   addComment, editComment, deleteComment, react, setTag, removeTag,
-  subscribe, unsubscribe, viewDoc, extractMentions,
+  subscribe, unsubscribe, viewDoc, extractMentions, setData, removeData, dataOf,
 } from './document.js';
 import { newId } from '../util.js';
 import { TroveError } from '../errors.js';
@@ -115,6 +115,33 @@ export class SidecarService {
     return this.view(nodeId);
   }
 
+  // --- per-plugin item data --------------------------------------------------
+  //
+  // A viewer's state about an ITEM, which is neither a contribution nor a setting. A
+  // contribution is derived from the file and is rewritten whenever it is re-indexed; a
+  // setting is per-device, and a listening position that does not follow you to your phone
+  // is the one people notice missing. This is the third thing, and the sidecar is where it
+  // belongs because the sidecar is already a per-item CRDT.
+  //
+  // `scope` is passed in by the caller and is NOT the plugin's to choose — the route
+  // derives it from the plugin it has already authenticated, the same way
+  // `/api/index/:indexerId` derives a contributor namespace.
+
+  async setData(nodeId, scope, key, value, principal) {
+    await this.manager.mutate(nodeId, (doc) => setData(doc, scope, key, value, { actor: principal?.id }));
+    return { ok: true };
+  }
+
+  async removeData(nodeId, scope, key, principal) {
+    await this.manager.mutate(nodeId, (doc) => removeData(doc, scope, key, { actor: principal?.id }));
+    return { ok: true };
+  }
+
+  /** One scope's present keys. Never another scope's — see `dataOf`. */
+  async data(nodeId, scope) {
+    return dataOf(await this.manager.get(nodeId), scope);
+  }
+
   // --- subscriptions ---------------------------------------------------------
 
   async subscribe(nodeId, principal, muted = false) {

package/packages/core/src/vfs.js

@@ -345,8 +345,9 @@ export class Vfs {
       collectionId, name, storageKey, size: plaintextSize ?? info.size, contentType: ct,
       etag: info.etag, encryption,
     });
-    // Small server-side writes index synchronously (search is ready on return);
-    // large client uploads (completeUpload) index in the background instead.
+    // Indexing is awaited on BOTH write paths — here and in `completeUpload` — so search
+    // is ready when the call returns. See the note there for why the background variant
+    // could not work on a request-scoped runtime.
     await this.indexing.indexNode(node).catch((e) => console.error('index error', e));
     return node;
   }
@@ -763,7 +764,17 @@ export class Vfs {
         size: obj.size, contentType: obj.contentType, etag: obj.etag,
         overwrite: obj.overwrite, encryption: obj.encryption || null,
       });
-      this.indexing.indexNode(node).catch((e) => console.error('index error', e));
+      // AWAITED, like the small-write path above. It used to be fire-and-forget, and the
+      // reasoning ("large client uploads index in the background") only holds on a
+      // runtime with a process that outlives the request. On Workers there is none: the
+      // response returns, the isolate goes, and the indexing is cut off mid-read — so an
+      // uploaded file got no contributions, no search entry and NO ISSUE either, because
+      // nothing failed. It simply never ran.
+      //
+      // Awaiting is bounded rather than open-ended: an indexer reads through `readRange`,
+      // capped at `maxIndexBytes` (2 MiB by default), so this costs a couple of megabytes
+      // of reads regardless of whether the file is 400 KB or 400 MB.
+      await this.indexing.indexNode(node).catch((e) => console.error('index error', e));
       return node;
     } catch (err) {
       await (await this.storageFor(obj.collectionId)).delete(obj.storageKey).catch(() => {});