@matatbread/matbot-core 0.4.3 → 0.4.5

package/package.json

@@ -4,7 +4,7 @@
     "node",
     "browser"
   ],
-  "version": "0.4.3",
+  "version": "0.4.5",
   "engines": {
     "node": ">=24.0.0"
   },
@@ -35,7 +35,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@matatbread/matbot-plugin-api": "^0.4.3"
+    "@matatbread/matbot-plugin-api": "^0.4.5"
   },
   "devDependencies": {
     "typescript": "^6.0.3"

package/src/index.ts

@@ -28,13 +28,13 @@ export {
   unifyServices, forwardingProxy, makeSwappable, singleTurnRequest, createMountTable,
   createBroadcaster, subscribable, createNotifier, scopedNotifier,
   installPrincipalCarrier, enterPrincipal, createConstantPrincipalCarrier,
-  contextSwitch, onContextQuiesce, flushIfQuiescent,
+  contextSwitch, machineBusy, quiesced, onContextQuiesce, scheduleAtEdge,
   installUsageCarrier, createSerialUsageCarrier, recordUsage, recordSpan, currentUsageSink,
   currentUsageScope, currentUsageSite, withUsageScope, withUsageSite,
 } from '@matatbread/matbot-plugin-api/host';
 export type {
   Subscribable, Broadcaster, Routed, RoutedFilter, SwapFn, MountTable, PrincipalCarrier, UsageCarrier,
-  UsageScope,
+  UsageScope, Quiescer,
 } from '@matatbread/matbot-plugin-api/host';
 export * from './session.js';
 export * from './usage.js';

package/src/loader.ts

@@ -62,8 +62,17 @@ let freshSeq = 0;
  *   - `name` — the canonical plugin name, when the host has already derived it (e.g. by
  *     reading the resolved package.json). Bypasses `resolver.identify(spec)` — necessary
  *     because `spec` (a remote URL, a versioned npm range) may not itself identify the name.
+ *   - `version` — the package.json `version`, when the host has already read it. Same reason as
+ *     `name`: `resolver.version(spec)` starts from `spec`, which for a bare name, a URL or a
+ *     version range locates no manifest at all.
  *   - `runtimes` — the package.json `matbotRuntime`, when the host has already read it; used
  *     for the pre-import gate in place of `resolver.runtimes(spec)`.
+ *   - `notes` — what the host noticed while resolving that would explain a failure but does not
+ *     itself cause one: a dependency it could not satisfy, an import shape it cannot fetch, a fetch
+ *     that failed before the import was even attempted. Appended to any failure recorded for this
+ *     spec, because the failure that surfaces is downstream of the cause (a missing dependency
+ *     arrives as ERR_MODULE_NOT_FOUND naming a file) — and dropped entirely if the plugin loads,
+ *     since a host cannot tell an unused declaration from a fatal one.
  *
  * @param bustCache When true, the URL imported for each specifier gets a unique
  *   `${FRESH_PARAM}` query stamp appended before importing — either a host-pre-resolved
@@ -91,13 +100,22 @@ let freshSeq = 0;
  *   only as a confusing empty-result error downstream — fail loudly with the reason instead.
  */
 export async function loadPlugins(
-  specifiers: readonly (string | { spec: string; importSpec?: string; name?: string; runtimes?: readonly Runtime[] })[],
+  specifiers: readonly (string | { spec: string; importSpec?: string; name?: string; version?: string; runtimes?: readonly Runtime[]; notes?: readonly string[] })[],
   services:   MatbotMachine,
   bustCache = false,
   prompt?:    PromptFn,
   onLoadError: 'skip' | 'throw' = 'skip',
 ): Promise<MatbotPlugin[]> {
-  const reqs = specifiers.map(s => (typeof s === 'string' ? { spec: s } : s) as { spec: string; importSpec?: string; name?: string; runtimes?: readonly Runtime[] });
+  const reqs = specifiers.map(s => (typeof s === 'string' ? { spec: s } : s) as { spec: string; importSpec?: string; name?: string; version?: string; runtimes?: readonly Runtime[]; notes?: readonly string[] });
+
+  // A reason reaching a user, decorated with what the host noticed while resolving this spec. The
+  // failure a load reports is usually downstream of its cause, and the host is the only layer that
+  // saw the cause.
+  const notesBySpec = new Map(reqs.filter(r => r.notes !== undefined && r.notes.length > 0).map(r => [r.spec, r.notes!]));
+  const withNotes = (spec: string, reason: string): string => {
+    const notes = notesBySpec.get(spec);
+    return notes === undefined ? reason : `${reason}\nNoted while resolving it:\n${notes.map(n => `  - ${n}`).join('\n')}`;
+  };
   if (bustCache) {
     console.debug(
       `[matbot] cache-bust requested for ${specifiers.length} plugin(s); the entry is re-evaluated, ` +
@@ -118,9 +136,9 @@ export async function loadPlugins(
     ),
   );
 
-  const toLoad: { spec: string; importSpec: string; resolvedUrl: string; name?: string; runtimes?: readonly Runtime[] }[] = [];
+  const toLoad: { spec: string; importSpec: string; resolvedUrl: string; name?: string; version?: string; runtimes?: readonly Runtime[] }[] = [];
   for (let i = 0; i < reqs.length; i++) {
-    const { spec, importSpec, name } = reqs[i]!;
+    const { spec, importSpec, name, version } = reqs[i]!;
     const runtimes = declared[i];
     if (runtimes !== undefined && runtimes.length > 0 && !runtimes.includes(CURRENT_RUNTIME)) {
       if (onLoadError === 'throw') {
@@ -139,6 +157,7 @@ export async function loadPlugins(
       // live plugin set). Falls back to the bare spec when the host pre-resolved nothing.
       resolvedUrl: importSpec ?? spec,
       ...(name     !== undefined ? { name }     : {}),
+      ...(version  !== undefined ? { version }  : {}),
       ...(runtimes !== undefined ? { runtimes } : {}),
     });
   }
@@ -157,7 +176,8 @@ export async function loadPlugins(
   // from a `'load'` failure (import rejection — a bad path, a syntax error). Only the former is
   // permanently-not-a-plugin: it throws NotAPluginError so the `add` flow can roll back the config
   // write, where an import failure may be a fixable typo and is left in config.
-  const failLoad = (spec: string, reason: string, kind: 'load' | 'shape' = 'load'): void => {
+  const failLoad = (spec: string, rawReason: string, kind: 'load' | 'shape' = 'load'): void => {
+    const reason = withNotes(spec, rawReason);
     if (onLoadError === 'throw') throw kind === 'shape' ? notAPluginError(spec, reason) : new Error(reason);
     console.warn(`[matbot] Skipping plugin "${spec}": ${reason}`);
     recordFailedPlugin({ specifier: spec, error: reason });
@@ -216,7 +236,11 @@ export async function loadPlugins(
     const name     = toLoad[i]!.name ?? (services.resolver !== undefined ? await services.resolver.identify(spec) : defaultIdentify(spec));
     const source   = sourceOf(spec);
     const runtimes = toLoad[i]!.runtimes;
-    const version  = services.resolver?.version !== undefined ? await services.resolver.version(spec) : undefined;
+    // Like `name`: the host's own reading wins, because it read the package.json it actually
+    // imported. `resolver.version(spec)` cannot — `spec` may be a bare name, a URL or a version
+    // range, none of which locate a manifest, so a plugin configured by name reported no version.
+    const version  = toLoad[i]!.version
+      ?? (services.resolver?.version !== undefined ? await services.resolver.version(spec) : undefined);
     const plugin: MatbotPlugin = {
       ...spec_obj,
       name,
@@ -248,7 +272,7 @@ export async function loadPlugins(
       // Graceful but not silent: record the setup failure (skip mode — the boot batch) so the `plugin`
       // tool and web UI can show it. Explicit single loads (`onLoadError: 'throw'`) surface the error to
       // the caller directly, so they are not added to the persistent failed list.
-      if (onLoadError === 'skip') recordFailedPlugin({ specifier: spec, name: plugin.name, error: reason });
+      if (onLoadError === 'skip') recordFailedPlugin({ specifier: spec, name: plugin.name, error: withNotes(spec, reason) });
       // Roll back partial registration so a failed load leaves no ghost state. setupPlugin
       // can throw after the plugin is already in the registry with some of its tools / hooks /
       // services / provider / storage wired (a wrong-platform plugin touching a missing

package/src/session-runner.ts

@@ -8,8 +8,8 @@ import type { MatbotPlugin } from './plugin.js';
 import type { HookRegistry } from './hooks.js';
 import type { ToolTypeIndex, ToolPresenter } from '@matatbread/matbot-plugin-api';
 import { appendMessage, createMessage } from './session.js';
-import { isReadOnlyError, foldOntoUserTurn, lastUserIndex } from '@matatbread/matbot-plugin-api';
-import { contextSwitch, withUsageScope } from '@matatbread/matbot-plugin-api/host';
+import { isReadOnlyError, foldOntoUserTurn, lastUserIndex, runAs } from '@matatbread/matbot-plugin-api';
+import { machineBusy, withUsageScope } from '@matatbread/matbot-plugin-api/host';
 import { runSession } from './runner.js';
 
 export interface SessionRunnerDeps {
@@ -255,6 +255,22 @@ export function createSessionRunner(deps: SessionRunnerDeps): SessionRunner {
   const pump = async (id: string, s: SessionState): Promise<void> => {
     if (s.running) return;
     s.running = true;
+    // Hold the machine for the WHOLE queue, not per turn. A deferred machine mutation (the
+    // StorageBackend swap, a plugin's deferred edit of this very session) may only land where no
+    // operation spans it — and the pump's own store work does not stop at a turn's end: it reads the
+    // committed document back for followup, appends markers to it, rewrites it for a retract, and
+    // persists the next turn's user message before that turn opens. All of that sits between turns,
+    // and a per-turn hold declared it idle. The queue draining is the real boundary — the same one
+    // usage flushes at, for the same reason. `machineBusy` releases on every exit including a throw,
+    // so no early return out of the loop below can leave the machine held.
+    //
+    // It also waits for deferred work to land before taking the hold, which is why there is no separate
+    // `quiesced()` here any more: a flusher that rewrites this very session (a `session_edit` deferred
+    // out of the last turn) finishes before the loop below reads its copy, or the read would take the
+    // pre-edit document and the write-back would put it back. That wait used to be spelled at this call
+    // site, and spelling it here made it something the next caller of `machineBusy` could omit without
+    // any symptom — so it moved inside the hold it guards.
+    return machineBusy(async () => {
     try {
       while (s.queue.length > 0) {
         // Per-submission concat: the head always runs; if the head is a concat submission it absorbs
@@ -350,8 +366,8 @@ export function createSessionRunner(deps: SessionRunnerDeps): SessionRunner {
           // turn's async root. Everything downstream (hooks, tools, and any Store/FileStore/Vault
           // access they trigger) reads it via currentPrincipal(). The consumption MUST happen inside
           // the callback: an async iterator returned out of the scope would lose it before it pulls.
-          // contextSwitch (not bare runAs): a turn is the transactional unit, so a StorageBackend swap
-          // deferred during it lands at this scope's quiescent edge — never mid-CAS.
+          // runAs, not a context switch of its own: the pump holds the machine across the whole queue
+          // (see above), so a turn declares only its owner. Each item carries its own submitter.
           // The terminal event this turn ended on. `followup` is post-COMMIT, and only `done` is a
           // commit: an `aborted` turn was cut short (a steer, a user cancel, a screen/toolcall hook
           // refusing it, a provider round ceiling) and an `error` turn produced no response at all, so
@@ -367,7 +383,7 @@ export function createSessionRunner(deps: SessionRunnerDeps): SessionRunner {
           // in something already tracked and needs no coordination of its own.
           await withUsageScope(async usage => {
           s.pendingUsage.push(usage.entries);
-          await contextSwitch(head.principal, async () => {
+          await runAs(head.principal, async () => {
             for await (const ev of runSession({
               session,
               config:         { provider: head.provider, traceId: head.traceId },
@@ -403,7 +419,7 @@ export function createSessionRunner(deps: SessionRunnerDeps): SessionRunner {
             const committed = await deps.store.get(id);
             if (committed && head.resubmitDepth < MAX_RESUBMIT_DEPTH) {
               let followup: { resubmits: MessageContent[][]; markers: MessageContent[]; retract?: { context: MessageContent[]; durable: MessageContent[] } } = { resubmits: [], markers: [] };
-              await contextSwitch(head.principal, async () => {
+              await runAs(head.principal, async () => {
                 followup = await hooks.runFollowup({
                   session:       committed,
                   resubmitDepth: head.resubmitDepth,
@@ -530,6 +546,7 @@ export function createSessionRunner(deps: SessionRunnerDeps): SessionRunner {
       notify(s, { type: 'idle', sessionId: id });
       maybeCleanup(id, s);
     }
+    });
   };
 
   return {

package/src/storage-base/caching-backend.ts

@@ -138,9 +138,22 @@ export class CachingStorageBackend implements StorageBackend {
   private readonly ttlMs:  number | undefined;
   private readonly stores = new Map<string, CachingStore<{ id: string; version: string }>>();
 
+  /**
+   * Forwarded only when the wrapped backend has it. `namespaces` is optional, and its ABSENCE is the
+   * signal a caller reads ("this backend cannot be enumerated") — so a decorator that always declared
+   * the method would answer for backends that cannot, turning a degradable capability into a runtime
+   * failure. Assigned per instance rather than declared as a method so `'namespaces' in backend` stays
+   * truthful through the wrapper.
+   *
+   * Not cached: the cache is per-namespace and populated by reads, so it knows only what has been
+   * asked for — never the set of namespaces that exist.
+   */
+  readonly namespaces?: () => Promise<string[]>;
+
   constructor(inner: StorageBackend, options: CachingOptions = {}) {
     this.inner = inner;
     this.ttlMs = options.ttlMs;
+    if (inner.namespaces !== undefined) this.namespaces = () => inner.namespaces!();
   }
 
   createStore<T extends { id: string; version: string }>(namespace: string): Store<T> {

package/src/storage-base/index.ts

@@ -1,2 +1,3 @@
 export * from './query/index.js';
 export { CachingStorageBackend, type CachingOptions, type CacheNamespaceStats } from './caching-backend.js';
+export { mediumGuard } from './medium-guard.js';

package/src/storage-base/medium-guard.ts

@@ -0,0 +1,114 @@
+import type { Store, CASResult, StoreQuery, QueryResult } from '@matatbread/matbot-plugin-api';
+
+/**
+ * Fail a write whose read came from a different storage medium.
+ *
+ * Exactly one `StorageBackend` is active, nothing is migrated between them, and a swap repoints every
+ * store proxy at once. A caller that read a document before the swap and writes it after is therefore
+ * addressing two different media with one read-modify-write, and neither end can tell: `cas` asks
+ * "did this document change?", which the new backend answers about a document it never issued — most
+ * often "there is nothing here", at which point an unconditional `set` writes the old backend's
+ * document into the new one and the session has silently migrated. Compare-and-swap cannot see this,
+ * because the medium is not what it compares.
+ *
+ * The fix belongs here rather than at whatever happened to be running: the exposure is not particular
+ * to the quiescent edge's deferred work — an HTTP tool call has always been able to straddle a swap
+ * the same way — so pushing the check up to every caller would be asking each of them to know
+ * something only storage knows.
+ *
+ * The version is the only token tying a read to its write, so it carries the medium: stamped on the
+ * way out, checked and stripped on the way in. Stripping is what keeps it invisible — a caller that
+ * writes a document straight back (`store.set(id, { ...doc, title })`, which does not bump the
+ * version) must not persist a stamp, or the next read would stamp it twice.
+ *
+ * An UNstamped version is always accepted: that is a document the caller minted rather than read, and
+ * a create has no earlier medium to disagree with.
+ */
+
+const STAMP = /^m(\d+)~([\s\S]*)$/;
+
+interface Stamped { id: string; version: string }
+
+function stamp<T extends Stamped>(doc: T, generation: number): T {
+  return { ...doc, version: `m${generation}~${doc.version}` };
+}
+
+type Checked =
+  | { ok: true; version: string }
+  | { ok: false; from: number };
+
+function check(version: string, generation: number): Checked {
+  const m = STAMP.exec(version);
+  if (m === null)                      return { ok: true, version };            // minted, not read
+  if (Number(m[1]) === generation)     return { ok: true, version: m[2]! };
+  return { ok: false, from: Number(m[1]) };
+}
+
+/** Strip a stamp wherever one appears, so nothing a caller hands back is persisted with one. */
+function strip<T extends Stamped>(doc: T): T {
+  const m = STAMP.exec(doc.version);
+  return m === null ? doc : { ...doc, version: m[2]! };
+}
+
+/**
+ * Wrap `inner` so every document it hands out is stamped with the medium generation it came from, and
+ * every write is checked against the generation current at the time of the write. `generation` must
+ * increase on every backend swap; the wrapper is placed OUTSIDE the swap proxy, so it is one stable
+ * object across every swap and a captured store reference keeps working.
+ */
+export function mediumGuard<T extends { id: string; version: string }>(
+  inner:      Store<T>,
+  generation: () => number,
+  namespace:  string,
+): Store<T> {
+  const stale = (op: string, id: string, from: number): string =>
+    `${op} on "${namespace}/${id}" read from storage backend generation ${from}, but generation ${generation()} is now active. ` +
+    'Nothing is migrated between backends, so this write would put a document from the previous one into its replacement. Re-read and retry.';
+
+  return {
+    async get(id) {
+      const doc = await inner.get(id);
+      return doc === null ? null : stamp(doc, generation());
+    },
+
+    async query(q: StoreQuery): Promise<QueryResult<T>> {
+      const page = await inner.query(q);
+      const gen  = generation();
+      return { ...page, items: page.items.map(d => stamp(d, gen)) };
+    },
+
+    async set(id, value) {
+      const seen = check(value.version, generation());
+      // `set` has no failure channel, and silence is what makes the migration invisible — so it
+      // throws. It is reachable only after a swap, holding a document read before it.
+      if (!seen.ok) throw new Error(stale('set', id, seen.from));
+      await inner.set(id, strip(value));
+    },
+
+    async cas(id, expected, next): Promise<CASResult<T>> {
+      const seen = check(expected, generation());
+      // Reported as a lost CAS rather than thrown: the caller already has a path for "someone else got
+      // there first", and this is that, with the medium in the role of the other writer. `current` is
+      // read from the backend now in force — the document the caller must reconcile against.
+      if (!seen.ok) {
+        console.warn(`[matbot] ${stale('cas', id, seen.from)}`);
+        const current = await inner.get(id);
+        return { ok: false, current: current === null ? null : stamp(current, generation()) };
+      }
+      const res = await inner.cas(id, seen.version, strip(next));
+      const gen = generation();
+      if (res.ok) return { ok: true, doc: stamp(res.doc, gen) };
+      return { ok: false, current: res.current === null ? null : stamp(res.current, gen) };
+    },
+
+    async delete(id, expectedVersion) {
+      if (expectedVersion === undefined) return inner.delete(id);
+      const seen = check(expectedVersion, generation());
+      if (!seen.ok) {
+        console.warn(`[matbot] ${stale('delete', id, seen.from)}`);
+        return false;
+      }
+      return inner.delete(id, seen.version);
+    },
+  };
+}