@peers-app/peers-sdk 0.30.5 → 0.30.6

package/dist/contracts/tool-catalog.d.ts

@@ -49,11 +49,42 @@ export declare class ToolCatalog {
     private subscription?;
     private disposed;
     private reconciliationQueued;
+    /**
+     * This context's Tools rows, read once by {@link ready} and kept current from the table's
+     * change events (local writes and synced changes alike). Publication checks against it:
+     * every start publishes every provider in every context, and reading rows for each one
+     * cost thousands of queries per start on a phone.
+     */
+    private rows?;
+    /** Rows changed since they were last reconciled. */
+    private readonly changed;
+    private readonly pendingRows;
+    private readonly revisions;
     /** Bind to the registry shared by host and isolated providers in this context. */
     constructor(dataContext: DataContext, registry: ContractRegistry);
     private enqueue;
     /** Await successful backfill and system publication. Failed initialization can be retried. */
     ready(): Promise<void>;
+    private initialized;
+    /** True while change events keep {@link rows} current. */
+    private live;
+    /**
+     * Read this context's rows into memory, after subscribing to the table's changes so none
+     * is missed while the read is in flight. A row that changes during the read keeps the
+     * changed version. Where the table can't report changes (a client process whose table
+     * events are off), the rows are read again before each use instead.
+     */
+    private load;
+    /** Read partial notifications in batches, retrying only rows changed during the read. */
+    private loadPendingRows;
+    /** Stop watching the table and forget its rows (after a failed start, or on dispose). */
+    private forget;
+    /**
+     * Record silent writes, but never replace a newer event with an older write result.
+     */
+    private write;
+    private get rowMap();
+    private queueReconciliation;
     private assertLive;
     private validate;
     /**
@@ -80,6 +111,7 @@ export declare class ToolCatalog {
      * leaves every other column, including concurrently synced policy, untouched.
      */
     private refresh;
+    /** Re-derive catalog state for these rows (all of them at startup, then those that change). */
     private reconcile;
     /**
      * Patch supplemental metadata of existing system members, preserving identity,

package/dist/contracts/tool-catalog.js

@@ -6,6 +6,7 @@ exports.retainToolCatalogFields = retainToolCatalogFields;
 exports.validateToolCatalogWrite = validateToolCatalogWrite;
 const lodash_1 = require("lodash");
 const tools_1 = require("../data/tools");
+const rpc_types_1 = require("../rpc-types");
 const registry_1 = require("./registry");
 const system_contracts_1 = require("./system-contracts");
 const tool_catalog_identity_1 = require("./tool-catalog-identity");
@@ -238,6 +239,14 @@ function resolvedFields(target, toolId) {
         catalogReason: undefined,
     };
 }
+/** Table writes normally report full rows; incomplete notifications need a storage read. */
+function isCompleteRow(row) {
+    return (row.name !== undefined &&
+        row.usageDescription !== undefined &&
+        row.inputSchema !== undefined &&
+        row.outputSchema !== undefined &&
+        row.code !== undefined);
+}
 /**
  * Per-context runtime owner of permanent Tools identities. Construction is inert;
  * ready() explicitly initializes storage after UserContext construction completes.
@@ -251,6 +260,17 @@ class ToolCatalog {
     subscription;
     disposed = false;
     reconciliationQueued = false;
+    /**
+     * This context's Tools rows, read once by {@link ready} and kept current from the table's
+     * change events (local writes and synced changes alike). Publication checks against it:
+     * every start publishes every provider in every context, and reading rows for each one
+     * cost thousands of queries per start on a phone.
+     */
+    rows;
+    /** Rows changed since they were last reconciled. */
+    changed = new Set();
+    pendingRows = new Set();
+    revisions = new Map();
     /** Bind to the registry shared by host and isolated providers in this context. */
     constructor(dataContext, registry) {
         this.dataContext = dataContext;
@@ -274,7 +294,8 @@ class ToolCatalog {
             return Promise.reject(new Error("Tool catalog disposed"));
         if (!this.initialization) {
             this.initialization = this.enqueue(async () => {
-                await this.reconcile();
+                await this.load();
+                await this.reconcile([...this.rowMap.keys()]);
                 const groups = new Map();
                 for (const entry of (0, system_contracts_1.listSystemContracts)()) {
                     const entries = groups.get(entry.providerPackageId) ?? [];
@@ -295,25 +316,148 @@ class ToolCatalog {
                         this.registry.replaceBatch(provider, registrations);
                     }
                 });
-                // dispose() may have run while initialization awaited storage; it has
-                // already unsubscribed, so a late subscription would never be released.
+                // dispose() may have run while initialization awaited storage.
                 this.assertLive();
-                this.subscription ??= (0, tools_1.Tools)(this.dataContext).dataChanged.subscribe(() => {
-                    if (this.reconciliationQueued || this.disposed)
-                        return;
-                    this.reconciliationQueued = true;
-                    void this.enqueue(async () => {
-                        this.reconciliationQueued = false;
-                        await this.reconcile();
-                    }).catch((error) => console.error("Tool catalog reconciliation failed", error));
-                });
+                this.initialized = true;
+                this.queueReconciliation();
             }).catch((error) => {
                 this.initialization = undefined;
+                this.forget();
                 throw error;
             });
         }
         return this.initialization;
     }
+    initialized = false;
+    /** True while change events keep {@link rows} current. */
+    live = false;
+    /**
+     * Read this context's rows into memory, after subscribing to the table's changes so none
+     * is missed while the read is in flight. A row that changes during the read keeps the
+     * changed version. Where the table can't report changes (a client process whose table
+     * events are off), the rows are read again before each use instead.
+     */
+    async load() {
+        // Work queued before dispose() would otherwise subscribe again, and nothing would release it.
+        this.assertLive();
+        if (this.rows && this.live)
+            return this.loadPendingRows();
+        const table = (0, tools_1.Tools)(this.dataContext);
+        const rows = new Map();
+        const seen = new Set();
+        if (!this.subscription && table.dataChanged && !(rpc_types_1.isClient && !(0, rpc_types_1.isSingleProcessClient)())) {
+            this.subscription = table.dataChanged.subscribe((event) => {
+                const toolId = event.dataObject.toolId;
+                this.revisions.set(toolId, (this.revisions.get(toolId) ?? 0) + 1);
+                if (!this.rows)
+                    seen.add(toolId);
+                const target = this.rows ?? rows;
+                if (event.op === "delete")
+                    target.delete(toolId);
+                // Full replacements clear omitted optional fields in SQL. Merging would retain
+                // removed provenance, descriptions or quarantine evidence indefinitely.
+                else if (isCompleteRow(event.dataObject))
+                    target.set(toolId, (0, lodash_1.cloneDeep)(event.dataObject));
+                else
+                    this.pendingRows.add(toolId);
+                this.changed.add(toolId);
+                if (this.initialized)
+                    this.queueReconciliation();
+            });
+            this.live = true;
+        }
+        for (const row of await table.list()) {
+            if (!seen.has(row.toolId))
+                rows.set(row.toolId, row);
+        }
+        this.assertLive();
+        this.rows = rows;
+        if (this.pendingRows.size)
+            await this.loadPendingRows();
+    }
+    /** Read partial notifications in batches, retrying only rows changed during the read. */
+    async loadPendingRows() {
+        while (this.pendingRows.size) {
+            const ids = [...this.pendingRows].slice(0, 500);
+            const revisions = ids.map((id) => this.revisions.get(id));
+            const rows = new Map((await (0, tools_1.Tools)(this.dataContext).list({ toolId: { $in: ids } })).map((row) => [
+                row.toolId,
+                row,
+            ]));
+            this.assertLive();
+            ids.forEach((id, index) => {
+                if (this.revisions.get(id) !== revisions[index])
+                    return;
+                const row = rows.get(id);
+                if (row)
+                    this.rowMap.set(id, row);
+                else
+                    this.rowMap.delete(id);
+                this.pendingRows.delete(id);
+            });
+        }
+    }
+    /** Stop watching the table and forget its rows (after a failed start, or on dispose). */
+    forget() {
+        this.subscription?.unsubscribe();
+        this.subscription = undefined;
+        this.live = false;
+        this.rows = undefined;
+        this.changed.clear();
+        this.pendingRows.clear();
+        this.revisions.clear();
+        this.initialized = false;
+    }
+    /**
+     * Record silent writes, but never replace a newer event with an older write result.
+     */
+    async write(toolId, save) {
+        const revision = this.revisions.get(toolId);
+        let row;
+        try {
+            row = await save();
+        }
+        catch (error) {
+            // The row write can succeed before history persistence fails and suppresses its event.
+            if (this.rows)
+                this.pendingRows.add(toolId);
+            throw error;
+        }
+        if (this.rows && this.revisions.get(toolId) === revision) {
+            // Table returns and event payloads can still be held and edited by their caller.
+            if (isCompleteRow(row))
+                this.rows.set(toolId, (0, lodash_1.cloneDeep)(row));
+            else
+                this.pendingRows.add(toolId);
+        }
+        return row;
+    }
+    get rowMap() {
+        if (!this.rows)
+            throw new Error("Tool catalog is not ready");
+        return this.rows;
+    }
+    queueReconciliation() {
+        if (this.reconciliationQueued || this.disposed || !this.changed.size)
+            return;
+        this.reconciliationQueued = true;
+        void this.enqueue(async () => {
+            this.reconciliationQueued = false;
+            const ids = [...this.changed];
+            this.changed.clear();
+            try {
+                if (this.pendingRows.size)
+                    await this.loadPendingRows();
+                await this.reconcile(ids);
+            }
+            catch (error) {
+                // Keep failed and unvisited rows for the next event, like the old full pass.
+                for (const id of ids)
+                    this.changed.add(id);
+                throw error;
+            }
+        }).catch((error) => console.error("Tool catalog reconciliation failed", error));
+    }
     assertLive() {
         if (this.disposed)
             throw new Error("Tool catalog disposed");
@@ -349,6 +493,7 @@ class ToolCatalog {
         }
         await this.ready();
         return this.enqueue(async () => {
+            await this.load();
             this.validate(provider, registrations);
             await this.materialize(registrations, false);
             await prepare();
@@ -356,7 +501,9 @@ class ToolCatalog {
             await this.materialize(registrations, true, provider);
             for (const metadata of toolMetadata) {
                 for (const toolId of (0, tool_catalog_migrations_1.toolCatalogIds)(metadata)) {
-                    const stored = await (0, tools_1.Tools)(this.dataContext).get(toolId);
+                    if (this.pendingRows.size)
+                        await this.loadPendingRows();
+                    const stored = this.rowMap.get(toolId);
                     if (!stored || stored.catalogState !== "resolved")
                         throw new Error(`Missing or quarantined tool identity: ${toolId}`);
                     await this.refresh({ ...metadata, toolId }, stored, tools_1.toolCatalogSupplementalFields);
@@ -392,6 +539,7 @@ class ToolCatalog {
             if (!expected.has(toolId))
                 return;
             try {
+                // Validate the payload as written: a partial row that omits identity fails closed.
                 validateRow(toolId, event.op === "delete" ? undefined : event.dataObject);
             }
             catch (error) {
@@ -405,8 +553,10 @@ class ToolCatalog {
                 this.validate(provider, registrations);
         };
         try {
+            // Clients without table events need the same final storage check as before caching.
+            await this.load();
             for (const toolId of expected.keys())
-                validateRow(toolId, await table.get(toolId));
+                validateRow(toolId, this.rowMap.get(toolId));
             assertCatalogCurrent();
             return await activate(assertCatalogCurrent);
         }
@@ -443,8 +593,9 @@ class ToolCatalog {
      * definition's name, usage description and IO schemas.
      */
     async materialize(registrations, refresh, provider) {
+        await this.load();
         const table = (0, tools_1.Tools)(this.dataContext);
-        const existing = new Map((await table.list()).map((row) => [row.toolId, row]));
+        const existing = this.rowMap;
         const identities = new Map();
         // Reserve the entire reviewed inventory, even on an empty database.
         for (const entry of tool_catalog_migrations_1.toolCatalogMigrations)
@@ -486,23 +637,32 @@ class ToolCatalog {
         // Validate the entire batch before the first write. Catalog metadata never
         // overwrites provider-specific policy or another provider's descriptions.
         for (const row of candidates.values()) {
+            if (this.pendingRows.size)
+                await this.loadPendingRows();
             const previous = existing.get(row.toolId);
+            let restored;
             // Restore missing identity columns only after validating the entire batch.
             // This marked patch leaves concurrent policy and historical fields intact.
             if (previous &&
                 (canRestoreBareDeterministicRow(previous, row) || canResolveMissingTarget(previous, row))) {
-                const restored = await table.update((0, tools_1.toolCatalogPatch)({
+                restored = await this.write(row.toolId, () => table.update((0, tools_1.toolCatalogPatch)({
                     toolId: row.toolId,
                     ...resolvedFields(toolCatalogTarget(row), row.toolId),
-                }));
+                })));
                 validateToolCatalogWrite(row, restored);
                 if (restored.catalogState === "quarantined")
                     throw new Error(`Quarantined tool identity: ${row.toolId}`);
             }
-            // Older runtimes allowed provider removal to leave tombstones. Only a
-            // validated public target may restore one; insert-only still preserves any
-            // row that arrived concurrently, and the returned mapping is checked below.
-            const stored = await table.save(row, { insertOnly: true, restoreIfDeleted: true });
+            // A row already in memory needs no insert-only save, which would only read it back:
+            // every start would cost a query per tool per context. A row that changes later is
+            // caught by guardActivation, which checks again under a subscription before anything
+            // activates. Older runtimes allowed provider removal to leave tombstones, which aren't
+            // in memory: only a validated public target may restore one, and insert-only still
+            // preserves a row that arrived concurrently.
+            let stored = restored ?? previous;
+            if (!stored) {
+                stored = await this.write(row.toolId, () => table.save(row, { insertOnly: true, restoreIfDeleted: true }));
+            }
             validateToolCatalogWrite(row, stored);
             if (!stored || stored.catalogState === "quarantined")
                 throw new Error(`Catalog insert did not succeed: ${row.toolId}`);
@@ -521,14 +681,20 @@ class ToolCatalog {
         const patch = { toolId: desired.toolId };
         for (const field of changed)
             Object.assign(patch, { [field]: desired[field] });
-        const refreshed = await (0, tools_1.Tools)(this.dataContext).update((0, tools_1.toolCatalogPatch)(patch));
+        const refreshed = await this.write(desired.toolId, () => (0, tools_1.Tools)(this.dataContext).update((0, tools_1.toolCatalogPatch)(patch)));
         validateToolCatalogWrite(stored, refreshed);
         if (refreshed.catalogState === "quarantined")
             throw new Error(`Quarantined tool identity: ${desired.toolId}`);
     }
-    async reconcile() {
+    /** Re-derive catalog state for these rows (all of them at startup, then those that change). */
+    async reconcile(toolIds) {
         const table = (0, tools_1.Tools)(this.dataContext);
-        for (const row of await table.list()) {
+        for (const toolId of toolIds) {
+            if (this.pendingRows.size)
+                await this.loadPendingRows();
+            const row = this.rowMap.get(toolId);
+            if (!row)
+                continue;
             let fields;
             // Quarantine is sticky: conflicting evidence is never silently rehabilitated. The one
             // exception is a row quarantined for its missing target that has since received the
@@ -540,7 +706,10 @@ class ToolCatalog {
                 const resolved = { ...row, ...resolvedFields(target, row.toolId) };
                 if (!canResolveMissingTarget(row, resolved))
                     continue;
-                await table.update((0, tools_1.toolCatalogPatch)({ toolId: row.toolId, ...resolvedFields(target, row.toolId) }));
+                await this.write(row.toolId, () => table.update((0, tools_1.toolCatalogPatch)({
+                    toolId: row.toolId,
+                    ...resolvedFields(target, row.toolId),
+                })));
                 continue;
             }
             try {
@@ -554,7 +723,7 @@ class ToolCatalog {
             if (Object.entries(fields).some(([key, value]) => row[key] !== value)) {
                 // Update only catalog-owned fields. A sync write may have changed legacy
                 // metadata since list() returned; writing the snapshot back would lose it.
-                await table.update((0, tools_1.toolCatalogPatch)({ toolId: row.toolId, ...fields }));
+                await this.write(row.toolId, () => table.update((0, tools_1.toolCatalogPatch)({ toolId: row.toolId, ...fields })));
             }
         }
     }
@@ -566,6 +735,7 @@ class ToolCatalog {
     async saveToolMetadata(entries) {
         await this.ready();
         await this.enqueue(async () => {
+            await this.load();
             const members = (0, system_contracts_1.listSystemContracts)().flatMap((entry) => {
                 const definition = entry.getDefinition();
                 return definition.tools.map((tool) => ({
@@ -578,7 +748,7 @@ class ToolCatalog {
             for (const entry of entries) {
                 const target = (0, tool_catalog_migrations_1.resolveLegacyToolId)(entry.toolId) ??
                     members.find((member) => (0, tool_catalog_migrations_1.toolCatalogIds)(member).includes(entry.toolId));
-                const stored = await (0, tools_1.Tools)(this.dataContext).get(entry.toolId);
+                const stored = this.rowMap.get(entry.toolId);
                 if (!target ||
                     !members.some((member) => (0, tool_catalog_identity_1.toolCatalogTargetKey)(member) === (0, tool_catalog_identity_1.toolCatalogTargetKey)(target)) ||
                     !stored ||
@@ -626,7 +796,7 @@ class ToolCatalog {
     /** Stop subscriptions and queued lifecycle work; permanent rows are retained. */
     async dispose() {
         this.disposed = true;
-        this.subscription?.unsubscribe();
+        this.forget();
         await this.tail;
     }
 }

package/dist/data/orm/table.d.ts

@@ -21,6 +21,8 @@ export declare class Table<T extends {
     protected readonly dataChangedEmitter: Emitter<IDataChangedEvent<T>>;
     readonly dataChanged: Event<IDataChangedEvent<T>>;
     private readonly localOperationCache;
+    /** Rows committed by an in-progress local write, for {@link reportCommittedLocalWrite}. */
+    private readonly committedLocalEvents;
     readonly dataSource: IDataSource<T>;
     readonly metaData: ITableMetaData;
     readonly groupId?: string;
@@ -28,6 +30,12 @@ export declare class Table<T extends {
     private getRecordId;
     private setLocalOperation;
     private clearLocalOperation;
+    /**
+     * A local write failed after its row was committed (for example, a tracked table's history
+     * write failed). Tell subscribers about the committed row, as a successful write would, so
+     * a cache of this table (the Tools catalog's) doesn't miss it.
+     */
+    private reportCommittedLocalWrite;
     private cachedRecords;
     get(id: string, opts?: {
         useCache?: boolean;

package/dist/data/orm/table.js

@@ -28,6 +28,8 @@ class Table {
     dataChangedEmitter;
     dataChanged;
     localOperationCache = new Map();
+    /** Rows committed by an in-progress local write, for {@link reportCommittedLocalWrite}. */
+    committedLocalEvents = new Map();
     dataSource;
     metaData;
     groupId;
@@ -79,6 +81,12 @@ class Table {
                         source: "remote",
                     });
                 }
+                else {
+                    // A local write's row is committed, but the write can still fail after it (a
+                    // tracked table's history write). Keep the event, so a failed write still reports
+                    // the committed row; a successful one emits its own event instead.
+                    this.committedLocalEvents.set(recordId, event);
+                }
             });
         }
     }
@@ -92,6 +100,17 @@ class Table {
     clearLocalOperation(recordId, operation) {
         const cacheKey = `${recordId}-${operation}`;
         this.localOperationCache.delete(cacheKey);
+        this.committedLocalEvents.delete(recordId);
+    }
+    /**
+     * A local write failed after its row was committed (for example, a tracked table's history
+     * write failed). Tell subscribers about the committed row, as a successful write would, so
+     * a cache of this table (the Tools catalog's) doesn't miss it.
+     */
+    reportCommittedLocalWrite(recordId) {
+        const event = this.committedLocalEvents.get(recordId);
+        if (event)
+            this.dataChangedEmitter.emit({ ...event, source: "local" });
     }
     cachedRecords = {};
     async get(id, opts = {}) {
@@ -161,6 +180,10 @@ class Table {
             // TODO: try to avoid the double-read, probably by passing the just-read result as `opts.dbData`
             result = await this.dataSource.save(record, opts);
         }
+        catch (error) {
+            this.reportCommittedLocalWrite(recordId);
+            throw error;
+        }
         finally {
             this.clearLocalOperation(recordId, operation);
         }
@@ -178,6 +201,10 @@ class Table {
         try {
             result = await this.dataSource.insert(record);
         }
+        catch (error) {
+            this.reportCommittedLocalWrite(recordId);
+            throw error;
+        }
         finally {
             this.clearLocalOperation(recordId, "insert");
         }
@@ -196,6 +223,10 @@ class Table {
         try {
             result = await this.dataSource.update(record);
         }
+        catch (error) {
+            this.reportCommittedLocalWrite(recordId);
+            throw error;
+        }
         finally {
             this.clearLocalOperation(recordId, "update");
         }
@@ -217,6 +248,10 @@ class Table {
         try {
             await this.dataSource.delete(idOrRecord);
         }
+        catch (error) {
+            this.reportCommittedLocalWrite(primaryKey);
+            throw error;
+        }
         finally {
             this.clearLocalOperation(primaryKey, "delete");
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peers-app/peers-sdk",
-  "version": "0.30.5",
+  "version": "0.30.6",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/peers-app/peers-sdk.git"