@pmate/chat-sdk 0.12.0-next.44 → 0.12.0-next.46

package/lib/runtime/batchedReadIndex.d.ts

@@ -1,5 +1,7 @@
-import { type ReadIndexPersistence, type ReadIndexState, type ReadIndexTransform } from "./readIndex.js";
+import { type ReadIndexPersistence, type ReadIndexState, type ReadIndexTransform, type ReadObservationScope } from "./readIndex.js";
 type Change = {
+    weight: number;
+    observation?: ReadObservationScope;
     transform: ReadIndexTransform;
     resolve: (state: ReadIndexState) => void;
     reject: (error: unknown) => void;
@@ -13,6 +15,7 @@ export declare class BatchedReadIndex implements ReadIndexPersistence {
     private queueLimit;
     private queues;
     private closed;
+    private sourceCounts;
     constructor(storage: ReadIndexPersistence, windowMs?: number, batchSize?: number, queueLimit?: number);
     load(appId: string, profileId: string): Promise<ReadIndexState | undefined>;
     loadVersioned(appId: string, profileId: string): Promise<{
@@ -24,7 +27,8 @@ export declare class BatchedReadIndex implements ReadIndexPersistence {
     }>;
     loadVersions(appId: string, profileId: string): Promise<{}>;
     invalidateTeam(appId: string, profileId: string, teamId: string, version: number, knownThreadIds?: readonly string[]): Promise<ReadIndexState>;
-    update(appId: string, profileId: string, transform: Change["transform"]): Promise<ReadIndexState>;
+    private weight;
+    update(appId: string, profileId: string, transform: Change["transform"], observation?: ReadObservationScope): Promise<ReadIndexState>;
     private drain;
     flush(appId: string, profileId: string): Promise<void>;
     close(): Promise<void>;

package/lib/runtime/batchedReadIndex.js

@@ -8,6 +8,7 @@ export class BatchedReadIndex {
     queueLimit;
     queues = new Map();
     closed = false;
+    sourceCounts = new WeakMap();
     constructor(storage, windowMs = 16, batchSize = 128, queueLimit = 4096) {
         this.storage = storage;
         this.windowMs = windowMs;
@@ -42,7 +43,22 @@ export class BatchedReadIndex {
         // barrier; any queued transform sees it inside its eventual transaction.
         return this.storage.invalidateTeam(appId, profileId, teamId, version, knownThreadIds);
     }
-    update(appId, profileId, transform) {
+    weight(observation) {
+        if (!observation)
+            return 1;
+        let counts = this.sourceCounts.get(observation.base);
+        if (!counts) {
+            counts = new Map();
+            for (const thread of observation.base.threads) {
+                counts.set(thread.id, (counts.get(thread.id) ?? 0) + 1);
+                if (thread.conversationId && thread.conversationId !== thread.id)
+                    counts.set(thread.conversationId, (counts.get(thread.conversationId) ?? 0) + 1);
+            }
+            this.sourceCounts.set(observation.base, counts);
+        }
+        return observation.targets.reduce((sum, target) => sum + (counts.get(target.threadId) ?? 1), 0);
+    }
+    update(appId, profileId, transform, observation) {
         if (this.closed)
             return Promise.reject(new Error("READ_STORAGE_CLOSED"));
         const key = JSON.stringify([appId, profileId]);
@@ -53,7 +69,7 @@ export class BatchedReadIndex {
         }
         if (queue.pending.length >= this.queueLimit)
             return Promise.reject(new Error("READ_STORAGE_BACKPRESSURE"));
-        const promise = new Promise((resolve, reject) => queue.pending.push({ transform, resolve, reject }));
+        const promise = new Promise((resolve, reject) => queue.pending.push({ transform, observation, weight: this.weight(observation), resolve, reject }));
         if (!queue.running && queue.timer === undefined)
             queue.timer = setTimeout(() => { queue.timer = undefined; void this.drain(key, queue); }, this.windowMs);
         return promise;
@@ -63,12 +79,22 @@ export class BatchedReadIndex {
             return queue.running;
         clearTimeout(queue.timer);
         queue.timer = undefined;
-        const batch = queue.pending.splice(0, this.batchSize);
+        const batch = [];
+        let weight = 0;
+        while (queue.pending.length && batch.length < this.batchSize) {
+            const nextWeight = queue.pending[0].weight;
+            if (batch.length && weight + nextWeight > 128)
+                break;
+            weight += nextWeight;
+            batch.push(queue.pending.shift());
+        }
         if (!batch.length)
             return;
         const run = (async () => {
             try {
-                const next = await this.storage.update(queue.appId, queue.profileId, (state, versions) => batch.reduce((current, change) => change.transform(current, versions), state));
+                const first = batch[0].observation;
+                const observation = first && batch.every(change => change.observation?.base === first.base) ? { base: first.base, targets: batch.flatMap(change => change.observation.targets) } : undefined;
+                const next = await this.storage.update(queue.appId, queue.profileId, (state, versions) => batch.reduce((current, change) => change.transform(current, versions), state), observation);
                 batch.forEach(change => change.resolve(next));
             }
             catch (error) {

package/lib/runtime/durableSender.d.ts

@@ -8,6 +8,7 @@ export type OutgoingMessage = {
     message: ChatRuntimeMessage;
     status: "sending" | "failed" | "sent";
     confirmed?: boolean;
+    pendingState?: 0 | 1;
     accessVersion?: number;
     voiceBlob?: Blob;
     voiceData?: {

package/lib/runtime/durableSender.js

@@ -39,8 +39,11 @@ export class DurableChatSender {
         this.db = new Dexie(options.name ?? "pmate-chat-outbox", { indexedDB: options.indexedDB ?? globalThis.indexedDB, IDBKeyRange: globalThis.IDBKeyRange });
         this.db.version(1).stores({ outbox: "[scope+hash],scope,[scope+threadId]" });
         this.db.version(2).stores({ scopeEpochs: "scope" });
+        this.db.version(3).stores({ outbox: "[scope+hash],scope,[scope+threadId],[scope+pendingState+hash]" });
         this.outbox = this.db.table("outbox");
         this.epochs = this.db.table("scopeEpochs");
+        this.outbox.hook("creating", (_key, row) => { row.pendingState = row.confirmed ? 0 : 1; });
+        this.outbox.hook("updating", (changes, _key, row) => ({ pendingState: (Object.hasOwn(changes, "confirmed") ? changes.confirmed : row.confirmed) ? 0 : 1 }));
         this.db.on("versionchange", () => { this.db.close({ disableAutoOpen: false }); return false; });
         this.ready = this.restore();
         void this.ready.catch(() => { });
@@ -61,6 +64,7 @@ export class DurableChatSender {
         const key = messageScopeKey(scope), db = new Dexie(options.name ?? "pmate-chat-outbox", { indexedDB: options.indexedDB ?? globalThis.indexedDB, IDBKeyRange: globalThis.IDBKeyRange });
         db.version(1).stores({ outbox: "[scope+hash],scope,[scope+threadId]" });
         db.version(2).stores({ scopeEpochs: "scope" });
+        db.version(3).stores({ outbox: "[scope+hash],scope,[scope+threadId],[scope+pendingState+hash]" });
         try {
             return await db.transaction("r", db.table("outbox"), db.table("scopeEpochs"), async () => {
                 if (((await db.table("scopeEpochs").get(key))?.version ?? 0) !== version)
@@ -90,7 +94,9 @@ export class DurableChatSender {
                 await this.epochs.put({ scope: this.scope, version });
                 await this.deleteRows();
             }
-            return { version, advanced: version > current, records: access?.allowed === false ? [] : await this.outbox.where("scope").equals(this.scope).toArray() };
+            const total = await this.outbox.where("scope").equals(this.scope).count();
+            const indexed = await this.outbox.where("[scope+pendingState+hash]").between([this.scope, 0, Dexie.minKey], [this.scope, 1, Dexie.maxKey], true, true).count();
+            return { version, advanced: version > current, legacy: total !== indexed };
         });
         this.version = loaded.version;
         if (loaded.advanced)
@@ -100,7 +106,45 @@ export class DurableChatSender {
         const current = await this.readAccess();
         if (current && (!current.allowed || current.version !== this.version))
             return;
-        loaded.records.filter(record => !record.confirmed && (record.accessVersion ?? 0) === this.version).forEach(record => this.values.set(record.hash, record));
+        if (access?.allowed === false)
+            return;
+        let after;
+        const restored = new Map();
+        while (!this.closed && !this.withdrawn) {
+            const batch = await this.db.transaction("rw", this.outbox, this.epochs, async () => {
+                if (((await this.epochs.get(this.scope))?.version ?? 0) !== this.version)
+                    throw new Error("OUTBOX_SCOPE_CHANGED");
+                const keys = await (loaded.legacy
+                    ? this.outbox.where("[scope+hash]").between([this.scope, after ?? Dexie.minKey], [this.scope, Dexie.maxKey], after === undefined, true)
+                    : this.outbox.where("[scope+pendingState+hash]").between([this.scope, 1, after ?? Dexie.minKey], [this.scope, 1, Dexie.maxKey], after === undefined, true)).limit(128).primaryKeys();
+                if (!keys.length)
+                    return { last: undefined, records: [] };
+                const last = keys.at(-1)[1];
+                // Key-only metadata covers confirmed history without reading wire bodies.
+                // Missing metadata identifies rows written by an older SDK after upgrade.
+                const confirmed = loaded.legacy ? await this.outbox.where("[scope+pendingState+hash]").between([this.scope, 0, keys[0][1]], [this.scope, 0, last], true, true).primaryKeys() : [];
+                const confirmedHashes = new Set(confirmed.map(key => key[1]));
+                const needed = keys.filter(key => !confirmedHashes.has(key[1]));
+                const rows = (await this.outbox.bulkGet(needed)).filter((row) => Boolean(row));
+                const repairs = rows.filter(row => row.pendingState !== (row.confirmed ? 0 : 1));
+                if (repairs.length)
+                    await this.outbox.bulkPut(repairs.map(row => ({ ...row, pendingState: row.confirmed ? 0 : 1 })));
+                return { last, records: rows.filter(row => !row.confirmed && (row.accessVersion ?? 0) === this.version) };
+            });
+            for (const row of batch.records)
+                restored.set(row.hash, row);
+            if (batch.last === undefined)
+                break;
+            after = batch.last;
+            // Leave the database transaction before yielding to input/rendering work.
+            await new Promise(resolve => setTimeout(resolve, 0));
+        }
+        if (this.closed || this.withdrawn)
+            return;
+        const latest = await this.readAccess();
+        if (latest && (!latest.allowed || latest.version !== this.version))
+            return;
+        this.values = restored;
         this.emit();
     }
     async checkAccess() {

package/lib/runtime/readIndex.d.ts

@@ -90,6 +90,14 @@ export declare function assertReadIndexVersions(expected: ReadIndexVersions, cur
 /** Remove only the withdrawn team's original sources, logical conversations and
  * queued observations. Other teams' pending operations must survive. */
 export declare function removeReadTeam(state: ReadIndexState, teamId: string, knownThreadIds?: readonly string[]): ReadIndexState;
+/** Internal hint for typed reading observations; arbitrary transforms omit it. */
+export type ReadObservationScope = {
+    base: ReadIndexState;
+    targets: readonly {
+        threadId: string;
+        messageId: string;
+    }[];
+};
 export interface ReadIndexPersistence {
     load(appId: string, profileId: string): Promise<ReadIndexState | undefined>;
     loadVersioned?(appId: string, profileId: string): Promise<{
@@ -99,7 +107,7 @@ export interface ReadIndexPersistence {
     loadVersions?(appId: string, profileId: string): Promise<ReadIndexVersions>;
     invalidateTeam?(appId: string, profileId: string, teamId: string, version: number, knownThreadIds?: readonly string[]): Promise<ReadIndexState>;
     /** Transform and persist atomically; UI publishes only the committed result. */
-    update(appId: string, profileId: string, transform: ReadIndexTransform): Promise<ReadIndexState>;
+    update(appId: string, profileId: string, transform: ReadIndexTransform, observation?: ReadObservationScope): Promise<ReadIndexState>;
 }
 /** Dexie preserves the legacy database/store and outbound [appId, profileId] keys. */
 export declare class IndexedDbReadIndex implements ReadIndexPersistence {
@@ -112,6 +120,7 @@ export declare class IndexedDbReadIndex implements ReadIndexPersistence {
     private messageRows;
     private positionRows;
     private normalizedStates;
+    private observationMessages;
     private versions;
     private closed;
     constructor(factory?: IDBFactory, name?: string, storageScope?: string | undefined, migrateLegacy?: boolean, metadataOnly?: boolean);
@@ -130,7 +139,8 @@ export declare class IndexedDbReadIndex implements ReadIndexPersistence {
     }>;
     loadVersions(appId: string, profileId: string): Promise<ReadIndexVersions>;
     invalidateTeam(appId: string, profileId: string, teamId: string, version: number, knownThreadIds?: readonly string[]): Promise<ReadIndexState>;
-    update(appId: string, profileId: string, transform: ReadIndexTransform): Promise<ReadIndexState>;
+    private observe;
+    update(appId: string, profileId: string, transform: ReadIndexTransform, observation?: ReadObservationScope): Promise<ReadIndexState>;
     close(): Promise<void>;
 }
 export declare function sourcesInConversation(state: ReadIndexState, conversationId: string): IndexedReadThread[];

package/lib/runtime/readIndex.js

@@ -214,6 +214,7 @@ export class IndexedDbReadIndex {
     messageRows;
     positionRows;
     normalizedStates = new WeakSet();
+    observationMessages = new WeakMap();
     versions;
     closed = false;
     constructor(factory = globalThis.indexedDB, name = "pmate-social-read-index", storageScope, migrateLegacy = false, metadataOnly = false) {
@@ -227,6 +228,9 @@ export class IndexedDbReadIndex {
         this.database.version(1).stores({ profiles: "" });
         this.database.version(2).stores({ profiles: "", versions: "" });
         this.database.version(3).stores({ readThreads: "[owner+id],owner", readMessages: "[owner+thread+id],[owner+thread],owner", readPositions: "[owner+id],owner" });
+        // Native indexes track scalar source identity even when an older writer
+        // changes rows. Reading these keys does not deserialize messageIds arrays.
+        this.database.version(4).stores({ readThreads: "[owner+id],owner,[owner+id+value.kind+value.teamId],[owner+value.conversationId+value.kind+value.teamId+id],[owner+id+value.conversationId]" });
         this.positionRows = this.database.table("readPositions");
         this.threadRows = this.database.table("readThreads");
         this.messageRows = this.database.table("readMessages");
@@ -435,8 +439,113 @@ export class IndexedDbReadIndex {
             return next;
         });
     }
-    async update(appId, profileId, transform) {
+    async observe(appId, profileId, transform, scope) {
+        if (!this.metadataOnly || scope.targets.length > 128)
+            return undefined;
+        const key = this.key(appId, profileId), owner = JSON.stringify(key), base = scope.base;
+        const header = await this.profiles.get(key);
+        if (header && (header.appId !== appId || header.profileId !== profileId || header.schemaVersion !== 1))
+            throw new Error("READ_IDENTITY_MISMATCH");
+        if (!header?.normalized || base.schemaVersion !== 1 || base.appId !== appId || base.profileId !== profileId || base.syncCursor !== header.syncCursor ||
+            header.threadOrder?.length !== base.threads.length || header.threadOrder.some((id, i) => id !== base.threads[i].id))
+            return undefined;
+        const sources = new Map();
+        for (const threadId of new Set(scope.targets.map(target => target.threadId))) {
+            const direct = await this.threadRows.where("[owner+id+value.kind+value.teamId]").between([owner, threadId, Dexie.minKey], [owner, threadId, Dexie.maxKey], true, true).limit(129).keys();
+            const assigned = await this.threadRows.where("[owner+value.conversationId+value.kind+value.teamId+id]").between([owner, threadId, Dexie.minKey], [owner, threadId, Dexie.maxKey], true, true).limit(129).keys();
+            if (direct.length + assigned.length > 128)
+                return undefined;
+            const identities = [...direct.map(value => ({ id: value[1], kind: value[2], teamId: value[3] })), ...assigned.map(value => ({ id: value[4], kind: value[2], teamId: value[3] }))];
+            if (!identities.length && base.threads.some(thread => thread.id === threadId || thread.conversationId === threadId))
+                return undefined;
+            for (const identity of identities) {
+                const cached = base.threads.find(thread => thread.id === identity.id);
+                if (!cached || cached.kind !== identity.kind || cached.teamId !== identity.teamId)
+                    return undefined;
+                const mapping = await this.threadRows.where("[owner+id+value.conversationId]").between([owner, identity.id, Dexie.minKey], [owner, identity.id, Dexie.maxKey], true, true).limit(2).keys();
+                if (mapping.length > 1 || cached.conversationId !== mapping[0]?.[2])
+                    return undefined;
+                let source = sources.get(identity.id);
+                if (!source) {
+                    source = { cached, ids: new Set() };
+                    sources.set(identity.id, source);
+                }
+                for (const target of scope.targets)
+                    if (target.threadId === threadId)
+                        source.ids.add(target.messageId);
+            }
+        }
+        if ([...sources.values()].reduce((sum, source) => sum + source.ids.size, 0) > 128)
+            return undefined;
+        const sourceIds = [...sources.keys()];
+        // Cursors are the mutable cross-document view. Refresh their small rows,
+        // while message metadata remains targeted; otherwise a later batch based
+        // on an older snapshot could regress a different conversation's progress.
+        const allPositions = await this.positionRows.where("owner").equals(owner).toArray();
+        const positionById = new Map(allPositions.map(position => [position.id, position]));
+        if (base.threads.some(thread => !positionById.has(thread.id)))
+            return undefined;
+        const positions = sourceIds.map(id => positionById.get(id));
+        const partialThreads = [];
+        for (let i = 0; i < sourceIds.length; i++) {
+            const id = sourceIds[i], source = sources.get(id), position = positions[i];
+            if (!position)
+                return undefined;
+            let cached = this.observationMessages.get(source.cached.messages);
+            if (!cached) {
+                cached = new Map(source.cached.messages.map(message => [message.id, message]));
+                this.observationMessages.set(source.cached.messages, cached);
+            }
+            const ids = [...source.ids], rows = await this.messageRows.bulkGet(ids.map(messageId => [owner, id, messageId]));
+            const messages = [];
+            for (let j = 0; j < ids.length; j++) {
+                const previous = cached.get(ids[j]), message = rows[j]?.value;
+                if (message && (message.payload !== null || typeof message.readingControl !== "boolean"))
+                    return undefined;
+                if (JSON.stringify(previous) !== JSON.stringify(message))
+                    return undefined;
+                if (message)
+                    messages.push(previous);
+            }
+            partialThreads.push({ ...source.cached, messages, readThroughSeq: position.readThroughSeq, explicitReadMessageIds: position.explicitReadMessageIds });
+        }
+        const partial = { schemaVersion: 1, appId, profileId, syncCursor: header.syncCursor, pending: header.pending, observedReadIntents: header.observedReadIntents, threads: partialThreads };
+        const result = transform(partial, (await this.versions.get(key))?.teams ?? {});
+        if (result.appId !== appId || result.profileId !== profileId || result.schemaVersion !== 1)
+            throw new Error("READ_IDENTITY_MISMATCH");
+        if (result.syncCursor !== partial.syncCursor || result.threads.length !== partialThreads.length || result.threads.some((thread, i) => thread.id !== partialThreads[i].id || thread.kind !== partialThreads[i].kind || thread.teamId !== partialThreads[i].teamId || thread.conversationId !== partialThreads[i].conversationId || thread.messages !== partialThreads[i].messages))
+            return undefined;
+        const writes = [];
+        for (let i = 0; i < result.threads.length; i++) {
+            const thread = result.threads[i], before = positions[i];
+            if (thread.readThroughSeq !== before.readThroughSeq || JSON.stringify(thread.explicitReadMessageIds) !== JSON.stringify(before.explicitReadMessageIds))
+                writes.push({ owner, id: thread.id, readThroughSeq: thread.readThroughSeq, explicitReadMessageIds: thread.explicitReadMessageIds });
+        }
+        if (writes.length)
+            await this.positionRows.bulkPut(writes);
+        if (JSON.stringify(header.pending) !== JSON.stringify(result.pending) || JSON.stringify(header.observedReadIntents) !== JSON.stringify(result.observedReadIntents))
+            await this.profiles.put({ ...header, pending: result.pending, observedReadIntents: result.observedReadIntents }, key);
+        const byId = new Map(allPositions.map(position => [position.id, position]));
+        result.threads.forEach(thread => byId.set(thread.id, thread));
+        let changed = false;
+        const threads = base.threads.map(thread => {
+            const position = byId.get(thread.id);
+            if (!position || thread.readThroughSeq === position.readThroughSeq && JSON.stringify(thread.explicitReadMessageIds) === JSON.stringify(position.explicitReadMessageIds))
+                return thread;
+            changed = true;
+            return { ...thread, readThroughSeq: position.readThroughSeq, explicitReadMessageIds: position.explicitReadMessageIds };
+        });
+        if (!changed && JSON.stringify(base.pending) === JSON.stringify(result.pending) && JSON.stringify(base.observedReadIntents) === JSON.stringify(result.observedReadIntents))
+            return base;
+        return { ...base, threads, pending: result.pending, observedReadIntents: result.observedReadIntents };
+    }
+    async update(appId, profileId, transform, observation) {
         return this.transaction("rw", async () => {
+            if (observation) {
+                const next = await this.observe(appId, profileId, transform, observation);
+                if (next)
+                    return next;
+            }
             const previous = await this.loadCurrent(appId, profileId);
             const transformed = transform(previous ?? createReadIndex(appId, profileId), (await this.versions.get(this.key(appId, profileId)))?.teams ?? {});
             const next = this.metadataOnly ? readingMetadata(transformed) : transformed;

package/lib/runtime/readingClient.js

@@ -39,7 +39,7 @@ export class ReadingClient {
     getSnapshot = () => this.state;
     subscribe = (listener) => { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; };
     publish(state) {
-        if (this.closed || JSON.stringify(state) === JSON.stringify(this.state))
+        if (this.closed || state === this.state || JSON.stringify(state) === JSON.stringify(this.state))
             return;
         this.state = state;
         this.listeners.forEach(listener => listener());
@@ -102,7 +102,7 @@ export class ReadingClient {
         // methods below only derive changes for their configured team.
         return this.updateScoped(transform);
     }
-    updateScoped(transform, teamId) {
+    updateScoped(transform, teamId, observation) {
         if (this.closed)
             return Promise.reject(new Error("READING_CLIENT_CLOSED"));
         // One shared metadata read per local authority generation, rather than one
@@ -114,7 +114,7 @@ export class ReadingClient {
                 assertReadIndexVersions(expected, currentVersions, teamId);
                 const next = transform(state);
                 return this.options.metadataOnly ? readingMetadata(next) : next;
-            });
+            }, observation);
             if (!this.closed) {
                 this.publish(next);
                 if (next.pending.length || next.observedReadIntents?.length)
@@ -162,7 +162,16 @@ export class ReadingClient {
     recordExplicit(threadId, messageIds) { return this.record(threadId, messageIds, "explicit"); }
     record(threadId, ids, readPolicy) {
         const messageIds = [...new Set(ids)];
-        return this.updateScoped(state => { this.assertThreadTeam(state, threadId); return messageIds.reduce((next, messageId) => recordReadIntent(next, { threadId, messageId, readPolicy, operationId: crypto.randomUUID() }), state); }, this.options.teamId);
+        if (!messageIds.length && this.state) {
+            try {
+                this.assertThreadTeam(this.state, threadId);
+                return Promise.resolve(this.state);
+            }
+            catch (error) {
+                return Promise.reject(error);
+            }
+        }
+        return this.updateScoped(state => { this.assertThreadTeam(state, threadId); return messageIds.reduce((next, messageId) => recordReadIntent(next, { threadId, messageId, readPolicy, operationId: crypto.randomUUID() }), state); }, this.options.teamId, this.state && messageIds.length ? { base: this.state, targets: messageIds.map(messageId => ({ threadId, messageId })) } : undefined);
     }
     withdraw() { this.stop(); }
     stop() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pmate/chat-sdk",
-  "version": "0.12.0-next.44",
+  "version": "0.12.0-next.46",
   "description": "Cross-platform PMate chat input components, voice input primitives, and ASR helpers",
   "type": "module",
   "main": "./lib/index.js",