@agentshouse/mdruntime 0.1.0-c5d3fa6fb67bc → 0.1.0-c5f333dfa0583

package/dist/search.d.ts

@@ -6,8 +6,11 @@ export type DocumentPart = Open<'fields' | 'sections' | 'preamble' | 'path'>;
 export type SearchRequest = {
     text: string;
     documents?: Filter;
+    collection?: true;
     conversations?: readonly string[];
     limit?: number;
+    after?: string;
+    total?: true;
 };
 export type DocumentHit = {
     room: string;
@@ -36,6 +39,13 @@ export type SearchHits = {
     documents: readonly DocumentHit[];
     messages: readonly MessageHit[];
 };
+export type SearchWindow = SearchHits & {
+    next?: string;
+    total?: {
+        documents: number;
+        messages: number;
+    };
+};
 export type SearchPending = SearchHits & {
     pending: readonly (PendingRoom | PendingConversation)[];
     coverage: SearchCoverage;
@@ -43,7 +53,7 @@ export type SearchPending = SearchHits & {
 export type SearchUndeclared = ProviderRefusal & {
     kind: 'undeclared';
 };
-export type SearchResult = (SearchHits & {
+export type SearchResult = (SearchWindow & {
     coverage: SearchCoverage;
 }) | SearchPending | SearchUndeclared;
 export declare function undeclaredSearch(): SearchUndeclared;
@@ -61,14 +71,18 @@ export type SearchedMessage = {
     text: string;
 };
 export type SearchSource = {
-    room(room: string): readonly string[];
-    document(room: string, path: string): SearchedDocument | null;
+    room(room: string): AsyncIterable<readonly SearchedDocument[]>;
+    documents(room: string, paths: readonly string[]): Promise<readonly SearchedDocument[]>;
 };
+type HitCursor = readonly [number, string, string] | readonly [] | null;
 export type SearchPlan = {
     text: string;
     documents: unknown;
+    collection: boolean;
     conversations: readonly string[] | null;
     limit: number | null;
+    after: readonly [HitCursor, HitCursor] | null;
+    total: boolean;
 };
 export type SearchIndex = {
     generation(): string;
@@ -83,8 +97,9 @@ export type SearchIndex = {
     search(plan: SearchPlan, documents: readonly {
         room: string;
         path: string;
-    }[] | null, conversations: readonly string[], rooms: ReadonlySet<string> | null): SearchHits;
+    }[] | null, conversations: readonly string[], rooms: ReadonlySet<string> | null, selections: ReadonlyMap<string, readonly string[]> | null): SearchWindow;
 };
 export declare function searchPlanOf(input: unknown): SearchPlan;
 export declare function createSearchIndex(source: SearchSource): SearchIndex;
+export {};
 //# sourceMappingURL=search.d.ts.map

package/dist/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAKL,KAAK,WAAW,EAChB,KAAK,MAAM,EACX,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC;AAE7E,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC,CAAC,CAAC;CAC5D,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;CAC5D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,SAAS,WAAW,EAAE,CAAC;IAC9B,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;CAAE,CAAC;AAElF,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,SAAS,WAAW,EAAE,CAAC;IAClC,QAAQ,EAAE,SAAS,UAAU,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG;IACvC,OAAO,EAAE,SAAS,CAAC,WAAW,GAAG,mBAAmB,CAAC,EAAE,CAAC;IACxD,QAAQ,EAAE,cAAc,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,CAAC;AAExE,MAAM,MAAM,YAAY,GACpB,CAAC,UAAU,GAAG;IAAE,QAAQ,EAAE,cAAc,CAAA;CAAE,CAAC,GAC3C,aAAa,GACb,gBAAgB,CAAC;AAErB,wBAAgB,gBAAgB,IAAI,gBAAgB,CAOnD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAAC;CAC/D,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACxC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,UAAU,IAAI,MAAM,CAAC;IACrB,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7B,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;IACrC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,eAAe,EAAE,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAChG,MAAM,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACvC,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B,OAAO,IAAI,MAAM,CAAC;IAClB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,CACJ,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,SAAS;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,EAC3D,aAAa,EAAE,SAAS,MAAM,EAAE,EAChC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,IAAI,GAChC,UAAU,CAAC;CACf,CAAC;AA4BF,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAiBvD;AAMD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,WAAW,CAkKnE"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAML,KAAK,WAAW,EAChB,KAAK,MAAM,EACX,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAKtC,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC;AAE7E,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,IAAI,CAAC;CACd,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC,CAAC,CAAC;CAC5D,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;CAC5D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,SAAS,WAAW,EAAE,CAAC;IAC9B,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;CAAE,CAAC;AAElF,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,SAAS,WAAW,EAAE,CAAC;IAClC,QAAQ,EAAE,SAAS,UAAU,EAAE,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG;IACvC,OAAO,EAAE,SAAS,CAAC,WAAW,GAAG,mBAAmB,CAAC,EAAE,CAAC;IACxD,QAAQ,EAAE,cAAc,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,YAAY,CAAA;CAAE,CAAC;AAExE,MAAM,MAAM,YAAY,GACpB,CAAC,YAAY,GAAG;IAAE,QAAQ,EAAE,cAAc,CAAA;CAAE,CAAC,GAC7C,aAAa,GACb,gBAAgB,CAAC;AAErB,wBAAgB,gBAAgB,IAAI,gBAAgB,CAOnD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;IAC/D,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,gBAAgB,EAAE,CAAC,CAAC;CACzF,CAAC;AAEF,KAAK,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC;AAExE,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACxC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,IAAI,CAAC;IAC9C,KAAK,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,UAAU,IAAI,MAAM,CAAC;IACrB,KAAK,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7B,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;IACrC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,eAAe,EAAE,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAChG,MAAM,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACvC,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC;IAC9B,OAAO,IAAI,MAAM,CAAC;IAClB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,CACJ,IAAI,EAAE,UAAU,EAChB,SAAS,EAAE,SAAS;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,IAAI,EAC3D,aAAa,EAAE,SAAS,MAAM,EAAE,EAChC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,IAAI,EACjC,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,GAAG,IAAI,GACxD,YAAY,CAAC;CACjB,CAAC;AAwDF,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,UAAU,CAoBvD;AAoCD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,YAAY,GAAG,WAAW,CAwQnE"}

package/dist/search.js

@@ -1,7 +1,9 @@
 import MiniSearch from 'minisearch';
 import { indexGeneration } from './identity.js';
-import { invalid, known, limitOf, plain, } from './query.js';
-import { inUnits, ROW_WEIGHT } from './turn.js';
+import { flag, invalid, known, limitOf, plain, } from './query.js';
+import { beneath } from './paths.js';
+import { inUnits, ROW_WEIGHT, yielded } from './turn.js';
+import { decodedCursor } from './windows.js';
 export function undeclaredSearch() {
     return {
         kind: 'undeclared',
@@ -11,6 +13,8 @@ export function undeclaredSearch() {
     };
 }
 const FIELDS = ['fields', 'sections', 'preamble', 'path', 'text'];
+const VACUUM_CONDITIONS = { minDirtCount: 20, minDirtFactor: 0.1 };
+const CHANGED_BATCH = 256;
 function documentId(room, path) {
     return JSON.stringify(['document', room, path]);
 }
@@ -26,10 +30,36 @@ function scalars(value) {
         return Object.values(value).flatMap(scalars);
     return [String(value)];
 }
+function hitCursor(input) {
+    if (input === null)
+        return null;
+    if (!Array.isArray(input))
+        return undefined;
+    if (input.length === 0)
+        return [];
+    const [score, first, second] = input;
+    return input.length === 3 &&
+        typeof score === 'number' &&
+        typeof first === 'string' &&
+        typeof second === 'string'
+        ? [score, first, second]
+        : undefined;
+}
+function searchAfter(input) {
+    if (input === undefined)
+        return null;
+    const decoded = typeof input === 'string' ? decodedCursor(input) : null;
+    const documents = decoded?.length === 2 ? hitCursor(decoded[0]) : undefined;
+    const messages = decoded?.length === 2 ? hitCursor(decoded[1]) : undefined;
+    if (documents === undefined || messages === undefined) {
+        invalid('after', 'expects the next an earlier search returned');
+    }
+    return [documents, messages];
+}
 export function searchPlanOf(input) {
     if (!plain(input))
         invalid('search', 'expects a search object');
-    known(input, ['text', 'documents', 'conversations', 'limit'], '');
+    known(input, ['text', 'documents', 'collection', 'conversations', 'limit', 'after', 'total'], '');
     if (typeof input.text !== 'string')
         invalid('text', 'expects text');
     const conversations = input.conversations;
@@ -40,13 +70,37 @@ export function searchPlanOf(input) {
     return {
         text: input.text,
         documents: input.documents,
+        collection: flag(input.collection, 'collection'),
         conversations: conversations ?? null,
         limit: limitOf(input.limit),
+        after: searchAfter(input.after),
+        total: flag(input.total, 'total'),
     };
 }
+function pair(left, right) {
+    if (left[0] !== right[0])
+        return left[0] < right[0] ? -1 : 1;
+    if (left[1] !== right[1])
+        return left[1] < right[1] ? -1 : 1;
+    return 0;
+}
+function hitWindow(found, cursor, limit, tie) {
+    if (cursor === null)
+        return { kept: [], cursor: null };
+    const start = cursor.length === 0
+        ? 0
+        : found.findIndex((hit) => hit.score < cursor[0] ||
+            (hit.score === cursor[0] && pair(tie(hit), [cursor[1], cursor[2]]) > 0));
+    const following = start === -1 ? [] : found.slice(start);
+    if (limit === null || following.length <= limit)
+        return { kept: following, cursor: null };
+    const kept = following.slice(0, limit);
+    const last = kept[kept.length - 1];
+    return { kept, cursor: last === undefined ? cursor : [last.score, ...tie(last)] };
+}
 export function createSearchIndex(source) {
     const generation = indexGeneration();
-    const index = new MiniSearch({ fields: FIELDS });
+    const index = new MiniSearch({ fields: FIELDS, autoVacuum: false });
     const documents = new Map();
     const messages = new Map();
     const rooms = new Set();
@@ -54,11 +108,20 @@ export function createSearchIndex(source) {
     const conversations = new Set();
     const acquiring = new Set();
     const changedDocuments = new Map();
+    const refilling = new Set();
+    const arrived = new Map();
+    let vacuuming = null;
     let changes = 0;
+    function added(entry) {
+        if (vacuuming === null)
+            index.add(entry);
+        else
+            arrived.set(entry.id, entry);
+    }
     function addDocument(document) {
         const id = documentId(document.room, document.path);
         discard(id, documents);
-        index.add({
+        added({
             id,
             fields: scalars(document.fields).join('\n'),
             sections: Object.entries(document.sections)
@@ -72,49 +135,114 @@ export function createSearchIndex(source) {
     function addMessage(message) {
         const id = messageId(message.conversation, message.identity);
         discard(id, messages);
-        index.add({ id, text: message.text });
+        added({ id, text: message.text });
         messages.set(id, { conversation: message.conversation, identity: message.identity });
     }
     function discard(id, entries) {
-        if (entries.delete(id))
-            index.discard(id);
+        if (!entries.delete(id) || arrived.delete(id))
+            return;
+        index.discard(id);
+        vacuum();
     }
-    function* pending() {
+    function vacuum() {
+        if (vacuuming !== null ||
+            index.dirtCount < VACUUM_CONDITIONS.minDirtCount ||
+            index.dirtFactor < VACUUM_CONDITIONS.minDirtFactor) {
+            return;
+        }
+        changes += 1;
+        vacuuming = index
+            .vacuum()
+            .then(() => inUnits(arrivals(), (id) => {
+            const entry = arrived.get(id);
+            if (entry === undefined)
+                return ROW_WEIGHT;
+            arrived.delete(id);
+            index.add(entry);
+            return Object.values(entry).reduce((total, value) => total + value.length, ROW_WEIGHT);
+        }))
+            .finally(() => {
+            vacuuming = null;
+            vacuum();
+        });
+    }
+    function* arrivals() {
         for (;;) {
-            const next = changedDocuments.entries().next();
+            const next = arrived.keys().next();
             if (next.done === true)
                 return;
-            changedDocuments.delete(next.value[0]);
             yield next.value;
         }
     }
+    async function vacuumed() {
+        while (vacuuming !== null)
+            await vacuuming;
+    }
+    async function paused() {
+        await yielded();
+        await vacuumed();
+    }
     let catching = null;
+    async function refill(room) {
+        for (const [id, entry] of documents)
+            if (entry.room === room)
+                discard(id, documents);
+        for await (const batch of source.room(room)) {
+            if (!rooms.has(room) && !filling.has(room))
+                return;
+            for (const document of batch)
+                addDocument(document);
+            await paused();
+        }
+    }
+    async function catchUp() {
+        for (const room of refilling) {
+            refilling.delete(room);
+            await refill(room);
+        }
+        while (changedDocuments.size > 0) {
+            await paused();
+            const batch = new Map();
+            for (const [id, { room, path }] of changedDocuments) {
+                changedDocuments.delete(id);
+                const held = batch.get(room) ?? new Map();
+                held.set(path, id);
+                batch.set(room, held);
+                if (held.size >= CHANGED_BATCH)
+                    break;
+            }
+            for (const [room, held] of batch) {
+                for (const id of held.values())
+                    discard(id, documents);
+                for (const document of await source.documents(room, [...held.keys()])) {
+                    if (rooms.has(room) || filling.has(room))
+                        addDocument(document);
+                }
+            }
+        }
+        await vacuumed();
+    }
     function caughtUp() {
         if (catching !== null)
             return catching;
-        const running = inUnits(pending(), ([id, { room, path }]) => {
-            discard(id, documents);
-            const document = source.document(room, path);
-            if (document === null)
-                return ROW_WEIGHT;
-            addDocument(document);
-            return Object.values(document.sections).reduce((total, text) => total + text.length, ROW_WEIGHT + document.preamble.length);
-        });
-        if (!(running instanceof Promise))
+        if (changedDocuments.size === 0 && refilling.size === 0 && vacuuming === null) {
             return Promise.resolve();
-        catching = running.finally(() => {
+        }
+        catching = catchUp().finally(() => {
             catching = null;
         });
         return catching;
     }
-    function ranked(text, entries, keep, limit) {
+    function ranked(text, entries, keep, tie) {
         const found = index.search(text, {
             filter: ({ id }) => {
                 const entry = entries.get(id);
                 return entry !== undefined && keep(entry, id);
             },
         });
-        return (limit === null ? found : found.slice(0, limit)).map(({ id, score, match }) => ({
+        found.sort((left, right) => right.score - left.score ||
+            pair(tie(entries.get(left.id)), tie(entries.get(right.id))));
+        return found.map(({ id, score, match }) => ({
             ...entries.get(id),
             score,
             match,
@@ -129,13 +257,13 @@ export function createSearchIndex(source) {
                 return;
             filling.add(room);
             try {
-                await inUnits(source.room(room), (path) => {
-                    const document = filling.has(room) ? source.document(room, path) : null;
-                    if (document === null)
-                        return ROW_WEIGHT;
-                    addDocument(document);
-                    return Object.values(document.sections).reduce((total, text) => total + text.length, ROW_WEIGHT + document.preamble.length);
-                });
+                for await (const batch of source.room(room)) {
+                    if (!filling.has(room))
+                        return;
+                    for (const document of batch)
+                        addDocument(document);
+                    await paused();
+                }
                 if (filling.has(room))
                     rooms.add(room);
             }
@@ -161,14 +289,23 @@ export function createSearchIndex(source) {
             for (const room of leaving) {
                 rooms.delete(room);
                 filling.delete(room);
+                refilling.delete(room);
             }
         },
         changed(set) {
             changes += 1;
             for (const { locator, path } of set.documents) {
-                if (rooms.has(locator) || filling.has(locator)) {
-                    changedDocuments.set(documentId(locator, path), { room: locator, path });
+                if (!rooms.has(locator) && !filling.has(locator))
+                    continue;
+                if (path === null) {
+                    for (const [id, entry] of changedDocuments) {
+                        if (entry.room === locator)
+                            changedDocuments.delete(id);
+                    }
+                    refilling.add(locator);
+                    continue;
                 }
+                changedDocuments.set(documentId(locator, path), { room: locator, path });
             }
             for (const { conversation, identity, text } of set.messages) {
                 if (!conversations.has(conversation) && !acquiring.has(conversation))
@@ -182,14 +319,34 @@ export function createSearchIndex(source) {
         },
         changes: () => changes,
         caughtUp,
-        search({ text, limit }, selected, named, within) {
+        search({ text, limit, after, total }, selected, named, within, selections) {
             const chosen = selected === null
                 ? null
                 : new Set(selected.map(({ room, path }) => documentId(room, path)));
             const asked = new Set(named);
+            const documentTie = ({ room, path }) => [room, path];
+            const messageTie = ({ conversation, identity }) => [conversation, identity];
+            const foundDocuments = ranked(text, documents, ({ room, path }, id) => (chosen === null || chosen.has(id)) &&
+                (within === null || within.has(room)) &&
+                !(selections?.get(room) ?? []).some((root) => beneath(root, path)), documentTie);
+            const foundMessages = ranked(text, messages, ({ conversation }) => asked.has(conversation), messageTie);
+            const [documentsAfter, messagesAfter] = after ?? [[], []];
+            const documentWindow = hitWindow(foundDocuments, documentsAfter, limit, documentTie);
+            const messageWindow = hitWindow(foundMessages, messagesAfter, limit, messageTie);
+            const more = documentWindow.cursor !== null || messageWindow.cursor !== null;
             return {
-                documents: ranked(text, documents, ({ room }, id) => (chosen === null || chosen.has(id)) && (within === null || within.has(room)), limit).map(({ match, ...hit }) => ({ ...hit, matches: match })),
-                messages: ranked(text, messages, ({ conversation }) => asked.has(conversation), limit).map(({ match, ...hit }) => ({ ...hit, matches: match })),
+                documents: documentWindow.kept.map(({ match, ...hit }) => ({
+                    ...hit,
+                    matches: match,
+                })),
+                messages: messageWindow.kept.map(({ match, ...hit }) => ({
+                    ...hit,
+                    matches: match,
+                })),
+                ...(more ? { next: JSON.stringify([documentWindow.cursor, messageWindow.cursor]) } : {}),
+                ...(total
+                    ? { total: { documents: foundDocuments.length, messages: foundMessages.length } }
+                    : {}),
             };
         },
     };

package/dist/seed.d.ts

@@ -0,0 +1,16 @@
+import type { KeyedResult } from './observation.js';
+import type { QueryRows } from './query.js';
+export type EntrySeed = {
+    readonly at: Readonly<Record<string, string>>;
+    readonly queries: readonly {
+        readonly query: unknown;
+        readonly rows: QueryRows;
+    }[];
+};
+export type Seeded = {
+    readonly rows: QueryRows;
+    readonly at: Readonly<Record<string, string>>;
+};
+export declare function entrySeedOf(value: unknown): EntrySeed | null;
+export declare function seededResult(seed: Seeded): KeyedResult;
+//# sourceMappingURL=seed.d.ts.map

package/dist/seed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"seed.d.ts","sourceRoot":"","sources":["../src/seed.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,KAAK,EAA4C,SAAS,EAAE,MAAM,YAAY,CAAC;AAGtF,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAA;KAAE,EAAE,CAAC;CACpF,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;CAAE,CAAC;AAwCjG,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,IAAI,CAe5D;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAoBtD"}

package/dist/seed.js

@@ -0,0 +1,75 @@
+import { documentKey } from './rows.js';
+function plain(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function windowOf(value) {
+    const { next, total } = value;
+    if (next !== undefined && typeof next !== 'string')
+        return null;
+    if (total !== undefined && !Number.isSafeInteger(total))
+        return null;
+    return {
+        ...(next === undefined ? {} : { next }),
+        ...(total === undefined ? {} : { total: total }),
+    };
+}
+function rowsOf(value) {
+    if (!plain(value))
+        return null;
+    const window = windowOf(value);
+    if (window === null)
+        return null;
+    if (Array.isArray(value.groups)) {
+        const groups = value.groups;
+        return groups.every((group) => plain(group) && Array.isArray(group.key) && typeof group.count === 'number')
+            ? { groups: groups, ...window }
+            : null;
+    }
+    if (!Array.isArray(value.documents))
+        return null;
+    const documents = value.documents;
+    return documents.every((document) => plain(document) && typeof document.room === 'string' && typeof document.path === 'string')
+        ? { documents: documents, ...window }
+        : null;
+}
+export function entrySeedOf(value) {
+    if (!plain(value) || !plain(value.at) || !Array.isArray(value.queries))
+        return null;
+    const at = {};
+    for (const [room, version] of Object.entries(value.at)) {
+        if (typeof version !== 'string')
+            return null;
+        at[room] = version;
+    }
+    const queries = [];
+    for (const entry of value.queries) {
+        if (!plain(entry) || !plain(entry.query))
+            return null;
+        const rows = rowsOf(entry.rows);
+        if (rows === null)
+            return null;
+        queries.push({ query: entry.query, rows });
+    }
+    return { at, queries };
+}
+export function seededResult(seed) {
+    const coverage = {
+        rooms: Object.entries(seed.at).map(([room, generation]) => ({
+            room,
+            generation,
+            current: false,
+        })),
+        indexes: [],
+        overlay: [],
+    };
+    if ('groups' in seed.rows) {
+        return {
+            result: { ...seed.rows, coverage },
+            keys: seed.rows.groups.map((_group, index) => index),
+        };
+    }
+    return {
+        result: { ...seed.rows, coverage },
+        keys: seed.rows.documents.map((document) => documentKey(document.room, document.path)),
+    };
+}

package/dist/shared.d.ts

@@ -1,11 +1,16 @@
 export type Paths<V> = Readonly<Record<string, V>>;
-type Weigh<V> = (key: string, value: V) => number;
+export declare function entriesIn<V>(map: Paths<V>): Generator<readonly [string, V]>;
+export declare function keysIn(...maps: readonly Paths<unknown>[]): Generator<number, Set<string>, void>;
+export declare function sortedKeys(map: Paths<unknown>): Generator<number, readonly string[], void>;
+export declare function sorting(keys: Iterable<string>): Generator<number, readonly string[], void>;
+export declare function sharing<V>(map: Paths<V>): Generator<number, Paths<V>, void>;
 export declare function shared<V>(map: Paths<V>): Paths<V>;
 export declare function patched<V>(base: Paths<V>, set: Iterable<readonly [string, V]>, removed?: Iterable<string>): Paths<V>;
-export declare function tally<V>(map: Paths<V>, weigh: Weigh<V>): number;
+export declare function patching<V>(base: Paths<V>, set: Iterable<readonly [string, V]>, removed?: Iterable<string>): Generator<number, Paths<V>, void>;
 export declare function sizeOf<V>(map: Paths<V>): number;
+export declare function differences<V>(left: Paths<V>, right: Paths<V>, same?: (left: V, right: V) => boolean): Generator<number, ReadonlySet<string>, void>;
 export declare function differing<V>(left: Paths<V>, right: Paths<V>, same?: (left: V, right: V) => boolean): ReadonlySet<string>;
 export declare function drafted<V>(base: Paths<V>): Record<string, V>;
 export declare function settled<V>(draft: Record<string, V>): Paths<V>;
-export {};
+export declare function settling<V>(draft: Record<string, V>): Generator<number, Paths<V>, void>;
 //# sourceMappingURL=shared.d.ts.map

package/dist/shared.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../src/shared.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;AAkBnD,KAAK,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,MAAM,CAAC;AA6LlD,wBAAgB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAEjD;AAED,wBAAgB,OAAO,CAAC,CAAC,EACvB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EACd,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EACnC,OAAO,GAAE,QAAQ,CAAC,MAAM,CAAM,GAC7B,KAAK,CAAC,CAAC,CAAC,CA8BV;AAED,wBAAgB,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAY/D;AAED,wBAAgB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAE/C;AAuCD,wBAAgB,SAAS,CAAC,CAAC,EACzB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EACf,IAAI,GAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,OAAmB,GAC/C,WAAW,CAAC,MAAM,CAAC,CAcrB;AAMD,wBAAgB,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAiD5D;AAED,wBAAgB,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAS7D"}
+{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../src/shared.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;AA+LnD,wBAAiB,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAO5E;AAeD,wBAAiB,MAAM,CAAC,GAAG,IAAI,EAAE,SAAS,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAShG;AA0BD,wBAAiB,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,CAAC,CAE3F;AAED,wBAAiB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,CAAC,CAoB3F;AAED,wBAAiB,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAE5E;AAED,wBAAgB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAEjD;AAED,wBAAgB,OAAO,CAAC,CAAC,EACvB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EACd,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EACnC,OAAO,GAAE,QAAQ,CAAC,MAAM,CAAM,GAC7B,KAAK,CAAC,CAAC,CAAC,CAEV;AAED,wBAAiB,QAAQ,CAAC,CAAC,EACzB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EACd,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EACnC,OAAO,GAAE,QAAQ,CAAC,MAAM,CAAM,GAC7B,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAoBnC;AAED,wBAAgB,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAE/C;AA8CD,wBAAiB,WAAW,CAAC,CAAC,EAC5B,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EACf,IAAI,GAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,OAAmB,GAC/C,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAkB9C;AAED,wBAAgB,SAAS,CAAC,CAAC,EACzB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EACd,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EACf,IAAI,GAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,OAAmB,GAC/C,WAAW,CAAC,MAAM,CAAC,CAErB;AAMD,wBAAgB,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAiD5D;AAED,wBAAgB,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAE7D;AAED,wBAAiB,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAUxF"}