@tearleads/client-sdk 0.0.0-stage → 0.1.13

package/workflows/container-contents/documentDiscovery.d.ts

@@ -0,0 +1,12 @@
+import type { DocumentSummary } from "../../data/documents/documentSummary.js";
+import type { ContainerDocumentDiscoveryApi, DiscoverAllContainerDocumentsOptions, DiscoverContainerDocumentsOptions, RefreshAllContainerDocumentsFromApiOptions } from "./documentDiscoveryTypes.js";
+export type { RefreshAllContainerDocumentsFromApiOptions } from "./documentDiscoveryTypes.js";
+export declare function hasUndiscoveredDocumentUpdateEvent(events: ReadonlyArray<unknown>, knownDocumentIds: ReadonlySet<string>): boolean;
+export declare function listAllRemoteContainerIds(listContainerParentLanes: ContainerDocumentDiscoveryApi["listContainerParentLanes"]): Promise<ReadonlyArray<string> | null>;
+export declare function listAllRemoteContainerIdsFromApi(apiClient: Pick<ContainerDocumentDiscoveryApi, "listContainerParentLanes">): Promise<ReadonlyArray<string> | null>;
+export declare function discoverContainerDocuments(options: DiscoverContainerDocumentsOptions): Promise<ReadonlyArray<DocumentSummary> | null>;
+export declare function discoverContainerDocumentsFromApi({ apiClient, ...input }: Omit<DiscoverContainerDocumentsOptions, "listContainerDocuments"> & {
+    readonly apiClient: Pick<ContainerDocumentDiscoveryApi, "listContainerDocuments" | "listContainerDocumentsResult">;
+}): Promise<ReadonlyArray<DocumentSummary> | null>;
+export declare function discoverAllContainerDocuments(options: DiscoverAllContainerDocumentsOptions): Promise<ReadonlyArray<DocumentSummary> | null>;
+export declare function refreshAllContainerDocumentsFromApi({ apiClient, ...input }: RefreshAllContainerDocumentsFromApiOptions): Promise<ReadonlyArray<DocumentSummary> | null>;

package/workflows/container-contents/documentDiscovery.js

@@ -0,0 +1,255 @@
+import { isDocumentUpdateCreatedEvent } from "../../data/documents/documentSync.js";
+import { dedupeReferencedPrincipalStates } from "../../data/keyingProjectionVerification/principalPolicyCache.js";
+import { listAllContainerDocuments, listContainerDocumentsFromApi, } from "./containerDocumentListing.js";
+import { withoutDeferredDocumentLinks } from "./deferredDocumentLinks.js";
+import { collectApplicableDocumentTombstones, collectDiscoveredDocumentInputs, getApplicableDocumentTombstones, } from "./documentDiscoveryInputs.js";
+import { settleContainerDocumentTombstones } from "./documentTombstoneGate.js";
+const CONTAINER_PARENT_DISCOVERY_BATCH_SIZE = 4;
+const CONTAINER_DOCUMENT_DISCOVERY_CONCURRENCY = 4;
+async function listContainerParentLaneBatch(listContainerParentLanes, requestedLanes) {
+    const response = await listContainerParentLanes({
+        lanes: requestedLanes.map(({ request }) => request),
+    });
+    if (!response || response.results.length !== requestedLanes.length) {
+        return null;
+    }
+    const requestedLaneIds = new Set(requestedLanes.map(({ request }) => request.laneId));
+    const pagesByLaneId = new Map();
+    for (const result of response.results) {
+        if (!requestedLaneIds.has(result.laneId) ||
+            pagesByLaneId.has(result.laneId)) {
+            return null;
+        }
+        pagesByLaneId.set(result.laneId, result.page);
+    }
+    return requestedLanes.map(({ lane, request }) => ({
+        lane,
+        response: pagesByLaneId.get(request.laneId),
+    }));
+}
+function collectRemoteContainerIds(items, seenContainerIds, containerIds) {
+    for (const { id } of items) {
+        if (!seenContainerIds.has(id)) {
+            seenContainerIds.add(id);
+            containerIds.push(id);
+        }
+    }
+    return items.map(({ id }) => id);
+}
+async function saveAppliedContainerDocumentWatermark(input) {
+    const { containerId, listedDocuments, saveContainerDocumentWatermark } = input;
+    if (!listedDocuments.nextWatermark) {
+        return;
+    }
+    await saveContainerDocumentWatermark(containerId, listedDocuments.nextWatermark);
+}
+async function listContainerDocumentLanes(input) {
+    const { containerIds, loadContainerDocumentWatermark, listContainerDocuments, } = input;
+    const listedDocumentsByContainer = [];
+    let nextContainerIndex = 0;
+    async function worker() {
+        while (nextContainerIndex < containerIds.length) {
+            const containerIndex = nextContainerIndex;
+            nextContainerIndex += 1;
+            const containerId = containerIds[containerIndex];
+            if (!containerId) {
+                throw new Error("Container document discovery received an empty lane");
+            }
+            listedDocumentsByContainer[containerIndex] = {
+                containerId,
+                listedDocuments: await listAllContainerDocuments({
+                    containerId,
+                    loadContainerDocumentWatermark,
+                    listContainerDocuments,
+                }),
+            };
+        }
+    }
+    await Promise.all(Array.from({
+        length: Math.min(CONTAINER_DOCUMENT_DISCOVERY_CONCURRENCY, containerIds.length),
+    }, () => worker()));
+    return listedDocumentsByContainer;
+}
+export function hasUndiscoveredDocumentUpdateEvent(events, knownDocumentIds) {
+    return events.some((event) => isDocumentUpdateCreatedEvent(event) &&
+        !knownDocumentIds.has(event.documentId));
+}
+export async function listAllRemoteContainerIds(listContainerParentLanes) {
+    const containerIds = [];
+    const queuedParentIds = new Set();
+    const seenContainerIds = new Set();
+    const lanes = [];
+    let nextLaneId = 0;
+    const queueParentLane = (parentId) => {
+        if (queuedParentIds.has(parentId)) {
+            return;
+        }
+        queuedParentIds.add(parentId);
+        lanes.push({ parentId, watermark: null });
+    };
+    queueParentLane(null);
+    while (lanes.length > 0) {
+        const laneBatch = lanes.splice(0, CONTAINER_PARENT_DISCOVERY_BATCH_SIZE);
+        const requestedLanes = laneBatch.map((lane) => ({
+            lane,
+            request: {
+                laneId: `container-parent-${nextLaneId++}`,
+                parentId: lane.parentId,
+                watermark: lane.watermark,
+            },
+        }));
+        const listedLanes = await listContainerParentLaneBatch(listContainerParentLanes, requestedLanes);
+        if (!listedLanes) {
+            return null;
+        }
+        const continuationLanes = [];
+        for (const { lane, response } of listedLanes) {
+            if (!response) {
+                return null;
+            }
+            for (const containerId of collectRemoteContainerIds(response.items, seenContainerIds, containerIds)) {
+                queueParentLane(containerId);
+            }
+            if (response.hasMore) {
+                if (!response.nextWatermark) {
+                    return null;
+                }
+                continuationLanes.push({
+                    parentId: lane.parentId,
+                    watermark: response.nextWatermark,
+                });
+            }
+        }
+        lanes.unshift(...continuationLanes);
+    }
+    return containerIds;
+}
+export function listAllRemoteContainerIdsFromApi(apiClient) {
+    return listAllRemoteContainerIds((input) => apiClient.listContainerParentLanes(input));
+}
+export async function discoverContainerDocuments(options) {
+    const { cacheReferencedPrincipalPolicies, containerId, loadContainerDocumentWatermark, listContainerDocuments, onFullListing, replaceDocumentLinksBatch, saveContainerDocumentWatermark, upsertDiscoveredDocuments, } = options;
+    const generation = await options.beginDocumentDiscovery();
+    const listedDocuments = await listAllContainerDocuments({
+        containerId,
+        loadContainerDocumentWatermark,
+        listContainerDocuments,
+    });
+    if (!listedDocuments) {
+        return null;
+    }
+    await cacheReferencedPrincipalPolicies?.(dedupeReferencedPrincipalStates(listedDocuments.items.flatMap((document) => document.referencedPrincipals)));
+    const verification = await options.verifyDiscoveredDocuments(listedDocuments.items.map((document) => ({
+        accessEpoch: document.currentAccessEpoch,
+        accessStateHash: document.currentAccessStateHash,
+        containerId,
+        listedContainerIds: [containerId],
+        createdAt: document.createdAt,
+        documentId: document.id,
+        effectiveAccessLevel: document.effectiveAccessLevel,
+        linkedContainerIds: document.linkedContainerIds,
+    })), [containerId], generation, getApplicableDocumentTombstones(listedDocuments));
+    if (!(await verification.isCurrent()))
+        return null;
+    const verifiedInputs = verification.inputs;
+    const discoveredDocuments = verifiedInputs.length
+        ? await upsertDiscoveredDocuments(verifiedInputs)
+        : [];
+    if (verifiedInputs.length)
+        await replaceDocumentLinksBatch(await withoutDeferredDocumentLinks(verifiedInputs.map((document) => ({
+            documentId: document.documentId,
+            accessEpoch: document.accessEpoch,
+            containerIds: document.linkedContainerIds,
+        })), discoveredDocuments));
+    const tombstoneDocumentSummaries = await settleContainerDocumentTombstones({
+        containerIds: [containerId],
+        store: options,
+        tombstones: getApplicableDocumentTombstones(listedDocuments),
+    });
+    const complete = await verification.commit();
+    if (!(await verification.isCurrent()))
+        return null;
+    await saveAppliedContainerDocumentWatermark({
+        containerId,
+        listedDocuments,
+        saveContainerDocumentWatermark,
+    });
+    if (listedDocuments.isFullListing) {
+        // Initial probes consume ids as discovery hints and verify their own heads.
+        onFullListing?.(listedDocuments.items.map((document) => document.id));
+    }
+    return !complete &&
+        discoveredDocuments.length === 0 &&
+        tombstoneDocumentSummaries.length === 0
+        ? null
+        : [...discoveredDocuments, ...tombstoneDocumentSummaries];
+}
+export function discoverContainerDocumentsFromApi({ apiClient, ...input }) {
+    return discoverContainerDocuments({
+        ...input,
+        listContainerDocuments: (containerId, options) => listContainerDocumentsFromApi(apiClient, containerId, options),
+    });
+}
+export async function discoverAllContainerDocuments(options) {
+    const { cacheReferencedPrincipalPolicies, containerIds, loadContainerDocumentWatermark, listContainerDocuments, replaceDocumentLinksBatch, saveContainerDocumentWatermark, upsertDiscoveredDocuments, } = options;
+    const uniqueContainerIds = Array.from(new Set(containerIds)).filter((containerId) => typeof containerId === "string" && containerId.length > 0);
+    const generation = await options.beginDocumentDiscovery();
+    const listedDocumentsByContainer = await listContainerDocumentLanes({
+        containerIds: uniqueContainerIds,
+        loadContainerDocumentWatermark,
+        listContainerDocuments,
+    });
+    await cacheReferencedPrincipalPolicies?.(dedupeReferencedPrincipalStates(listedDocumentsByContainer.flatMap(({ listedDocuments }) => listedDocuments?.items.flatMap((document) => document.referencedPrincipals) ?? [])));
+    const verification = await options.verifyDiscoveredDocuments(collectDiscoveredDocumentInputs(listedDocumentsByContainer), uniqueContainerIds, generation, collectApplicableDocumentTombstones(listedDocumentsByContainer));
+    if (!(await verification.isCurrent()))
+        return null;
+    const discoveredDocumentInputs = verification.inputs;
+    const discoveredDocuments = discoveredDocumentInputs.length === 0
+        ? []
+        : await upsertDiscoveredDocuments(discoveredDocumentInputs);
+    if (discoveredDocumentInputs.length > 0) {
+        await replaceDocumentLinksBatch(await withoutDeferredDocumentLinks(discoveredDocumentInputs.map((input) => ({
+            documentId: input.documentId,
+            accessEpoch: input.accessEpoch,
+            containerIds: input.linkedContainerIds,
+        })), discoveredDocuments));
+    }
+    const tombstoneDocumentSummaries = await settleContainerDocumentTombstones({
+        containerIds: listedDocumentsByContainer.flatMap(({ containerId, listedDocuments }) => listedDocuments ? [containerId] : []),
+        store: options,
+        tombstones: collectApplicableDocumentTombstones(listedDocumentsByContainer),
+    });
+    const complete = await verification.commit();
+    if (!(await verification.isCurrent()))
+        return null;
+    await Promise.all(listedDocumentsByContainer.map(({ containerId, listedDocuments }) => listedDocuments
+        ? saveAppliedContainerDocumentWatermark({
+            containerId,
+            listedDocuments,
+            saveContainerDocumentWatermark,
+        })
+        : undefined));
+    return !complete &&
+        discoveredDocuments.length === 0 &&
+        tombstoneDocumentSummaries.length === 0
+        ? null
+        : [...discoveredDocuments, ...tombstoneDocumentSummaries];
+}
+async function refreshAllContainerDocuments({ listContainerParentLanes, ...input }) {
+    const containerIds = await listAllRemoteContainerIds(listContainerParentLanes);
+    if (!containerIds) {
+        return null;
+    }
+    return discoverAllContainerDocuments({
+        ...input,
+        containerIds,
+    });
+}
+export function refreshAllContainerDocumentsFromApi({ apiClient, ...input }) {
+    return refreshAllContainerDocuments({
+        ...input,
+        listContainerDocuments: (containerId, options) => listContainerDocumentsFromApi(apiClient, containerId, options),
+        listContainerParentLanes: (input) => apiClient.listContainerParentLanes(input),
+    });
+}
+//# sourceMappingURL=documentDiscovery.js.map

package/workflows/container-contents/documentDiscovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"documentDiscovery.js","sourceRoot":"","sources":["documentDiscovery.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mCAAmC,CAAC;AACjF,OAAO,EAAE,+BAA+B,EAAE,MAAM,8DAA8D,CAAC;AAC/G,OAAO,EACL,yBAAyB,EACzB,6BAA6B,GAC9B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EACL,mCAAmC,EACnC,+BAA+B,EAC/B,+BAA+B,GAChC,MAAM,2BAA2B,CAAC;AAWnC,OAAO,EAAE,iCAAiC,EAAE,MAAM,yBAAyB,CAAC;AAI5E,MAAM,qCAAqC,GAAG,CAAC,CAAC;AAChD,MAAM,wCAAwC,GAAG,CAAC,CAAC;AAWnD,KAAK,UAAU,4BAA4B,CACzC,wBAAmF,EACnF,cAA2D;IAE3D,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAAC;QAC9C,KAAK,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC;KACpD,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM,EAAE,CAAC;QACnE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CACpD,CAAC;IACF,MAAM,aAAa,GAAG,IAAI,GAAG,EAG1B,CAAC;IACJ,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,IACE,CAAC,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;YACpC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAChC,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QAChD,IAAI;QACJ,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;KAC5C,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,yBAAyB,CAChC,KAAoC,EACpC,gBAA6B,EAC7B,YAAsB;IAEtB,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAC9B,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,qCAAqC,CAAC,KAIpD;IACC,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,8BAA8B,EAAE,GACpE,KAAK,CAAC;IACR,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;QACnC,OAAO;IACT,CAAC;IAED,MAAM,8BAA8B,CAClC,WAAW,EACX,eAAe,CAAC,aAAa,CAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,0BAA0B,CAAC,KAIzC;IACC,MAAM,EACJ,YAAY,EACZ,8BAA8B,EAC9B,sBAAsB,GACvB,GAAG,KAAK,CAAC;IACV,MAAM,0BAA0B,GAAmC,EAAE,CAAC;IACtE,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAE3B,KAAK,UAAU,MAAM;QACnB,OAAO,kBAAkB,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CAAC;YAC1C,kBAAkB,IAAI,CAAC,CAAC;YACxB,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;YACzE,CAAC;YAED,0BAA0B,CAAC,cAAc,CAAC,GAAG;gBAC3C,WAAW;gBACX,eAAe,EAAE,MAAM,yBAAyB,CAAC;oBAC/C,WAAW;oBACX,8BAA8B;oBAC9B,sBAAsB;iBACvB,CAAC;aACH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CACR;QACE,MAAM,EAAE,IAAI,CAAC,GAAG,CACd,wCAAwC,EACxC,YAAY,CAAC,MAAM,CACpB;KACF,EACD,GAAG,EAAE,CAAC,MAAM,EAAE,CACf,CACF,CAAC;IAEF,OAAO,0BAA0B,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,MAA8B,EAC9B,gBAAqC;IAErC,OAAO,MAAM,CAAC,IAAI,CAChB,CAAC,KAAK,EAAE,EAAE,CACR,4BAA4B,CAAC,KAAK,CAAC;QACnC,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAC1C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,wBAAmF;IAEnF,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAiB,CAAC;IACjD,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC3C,MAAM,KAAK,GAAmC,EAAE,CAAC;IACjD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,eAAe,GAAG,CAAC,QAAuB,EAAE,EAAE;QAClD,IAAI,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,eAAe,CAAC,IAAI,CAAC,CAAC;IAEtB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,qCAAqC,CAAC,CAAC;QACzE,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC9C,IAAI;YACJ,OAAO,EAAE;gBACP,MAAM,EAAE,oBAAoB,UAAU,EAAE,EAAE;gBAC1C,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B;SACF,CAAC,CAAC,CAAC;QACJ,MAAM,WAAW,GAAG,MAAM,4BAA4B,CACpD,wBAAwB,EACxB,cAAc,CACf,CAAC;QACF,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,iBAAiB,GAAmC,EAAE,CAAC;QAE7D,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,WAAW,EAAE,CAAC;YAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC;YACd,CAAC;YAED,KAAK,MAAM,WAAW,IAAI,yBAAyB,CACjD,QAAQ,CAAC,KAAK,EACd,gBAAgB,EAChB,YAAY,CACb,EAAE,CAAC;gBACF,eAAe,CAAC,WAAW,CAAC,CAAC;YAC/B,CAAC;YAED,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACrB,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;oBAC5B,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,iBAAiB,CAAC,IAAI,CAAC;oBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,SAAS,EAAE,QAAQ,CAAC,aAAa;iBAClC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,SAA0E;IAE1E,OAAO,yBAAyB,CAAC,CAAC,KAAK,EAAE,EAAE,CACzC,SAAS,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAC1C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,OAA0C;IAE1C,MAAM,EACJ,gCAAgC,EAChC,WAAW,EACX,8BAA8B,EAC9B,sBAAsB,EACtB,aAAa,EACb,yBAAyB,EACzB,8BAA8B,EAC9B,yBAAyB,GAC1B,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAC1D,MAAM,eAAe,GAAG,MAAM,yBAAyB,CAAC;QACtD,WAAW;QACX,8BAA8B;QAC9B,sBAAsB;KACvB,CAAC,CAAC;IACH,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,gCAAgC,EAAE,CACtC,+BAA+B,CAC7B,eAAe,CAAC,KAAK,CAAC,OAAO,CAC3B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAC5C,CACF,CACF,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,yBAAyB,CAC1D,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACvC,WAAW,EAAE,QAAQ,CAAC,kBAAkB;QACxC,eAAe,EAAE,QAAQ,CAAC,sBAAsB;QAChD,WAAW;QACX,kBAAkB,EAAE,CAAC,WAAW,CAAC;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,QAAQ,CAAC,EAAE;QACvB,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;QACnD,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;KAChD,CAAC,CAAC,EACH,CAAC,WAAW,CAAC,EACb,UAAU,EACV,+BAA+B,CAAC,eAAe,CAAC,CACjD,CAAC;IACF,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC;IAC3C,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM;QAC/C,CAAC,CAAC,MAAM,yBAAyB,CAAC,cAAc,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC;IAEP,IAAI,cAAc,CAAC,MAAM;QACvB,MAAM,yBAAyB,CAC7B,MAAM,4BAA4B,CAChC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAChC,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,YAAY,EAAE,QAAQ,CAAC,kBAAkB;SAC1C,CAAC,CAAC,EACH,mBAAmB,CACpB,CACF,CAAC;IAEJ,MAAM,0BAA0B,GAAG,MAAM,iCAAiC,CAAC;QACzE,YAAY,EAAE,CAAC,WAAW,CAAC;QAC3B,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,+BAA+B,CAAC,eAAe,CAAC;KAC7D,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,CAAC;IAC7C,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,qCAAqC,CAAC;QAC1C,WAAW;QACX,eAAe;QACf,8BAA8B;KAC/B,CAAC,CAAC;IAEH,IAAI,eAAe,CAAC,aAAa,EAAE,CAAC;QAClC,4EAA4E;QAC5E,aAAa,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,CAAC,QAAQ;QACd,mBAAmB,CAAC,MAAM,KAAK,CAAC;QAChC,0BAA0B,CAAC,MAAM,KAAK,CAAC;QACvC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,0BAA0B,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,EAChD,SAAS,EACT,GAAG,KAAK,EAMT;IACC,OAAO,0BAA0B,CAAC;QAChC,GAAG,KAAK;QACR,sBAAsB,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAC/C,6BAA6B,CAAC,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;KACjE,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAA6C;IAE7C,MAAM,EACJ,gCAAgC,EAChC,YAAY,EACZ,8BAA8B,EAC9B,sBAAsB,EACtB,yBAAyB,EACzB,8BAA8B,EAC9B,yBAAyB,GAC1B,GAAG,OAAO,CAAC;IACZ,MAAM,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CACjE,CAAC,WAAW,EAAyB,EAAE,CACrC,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAC5D,CAAC;IACF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAC1D,MAAM,0BAA0B,GAAG,MAAM,0BAA0B,CAAC;QAClE,YAAY,EAAE,kBAAkB;QAChC,8BAA8B;QAC9B,sBAAsB;KACvB,CAAC,CAAC;IACH,MAAM,gCAAgC,EAAE,CACtC,+BAA+B,CAC7B,0BAA0B,CAAC,OAAO,CAChC,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,CACtB,eAAe,EAAE,KAAK,CAAC,OAAO,CAC5B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAC5C,IAAI,EAAE,CACV,CACF,CACF,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,yBAAyB,CAC1D,+BAA+B,CAAC,0BAA0B,CAAC,EAC3D,kBAAkB,EAClB,UAAU,EACV,mCAAmC,CAAC,0BAA0B,CAAC,CAChE,CAAC;IACF,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,wBAAwB,GAAG,YAAY,CAAC,MAAM,CAAC;IACrD,MAAM,mBAAmB,GACvB,wBAAwB,CAAC,MAAM,KAAK,CAAC;QACnC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,yBAAyB,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,wBAAwB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,yBAAyB,CAC7B,MAAM,4BAA4B,CAChC,wBAAwB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACvC,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,YAAY,EAAE,KAAK,CAAC,kBAAkB;SACvC,CAAC,CAAC,EACH,mBAAmB,CACpB,CACF,CAAC;IACJ,CAAC;IAED,MAAM,0BAA0B,GAAG,MAAM,iCAAiC,CAAC;QACzE,YAAY,EAAE,0BAA0B,CAAC,OAAO,CAC9C,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,EAAE,EAAE,CACnC,eAAe,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CACvC;QACD,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,mCAAmC,CAAC,0BAA0B,CAAC;KAC5E,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,CAAC;IAC7C,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnD,MAAM,OAAO,CAAC,GAAG,CACf,0BAA0B,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,EAAE,EAAE,CAClE,eAAe;QACb,CAAC,CAAC,qCAAqC,CAAC;YACpC,WAAW;YACX,eAAe;YACf,8BAA8B;SAC/B,CAAC;QACJ,CAAC,CAAC,SAAS,CACd,CACF,CAAC;IAEF,OAAO,CAAC,QAAQ;QACd,mBAAmB,CAAC,MAAM,KAAK,CAAC;QAChC,0BAA0B,CAAC,MAAM,KAAK,CAAC;QACvC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,0BAA0B,CAAC,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,EAC1C,wBAAwB,EACxB,GAAG,KAAK,EAC4B;IACpC,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAClD,wBAAwB,CACzB,CAAC;IACF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,6BAA6B,CAAC;QACnC,GAAG,KAAK;QACR,YAAY;KACb,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,EAClD,SAAS,EACT,GAAG,KAAK,EACmC;IAC3C,OAAO,4BAA4B,CAAC;QAClC,GAAG,KAAK;QACR,sBAAsB,EAAE,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAC/C,6BAA6B,CAAC,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC;QAChE,wBAAwB,EAAE,CAAC,KAAK,EAAE,EAAE,CAClC,SAAS,CAAC,wBAAwB,CAAC,KAAK,CAAC;KAC5C,CAAC,CAAC;AACL,CAAC","sourcesContent":["import type { DocumentSummary } from \"../../data/documents/documentSummary\";\nimport { isDocumentUpdateCreatedEvent } from \"../../data/documents/documentSync\";\nimport { dedupeReferencedPrincipalStates } from \"../../data/keyingProjectionVerification/principalPolicyCache\";\nimport {\n  listAllContainerDocuments,\n  listContainerDocumentsFromApi,\n} from \"./containerDocumentListing\";\nimport { withoutDeferredDocumentLinks } from \"./deferredDocumentLinks\";\nimport {\n  collectApplicableDocumentTombstones,\n  collectDiscoveredDocumentInputs,\n  getApplicableDocumentTombstones,\n} from \"./documentDiscoveryInputs\";\nimport type {\n  ContainerDocumentDiscoveryApi,\n  ContainerParentDiscoveryLane,\n  DiscoverAllContainerDocumentsOptions,\n  DiscoverContainerDocumentsOptions,\n  ListedContainerDocuments,\n  ListedContainerDocumentsLane,\n  RefreshAllContainerDocumentsFromApiOptions,\n  RefreshAllContainerDocumentsOptions,\n} from \"./documentDiscoveryTypes\";\nimport { settleContainerDocumentTombstones } from \"./documentTombstoneGate\";\n\nexport type { RefreshAllContainerDocumentsFromApiOptions } from \"./documentDiscoveryTypes\";\n\nconst CONTAINER_PARENT_DISCOVERY_BATCH_SIZE = 4;\nconst CONTAINER_DOCUMENT_DISCOVERY_CONCURRENCY = 4;\n\ntype ContainerParentLaneRequest = Parameters<\n  ContainerDocumentDiscoveryApi[\"listContainerParentLanes\"]\n>[0][\"lanes\"][number];\n\ninterface RequestedContainerParentLane {\n  lane: ContainerParentDiscoveryLane;\n  request: ContainerParentLaneRequest;\n}\n\nasync function listContainerParentLaneBatch(\n  listContainerParentLanes: ContainerDocumentDiscoveryApi[\"listContainerParentLanes\"],\n  requestedLanes: ReadonlyArray<RequestedContainerParentLane>,\n) {\n  const response = await listContainerParentLanes({\n    lanes: requestedLanes.map(({ request }) => request),\n  });\n  if (!response || response.results.length !== requestedLanes.length) {\n    return null;\n  }\n\n  const requestedLaneIds = new Set(\n    requestedLanes.map(({ request }) => request.laneId),\n  );\n  const pagesByLaneId = new Map<\n    string,\n    (typeof response.results)[number][\"page\"]\n  >();\n  for (const result of response.results) {\n    if (\n      !requestedLaneIds.has(result.laneId) ||\n      pagesByLaneId.has(result.laneId)\n    ) {\n      return null;\n    }\n    pagesByLaneId.set(result.laneId, result.page);\n  }\n\n  return requestedLanes.map(({ lane, request }) => ({\n    lane,\n    response: pagesByLaneId.get(request.laneId),\n  }));\n}\n\nfunction collectRemoteContainerIds(\n  items: ReadonlyArray<{ id: string }>,\n  seenContainerIds: Set<string>,\n  containerIds: string[],\n): ReadonlyArray<string> {\n  for (const { id } of items) {\n    if (!seenContainerIds.has(id)) {\n      seenContainerIds.add(id);\n      containerIds.push(id);\n    }\n  }\n\n  return items.map(({ id }) => id);\n}\n\nasync function saveAppliedContainerDocumentWatermark(input: {\n  containerId: string;\n  listedDocuments: ListedContainerDocuments;\n  saveContainerDocumentWatermark: DiscoverContainerDocumentsOptions[\"saveContainerDocumentWatermark\"];\n}) {\n  const { containerId, listedDocuments, saveContainerDocumentWatermark } =\n    input;\n  if (!listedDocuments.nextWatermark) {\n    return;\n  }\n\n  await saveContainerDocumentWatermark(\n    containerId,\n    listedDocuments.nextWatermark,\n  );\n}\n\nasync function listContainerDocumentLanes(input: {\n  containerIds: ReadonlyArray<string>;\n  loadContainerDocumentWatermark: DiscoverContainerDocumentsOptions[\"loadContainerDocumentWatermark\"];\n  listContainerDocuments: DiscoverContainerDocumentsOptions[\"listContainerDocuments\"];\n}): Promise<ListedContainerDocumentsLane[]> {\n  const {\n    containerIds,\n    loadContainerDocumentWatermark,\n    listContainerDocuments,\n  } = input;\n  const listedDocumentsByContainer: ListedContainerDocumentsLane[] = [];\n  let nextContainerIndex = 0;\n\n  async function worker() {\n    while (nextContainerIndex < containerIds.length) {\n      const containerIndex = nextContainerIndex;\n      nextContainerIndex += 1;\n      const containerId = containerIds[containerIndex];\n      if (!containerId) {\n        throw new Error(\"Container document discovery received an empty lane\");\n      }\n\n      listedDocumentsByContainer[containerIndex] = {\n        containerId,\n        listedDocuments: await listAllContainerDocuments({\n          containerId,\n          loadContainerDocumentWatermark,\n          listContainerDocuments,\n        }),\n      };\n    }\n  }\n\n  await Promise.all(\n    Array.from(\n      {\n        length: Math.min(\n          CONTAINER_DOCUMENT_DISCOVERY_CONCURRENCY,\n          containerIds.length,\n        ),\n      },\n      () => worker(),\n    ),\n  );\n\n  return listedDocumentsByContainer;\n}\n\nexport function hasUndiscoveredDocumentUpdateEvent(\n  events: ReadonlyArray<unknown>,\n  knownDocumentIds: ReadonlySet<string>,\n): boolean {\n  return events.some(\n    (event) =>\n      isDocumentUpdateCreatedEvent(event) &&\n      !knownDocumentIds.has(event.documentId),\n  );\n}\n\nexport async function listAllRemoteContainerIds(\n  listContainerParentLanes: ContainerDocumentDiscoveryApi[\"listContainerParentLanes\"],\n): Promise<ReadonlyArray<string> | null> {\n  const containerIds: string[] = [];\n  const queuedParentIds = new Set<string | null>();\n  const seenContainerIds = new Set<string>();\n  const lanes: ContainerParentDiscoveryLane[] = [];\n  let nextLaneId = 0;\n  const queueParentLane = (parentId: string | null) => {\n    if (queuedParentIds.has(parentId)) {\n      return;\n    }\n\n    queuedParentIds.add(parentId);\n    lanes.push({ parentId, watermark: null });\n  };\n\n  queueParentLane(null);\n\n  while (lanes.length > 0) {\n    const laneBatch = lanes.splice(0, CONTAINER_PARENT_DISCOVERY_BATCH_SIZE);\n    const requestedLanes = laneBatch.map((lane) => ({\n      lane,\n      request: {\n        laneId: `container-parent-${nextLaneId++}`,\n        parentId: lane.parentId,\n        watermark: lane.watermark,\n      },\n    }));\n    const listedLanes = await listContainerParentLaneBatch(\n      listContainerParentLanes,\n      requestedLanes,\n    );\n    if (!listedLanes) {\n      return null;\n    }\n    const continuationLanes: ContainerParentDiscoveryLane[] = [];\n\n    for (const { lane, response } of listedLanes) {\n      if (!response) {\n        return null;\n      }\n\n      for (const containerId of collectRemoteContainerIds(\n        response.items,\n        seenContainerIds,\n        containerIds,\n      )) {\n        queueParentLane(containerId);\n      }\n\n      if (response.hasMore) {\n        if (!response.nextWatermark) {\n          return null;\n        }\n        continuationLanes.push({\n          parentId: lane.parentId,\n          watermark: response.nextWatermark,\n        });\n      }\n    }\n\n    lanes.unshift(...continuationLanes);\n  }\n\n  return containerIds;\n}\n\nexport function listAllRemoteContainerIdsFromApi(\n  apiClient: Pick<ContainerDocumentDiscoveryApi, \"listContainerParentLanes\">,\n): Promise<ReadonlyArray<string> | null> {\n  return listAllRemoteContainerIds((input) =>\n    apiClient.listContainerParentLanes(input),\n  );\n}\n\nexport async function discoverContainerDocuments(\n  options: DiscoverContainerDocumentsOptions,\n): Promise<ReadonlyArray<DocumentSummary> | null> {\n  const {\n    cacheReferencedPrincipalPolicies,\n    containerId,\n    loadContainerDocumentWatermark,\n    listContainerDocuments,\n    onFullListing,\n    replaceDocumentLinksBatch,\n    saveContainerDocumentWatermark,\n    upsertDiscoveredDocuments,\n  } = options;\n  const generation = await options.beginDocumentDiscovery();\n  const listedDocuments = await listAllContainerDocuments({\n    containerId,\n    loadContainerDocumentWatermark,\n    listContainerDocuments,\n  });\n  if (!listedDocuments) {\n    return null;\n  }\n\n  await cacheReferencedPrincipalPolicies?.(\n    dedupeReferencedPrincipalStates(\n      listedDocuments.items.flatMap(\n        (document) => document.referencedPrincipals,\n      ),\n    ),\n  );\n\n  const verification = await options.verifyDiscoveredDocuments(\n    listedDocuments.items.map((document) => ({\n      accessEpoch: document.currentAccessEpoch,\n      accessStateHash: document.currentAccessStateHash,\n      containerId,\n      listedContainerIds: [containerId],\n      createdAt: document.createdAt,\n      documentId: document.id,\n      effectiveAccessLevel: document.effectiveAccessLevel,\n      linkedContainerIds: document.linkedContainerIds,\n    })),\n    [containerId],\n    generation,\n    getApplicableDocumentTombstones(listedDocuments),\n  );\n  if (!(await verification.isCurrent())) return null;\n  const verifiedInputs = verification.inputs;\n  const discoveredDocuments = verifiedInputs.length\n    ? await upsertDiscoveredDocuments(verifiedInputs)\n    : [];\n\n  if (verifiedInputs.length)\n    await replaceDocumentLinksBatch(\n      await withoutDeferredDocumentLinks(\n        verifiedInputs.map((document) => ({\n          documentId: document.documentId,\n          accessEpoch: document.accessEpoch,\n          containerIds: document.linkedContainerIds,\n        })),\n        discoveredDocuments,\n      ),\n    );\n\n  const tombstoneDocumentSummaries = await settleContainerDocumentTombstones({\n    containerIds: [containerId],\n    store: options,\n    tombstones: getApplicableDocumentTombstones(listedDocuments),\n  });\n\n  const complete = await verification.commit();\n  if (!(await verification.isCurrent())) return null;\n  await saveAppliedContainerDocumentWatermark({\n    containerId,\n    listedDocuments,\n    saveContainerDocumentWatermark,\n  });\n\n  if (listedDocuments.isFullListing) {\n    // Initial probes consume ids as discovery hints and verify their own heads.\n    onFullListing?.(listedDocuments.items.map((document) => document.id));\n  }\n\n  return !complete &&\n    discoveredDocuments.length === 0 &&\n    tombstoneDocumentSummaries.length === 0\n    ? null\n    : [...discoveredDocuments, ...tombstoneDocumentSummaries];\n}\n\nexport function discoverContainerDocumentsFromApi({\n  apiClient,\n  ...input\n}: Omit<DiscoverContainerDocumentsOptions, \"listContainerDocuments\"> & {\n  readonly apiClient: Pick<\n    ContainerDocumentDiscoveryApi,\n    \"listContainerDocuments\" | \"listContainerDocumentsResult\"\n  >;\n}): Promise<ReadonlyArray<DocumentSummary> | null> {\n  return discoverContainerDocuments({\n    ...input,\n    listContainerDocuments: (containerId, options) =>\n      listContainerDocumentsFromApi(apiClient, containerId, options),\n  });\n}\n\nexport async function discoverAllContainerDocuments(\n  options: DiscoverAllContainerDocumentsOptions,\n): Promise<ReadonlyArray<DocumentSummary> | null> {\n  const {\n    cacheReferencedPrincipalPolicies,\n    containerIds,\n    loadContainerDocumentWatermark,\n    listContainerDocuments,\n    replaceDocumentLinksBatch,\n    saveContainerDocumentWatermark,\n    upsertDiscoveredDocuments,\n  } = options;\n  const uniqueContainerIds = Array.from(new Set(containerIds)).filter(\n    (containerId): containerId is string =>\n      typeof containerId === \"string\" && containerId.length > 0,\n  );\n  const generation = await options.beginDocumentDiscovery();\n  const listedDocumentsByContainer = await listContainerDocumentLanes({\n    containerIds: uniqueContainerIds,\n    loadContainerDocumentWatermark,\n    listContainerDocuments,\n  });\n  await cacheReferencedPrincipalPolicies?.(\n    dedupeReferencedPrincipalStates(\n      listedDocumentsByContainer.flatMap(\n        ({ listedDocuments }) =>\n          listedDocuments?.items.flatMap(\n            (document) => document.referencedPrincipals,\n          ) ?? [],\n      ),\n    ),\n  );\n  const verification = await options.verifyDiscoveredDocuments(\n    collectDiscoveredDocumentInputs(listedDocumentsByContainer),\n    uniqueContainerIds,\n    generation,\n    collectApplicableDocumentTombstones(listedDocumentsByContainer),\n  );\n  if (!(await verification.isCurrent())) return null;\n  const discoveredDocumentInputs = verification.inputs;\n  const discoveredDocuments =\n    discoveredDocumentInputs.length === 0\n      ? []\n      : await upsertDiscoveredDocuments(discoveredDocumentInputs);\n\n  if (discoveredDocumentInputs.length > 0) {\n    await replaceDocumentLinksBatch(\n      await withoutDeferredDocumentLinks(\n        discoveredDocumentInputs.map((input) => ({\n          documentId: input.documentId,\n          accessEpoch: input.accessEpoch,\n          containerIds: input.linkedContainerIds,\n        })),\n        discoveredDocuments,\n      ),\n    );\n  }\n\n  const tombstoneDocumentSummaries = await settleContainerDocumentTombstones({\n    containerIds: listedDocumentsByContainer.flatMap(\n      ({ containerId, listedDocuments }) =>\n        listedDocuments ? [containerId] : [],\n    ),\n    store: options,\n    tombstones: collectApplicableDocumentTombstones(listedDocumentsByContainer),\n  });\n\n  const complete = await verification.commit();\n  if (!(await verification.isCurrent())) return null;\n\n  await Promise.all(\n    listedDocumentsByContainer.map(({ containerId, listedDocuments }) =>\n      listedDocuments\n        ? saveAppliedContainerDocumentWatermark({\n            containerId,\n            listedDocuments,\n            saveContainerDocumentWatermark,\n          })\n        : undefined,\n    ),\n  );\n\n  return !complete &&\n    discoveredDocuments.length === 0 &&\n    tombstoneDocumentSummaries.length === 0\n    ? null\n    : [...discoveredDocuments, ...tombstoneDocumentSummaries];\n}\n\nasync function refreshAllContainerDocuments({\n  listContainerParentLanes,\n  ...input\n}: RefreshAllContainerDocumentsOptions): Promise<ReadonlyArray<DocumentSummary> | null> {\n  const containerIds = await listAllRemoteContainerIds(\n    listContainerParentLanes,\n  );\n  if (!containerIds) {\n    return null;\n  }\n\n  return discoverAllContainerDocuments({\n    ...input,\n    containerIds,\n  });\n}\n\nexport function refreshAllContainerDocumentsFromApi({\n  apiClient,\n  ...input\n}: RefreshAllContainerDocumentsFromApiOptions): Promise<ReadonlyArray<DocumentSummary> | null> {\n  return refreshAllContainerDocuments({\n    ...input,\n    listContainerDocuments: (containerId, options) =>\n      listContainerDocumentsFromApi(apiClient, containerId, options),\n    listContainerParentLanes: (input) =>\n      apiClient.listContainerParentLanes(input),\n  });\n}\n"]}

package/workflows/container-contents/documentDiscoveryCandidates.js

@@ -0,0 +1,29 @@
+export function groupPendingDocumentDiscoveries(candidates) {
+    const byDocument = new Map();
+    for (const candidate of candidates) {
+        const group = byDocument.get(candidate.documentId) ?? [];
+        group.push(candidate);
+        byDocument.set(candidate.documentId, group);
+    }
+    return [...byDocument.values()].map((rows) => {
+        rows.sort((a, b) => b.accessEpoch - a.accessEpoch ||
+            (a.containerId < b.containerId
+                ? -1
+                : a.containerId > b.containerId
+                    ? 1
+                    : 0));
+        const first = rows[0];
+        if (!first)
+            throw new Error("Empty discovery candidate group");
+        return {
+            rows,
+            input: {
+                ...first,
+                listedContainerIds: [
+                    ...new Set(rows.flatMap((row) => row.listedContainerIds)),
+                ].sort(),
+            },
+        };
+    });
+}
+//# sourceMappingURL=documentDiscoveryCandidates.js.map

package/workflows/container-contents/documentDiscoveryCandidates.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"documentDiscoveryCandidates.js","sourceRoot":"","sources":["documentDiscoveryCandidates.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,+BAA+B,CAC7C,UAAkD;IAElD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyC,CAAC;IACpE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3C,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;YAC7B,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;gBAC5B,CAAC,CAAC,CAAC,CAAC;gBACJ,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;oBAC7B,CAAC,CAAC,CAAC;oBACH,CAAC,CAAC,CAAC,CAAC,CACX,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAC/D,OAAO;YACL,IAAI;YACJ,KAAK,EAAE;gBACL,GAAG,KAAK;gBACR,kBAAkB,EAAE;oBAClB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;iBAC1D,CAAC,IAAI,EAAE;aACT;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC","sourcesContent":["import type { DiscoveredDocumentCandidate } from \"../../data/persistence/documents/documentDiscoveryEvidencePersistence\";\n\nexport function groupPendingDocumentDiscoveries(\n  candidates: readonly DiscoveredDocumentCandidate[],\n) {\n  const byDocument = new Map<string, DiscoveredDocumentCandidate[]>();\n  for (const candidate of candidates) {\n    const group = byDocument.get(candidate.documentId) ?? [];\n    group.push(candidate);\n    byDocument.set(candidate.documentId, group);\n  }\n  return [...byDocument.values()].map((rows) => {\n    rows.sort(\n      (a, b) =>\n        b.accessEpoch - a.accessEpoch ||\n        (a.containerId < b.containerId\n          ? -1\n          : a.containerId > b.containerId\n            ? 1\n            : 0),\n    );\n    const first = rows[0];\n    if (!first) throw new Error(\"Empty discovery candidate group\");\n    return {\n      rows,\n      input: {\n        ...first,\n        listedContainerIds: [\n          ...new Set(rows.flatMap((row) => row.listedContainerIds)),\n        ].sort(),\n      },\n    };\n  });\n}\n"]}

package/workflows/container-contents/documentDiscoveryEvidence.js

@@ -0,0 +1,91 @@
+import { groupPendingDocumentDiscoveries } from "./documentDiscoveryCandidates.js";
+import { documentHeadMatchesOrganization } from "./documentHeadScope.js";
+import { HEAD_LINK_SET_LOAD_CONCURRENCY, HEAD_LINK_SET_LOADS_PER_RUN, } from "./documentTombstoneEvidence.js";
+async function verifyInput(input, loadHead, loadLocalEpoch, store, generation, organizationId) {
+    const localEpoch = await loadLocalEpoch(input.documentId).catch(() => Number.MAX_SAFE_INTEGER);
+    const cached = await store.loadHead(input.documentId, input.accessStateHash);
+    const minimumEpoch = Math.max(localEpoch, input.accessEpoch);
+    const head = cached && cached.accessEpoch >= minimumEpoch
+        ? cached
+        : await loadHead(input.documentId, input.accessEpoch >= localEpoch
+            ? (input.accessStateHash ?? undefined)
+            : undefined);
+    if (head === "not-found")
+        return null;
+    if (!head || head.accessEpoch < Math.max(localEpoch, input.accessEpoch))
+        return "unavailable";
+    if (!documentHeadMatchesOrganization(head, organizationId))
+        return "unavailable";
+    // A stale first lane must not hide a document present in another listed lane.
+    const containerId = input.listedContainerIds.find((id) => head.linkedContainerIds.includes(id));
+    if (!containerId) {
+        // A same-epoch head can predate a link addition. Only the named head or
+        // terminal signed purge evidence can settle a candidate without a retry.
+        return head.accessEpoch > input.accessEpoch ||
+            head.accessStateHash === input.accessStateHash ||
+            (!head.accessStateHash && head.accessEpoch === Number.MAX_SAFE_INTEGER)
+            ? null
+            : "unavailable";
+    }
+    if (!head.accessStateHash)
+        return "unavailable";
+    if (!(await store.saveHead(input.documentId, head, generation)))
+        return "unavailable";
+    const { listedContainerIds: _listedContainerIds, ...document } = input;
+    return {
+        ...document,
+        containerId,
+        accessEpoch: head.accessEpoch,
+        accessStateHash: head.accessStateHash,
+        linkedContainerIds: head.linkedContainerIds,
+    };
+}
+/**
+ * Persist untrusted candidates before advancing listing watermarks. Each pass
+ * checks a bounded batch; unavailable heads back off independently while verified
+ * siblings and tombstones still settle. Acknowledgement follows local apply so
+ * a crash cannot lose a candidate behind an already advanced listing watermark.
+ */
+export function createDiscoveredDocumentVerifier(organizationId, loadHead, loadLocalEpoch, store, onPendingDiscovery) {
+    return async (inputs, containerIds, generation, tombstones = []) => {
+        const isCurrent = () => store.isCurrent(generation);
+        const lanes = new Set(containerIds);
+        const removed = tombstones.filter((row) => lanes.has(row.containerId));
+        if (!(await store.stage(inputs, generation, removed)))
+            return { inputs: [], isCurrent, commit: async () => true };
+        const candidates = groupPendingDocumentDiscoveries(await store.pending(containerIds, HEAD_LINK_SET_LOADS_PER_RUN));
+        const verified = new Map();
+        const settled = [];
+        let next = 0;
+        const worker = async () => {
+            while (next < candidates.length) {
+                const group = candidates[next++];
+                if (!group)
+                    break;
+                const { input, rows } = group;
+                const result = await verifyInput(input, loadHead, loadLocalEpoch, store, generation, organizationId);
+                if (result === "unavailable") {
+                    await Promise.all(rows.map((row) => store.defer(row, generation)));
+                }
+                else {
+                    if (result)
+                        verified.set(result.documentId, result);
+                    settled.push(...rows);
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(HEAD_LINK_SET_LOAD_CONCURRENCY, candidates.length) }, worker));
+        return {
+            inputs: [...verified.values()],
+            isCurrent,
+            commit: async () => {
+                await store.acknowledge(settled, generation);
+                const retryDelay = await store.retryDelay(containerIds);
+                if (retryDelay !== null)
+                    onPendingDiscovery?.(retryDelay);
+                return retryDelay === null;
+            },
+        };
+    };
+}
+//# sourceMappingURL=documentDiscoveryEvidence.js.map

package/workflows/container-contents/documentDiscoveryEvidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"documentDiscoveryEvidence.js","sourceRoot":"","sources":["documentDiscoveryEvidence.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAEhF,OAAO,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAC;AAKtE,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,GAC5B,MAAM,6BAA6B,CAAC;AAErC,KAAK,UAAU,WAAW,CACxB,KAAkC,EAClC,QAAmC,EACnC,cAA8C,EAC9C,KAAqC,EACrC,UAAkB,EAClB,cAAkC;IAElC,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAC7D,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAC9B,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,IAAI,GACR,MAAM,IAAI,MAAM,CAAC,WAAW,IAAI,YAAY;QAC1C,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,MAAM,QAAQ,CACZ,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,WAAW,IAAI,UAAU;YAC7B,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,SAAS,CAAC;YACtC,CAAC,CAAC,SAAS,CACd,CAAC;IACR,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC;QACrE,OAAO,aAAa,CAAC;IACvB,IAAI,CAAC,+BAA+B,CAAC,IAAI,EAAE,cAAc,CAAC;QACxD,OAAO,aAAa,CAAC;IACvB,8EAA8E;IAC9E,MAAM,WAAW,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACvD,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC,CACrC,CAAC;IACF,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,wEAAwE;QACxE,yEAAyE;QACzE,OAAO,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW;YACzC,IAAI,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe;YAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,gBAAgB,CAAC;YACvE,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,aAAa,CAAC;IACpB,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,eAAe;QAAE,OAAO,aAAa,CAAC;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAC7D,OAAO,aAAa,CAAC;IACvB,MAAM,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAC;IACvE,OAAO;QACL,GAAG,QAAQ;QACX,WAAW;QACX,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;KAC5C,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,cAAkC,EAClC,QAAmC,EACnC,cAA8C,EAC9C,KAAqC,EACrC,kBAA4D;IAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,GAAG,EAAE,EAAE,EAAE;QACjE,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,+BAA+B,CAChD,MAAM,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,2BAA2B,CAAC,CAC/D,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC5D,MAAM,OAAO,GAAkC,EAAE,CAAC;QAClD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,IAAI,CAAC,KAAK;oBAAE,MAAM;gBAClB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,KAAK,EACL,QAAQ,EACR,cAAc,EACd,KAAK,EACL,UAAU,EACV,cAAc,CACf,CAAC;gBACF,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;gBACrE,CAAC;qBAAM,CAAC;oBACN,IAAI,MAAM;wBAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;oBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,8BAA8B,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EACvE,MAAM,CACP,CACF,CAAC;QACF,OAAO;YACL,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9B,SAAS;YACT,MAAM,EAAE,KAAK,IAAI,EAAE;gBACjB,MAAM,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;gBACxD,IAAI,UAAU,KAAK,IAAI;oBAAE,kBAAkB,EAAE,CAAC,UAAU,CAAC,CAAC;gBAC1D,OAAO,UAAU,KAAK,IAAI,CAAC;YAC7B,CAAC;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import type { DiscoveredDocumentInput } from \"../../data/documents/documentSummary\";\nimport type {\n  DiscoveredDocumentCandidate,\n  DocumentDiscoveryEvidenceStore,\n} from \"../../data/persistence/documents/documentDiscoveryEvidencePersistence\";\nimport { groupPendingDocumentDiscoveries } from \"./documentDiscoveryCandidates\";\nimport type { DiscoverContainerDocumentsOptions } from \"./documentDiscoveryTypes\";\nimport { documentHeadMatchesOrganization } from \"./documentHeadScope\";\nimport type {\n  DocumentHeadLinkSetLoader,\n  LocalDocumentAccessEpochLoader,\n} from \"./documentTombstoneEvidence\";\nimport {\n  HEAD_LINK_SET_LOAD_CONCURRENCY,\n  HEAD_LINK_SET_LOADS_PER_RUN,\n} from \"./documentTombstoneEvidence\";\n\nasync function verifyInput(\n  input: DiscoveredDocumentCandidate,\n  loadHead: DocumentHeadLinkSetLoader,\n  loadLocalEpoch: LocalDocumentAccessEpochLoader,\n  store: DocumentDiscoveryEvidenceStore,\n  generation: number,\n  organizationId: string | undefined,\n): Promise<DiscoveredDocumentInput | null | \"unavailable\"> {\n  const localEpoch = await loadLocalEpoch(input.documentId).catch(\n    () => Number.MAX_SAFE_INTEGER,\n  );\n  const cached = await store.loadHead(input.documentId, input.accessStateHash);\n  const minimumEpoch = Math.max(localEpoch, input.accessEpoch);\n  const head =\n    cached && cached.accessEpoch >= minimumEpoch\n      ? cached\n      : await loadHead(\n          input.documentId,\n          input.accessEpoch >= localEpoch\n            ? (input.accessStateHash ?? undefined)\n            : undefined,\n        );\n  if (head === \"not-found\") return null;\n  if (!head || head.accessEpoch < Math.max(localEpoch, input.accessEpoch))\n    return \"unavailable\";\n  if (!documentHeadMatchesOrganization(head, organizationId))\n    return \"unavailable\";\n  // A stale first lane must not hide a document present in another listed lane.\n  const containerId = input.listedContainerIds.find((id) =>\n    head.linkedContainerIds.includes(id),\n  );\n  if (!containerId) {\n    // A same-epoch head can predate a link addition. Only the named head or\n    // terminal signed purge evidence can settle a candidate without a retry.\n    return head.accessEpoch > input.accessEpoch ||\n      head.accessStateHash === input.accessStateHash ||\n      (!head.accessStateHash && head.accessEpoch === Number.MAX_SAFE_INTEGER)\n      ? null\n      : \"unavailable\";\n  }\n  if (!head.accessStateHash) return \"unavailable\";\n  if (!(await store.saveHead(input.documentId, head, generation)))\n    return \"unavailable\";\n  const { listedContainerIds: _listedContainerIds, ...document } = input;\n  return {\n    ...document,\n    containerId,\n    accessEpoch: head.accessEpoch,\n    accessStateHash: head.accessStateHash,\n    linkedContainerIds: head.linkedContainerIds,\n  };\n}\n\n/**\n * Persist untrusted candidates before advancing listing watermarks. Each pass\n * checks a bounded batch; unavailable heads back off independently while verified\n * siblings and tombstones still settle. Acknowledgement follows local apply so\n * a crash cannot lose a candidate behind an already advanced listing watermark.\n */\nexport function createDiscoveredDocumentVerifier(\n  organizationId: string | undefined,\n  loadHead: DocumentHeadLinkSetLoader,\n  loadLocalEpoch: LocalDocumentAccessEpochLoader,\n  store: DocumentDiscoveryEvidenceStore,\n  onPendingDiscovery?: ((delayMs: number) => void) | undefined,\n): DiscoverContainerDocumentsOptions[\"verifyDiscoveredDocuments\"] {\n  return async (inputs, containerIds, generation, tombstones = []) => {\n    const isCurrent = () => store.isCurrent(generation);\n    const lanes = new Set(containerIds);\n    const removed = tombstones.filter((row) => lanes.has(row.containerId));\n    if (!(await store.stage(inputs, generation, removed)))\n      return { inputs: [], isCurrent, commit: async () => true };\n    const candidates = groupPendingDocumentDiscoveries(\n      await store.pending(containerIds, HEAD_LINK_SET_LOADS_PER_RUN),\n    );\n    const verified = new Map<string, DiscoveredDocumentInput>();\n    const settled: DiscoveredDocumentCandidate[] = [];\n    let next = 0;\n    const worker = async () => {\n      while (next < candidates.length) {\n        const group = candidates[next++];\n        if (!group) break;\n        const { input, rows } = group;\n        const result = await verifyInput(\n          input,\n          loadHead,\n          loadLocalEpoch,\n          store,\n          generation,\n          organizationId,\n        );\n        if (result === \"unavailable\") {\n          await Promise.all(rows.map((row) => store.defer(row, generation)));\n        } else {\n          if (result) verified.set(result.documentId, result);\n          settled.push(...rows);\n        }\n      }\n    };\n    await Promise.all(\n      Array.from(\n        { length: Math.min(HEAD_LINK_SET_LOAD_CONCURRENCY, candidates.length) },\n        worker,\n      ),\n    );\n    return {\n      inputs: [...verified.values()],\n      isCurrent,\n      commit: async () => {\n        await store.acknowledge(settled, generation);\n        const retryDelay = await store.retryDelay(containerIds);\n        if (retryDelay !== null) onPendingDiscovery?.(retryDelay);\n        return retryDelay === null;\n      },\n    };\n  };\n}\n"]}

package/workflows/container-contents/documentDiscoveryInputs.js

@@ -0,0 +1,85 @@
+import { maxEffectiveAccessLevel, normalizeEffectiveAccessLevel, } from "../../data/accessLevel.js";
+import { uniqueSortedStrings } from "../../data/documents/shared/readers.js";
+function mergeDiscoveredDocumentInputs(current, next) {
+    // A lane page carries a complete link set for its epoch. Combining an old
+    // root page with a newer trash page must not manufacture a newer root link.
+    if (next.accessEpoch !== current.accessEpoch) {
+        return {
+            ...(next.accessEpoch > current.accessEpoch ? next : current),
+            listedContainerIds: uniqueSortedStrings([
+                ...current.listedContainerIds,
+                ...next.listedContainerIds,
+            ]),
+        };
+    }
+    const accessStateHash = next.accessEpoch >= current.accessEpoch
+        ? next.accessStateHash
+        : current.accessStateHash;
+    return {
+        accessEpoch: Math.max(current.accessEpoch, next.accessEpoch),
+        ...(accessStateHash === undefined ? {} : { accessStateHash }),
+        containerId: current.containerId,
+        listedContainerIds: uniqueSortedStrings([
+            ...current.listedContainerIds,
+            ...next.listedContainerIds,
+        ]),
+        createdAt: current.createdAt.localeCompare(next.createdAt) <= 0
+            ? current.createdAt
+            : next.createdAt,
+        documentId: current.documentId,
+        effectiveAccessLevel: maxEffectiveAccessLevel(normalizeEffectiveAccessLevel(current.effectiveAccessLevel), normalizeEffectiveAccessLevel(next.effectiveAccessLevel)),
+        linkedContainerIds: uniqueSortedStrings([
+            current.containerId,
+            next.containerId,
+            ...current.linkedContainerIds,
+            ...next.linkedContainerIds,
+        ]),
+    };
+}
+export function collectDiscoveredDocumentInputs(listedDocumentsByContainer) {
+    const discoveredDocumentInputsByDocumentId = new Map();
+    for (const { containerId, listedDocuments } of listedDocumentsByContainer) {
+        if (!listedDocuments) {
+            continue;
+        }
+        for (const document of listedDocuments.items) {
+            const discoveredDocumentInput = {
+                accessEpoch: document.currentAccessEpoch,
+                accessStateHash: document.currentAccessStateHash,
+                containerId,
+                listedContainerIds: [containerId],
+                createdAt: document.createdAt,
+                documentId: document.id,
+                effectiveAccessLevel: normalizeEffectiveAccessLevel(document.effectiveAccessLevel),
+                linkedContainerIds: uniqueSortedStrings([
+                    containerId,
+                    ...document.linkedContainerIds,
+                ]),
+            };
+            const currentDiscoveredDocumentCandidate = discoveredDocumentInputsByDocumentId.get(document.id);
+            discoveredDocumentInputsByDocumentId.set(document.id, currentDiscoveredDocumentCandidate
+                ? mergeDiscoveredDocumentInputs(currentDiscoveredDocumentCandidate, discoveredDocumentInput)
+                : discoveredDocumentInput);
+        }
+    }
+    return Array.from(discoveredDocumentInputsByDocumentId.values());
+}
+export function getApplicableDocumentTombstones(listedDocuments) {
+    const latestItemsByDocumentId = new Map();
+    for (const document of listedDocuments.items) {
+        const existingDocument = latestItemsByDocumentId.get(document.id);
+        if (!existingDocument ||
+            existingDocument.updatedAt.localeCompare(document.updatedAt) < 0) {
+            latestItemsByDocumentId.set(document.id, document);
+        }
+    }
+    return listedDocuments.tombstones.filter((tombstone) => {
+        const item = latestItemsByDocumentId.get(tombstone.documentId);
+        return (!item?.linkedContainerIds.includes(tombstone.containerId) ||
+            item.updatedAt.localeCompare(tombstone.updatedAt) < 0);
+    });
+}
+export function collectApplicableDocumentTombstones(lanes) {
+    return lanes.flatMap(({ listedDocuments }) => listedDocuments ? getApplicableDocumentTombstones(listedDocuments) : []);
+}
+//# sourceMappingURL=documentDiscoveryInputs.js.map

package/workflows/container-contents/documentDiscoveryInputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"documentDiscoveryInputs.js","sourceRoot":"","sources":["documentDiscoveryInputs.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAS1E,SAAS,6BAA6B,CACpC,OAAoC,EACpC,IAAiC;IAEjC,0EAA0E;IAC1E,4EAA4E;IAC5E,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC;QAC7C,OAAO;YACL,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;YAC5D,kBAAkB,EAAE,mBAAmB,CAAC;gBACtC,GAAG,OAAO,CAAC,kBAAkB;gBAC7B,GAAG,IAAI,CAAC,kBAAkB;aAC3B,CAAC;SACH,CAAC;IACJ,CAAC;IACD,MAAM,eAAe,GACnB,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW;QACrC,CAAC,CAAC,IAAI,CAAC,eAAe;QACtB,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;IAE9B,OAAO;QACL,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC;QAC5D,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC;QAC7D,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,kBAAkB,EAAE,mBAAmB,CAAC;YACtC,GAAG,OAAO,CAAC,kBAAkB;YAC7B,GAAG,IAAI,CAAC,kBAAkB;SAC3B,CAAC;QACF,SAAS,EACP,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;YAClD,CAAC,CAAC,OAAO,CAAC,SAAS;YACnB,CAAC,CAAC,IAAI,CAAC,SAAS;QACpB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,oBAAoB,EAAE,uBAAuB,CAC3C,6BAA6B,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAC3D,6BAA6B,CAAC,IAAI,CAAC,oBAAoB,CAAC,CACzD;QACD,kBAAkB,EAAE,mBAAmB,CAAC;YACtC,OAAO,CAAC,WAAW;YACnB,IAAI,CAAC,WAAW;YAChB,GAAG,OAAO,CAAC,kBAAkB;YAC7B,GAAG,IAAI,CAAC,kBAAkB;SAC3B,CAAC;KACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,0BAAuE;IAEvE,MAAM,oCAAoC,GAAG,IAAI,GAAG,EAGjD,CAAC;IAEJ,KAAK,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,IAAI,0BAA0B,EAAE,CAAC;QAC1E,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,uBAAuB,GAAgC;gBAC3D,WAAW,EAAE,QAAQ,CAAC,kBAAkB;gBACxC,eAAe,EAAE,QAAQ,CAAC,sBAAsB;gBAChD,WAAW;gBACX,kBAAkB,EAAE,CAAC,WAAW,CAAC;gBACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,UAAU,EAAE,QAAQ,CAAC,EAAE;gBACvB,oBAAoB,EAAE,6BAA6B,CACjD,QAAQ,CAAC,oBAAoB,CAC9B;gBACD,kBAAkB,EAAE,mBAAmB,CAAC;oBACtC,WAAW;oBACX,GAAG,QAAQ,CAAC,kBAAkB;iBAC/B,CAAC;aACH,CAAC;YACF,MAAM,kCAAkC,GACtC,oCAAoC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACxD,oCAAoC,CAAC,GAAG,CACtC,QAAQ,CAAC,EAAE,EACX,kCAAkC;gBAChC,CAAC,CAAC,6BAA6B,CAC3B,kCAAkC,EAClC,uBAAuB,CACxB;gBACH,CAAC,CAAC,uBAAuB,CAC5B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,MAAM,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,eAAyC;IAEzC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAmC,CAAC;IAC3E,KAAK,MAAM,QAAQ,IAAI,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7C,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAClE,IACE,CAAC,gBAAgB;YACjB,gBAAgB,CAAC,SAAS,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAChE,CAAC;YACD,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,uBAAuB,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC/D,OAAO,CACL,CAAC,IAAI,EAAE,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,WAAW,CAAC;YACzD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CACtD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mCAAmC,CACjD,KAAkD;IAElD,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,CAC3C,eAAe,CAAC,CAAC,CAAC,+BAA+B,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CACxE,CAAC;AACJ,CAAC","sourcesContent":["import {\n  maxEffectiveAccessLevel,\n  normalizeEffectiveAccessLevel,\n} from \"../../data/accessLevel\";\nimport { uniqueSortedStrings } from \"../../data/documents/shared/readers\";\nimport type { DiscoveredDocumentCandidate } from \"../../data/persistence/documents/documentDiscoveryEvidencePersistence\";\nimport type {\n  ContainerDocumentTombstone,\n  ListedContainerDocument,\n  ListedContainerDocuments,\n  ListedContainerDocumentsLane,\n} from \"./documentDiscoveryTypes\";\n\nfunction mergeDiscoveredDocumentInputs(\n  current: DiscoveredDocumentCandidate,\n  next: DiscoveredDocumentCandidate,\n): DiscoveredDocumentCandidate {\n  // A lane page carries a complete link set for its epoch. Combining an old\n  // root page with a newer trash page must not manufacture a newer root link.\n  if (next.accessEpoch !== current.accessEpoch) {\n    return {\n      ...(next.accessEpoch > current.accessEpoch ? next : current),\n      listedContainerIds: uniqueSortedStrings([\n        ...current.listedContainerIds,\n        ...next.listedContainerIds,\n      ]),\n    };\n  }\n  const accessStateHash =\n    next.accessEpoch >= current.accessEpoch\n      ? next.accessStateHash\n      : current.accessStateHash;\n\n  return {\n    accessEpoch: Math.max(current.accessEpoch, next.accessEpoch),\n    ...(accessStateHash === undefined ? {} : { accessStateHash }),\n    containerId: current.containerId,\n    listedContainerIds: uniqueSortedStrings([\n      ...current.listedContainerIds,\n      ...next.listedContainerIds,\n    ]),\n    createdAt:\n      current.createdAt.localeCompare(next.createdAt) <= 0\n        ? current.createdAt\n        : next.createdAt,\n    documentId: current.documentId,\n    effectiveAccessLevel: maxEffectiveAccessLevel(\n      normalizeEffectiveAccessLevel(current.effectiveAccessLevel),\n      normalizeEffectiveAccessLevel(next.effectiveAccessLevel),\n    ),\n    linkedContainerIds: uniqueSortedStrings([\n      current.containerId,\n      next.containerId,\n      ...current.linkedContainerIds,\n      ...next.linkedContainerIds,\n    ]),\n  };\n}\n\nexport function collectDiscoveredDocumentInputs(\n  listedDocumentsByContainer: ReadonlyArray<ListedContainerDocumentsLane>,\n): DiscoveredDocumentCandidate[] {\n  const discoveredDocumentInputsByDocumentId = new Map<\n    string,\n    DiscoveredDocumentCandidate\n  >();\n\n  for (const { containerId, listedDocuments } of listedDocumentsByContainer) {\n    if (!listedDocuments) {\n      continue;\n    }\n\n    for (const document of listedDocuments.items) {\n      const discoveredDocumentInput: DiscoveredDocumentCandidate = {\n        accessEpoch: document.currentAccessEpoch,\n        accessStateHash: document.currentAccessStateHash,\n        containerId,\n        listedContainerIds: [containerId],\n        createdAt: document.createdAt,\n        documentId: document.id,\n        effectiveAccessLevel: normalizeEffectiveAccessLevel(\n          document.effectiveAccessLevel,\n        ),\n        linkedContainerIds: uniqueSortedStrings([\n          containerId,\n          ...document.linkedContainerIds,\n        ]),\n      };\n      const currentDiscoveredDocumentCandidate =\n        discoveredDocumentInputsByDocumentId.get(document.id);\n      discoveredDocumentInputsByDocumentId.set(\n        document.id,\n        currentDiscoveredDocumentCandidate\n          ? mergeDiscoveredDocumentInputs(\n              currentDiscoveredDocumentCandidate,\n              discoveredDocumentInput,\n            )\n          : discoveredDocumentInput,\n      );\n    }\n  }\n\n  return Array.from(discoveredDocumentInputsByDocumentId.values());\n}\n\nexport function getApplicableDocumentTombstones(\n  listedDocuments: ListedContainerDocuments,\n): ContainerDocumentTombstone[] {\n  const latestItemsByDocumentId = new Map<string, ListedContainerDocument>();\n  for (const document of listedDocuments.items) {\n    const existingDocument = latestItemsByDocumentId.get(document.id);\n    if (\n      !existingDocument ||\n      existingDocument.updatedAt.localeCompare(document.updatedAt) < 0\n    ) {\n      latestItemsByDocumentId.set(document.id, document);\n    }\n  }\n\n  return listedDocuments.tombstones.filter((tombstone) => {\n    const item = latestItemsByDocumentId.get(tombstone.documentId);\n    return (\n      !item?.linkedContainerIds.includes(tombstone.containerId) ||\n      item.updatedAt.localeCompare(tombstone.updatedAt) < 0\n    );\n  });\n}\n\nexport function collectApplicableDocumentTombstones(\n  lanes: ReadonlyArray<ListedContainerDocumentsLane>,\n): ContainerDocumentTombstone[] {\n  return lanes.flatMap(({ listedDocuments }) =>\n    listedDocuments ? getApplicableDocumentTombstones(listedDocuments) : [],\n  );\n}\n"]}