@tearleads/client-sdk 0.1.253 → 0.1.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/containerContents/documentDiscovery.js +0 -2
- package/client/containerContents/documentDiscovery.js.map +1 -1
- package/data/persistence/container-contents/containerContentsPersistenceTypes.d.ts +7 -10
- package/data/persistence/container-contents/containerContentsPersistenceTypes.js.map +1 -1
- package/data/persistence/container-contents/containerDeletionPersistence.js +5 -1
- package/data/persistence/container-contents/containerDeletionPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerHydrationPersistence.js +1 -1
- package/data/persistence/container-contents/containerHydrationPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerMetadataMutationPersistence.js +11 -4
- package/data/persistence/container-contents/containerMetadataMutationPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerRemovalMetadata.js +47 -0
- package/data/persistence/container-contents/containerRemovalMetadata.js.map +1 -0
- package/data/persistence/container-contents/containerRemovalPlacements.js +22 -0
- package/data/persistence/container-contents/containerRemovalPlacements.js.map +1 -0
- package/data/persistence/container-contents/containerSyncWatermarkPersistence.d.ts +2 -1
- package/data/persistence/container-contents/containerSyncWatermarkPersistence.js +10 -2
- package/data/persistence/container-contents/containerSyncWatermarkPersistence.js.map +1 -1
- package/data/persistence/documents/documentDiscoveryCandidate.d.ts +1 -0
- package/data/persistence/documents/documentDiscoveryCandidate.js +3 -1
- package/data/persistence/documents/documentDiscoveryCandidate.js.map +1 -1
- package/data/persistence/documents/documentDiscoveryEvidencePersistence.d.ts +1 -1
- package/data/persistence/documents/documentDiscoveryEvidencePersistence.js +16 -4
- package/data/persistence/documents/documentDiscoveryEvidencePersistence.js.map +1 -1
- package/data/sqlite/migrations/localSchemaMigrations.js +11 -0
- package/data/sqlite/migrations/localSchemaMigrations.js.map +1 -1
- package/data/sqlite/schema/containerSyncWatermarkSchema.js +6 -3
- package/data/sqlite/schema/containerSyncWatermarkSchema.js.map +1 -1
- package/data/sqlite/schema/index.d.ts +17 -0
- package/internal/api-client/ApiClient.d.ts +1 -0
- package/internal/api-client/client/apiClientContext.d.ts +1 -0
- package/internal/api-client/client/containerReads.d.ts +1 -0
- package/internal/api-client/requestInternals.js +14 -6
- package/internal/api-client/requestInternals.js.map +1 -1
- package/internal/validators/operation/containerReads.d.ts +29 -2
- package/internal/validators/operation/containerReads.js +2 -1
- package/internal/validators/operation/containerReads.js.map +1 -1
- package/internal/validators/request/containerDocumentLanes.d.ts +7 -0
- package/internal/validators/request/containerDocumentLanes.js +6 -0
- package/internal/validators/request/containerDocumentLanes.js.map +1 -1
- package/internal/validators/response/container.d.ts +8 -0
- package/internal/validators/response/container.js +1 -0
- package/internal/validators/response/container.js.map +1 -1
- package/internal/validators/response/containerParentLanes.d.ts +2 -0
- package/package.json +1 -1
- package/workflows/container-contents/documentDiscovery/containerDocumentListing.js +52 -4
- package/workflows/container-contents/documentDiscovery/containerDocumentListing.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/documentDiscoveryApply.js +4 -2
- package/workflows/container-contents/documentDiscovery/documentDiscoveryApply.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/documentDiscoveryEvidence.js +17 -12
- package/workflows/container-contents/documentDiscovery/documentDiscoveryEvidence.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/documentDiscoveryTypes.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/publicationWatermark.js +15 -0
- package/workflows/container-contents/documentDiscovery/publicationWatermark.js.map +1 -0
- package/workflows/container-contents/remoteHydration/hydrateRemoteContainers.js +4 -4
- package/workflows/container-contents/remoteHydration/hydrateRemoteContainers.js.map +1 -1
- package/workflows/container-contents/remoteHydration/lanes/restorationReceipt.js +25 -0
- package/workflows/container-contents/remoteHydration/lanes/restorationReceipt.js.map +1 -0
- package/workflows/container-contents/remoteHydration/lanes/tombstoneAccess.js +107 -0
- package/workflows/container-contents/remoteHydration/lanes/tombstoneAccess.js.map +1 -0
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerPageHydration.js +29 -27
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerPageHydration.js.map +1 -1
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerState.js +17 -5
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerState.js.map +1 -1
- package/workflows/container-contents/remoteHydration/tombstoneApplication.js +67 -24
- package/workflows/container-contents/remoteHydration/tombstoneApplication.js.map +1 -1
- package/workflows/container-contents/remoteHydration/types.d.ts +3 -1
- package/workflows/container-contents/remoteHydration/types.js.map +1 -1
- package/workflows/container-contents/remoteReset/remoteResetCursorScope.js +3 -3
- package/workflows/container-contents/remoteReset/remoteResetCursorScope.js.map +1 -1
- package/workflows/container-contents/syncPause/discoveryGuard.js +6 -2
- package/workflows/container-contents/syncPause/discoveryGuard.js.map +1 -1
- package/workflows/container-contents/tombstones/documentTombstoneGate.js +6 -3
- package/workflows/container-contents/tombstones/documentTombstoneGate.js.map +1 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_CONTAINER_DOCUMENT_LANE_PAGE_LIMIT, MAX_CONTAINER_DOCUMENT_LANE_PAGE_TOTAL, MAX_CONTAINER_DOCUMENT_LANES, } from "../../../internal/validators/request/index.js";
|
|
2
|
+
import { changesDiscoveryOrder, isPublicationWatermark, } from "./publicationWatermark.js";
|
|
2
3
|
/**
|
|
3
4
|
* Rows each lane may return so a full batch stays within the single-page
|
|
4
5
|
* maximum; a lone lane keeps the ordinary page size.
|
|
@@ -28,6 +29,45 @@ export async function listContainerDocumentLanesFromApi(apiClient, lanes, report
|
|
|
28
29
|
return result && "page" in result ? result.page : null;
|
|
29
30
|
});
|
|
30
31
|
}
|
|
32
|
+
function removedDocumentPlacements(listed) {
|
|
33
|
+
const removals = new Map();
|
|
34
|
+
for (const lane of listed.values()) {
|
|
35
|
+
if (!lane ||
|
|
36
|
+
lane === "unavailable" ||
|
|
37
|
+
!isPublicationWatermark(lane.nextWatermark))
|
|
38
|
+
continue;
|
|
39
|
+
for (const row of lane.tombstones) {
|
|
40
|
+
const containers = removals.get(row.documentId) ?? new Set();
|
|
41
|
+
containers.add(row.containerId);
|
|
42
|
+
removals.set(row.documentId, containers);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return removals;
|
|
46
|
+
}
|
|
47
|
+
// Publication-ordered lanes exclude historical unlinks for live placements.
|
|
48
|
+
// Check the whole fetched batch before applying a lane: a linked document
|
|
49
|
+
// carries its placements in other containers as well as the queried one.
|
|
50
|
+
function discardConflictingDocumentPages(listed) {
|
|
51
|
+
const removals = removedDocumentPlacements(listed);
|
|
52
|
+
if (!removals.size)
|
|
53
|
+
return;
|
|
54
|
+
const conflicted = new Set();
|
|
55
|
+
for (const [id, lane] of listed) {
|
|
56
|
+
if (!lane || lane === "unavailable")
|
|
57
|
+
continue;
|
|
58
|
+
for (const item of lane.items) {
|
|
59
|
+
const removed = removals.get(item.id);
|
|
60
|
+
if (!removed)
|
|
61
|
+
continue;
|
|
62
|
+
for (const linked of item.linkedContainerIds.filter((container) => removed.has(container))) {
|
|
63
|
+
conflicted.add(id);
|
|
64
|
+
conflicted.add(linked);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
for (const id of conflicted)
|
|
69
|
+
listed.set(id, "unavailable");
|
|
70
|
+
}
|
|
31
71
|
/**
|
|
32
72
|
* Pages every lane to exhaustion, batching first pages and continuations
|
|
33
73
|
* alike. Each container keeps its own watermark, so one lane's failure or
|
|
@@ -45,6 +85,7 @@ export async function listAllContainerDocumentLanes(input) {
|
|
|
45
85
|
pending.push({
|
|
46
86
|
containerId,
|
|
47
87
|
isFullListing: watermark === null,
|
|
88
|
+
isContinuation: false,
|
|
48
89
|
items: [],
|
|
49
90
|
tombstones: [],
|
|
50
91
|
watermark,
|
|
@@ -53,16 +94,21 @@ export async function listAllContainerDocumentLanes(input) {
|
|
|
53
94
|
while (pending.length > 0) {
|
|
54
95
|
const batch = pending.splice(0, MAX_CONTAINER_DOCUMENT_LANES);
|
|
55
96
|
const pages = await input.listContainerDocumentLanes(batch.map(({ containerId, watermark }) => ({ containerId, watermark })));
|
|
56
|
-
|
|
57
|
-
|
|
97
|
+
if (!pages) {
|
|
98
|
+
for (const lane of batch)
|
|
58
99
|
listed.set(lane.containerId, "unavailable");
|
|
59
|
-
|
|
60
|
-
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
batch.forEach((lane, index) => {
|
|
61
103
|
const page = pages[index] ?? null;
|
|
62
104
|
if (!page) {
|
|
63
105
|
listed.set(lane.containerId, null);
|
|
64
106
|
return;
|
|
65
107
|
}
|
|
108
|
+
if (changesDiscoveryOrder(lane.watermark, page.nextWatermark, lane.isContinuation)) {
|
|
109
|
+
listed.set(lane.containerId, "unavailable");
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
66
112
|
lane.items.push(...page.items);
|
|
67
113
|
lane.tombstones.push(...page.tombstones);
|
|
68
114
|
lane.watermark = page.nextWatermark;
|
|
@@ -75,6 +121,7 @@ export async function listAllContainerDocumentLanes(input) {
|
|
|
75
121
|
});
|
|
76
122
|
}
|
|
77
123
|
else if (lane.watermark) {
|
|
124
|
+
lane.isContinuation = true;
|
|
78
125
|
pending.push(lane);
|
|
79
126
|
}
|
|
80
127
|
else {
|
|
@@ -82,6 +129,7 @@ export async function listAllContainerDocumentLanes(input) {
|
|
|
82
129
|
}
|
|
83
130
|
});
|
|
84
131
|
}
|
|
132
|
+
discardConflictingDocumentPages(listed);
|
|
85
133
|
return listed;
|
|
86
134
|
}
|
|
87
135
|
/** Lists the authoritative current ids by deliberately starting at no watermark. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"containerDocumentListing.js","sourceRoot":"","sources":["containerDocumentListing.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,0CAA0C,EAC1C,sCAAsC,EACtC,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AAevC;;;GAGG;AACH,SAAS,SAAS,CAAC,SAAiB;IAClC,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,EACD,IAAI,CAAC,GAAG,CACN,0CAA0C,EAC1C,IAAI,CAAC,KAAK,CAAC,sCAAsC,GAAG,SAAS,CAAC,CAC/D,CACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,SAAmC,EACnC,KAAiD,EACjD,YAAY,GAAG,IAAI;IAEnB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5C,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,MAAM,EAAE,QAAQ,KAAK,EAAE;QACvB,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC,CAAC;IACJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,0BAA0B,CACzD,EAAE,KAAK,EAAE,SAAS,EAAE,EACpB,EAAE,YAAY,EAAE,CACjB,CAAC;IACF,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,eAAe,GAAG,IAAI,GAAG,CAC7B,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAC1D,CAAC;IACF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,MAAM,IAAI,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC;AAUD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAMnD;IACC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwC,CAAC;IAC/D,MAAM,OAAO,GAAmC,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACrD,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YAAE,SAAS;QACtC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC;YACX,WAAW;YACX,aAAa,EAAE,SAAS,KAAK,IAAI;YACjC,KAAK,EAAE,EAAE;YACT,UAAU,EAAE,EAAE;YACd,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,4BAA4B,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,0BAA0B,CAClD,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CACxE,CAAC;QACF,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;gBAC5C,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;YAClC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;gBACnC,OAAO;YACT,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;YACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE;oBAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,aAAa,EAAE,IAAI,CAAC,SAAS;oBAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACrC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,kCAAkC,CAAC,KAIxD;IACC,MAAM,MAAM,GAAG,CACb,MAAM,6BAA6B,CAAC;QAClC,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC5D,0BAA0B,EAAE,CAAC,KAAK,EAAE,EAAE,CACpC,iCAAiC,CAC/B,KAAK,CAAC,SAAS,EACf,KAAK,EACL,KAAK,CAAC,YAAY,CACnB;KACJ,CAAC,CACH,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,MAAM,IAAI,MAAM,KAAK,aAAa;QACvC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,oCAAoC,CAAC,KAM1D;IACC,MAAM,KAAK,GACT,EAAE,CAAC;IACL,KAAK,MAAM,WAAW,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,WAAW;YACX,SAAS,EAAE,MAAM,KAAK,CAAC,8BAA8B,CAAC,WAAW,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IACD,OAAO,6BAA6B,CAAC;QACnC,KAAK;QACL,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;KAC7D,CAAC,CAAC;AACL,CAAC","sourcesContent":["import {\n DEFAULT_CONTAINER_DOCUMENT_LANE_PAGE_LIMIT,\n MAX_CONTAINER_DOCUMENT_LANE_PAGE_TOTAL,\n MAX_CONTAINER_DOCUMENT_LANES,\n} from \"@tearleads/validators/request\";\nimport type { SyncWatermark } from \"@tearleads/validators/response\";\nimport type {\n ContainerDocumentDiscoveryApi,\n ContainerDocumentLaneLister,\n ContainerDocumentLaneListing,\n ContainerDocumentTombstone,\n ListedContainerDocument,\n} from \"./documentDiscoveryTypes\";\n\ntype ContainerDocumentLaneApi = Pick<\n ContainerDocumentDiscoveryApi,\n \"listContainerDocumentLanes\"\n>;\n\n/**\n * Rows each lane may return so a full batch stays within the single-page\n * maximum; a lone lane keeps the ordinary page size.\n */\nfunction laneLimit(laneCount: number): number {\n return Math.max(\n 1,\n Math.min(\n DEFAULT_CONTAINER_DOCUMENT_LANE_PAGE_LIMIT,\n Math.floor(MAX_CONTAINER_DOCUMENT_LANE_PAGE_TOTAL / laneCount),\n ),\n );\n}\n\n/**\n * Lists one page per requested lane in a single batched request. A lane the\n * server rejected yields `null` for that lane only; a failed request yields\n * `null` for the batch.\n */\nexport async function listContainerDocumentLanesFromApi(\n apiClient: ContainerDocumentLaneApi,\n lanes: Parameters<ContainerDocumentLaneLister>[0],\n reportErrors = true,\n): ReturnType<ContainerDocumentLaneLister> {\n const limit = laneLimit(lanes.length);\n const requested = lanes.map((lane, index) => ({\n containerId: lane.containerId,\n laneId: `lane-${index}`,\n limit,\n watermark: lane.watermark,\n }));\n const response = await apiClient.listContainerDocumentLanes(\n { lanes: requested },\n { reportErrors },\n );\n if (!response) return null;\n const resultsByLaneId = new Map(\n response.results.map((result) => [result.laneId, result]),\n );\n return requested.map(({ laneId }) => {\n const result = resultsByLaneId.get(laneId);\n return result && \"page\" in result ? result.page : null;\n });\n}\n\ninterface PendingContainerDocumentLane {\n readonly containerId: string;\n readonly isFullListing: boolean;\n readonly items: ListedContainerDocument[];\n readonly tombstones: ContainerDocumentTombstone[];\n watermark: SyncWatermark | null;\n}\n\n/**\n * Pages every lane to exhaustion, batching first pages and continuations\n * alike. Each container keeps its own watermark, so one lane's failure or\n * continuation never changes another lane's result. A failed request marks\n * only the lanes it carried unavailable; the other batches still list.\n */\nexport async function listAllContainerDocumentLanes(input: {\n readonly lanes: ReadonlyArray<{\n readonly containerId: string;\n readonly watermark: SyncWatermark | null;\n }>;\n readonly listContainerDocumentLanes: ContainerDocumentLaneLister;\n}): Promise<ReadonlyMap<string, ContainerDocumentLaneListing>> {\n const listed = new Map<string, ContainerDocumentLaneListing>();\n const pending: PendingContainerDocumentLane[] = [];\n const queued = new Set<string>();\n for (const { containerId, watermark } of input.lanes) {\n if (queued.has(containerId)) continue;\n queued.add(containerId);\n pending.push({\n containerId,\n isFullListing: watermark === null,\n items: [],\n tombstones: [],\n watermark,\n });\n }\n\n while (pending.length > 0) {\n const batch = pending.splice(0, MAX_CONTAINER_DOCUMENT_LANES);\n const pages = await input.listContainerDocumentLanes(\n batch.map(({ containerId, watermark }) => ({ containerId, watermark })),\n );\n batch.forEach((lane, index) => {\n if (!pages) {\n listed.set(lane.containerId, \"unavailable\");\n return;\n }\n const page = pages[index] ?? null;\n if (!page) {\n listed.set(lane.containerId, null);\n return;\n }\n lane.items.push(...page.items);\n lane.tombstones.push(...page.tombstones);\n lane.watermark = page.nextWatermark;\n if (!page.hasMore) {\n listed.set(lane.containerId, {\n isFullListing: lane.isFullListing,\n items: lane.items,\n nextWatermark: lane.watermark,\n tombstones: lane.tombstones,\n });\n } else if (lane.watermark) {\n pending.push(lane);\n } else {\n listed.set(lane.containerId, null);\n }\n });\n }\n return listed;\n}\n\n/** Lists the authoritative current ids by deliberately starting at no watermark. */\nexport async function listAllContainerDocumentIdsFromApi(input: {\n readonly apiClient: ContainerDocumentLaneApi;\n readonly containerId: string;\n readonly reportErrors?: boolean;\n}): Promise<ReadonlyArray<string> | null> {\n const listed = (\n await listAllContainerDocumentLanes({\n lanes: [{ containerId: input.containerId, watermark: null }],\n listContainerDocumentLanes: (lanes) =>\n listContainerDocumentLanesFromApi(\n input.apiClient,\n lanes,\n input.reportErrors,\n ),\n })\n ).get(input.containerId);\n return listed && listed !== \"unavailable\"\n ? Array.from(new Set(listed.items.map((document) => document.id)))\n : null;\n}\n\n/** Loads each lane's persisted watermark, then lists every lane in batches. */\nexport async function listContainerDocumentsFromWatermarks(input: {\n readonly containerIds: ReadonlyArray<string>;\n readonly listContainerDocumentLanes: ContainerDocumentLaneLister;\n readonly loadContainerDocumentWatermark: (\n containerId: string,\n ) => Promise<SyncWatermark | null>;\n}): Promise<ReadonlyMap<string, ContainerDocumentLaneListing>> {\n const lanes: Array<{ containerId: string; watermark: SyncWatermark | null }> =\n [];\n for (const containerId of new Set(input.containerIds)) {\n lanes.push({\n containerId,\n watermark: await input.loadContainerDocumentWatermark(containerId),\n });\n }\n return listAllContainerDocumentLanes({\n lanes,\n listContainerDocumentLanes: input.listContainerDocumentLanes,\n });\n}\n"]}
|
|
1
|
+
{"version":3,"file":"containerDocumentListing.js","sourceRoot":"","sources":["containerDocumentListing.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,0CAA0C,EAC1C,sCAAsC,EACtC,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AASvC,OAAO,EACL,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,wBAAwB,CAAC;AAOhC;;;GAGG;AACH,SAAS,SAAS,CAAC,SAAiB;IAClC,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,EACD,IAAI,CAAC,GAAG,CACN,0CAA0C,EAC1C,IAAI,CAAC,KAAK,CAAC,sCAAsC,GAAG,SAAS,CAAC,CAC/D,CACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,SAAmC,EACnC,KAAiD,EACjD,YAAY,GAAG,IAAI;IAEnB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5C,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,MAAM,EAAE,QAAQ,KAAK,EAAE;QACvB,KAAK;QACL,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,CAAC,CAAC,CAAC;IACJ,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,0BAA0B,CACzD,EAAE,KAAK,EAAE,SAAS,EAAE,EACpB,EAAE,YAAY,EAAE,CACjB,CAAC;IACF,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,eAAe,GAAG,IAAI,GAAG,CAC7B,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAC1D,CAAC;IACF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,MAAM,IAAI,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC;AAWD,SAAS,yBAAyB,CAChC,MAAyD;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACnC,IACE,CAAC,IAAI;YACL,IAAI,KAAK,aAAa;YACtB,CAAC,sBAAsB,CAAC,IAAI,CAAC,aAAa,CAAC;YAE3C,SAAS;QACX,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;YACrE,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAChC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,4EAA4E;AAC5E,0EAA0E;AAC1E,yEAAyE;AACzE,SAAS,+BAA+B,CACtC,MAAiD;IAEjD,MAAM,QAAQ,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,QAAQ,CAAC,IAAI;QAAE,OAAO;IAC3B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,aAAa;YAAE,SAAS;QAC9C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAChE,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CACvB,EAAE,CAAC;gBACF,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACnB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,UAAU;QAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAMnD;IACC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwC,CAAC;IAC/D,MAAM,OAAO,GAAmC,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KAAK,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QACrD,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YAAE,SAAS;QACtC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC;YACX,WAAW;YACX,aAAa,EAAE,SAAS,KAAK,IAAI;YACjC,cAAc,EAAE,KAAK;YACrB,KAAK,EAAE,EAAE;YACT,UAAU,EAAE,EAAE;YACd,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,4BAA4B,CAAC,CAAC;QAC9D,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,0BAA0B,CAClD,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CACxE,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;YAClC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;gBACnC,OAAO;YACT,CAAC;YACD,IACE,qBAAqB,CACnB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,cAAc,CACpB,EACD,CAAC;gBACD,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;gBAC5C,OAAO;YACT,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;YACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE;oBAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,aAAa,EAAE,IAAI,CAAC,SAAS;oBAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACrC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD,+BAA+B,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,oFAAoF;AACpF,MAAM,CAAC,KAAK,UAAU,kCAAkC,CAAC,KAIxD;IACC,MAAM,MAAM,GAAG,CACb,MAAM,6BAA6B,CAAC;QAClC,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC5D,0BAA0B,EAAE,CAAC,KAAK,EAAE,EAAE,CACpC,iCAAiC,CAC/B,KAAK,CAAC,SAAS,EACf,KAAK,EACL,KAAK,CAAC,YAAY,CACnB;KACJ,CAAC,CACH,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzB,OAAO,MAAM,IAAI,MAAM,KAAK,aAAa;QACvC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,oCAAoC,CAAC,KAM1D;IACC,MAAM,KAAK,GACT,EAAE,CAAC;IACL,KAAK,MAAM,WAAW,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,WAAW;YACX,SAAS,EAAE,MAAM,KAAK,CAAC,8BAA8B,CAAC,WAAW,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IACD,OAAO,6BAA6B,CAAC;QACnC,KAAK;QACL,0BAA0B,EAAE,KAAK,CAAC,0BAA0B;KAC7D,CAAC,CAAC;AACL,CAAC","sourcesContent":["import {\n DEFAULT_CONTAINER_DOCUMENT_LANE_PAGE_LIMIT,\n MAX_CONTAINER_DOCUMENT_LANE_PAGE_TOTAL,\n MAX_CONTAINER_DOCUMENT_LANES,\n} from \"@tearleads/validators/request\";\nimport type { SyncWatermark } from \"@tearleads/validators/response\";\nimport type {\n ContainerDocumentDiscoveryApi,\n ContainerDocumentLaneLister,\n ContainerDocumentLaneListing,\n ContainerDocumentTombstone,\n ListedContainerDocument,\n} from \"./documentDiscoveryTypes\";\nimport {\n changesDiscoveryOrder,\n isPublicationWatermark,\n} from \"./publicationWatermark\";\n\ntype ContainerDocumentLaneApi = Pick<\n ContainerDocumentDiscoveryApi,\n \"listContainerDocumentLanes\"\n>;\n\n/**\n * Rows each lane may return so a full batch stays within the single-page\n * maximum; a lone lane keeps the ordinary page size.\n */\nfunction laneLimit(laneCount: number): number {\n return Math.max(\n 1,\n Math.min(\n DEFAULT_CONTAINER_DOCUMENT_LANE_PAGE_LIMIT,\n Math.floor(MAX_CONTAINER_DOCUMENT_LANE_PAGE_TOTAL / laneCount),\n ),\n );\n}\n\n/**\n * Lists one page per requested lane in a single batched request. A lane the\n * server rejected yields `null` for that lane only; a failed request yields\n * `null` for the batch.\n */\nexport async function listContainerDocumentLanesFromApi(\n apiClient: ContainerDocumentLaneApi,\n lanes: Parameters<ContainerDocumentLaneLister>[0],\n reportErrors = true,\n): ReturnType<ContainerDocumentLaneLister> {\n const limit = laneLimit(lanes.length);\n const requested = lanes.map((lane, index) => ({\n containerId: lane.containerId,\n laneId: `lane-${index}`,\n limit,\n watermark: lane.watermark,\n }));\n const response = await apiClient.listContainerDocumentLanes(\n { lanes: requested },\n { reportErrors },\n );\n if (!response) return null;\n const resultsByLaneId = new Map(\n response.results.map((result) => [result.laneId, result]),\n );\n return requested.map(({ laneId }) => {\n const result = resultsByLaneId.get(laneId);\n return result && \"page\" in result ? result.page : null;\n });\n}\n\ninterface PendingContainerDocumentLane {\n readonly containerId: string;\n readonly isFullListing: boolean;\n readonly items: ListedContainerDocument[];\n readonly tombstones: ContainerDocumentTombstone[];\n watermark: SyncWatermark | null;\n isContinuation: boolean;\n}\n\nfunction removedDocumentPlacements(\n listed: ReadonlyMap<string, ContainerDocumentLaneListing>,\n) {\n const removals = new Map<string, Set<string>>();\n for (const lane of listed.values()) {\n if (\n !lane ||\n lane === \"unavailable\" ||\n !isPublicationWatermark(lane.nextWatermark)\n )\n continue;\n for (const row of lane.tombstones) {\n const containers = removals.get(row.documentId) ?? new Set<string>();\n containers.add(row.containerId);\n removals.set(row.documentId, containers);\n }\n }\n return removals;\n}\n\n// Publication-ordered lanes exclude historical unlinks for live placements.\n// Check the whole fetched batch before applying a lane: a linked document\n// carries its placements in other containers as well as the queried one.\nfunction discardConflictingDocumentPages(\n listed: Map<string, ContainerDocumentLaneListing>,\n) {\n const removals = removedDocumentPlacements(listed);\n if (!removals.size) return;\n const conflicted = new Set<string>();\n for (const [id, lane] of listed) {\n if (!lane || lane === \"unavailable\") continue;\n for (const item of lane.items) {\n const removed = removals.get(item.id);\n if (!removed) continue;\n for (const linked of item.linkedContainerIds.filter((container) =>\n removed.has(container),\n )) {\n conflicted.add(id);\n conflicted.add(linked);\n }\n }\n }\n for (const id of conflicted) listed.set(id, \"unavailable\");\n}\n\n/**\n * Pages every lane to exhaustion, batching first pages and continuations\n * alike. Each container keeps its own watermark, so one lane's failure or\n * continuation never changes another lane's result. A failed request marks\n * only the lanes it carried unavailable; the other batches still list.\n */\nexport async function listAllContainerDocumentLanes(input: {\n readonly lanes: ReadonlyArray<{\n readonly containerId: string;\n readonly watermark: SyncWatermark | null;\n }>;\n readonly listContainerDocumentLanes: ContainerDocumentLaneLister;\n}): Promise<ReadonlyMap<string, ContainerDocumentLaneListing>> {\n const listed = new Map<string, ContainerDocumentLaneListing>();\n const pending: PendingContainerDocumentLane[] = [];\n const queued = new Set<string>();\n for (const { containerId, watermark } of input.lanes) {\n if (queued.has(containerId)) continue;\n queued.add(containerId);\n pending.push({\n containerId,\n isFullListing: watermark === null,\n isContinuation: false,\n items: [],\n tombstones: [],\n watermark,\n });\n }\n\n while (pending.length > 0) {\n const batch = pending.splice(0, MAX_CONTAINER_DOCUMENT_LANES);\n const pages = await input.listContainerDocumentLanes(\n batch.map(({ containerId, watermark }) => ({ containerId, watermark })),\n );\n if (!pages) {\n for (const lane of batch) listed.set(lane.containerId, \"unavailable\");\n continue;\n }\n batch.forEach((lane, index) => {\n const page = pages[index] ?? null;\n if (!page) {\n listed.set(lane.containerId, null);\n return;\n }\n if (\n changesDiscoveryOrder(\n lane.watermark,\n page.nextWatermark,\n lane.isContinuation,\n )\n ) {\n listed.set(lane.containerId, \"unavailable\");\n return;\n }\n lane.items.push(...page.items);\n lane.tombstones.push(...page.tombstones);\n lane.watermark = page.nextWatermark;\n if (!page.hasMore) {\n listed.set(lane.containerId, {\n isFullListing: lane.isFullListing,\n items: lane.items,\n nextWatermark: lane.watermark,\n tombstones: lane.tombstones,\n });\n } else if (lane.watermark) {\n lane.isContinuation = true;\n pending.push(lane);\n } else {\n listed.set(lane.containerId, null);\n }\n });\n }\n discardConflictingDocumentPages(listed);\n return listed;\n}\n\n/** Lists the authoritative current ids by deliberately starting at no watermark. */\nexport async function listAllContainerDocumentIdsFromApi(input: {\n readonly apiClient: ContainerDocumentLaneApi;\n readonly containerId: string;\n readonly reportErrors?: boolean;\n}): Promise<ReadonlyArray<string> | null> {\n const listed = (\n await listAllContainerDocumentLanes({\n lanes: [{ containerId: input.containerId, watermark: null }],\n listContainerDocumentLanes: (lanes) =>\n listContainerDocumentLanesFromApi(\n input.apiClient,\n lanes,\n input.reportErrors,\n ),\n })\n ).get(input.containerId);\n return listed && listed !== \"unavailable\"\n ? Array.from(new Set(listed.items.map((document) => document.id)))\n : null;\n}\n\n/** Loads each lane's persisted watermark, then lists every lane in batches. */\nexport async function listContainerDocumentsFromWatermarks(input: {\n readonly containerIds: ReadonlyArray<string>;\n readonly listContainerDocumentLanes: ContainerDocumentLaneLister;\n readonly loadContainerDocumentWatermark: (\n containerId: string,\n ) => Promise<SyncWatermark | null>;\n}): Promise<ReadonlyMap<string, ContainerDocumentLaneListing>> {\n const lanes: Array<{ containerId: string; watermark: SyncWatermark | null }> =\n [];\n for (const containerId of new Set(input.containerIds)) {\n lanes.push({\n containerId,\n watermark: await input.loadContainerDocumentWatermark(containerId),\n });\n }\n return listAllContainerDocumentLanes({\n lanes,\n listContainerDocumentLanes: input.listContainerDocumentLanes,\n });\n}\n"]}
|
|
@@ -17,6 +17,7 @@ async function saveAppliedContainerDocumentWatermark(input) {
|
|
|
17
17
|
export async function applyListedContainerDocuments(options, generation, listedDocuments) {
|
|
18
18
|
const { containerId, upsertDiscoveredDocuments } = options;
|
|
19
19
|
await options.cacheReferencedPrincipalPolicies?.(dedupeReferencedPrincipalStates(listedDocuments.items.flatMap((document) => document.referencedPrincipals)));
|
|
20
|
+
const tombstones = getApplicableDocumentTombstones(listedDocuments);
|
|
20
21
|
const verification = await options.verifyDiscoveredDocuments(listedDocuments.items.map((document) => ({
|
|
21
22
|
accessEpoch: document.currentAccessEpoch,
|
|
22
23
|
accessStateHash: document.currentAccessStateHash,
|
|
@@ -26,7 +27,7 @@ export async function applyListedContainerDocuments(options, generation, listedD
|
|
|
26
27
|
documentId: document.id,
|
|
27
28
|
effectiveAccessLevel: document.effectiveAccessLevel,
|
|
28
29
|
linkedContainerIds: document.linkedContainerIds,
|
|
29
|
-
})), [containerId], generation,
|
|
30
|
+
})), [containerId], generation, tombstones, new Set(tombstones.map((row) => row.documentId)));
|
|
30
31
|
if (!(await verification.isCurrent()))
|
|
31
32
|
return null;
|
|
32
33
|
const verifiedInputs = verification.inputs;
|
|
@@ -42,7 +43,8 @@ export async function applyListedContainerDocuments(options, generation, listedD
|
|
|
42
43
|
const tombstoneDocumentSummaries = await settleContainerDocumentTombstones({
|
|
43
44
|
containerIds: [containerId],
|
|
44
45
|
store: options,
|
|
45
|
-
|
|
46
|
+
isCurrent: verification.isCurrent,
|
|
47
|
+
tombstones,
|
|
46
48
|
});
|
|
47
49
|
const complete = await verification.commit();
|
|
48
50
|
if (!(await verification.isCurrent()))
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"documentDiscoveryApply.js","sourceRoot":"","sources":["documentDiscoveryApply.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,+BAA+B,EAAE,MAAM,iFAAiF,CAAC;AAClI,OAAO,EAAE,4BAA4B,EAAE,MAAM,wCAAwC,CAAC;AACtF,OAAO,EAAE,iCAAiC,EAAE,MAAM,qCAAqC,CAAC;AACxF,OAAO,EAAE,+BAA+B,EAAE,MAAM,2BAA2B,CAAC;AAc5E,KAAK,UAAU,qCAAqC,CAAC,KAKpD;IACC,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,8BAA8B,EAAE,GACpE,KAAK,CAAC;IACR,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;QACnC,OAAO;IACT,CAAC;IAED,MAAM,8BAA8B,CAClC,WAAW,EACX,eAAe,CAAC,aAAa,EAC7B,KAAK,CAAC,YAAY,CACnB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAAuC,EACvC,UAAkB,EAClB,eAAyC;IAEzC,MAAM,EAAE,WAAW,EAAE,yBAAyB,EAAE,GAAG,OAAO,CAAC;IAC3D,MAAM,OAAO,CAAC,gCAAgC,EAAE,CAC9C,+BAA+B,CAC7B,eAAe,CAAC,KAAK,CAAC,OAAO,CAC3B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAC5C,CACF,CACF,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,yBAAyB,CAC1D,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACvC,WAAW,EAAE,QAAQ,CAAC,kBAAkB;QACxC,eAAe,EAAE,QAAQ,CAAC,sBAAsB;QAChD,WAAW;QACX,kBAAkB,EAAE,CAAC,WAAW,CAAC;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,QAAQ,CAAC,EAAE;QACvB,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;QACnD,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;KAChD,CAAC,CAAC,EACH,CAAC,WAAW,CAAC,EACb,UAAU,EACV
|
|
1
|
+
{"version":3,"file":"documentDiscoveryApply.js","sourceRoot":"","sources":["documentDiscoveryApply.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,+BAA+B,EAAE,MAAM,iFAAiF,CAAC;AAClI,OAAO,EAAE,4BAA4B,EAAE,MAAM,wCAAwC,CAAC;AACtF,OAAO,EAAE,iCAAiC,EAAE,MAAM,qCAAqC,CAAC;AACxF,OAAO,EAAE,+BAA+B,EAAE,MAAM,2BAA2B,CAAC;AAc5E,KAAK,UAAU,qCAAqC,CAAC,KAKpD;IACC,MAAM,EAAE,WAAW,EAAE,eAAe,EAAE,8BAA8B,EAAE,GACpE,KAAK,CAAC;IACR,IAAI,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;QACnC,OAAO;IACT,CAAC;IAED,MAAM,8BAA8B,CAClC,WAAW,EACX,eAAe,CAAC,aAAa,EAC7B,KAAK,CAAC,YAAY,CACnB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAAuC,EACvC,UAAkB,EAClB,eAAyC;IAEzC,MAAM,EAAE,WAAW,EAAE,yBAAyB,EAAE,GAAG,OAAO,CAAC;IAC3D,MAAM,OAAO,CAAC,gCAAgC,EAAE,CAC9C,+BAA+B,CAC7B,eAAe,CAAC,KAAK,CAAC,OAAO,CAC3B,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAC5C,CACF,CACF,CAAC;IAEF,MAAM,UAAU,GAAG,+BAA+B,CAAC,eAAe,CAAC,CAAC;IACpE,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,yBAAyB,CAC1D,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACvC,WAAW,EAAE,QAAQ,CAAC,kBAAkB;QACxC,eAAe,EAAE,QAAQ,CAAC,sBAAsB;QAChD,WAAW;QACX,kBAAkB,EAAE,CAAC,WAAW,CAAC;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,QAAQ,CAAC,EAAE;QACvB,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;QACnD,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;KAChD,CAAC,CAAC,EACH,CAAC,WAAW,CAAC,EACb,UAAU,EACV,UAAU,EACV,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CACjD,CAAC;IACF,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC;IAC3C,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM;QAC/C,CAAC,CAAC,MAAM,yBAAyB,CAAC,cAAc,EAAE,YAAY,CAAC,SAAS,CAAC;QACzE,CAAC,CAAC,EAAE,CAAC;IAEP,IAAI,cAAc,CAAC,MAAM;QACvB,MAAM,OAAO,CAAC,yBAAyB,CACrC,MAAM,4BAA4B,CAChC,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAChC,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,YAAY,EAAE,QAAQ,CAAC,kBAAkB;SAC1C,CAAC,CAAC,EACH,mBAAmB,CACpB,EACD,YAAY,CAAC,SAAS,CACvB,CAAC;IAEJ,MAAM,0BAA0B,GAAG,MAAM,iCAAiC,CAAC;QACzE,YAAY,EAAE,CAAC,WAAW,CAAC;QAC3B,KAAK,EAAE,OAAO;QACd,SAAS,EAAE,YAAY,CAAC,SAAS;QACjC,UAAU;KACX,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,CAAC;IAC7C,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,qCAAqC,CAAC;QAC1C,WAAW;QACX,eAAe;QACf,8BAA8B,EAAE,OAAO,CAAC,8BAA8B;QACtE,YAAY,EAAE,YAAY,CAAC,SAAS;KACrC,CAAC,CAAC;IACH,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnD,IAAI,eAAe,CAAC,aAAa,EAAE,CAAC;QAClC,4EAA4E;QAC5E,OAAO,CAAC,aAAa,EAAE,CACrB,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CACrD,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,QAAQ;QACd,mBAAmB,CAAC,MAAM,KAAK,CAAC;QAChC,0BAA0B,CAAC,MAAM,KAAK,CAAC;QACvC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,0BAA0B,CAAC,CAAC;AAC9D,CAAC","sourcesContent":["import type { DocumentSummary } from \"../../../data/documents/documentSummary\";\nimport { dedupeReferencedPrincipalStates } from \"../../../data/keyingProjectionVerification/principalPolicy/principalPolicyCache\";\nimport { withoutDeferredDocumentLinks } from \"../documentLinks/deferredDocumentLinks\";\nimport { settleContainerDocumentTombstones } from \"../tombstones/documentTombstoneGate\";\nimport { getApplicableDocumentTombstones } from \"./documentDiscoveryInputs\";\nimport type {\n DiscoverContainerDocumentsOptions,\n ListedContainerDocuments,\n} from \"./documentDiscoveryTypes\";\n\n/** A discovery lane's local stores; the listing itself is supplied by the caller. */\nexport type ContainerDocumentDiscoveryLane = Omit<\n DiscoverContainerDocumentsOptions,\n | \"beginDocumentDiscovery\"\n | \"listContainerDocumentLanes\"\n | \"loadContainerDocumentWatermark\"\n>;\n\nasync function saveAppliedContainerDocumentWatermark(input: {\n containerId: string;\n listedDocuments: ListedContainerDocuments;\n saveContainerDocumentWatermark: DiscoverContainerDocumentsOptions[\"saveContainerDocumentWatermark\"];\n stillCurrent: () => Promise<boolean>;\n}) {\n const { containerId, listedDocuments, saveContainerDocumentWatermark } =\n input;\n if (!listedDocuments.nextWatermark) {\n return;\n }\n\n await saveContainerDocumentWatermark(\n containerId,\n listedDocuments.nextWatermark,\n input.stillCurrent,\n );\n}\n\n/**\n * Verifies and applies one container's completed listing under a discovery\n * generation captured before the listing request, then advances its\n * watermark. Returns `null` when nothing was applied.\n */\nexport async function applyListedContainerDocuments(\n options: ContainerDocumentDiscoveryLane,\n generation: number,\n listedDocuments: ListedContainerDocuments,\n): Promise<ReadonlyArray<DocumentSummary> | null> {\n const { containerId, upsertDiscoveredDocuments } = options;\n await options.cacheReferencedPrincipalPolicies?.(\n dedupeReferencedPrincipalStates(\n listedDocuments.items.flatMap(\n (document) => document.referencedPrincipals,\n ),\n ),\n );\n\n const tombstones = getApplicableDocumentTombstones(listedDocuments);\n const verification = await options.verifyDiscoveredDocuments(\n listedDocuments.items.map((document) => ({\n accessEpoch: document.currentAccessEpoch,\n accessStateHash: document.currentAccessStateHash,\n containerId,\n listedContainerIds: [containerId],\n createdAt: document.createdAt,\n documentId: document.id,\n effectiveAccessLevel: document.effectiveAccessLevel,\n linkedContainerIds: document.linkedContainerIds,\n })),\n [containerId],\n generation,\n tombstones,\n new Set(tombstones.map((row) => row.documentId)),\n );\n if (!(await verification.isCurrent())) return null;\n const verifiedInputs = verification.inputs;\n const discoveredDocuments = verifiedInputs.length\n ? await upsertDiscoveredDocuments(verifiedInputs, verification.isCurrent)\n : [];\n\n if (verifiedInputs.length)\n await options.replaceDocumentLinksBatch(\n await withoutDeferredDocumentLinks(\n verifiedInputs.map((document) => ({\n documentId: document.documentId,\n accessEpoch: document.accessEpoch,\n containerIds: document.linkedContainerIds,\n })),\n discoveredDocuments,\n ),\n verification.isCurrent,\n );\n\n const tombstoneDocumentSummaries = await settleContainerDocumentTombstones({\n containerIds: [containerId],\n store: options,\n isCurrent: verification.isCurrent,\n tombstones,\n });\n\n const complete = await verification.commit();\n if (!(await verification.isCurrent())) return null;\n await saveAppliedContainerDocumentWatermark({\n containerId,\n listedDocuments,\n saveContainerDocumentWatermark: options.saveContainerDocumentWatermark,\n stillCurrent: verification.isCurrent,\n });\n if (!(await verification.isCurrent())) return null;\n\n if (listedDocuments.isFullListing) {\n // Initial probes consume ids as discovery hints and verify their own heads.\n options.onFullListing?.(\n listedDocuments.items.map((document) => document.id),\n );\n }\n\n return !complete &&\n discoveredDocuments.length === 0 &&\n tombstoneDocumentSummaries.length === 0\n ? null\n : [...discoveredDocuments, ...tombstoneDocumentSummaries];\n}\n"]}
|
|
@@ -10,15 +10,20 @@ function unlistedHeadVerdict(head, input) {
|
|
|
10
10
|
? null
|
|
11
11
|
: "unavailable";
|
|
12
12
|
}
|
|
13
|
-
async function
|
|
14
|
-
const
|
|
15
|
-
|
|
13
|
+
async function readDiscoveryHead(input, loadHead, store, localEpoch, requiresCurrentHead) {
|
|
14
|
+
const cached = requiresCurrentHead
|
|
15
|
+
? null
|
|
16
|
+
: await store.loadHead(input.documentId, input.accessStateHash);
|
|
16
17
|
const minimumEpoch = Math.max(localEpoch, input.accessEpoch);
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
const usableCached = cached && cached.accessEpoch >= minimumEpoch ? cached : null;
|
|
19
|
+
return (usableCached ??
|
|
20
|
+
(await loadHead(input.documentId, !requiresCurrentHead && input.accessEpoch >= localEpoch
|
|
20
21
|
? (input.accessStateHash ?? undefined)
|
|
21
|
-
: undefined);
|
|
22
|
+
: undefined)));
|
|
23
|
+
}
|
|
24
|
+
async function verifyInput(input, loadHead, loadLocalEpoch, store, generation, organizationId, requiresCurrentHead) {
|
|
25
|
+
const localEpoch = await loadLocalEpoch(input.documentId).catch(() => Number.MAX_SAFE_INTEGER);
|
|
26
|
+
const head = await readDiscoveryHead(input, loadHead, store, localEpoch, requiresCurrentHead);
|
|
22
27
|
if (head === "not-found")
|
|
23
28
|
return null;
|
|
24
29
|
if (!head || head.accessEpoch < Math.max(localEpoch, input.accessEpoch))
|
|
@@ -33,7 +38,7 @@ async function verifyInput(input, loadHead, loadLocalEpoch, store, generation, o
|
|
|
33
38
|
return "unavailable";
|
|
34
39
|
if (!(await store.saveHead(input.documentId, head, generation)))
|
|
35
40
|
return "unavailable";
|
|
36
|
-
const { listedContainerIds: _listedContainerIds, ...document } = input;
|
|
41
|
+
const { listedContainerIds: _listedContainerIds, requiresCurrentHead: _requiresCurrentHead, ...document } = input;
|
|
37
42
|
return {
|
|
38
43
|
...document,
|
|
39
44
|
containerId,
|
|
@@ -49,11 +54,11 @@ async function verifyInput(input, loadHead, loadLocalEpoch, store, generation, o
|
|
|
49
54
|
* a crash cannot lose a candidate behind an already advanced listing watermark.
|
|
50
55
|
*/
|
|
51
56
|
export function createDiscoveredDocumentVerifier(organizationId, loadHead, loadLocalEpoch, store, onPendingDiscovery) {
|
|
52
|
-
return async (inputs, containerIds, generation, tombstones
|
|
57
|
+
return async (inputs, containerIds, generation, tombstones, conflictingDocumentIds) => {
|
|
53
58
|
const isCurrent = () => store.isCurrent(generation);
|
|
54
59
|
const lanes = new Set(containerIds);
|
|
55
|
-
const removed = tombstones.filter((row) => lanes.has(row.containerId));
|
|
56
|
-
if (!(await store.stage(inputs, generation, removed)))
|
|
60
|
+
const removed = (tombstones ?? []).filter((row) => lanes.has(row.containerId));
|
|
61
|
+
if (!(await store.stage(inputs, generation, removed, conflictingDocumentIds)))
|
|
57
62
|
return { inputs: [], isCurrent, commit: async () => true };
|
|
58
63
|
const candidates = groupPendingDocumentDiscoveries(await store.pending(containerIds, HEAD_LINK_SET_LOADS_PER_RUN));
|
|
59
64
|
const verified = new Map();
|
|
@@ -65,7 +70,7 @@ export function createDiscoveredDocumentVerifier(organizationId, loadHead, loadL
|
|
|
65
70
|
if (!group)
|
|
66
71
|
break;
|
|
67
72
|
const { input, rows } = group;
|
|
68
|
-
const result = await verifyInput(input, loadHead, loadLocalEpoch, store, generation, organizationId);
|
|
73
|
+
const result = await verifyInput(input, loadHead, loadLocalEpoch, store, generation, organizationId, rows.some((row) => row.requiresCurrentHead === true));
|
|
69
74
|
if (result === "unavailable") {
|
|
70
75
|
await Promise.all(rows.map((row) => store.defer(row, generation)));
|
|
71
76
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"documentDiscoveryEvidence.js","sourceRoot":"","sources":["documentDiscoveryEvidence.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,GAC5B,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAEhF,OAAO,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAC;AAEtE,wEAAwE;AACxE,yEAAyE;AACzE,SAAS,mBAAmB,CAC1B,IAAiC,EACjC,KAAkC;IAElC,OAAO,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW;QACzC,IAAI,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe;QAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,gBAAgB,CAAC;QACvE,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,aAAa,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAAkC,EAClC,QAAmC,EACnC,cAA8C,EAC9C,KAA0C,EAC1C,UAAkB,EAClB,cAAkC;IAElC,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAC7D,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAC9B,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAC7E,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,IAAI,GACR,MAAM,IAAI,MAAM,CAAC,WAAW,IAAI,YAAY;QAC1C,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,MAAM,QAAQ,CACZ,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,WAAW,IAAI,UAAU;YAC7B,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,SAAS,CAAC;YACtC,CAAC,CAAC,SAAS,CACd,CAAC;IACR,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC;QACrE,OAAO,aAAa,CAAC;IACvB,IAAI,CAAC,+BAA+B,CAAC,IAAI,EAAE,cAAc,CAAC;QACxD,OAAO,aAAa,CAAC;IACvB,8EAA8E;IAC9E,MAAM,WAAW,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACvD,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC,CACrC,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1D,IAAI,CAAC,IAAI,CAAC,eAAe;QAAE,OAAO,aAAa,CAAC;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAC7D,OAAO,aAAa,CAAC;IACvB,MAAM,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,GAAG,QAAQ,EAAE,GAAG,KAAK,CAAC;IACvE,OAAO;QACL,GAAG,QAAQ;QACX,WAAW;QACX,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;KAC5C,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,cAAkC,EAClC,QAAmC,EACnC,cAA8C,EAC9C,KAA0C,EAC1C,kBAA4D;IAE5D,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,GAAG,EAAE,EAAE,EAAE;QACjE,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,+BAA+B,CAChD,MAAM,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,2BAA2B,CAAC,CAC/D,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC5D,MAAM,OAAO,GAAkC,EAAE,CAAC;QAClD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,IAAI,CAAC,KAAK;oBAAE,MAAM;gBAClB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,KAAK,EACL,QAAQ,EACR,cAAc,EACd,KAAK,EACL,UAAU,EACV,cAAc,CACf,CAAC;gBACF,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;oBACnE,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM;oBAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,8BAA8B,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EACvE,MAAM,CACP,CACF,CAAC;QACF,OAAO;YACL,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9B,SAAS;YACT,MAAM,EAAE,KAAK,IAAI,EAAE;gBACjB,MAAM,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;gBACxD,IAAI,UAAU,KAAK,IAAI;oBAAE,kBAAkB,EAAE,CAAC,UAAU,CAAC,CAAC;gBAC1D,OAAO,UAAU,KAAK,IAAI,CAAC;YAC7B,CAAC;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import type { DiscoveredDocumentInput } from \"../../../data/documents/documentSummary\";\nimport type {\n DiscoveredDocumentCandidate,\n DocumentDiscoveryEvidenceRepository,\n} from \"../../../data/persistence/documents/documentDiscoveryEvidencePersistence\";\nimport type {\n DocumentHeadLinkSetLoader,\n LocalDocumentAccessEpochLoader,\n VerifiedDocumentHeadLinkSet,\n} from \"../tombstones/documentTombstoneEvidence\";\nimport {\n HEAD_LINK_SET_LOAD_CONCURRENCY,\n HEAD_LINK_SET_LOADS_PER_RUN,\n} from \"../tombstones/documentTombstoneEvidence\";\nimport { groupPendingDocumentDiscoveries } from \"./documentDiscoveryCandidates\";\nimport type { DiscoverContainerDocumentsOptions } from \"./documentDiscoveryTypes\";\nimport { documentHeadMatchesOrganization } from \"./documentHeadScope\";\n\n// A same-epoch head can predate a link addition. Only the named head or\n// terminal signed purge evidence can settle a candidate without a retry.\nfunction unlistedHeadVerdict(\n head: VerifiedDocumentHeadLinkSet,\n input: DiscoveredDocumentCandidate,\n): null | \"unavailable\" {\n return head.accessEpoch > input.accessEpoch ||\n head.accessStateHash === input.accessStateHash ||\n (!head.accessStateHash && head.accessEpoch === Number.MAX_SAFE_INTEGER)\n ? null\n : \"unavailable\";\n}\n\nasync function verifyInput(\n input: DiscoveredDocumentCandidate,\n loadHead: DocumentHeadLinkSetLoader,\n loadLocalEpoch: LocalDocumentAccessEpochLoader,\n store: DocumentDiscoveryEvidenceRepository,\n generation: number,\n organizationId: string | undefined,\n): Promise<DiscoveredDocumentInput | null | \"unavailable\"> {\n const localEpoch = await loadLocalEpoch(input.documentId).catch(\n () => Number.MAX_SAFE_INTEGER,\n );\n const cached = await store.loadHead(input.documentId, input.accessStateHash);\n const minimumEpoch = Math.max(localEpoch, input.accessEpoch);\n const head =\n cached && cached.accessEpoch >= minimumEpoch\n ? cached\n : await loadHead(\n input.documentId,\n input.accessEpoch >= localEpoch\n ? (input.accessStateHash ?? undefined)\n : undefined,\n );\n if (head === \"not-found\") return null;\n if (!head || head.accessEpoch < Math.max(localEpoch, input.accessEpoch))\n return \"unavailable\";\n if (!documentHeadMatchesOrganization(head, organizationId))\n return \"unavailable\";\n // A stale first lane must not hide a document present in another listed lane.\n const containerId = input.listedContainerIds.find((id) =>\n head.linkedContainerIds.includes(id),\n );\n if (!containerId) return unlistedHeadVerdict(head, input);\n if (!head.accessStateHash) return \"unavailable\";\n if (!(await store.saveHead(input.documentId, head, generation)))\n return \"unavailable\";\n const { listedContainerIds: _listedContainerIds, ...document } = input;\n return {\n ...document,\n containerId,\n accessEpoch: head.accessEpoch,\n accessStateHash: head.accessStateHash,\n linkedContainerIds: head.linkedContainerIds,\n };\n}\n\n/**\n * Persist untrusted candidates before advancing listing watermarks. Each pass\n * checks a bounded batch; unavailable heads back off independently while verified\n * siblings and tombstones still settle. Acknowledgement follows local apply so\n * a crash cannot lose a candidate behind an already advanced listing watermark.\n */\nexport function createDiscoveredDocumentVerifier(\n organizationId: string | undefined,\n loadHead: DocumentHeadLinkSetLoader,\n loadLocalEpoch: LocalDocumentAccessEpochLoader,\n store: DocumentDiscoveryEvidenceRepository,\n onPendingDiscovery?: ((delayMs: number) => void) | undefined,\n): DiscoverContainerDocumentsOptions[\"verifyDiscoveredDocuments\"] {\n return async (inputs, containerIds, generation, tombstones = []) => {\n const isCurrent = () => store.isCurrent(generation);\n const lanes = new Set(containerIds);\n const removed = tombstones.filter((row) => lanes.has(row.containerId));\n if (!(await store.stage(inputs, generation, removed)))\n return { inputs: [], isCurrent, commit: async () => true };\n const candidates = groupPendingDocumentDiscoveries(\n await store.pending(containerIds, HEAD_LINK_SET_LOADS_PER_RUN),\n );\n const verified = new Map<string, DiscoveredDocumentInput>();\n const settled: DiscoveredDocumentCandidate[] = [];\n let next = 0;\n const worker = async () => {\n while (next < candidates.length) {\n const group = candidates[next++];\n if (!group) break;\n const { input, rows } = group;\n const result = await verifyInput(\n input,\n loadHead,\n loadLocalEpoch,\n store,\n generation,\n organizationId,\n );\n if (result === \"unavailable\") {\n await Promise.all(rows.map((row) => store.defer(row, generation)));\n continue;\n }\n if (result) verified.set(result.documentId, result);\n settled.push(...rows);\n }\n };\n await Promise.all(\n Array.from(\n { length: Math.min(HEAD_LINK_SET_LOAD_CONCURRENCY, candidates.length) },\n worker,\n ),\n );\n return {\n inputs: [...verified.values()],\n isCurrent,\n commit: async () => {\n await store.acknowledge(settled, generation);\n const retryDelay = await store.retryDelay(containerIds);\n if (retryDelay !== null) onPendingDiscovery?.(retryDelay);\n return retryDelay === null;\n },\n };\n };\n}\n"]}
|
|
1
|
+
{"version":3,"file":"documentDiscoveryEvidence.js","sourceRoot":"","sources":["documentDiscoveryEvidence.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,8BAA8B,EAC9B,2BAA2B,GAC5B,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAEhF,OAAO,EAAE,+BAA+B,EAAE,MAAM,qBAAqB,CAAC;AAEtE,wEAAwE;AACxE,yEAAyE;AACzE,SAAS,mBAAmB,CAC1B,IAAiC,EACjC,KAAkC;IAElC,OAAO,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW;QACzC,IAAI,CAAC,eAAe,KAAK,KAAK,CAAC,eAAe;QAC9C,CAAC,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,WAAW,KAAK,MAAM,CAAC,gBAAgB,CAAC;QACvE,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,aAAa,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,KAAkC,EAClC,QAAmC,EACnC,KAA0C,EAC1C,UAAkB,EAClB,mBAA4B;IAE5B,MAAM,MAAM,GAAG,mBAAmB;QAChC,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC7D,MAAM,YAAY,GAChB,MAAM,IAAI,MAAM,CAAC,WAAW,IAAI,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,OAAO,CACL,YAAY;QACZ,CAAC,MAAM,QAAQ,CACb,KAAK,CAAC,UAAU,EAChB,CAAC,mBAAmB,IAAI,KAAK,CAAC,WAAW,IAAI,UAAU;YACrD,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,IAAI,SAAS,CAAC;YACtC,CAAC,CAAC,SAAS,CACd,CAAC,CACH,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,KAAkC,EAClC,QAAmC,EACnC,cAA8C,EAC9C,KAA0C,EAC1C,UAAkB,EAClB,cAAkC,EAClC,mBAA4B;IAE5B,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAC7D,GAAG,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAC9B,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAClC,KAAK,EACL,QAAQ,EACR,KAAK,EACL,UAAU,EACV,mBAAmB,CACpB,CAAC;IACF,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC;QACrE,OAAO,aAAa,CAAC;IACvB,IAAI,CAAC,+BAA+B,CAAC,IAAI,EAAE,cAAc,CAAC;QACxD,OAAO,aAAa,CAAC;IACvB,8EAA8E;IAC9E,MAAM,WAAW,GAAG,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACvD,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC,CACrC,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1D,IAAI,CAAC,IAAI,CAAC,eAAe;QAAE,OAAO,aAAa,CAAC;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAC7D,OAAO,aAAa,CAAC;IACvB,MAAM,EACJ,kBAAkB,EAAE,mBAAmB,EACvC,mBAAmB,EAAE,oBAAoB,EACzC,GAAG,QAAQ,EACZ,GAAG,KAAK,CAAC;IACV,OAAO;QACL,GAAG,QAAQ;QACX,WAAW;QACX,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;KAC5C,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAC9C,cAAkC,EAClC,QAAmC,EACnC,cAA8C,EAC9C,KAA0C,EAC1C,kBAA4D;IAE5D,OAAO,KAAK,EACV,MAAM,EACN,YAAY,EACZ,UAAU,EACV,UAAU,EACV,sBAAsB,EACtB,EAAE;QACF,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAChD,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAC3B,CAAC;QACF,IACE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,sBAAsB,CAAC,CAAC;YAEzE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,+BAA+B,CAChD,MAAM,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,2BAA2B,CAAC,CAC/D,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC5D,MAAM,OAAO,GAAkC,EAAE,CAAC;QAClD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,IAAI,CAAC,KAAK;oBAAE,MAAM;gBAClB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,KAAK,EACL,QAAQ,EACR,cAAc,EACd,KAAK,EACL,UAAU,EACV,cAAc,EACd,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,mBAAmB,KAAK,IAAI,CAAC,CACrD,CAAC;gBACF,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;oBACnE,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM;oBAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,8BAA8B,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EACvE,MAAM,CACP,CACF,CAAC;QACF,OAAO;YACL,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9B,SAAS;YACT,MAAM,EAAE,KAAK,IAAI,EAAE;gBACjB,MAAM,KAAK,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;gBACxD,IAAI,UAAU,KAAK,IAAI;oBAAE,kBAAkB,EAAE,CAAC,UAAU,CAAC,CAAC;gBAC1D,OAAO,UAAU,KAAK,IAAI,CAAC;YAC7B,CAAC;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC","sourcesContent":["import type { DiscoveredDocumentInput } from \"../../../data/documents/documentSummary\";\nimport type {\n DiscoveredDocumentCandidate,\n DocumentDiscoveryEvidenceRepository,\n} from \"../../../data/persistence/documents/documentDiscoveryEvidencePersistence\";\nimport type {\n DocumentHeadLinkSetLoader,\n LocalDocumentAccessEpochLoader,\n VerifiedDocumentHeadLinkSet,\n} from \"../tombstones/documentTombstoneEvidence\";\nimport {\n HEAD_LINK_SET_LOAD_CONCURRENCY,\n HEAD_LINK_SET_LOADS_PER_RUN,\n} from \"../tombstones/documentTombstoneEvidence\";\nimport { groupPendingDocumentDiscoveries } from \"./documentDiscoveryCandidates\";\nimport type { DiscoverContainerDocumentsOptions } from \"./documentDiscoveryTypes\";\nimport { documentHeadMatchesOrganization } from \"./documentHeadScope\";\n\n// A same-epoch head can predate a link addition. Only the named head or\n// terminal signed purge evidence can settle a candidate without a retry.\nfunction unlistedHeadVerdict(\n head: VerifiedDocumentHeadLinkSet,\n input: DiscoveredDocumentCandidate,\n): null | \"unavailable\" {\n return head.accessEpoch > input.accessEpoch ||\n head.accessStateHash === input.accessStateHash ||\n (!head.accessStateHash && head.accessEpoch === Number.MAX_SAFE_INTEGER)\n ? null\n : \"unavailable\";\n}\n\nasync function readDiscoveryHead(\n input: DiscoveredDocumentCandidate,\n loadHead: DocumentHeadLinkSetLoader,\n store: DocumentDiscoveryEvidenceRepository,\n localEpoch: number,\n requiresCurrentHead: boolean,\n) {\n const cached = requiresCurrentHead\n ? null\n : await store.loadHead(input.documentId, input.accessStateHash);\n const minimumEpoch = Math.max(localEpoch, input.accessEpoch);\n const usableCached =\n cached && cached.accessEpoch >= minimumEpoch ? cached : null;\n return (\n usableCached ??\n (await loadHead(\n input.documentId,\n !requiresCurrentHead && input.accessEpoch >= localEpoch\n ? (input.accessStateHash ?? undefined)\n : undefined,\n ))\n );\n}\n\nasync function verifyInput(\n input: DiscoveredDocumentCandidate,\n loadHead: DocumentHeadLinkSetLoader,\n loadLocalEpoch: LocalDocumentAccessEpochLoader,\n store: DocumentDiscoveryEvidenceRepository,\n generation: number,\n organizationId: string | undefined,\n requiresCurrentHead: boolean,\n): Promise<DiscoveredDocumentInput | null | \"unavailable\"> {\n const localEpoch = await loadLocalEpoch(input.documentId).catch(\n () => Number.MAX_SAFE_INTEGER,\n );\n const head = await readDiscoveryHead(\n input,\n loadHead,\n store,\n localEpoch,\n requiresCurrentHead,\n );\n if (head === \"not-found\") return null;\n if (!head || head.accessEpoch < Math.max(localEpoch, input.accessEpoch))\n return \"unavailable\";\n if (!documentHeadMatchesOrganization(head, organizationId))\n return \"unavailable\";\n // A stale first lane must not hide a document present in another listed lane.\n const containerId = input.listedContainerIds.find((id) =>\n head.linkedContainerIds.includes(id),\n );\n if (!containerId) return unlistedHeadVerdict(head, input);\n if (!head.accessStateHash) return \"unavailable\";\n if (!(await store.saveHead(input.documentId, head, generation)))\n return \"unavailable\";\n const {\n listedContainerIds: _listedContainerIds,\n requiresCurrentHead: _requiresCurrentHead,\n ...document\n } = input;\n return {\n ...document,\n containerId,\n accessEpoch: head.accessEpoch,\n accessStateHash: head.accessStateHash,\n linkedContainerIds: head.linkedContainerIds,\n };\n}\n\n/**\n * Persist untrusted candidates before advancing listing watermarks. Each pass\n * checks a bounded batch; unavailable heads back off independently while verified\n * siblings and tombstones still settle. Acknowledgement follows local apply so\n * a crash cannot lose a candidate behind an already advanced listing watermark.\n */\nexport function createDiscoveredDocumentVerifier(\n organizationId: string | undefined,\n loadHead: DocumentHeadLinkSetLoader,\n loadLocalEpoch: LocalDocumentAccessEpochLoader,\n store: DocumentDiscoveryEvidenceRepository,\n onPendingDiscovery?: ((delayMs: number) => void) | undefined,\n): DiscoverContainerDocumentsOptions[\"verifyDiscoveredDocuments\"] {\n return async (\n inputs,\n containerIds,\n generation,\n tombstones,\n conflictingDocumentIds,\n ) => {\n const isCurrent = () => store.isCurrent(generation);\n const lanes = new Set(containerIds);\n const removed = (tombstones ?? []).filter((row) =>\n lanes.has(row.containerId),\n );\n if (\n !(await store.stage(inputs, generation, removed, conflictingDocumentIds))\n )\n return { inputs: [], isCurrent, commit: async () => true };\n const candidates = groupPendingDocumentDiscoveries(\n await store.pending(containerIds, HEAD_LINK_SET_LOADS_PER_RUN),\n );\n const verified = new Map<string, DiscoveredDocumentInput>();\n const settled: DiscoveredDocumentCandidate[] = [];\n let next = 0;\n const worker = async () => {\n while (next < candidates.length) {\n const group = candidates[next++];\n if (!group) break;\n const { input, rows } = group;\n const result = await verifyInput(\n input,\n loadHead,\n loadLocalEpoch,\n store,\n generation,\n organizationId,\n rows.some((row) => row.requiresCurrentHead === true),\n );\n if (result === \"unavailable\") {\n await Promise.all(rows.map((row) => store.defer(row, generation)));\n continue;\n }\n if (result) verified.set(result.documentId, result);\n settled.push(...rows);\n }\n };\n await Promise.all(\n Array.from(\n { length: Math.min(HEAD_LINK_SET_LOAD_CONCURRENCY, candidates.length) },\n worker,\n ),\n );\n return {\n inputs: [...verified.values()],\n isCurrent,\n commit: async () => {\n await store.acknowledge(settled, generation);\n const retryDelay = await store.retryDelay(containerIds);\n if (retryDelay !== null) onPendingDiscovery?.(retryDelay);\n return retryDelay === null;\n },\n };\n };\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"documentDiscoveryTypes.js","sourceRoot":"","sources":["documentDiscoveryTypes.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"documentDiscoveryTypes.js","sourceRoot":"","sources":["documentDiscoveryTypes.ts"],"names":[],"mappings":"AAwOA;;;;GAIG;AACH,MAAM,OAAO,wCAAyC,SAAQ,KAAK;IAC5C;IAArB,YAAqB,WAAmB;QACtC,KAAK,CAAC,kCAAkC,WAAW,iBAAiB,CAAC,CAAC;QADnD,gBAAW,GAAX,WAAW,CAAQ;QAEtC,IAAI,CAAC,IAAI,GAAG,0CAA0C,CAAC;IACzD,CAAC;CACF","sourcesContent":["import type {\n ListContainerDocumentLanesRequest,\n ListContainerParentLanesRequest,\n} from \"@tearleads/validators/request\";\nimport type {\n EffectiveAccessLevel,\n ReferencedPrincipalStateResponse,\n SyncWatermark,\n} from \"@tearleads/validators/response\";\nimport type {\n DiscoveredDocumentInput,\n DocumentSummary,\n} from \"../../../data/documents/documentSummary\";\nimport type { DiscoveredDocumentCandidate } from \"../../../data/persistence/documents/documentDiscoveryEvidencePersistence\";\n\nexport interface ListedContainerDocument {\n createdAt: string;\n currentAccessEpoch: number;\n currentAccessStateHash: string;\n effectiveAccessLevel?: EffectiveAccessLevel | undefined;\n id: string;\n linkedContainerIds: string[];\n referencedPrincipals: ReferencedPrincipalStateResponse[];\n updatedAt: string;\n}\n\nexport interface ListContainerDocumentsResponse {\n hasMore: boolean;\n items: ListedContainerDocument[];\n nextWatermark: SyncWatermark | null;\n tombstones: ReadonlyArray<{\n containerId: string;\n documentId: string;\n updatedAt: string;\n }>;\n}\n\nexport interface ListedContainer {\n id: string;\n}\n\nexport interface ListContainersResponse {\n hasMore: boolean;\n items: ListedContainer[];\n nextWatermark: SyncWatermark | null;\n}\n\nexport interface ListContainerParentLanesResponse {\n results: ReadonlyArray<{\n laneId: string;\n page: ListContainersResponse;\n }>;\n}\n\nexport type ContainerDocumentLaneResult =\n | { readonly laneId: string; readonly page: ListContainerDocumentsResponse }\n | {\n readonly failure: { readonly code?: string | undefined };\n readonly laneId: string;\n };\n\nexport interface ContainerDocumentDiscoveryApi {\n readonly listContainerDocumentLanes: (\n input: ListContainerDocumentLanesRequest,\n options?: { readonly reportErrors?: boolean | undefined },\n ) => Promise<{\n readonly results: ReadonlyArray<ContainerDocumentLaneResult>;\n } | null>;\n readonly listContainerParentLanes: (\n input: ListContainerParentLanesRequest,\n ) => Promise<ListContainerParentLanesResponse | null>;\n}\n\n/**\n * One page per lane, aligned with the requested lanes: `null` for a lane the\n * server rejected, or `null` overall when the request itself failed.\n */\nexport type ContainerDocumentLaneLister = (\n lanes: ReadonlyArray<{\n readonly containerId: string;\n readonly watermark: SyncWatermark | null;\n }>,\n) => Promise<ReadonlyArray<ListContainerDocumentsResponse | null> | null>;\n\nexport interface DocumentLinkInput {\n accessEpoch?: number | undefined;\n containerIds: ReadonlyArray<string>;\n documentId: string;\n}\n\nexport type ContainerDocumentTombstone =\n ListContainerDocumentsResponse[\"tombstones\"][number];\n\nexport interface VerifiedContainerDocumentTombstone\n extends ContainerDocumentTombstone {\n /** The verified head's link-set epoch, not below the local document epoch. */\n readonly accessEpoch: number;\n /** The verified head link set; it omits `containerId`. */\n readonly linkedContainerIds: ReadonlyArray<string>;\n}\n\n/**\n * What the document's signed head says about a listing tombstone.\n * - `verified`: the verified head link set omits the container; apply.\n * - `refuted`: the verified head still links the container; keep visible and retry.\n * - `unverified`: no verified head is available; hold, hide, retry later.\n * - `deferred`: not attempted this run (the per-run head-load cap); hold\n * without counting an attempt, so the retry backoff does not grow.\n */\nexport type ContainerDocumentTombstoneVerdict =\n | {\n readonly kind: \"verified\";\n readonly tombstone: VerifiedContainerDocumentTombstone;\n }\n | {\n readonly kind: \"refuted\";\n /** The verified head link set, which contains the container. */\n readonly linkedContainerIds: ReadonlyArray<string>;\n readonly tombstone: ContainerDocumentTombstone;\n }\n | {\n readonly kind: \"unverified\";\n readonly tombstone: ContainerDocumentTombstone;\n }\n | {\n readonly kind: \"deferred\";\n readonly tombstone: ContainerDocumentTombstone;\n };\n\nexport interface HeldContainerDocumentTombstoneInput\n extends ContainerDocumentTombstone {\n /** Held without a verification attempt; the backoff does not advance. */\n readonly deferred?: boolean | undefined;\n /** A verified head still links it; retry the tombstone while keeping it visible. */\n readonly refuted?: boolean | undefined;\n}\n\nexport type ContainerDocumentTombstoneVerifier = (\n tombstones: ReadonlyArray<ContainerDocumentTombstone>,\n) => Promise<ReadonlyArray<ContainerDocumentTombstoneVerdict>>;\n\nexport type ContainerDocumentPlacement = Pick<\n ContainerDocumentTombstone,\n \"containerId\" | \"documentId\"\n>;\n\nexport interface ContainerDocumentTombstoneHoldStore {\n holdContainerDocumentTombstones: (\n tombstones: ReadonlyArray<HeldContainerDocumentTombstoneInput>,\n stillCurrent: () => Promise<boolean>,\n ) => Promise<void>;\n /** Held tombstones on these containers that are due for another attempt. */\n listHeldContainerDocumentTombstones: (\n containerIds: ReadonlyArray<string>,\n ) => Promise<ReadonlyArray<ContainerDocumentTombstone>>;\n /** The subset of placements that exist locally (link row or primary). */\n listKnownContainerDocumentPlacements: (\n placements: ReadonlyArray<ContainerDocumentPlacement>,\n ) => Promise<ReadonlyArray<ContainerDocumentPlacement>>;\n refuteContainerDocumentTombstoneHolds: (\n placements: ReadonlyArray<ContainerDocumentPlacement>,\n stillCurrent: () => Promise<boolean>,\n ) => Promise<void>;\n}\n\nexport interface DiscoverContainerDocumentsOptions\n extends ContainerDocumentTombstoneHoldStore {\n applyContainerDocumentTombstones: (\n tombstones: ReadonlyArray<VerifiedContainerDocumentTombstone>,\n stillCurrent: () => Promise<boolean>,\n ) => Promise<ReadonlyArray<DocumentSummary>>;\n verifyContainerDocumentTombstones: ContainerDocumentTombstoneVerifier;\n /** Allocate durable local order before fetching a listing. */\n beginDocumentDiscovery: () => Promise<number>;\n /** Persist pending candidates and return only signed placements. */\n verifyDiscoveredDocuments: (\n inputs: ReadonlyArray<DiscoveredDocumentCandidate>,\n containerIds: ReadonlyArray<string>,\n generation: number,\n tombstones?: ReadonlyArray<ContainerDocumentTombstone>,\n conflictingDocumentIds?: ReadonlySet<string>,\n ) => Promise<{\n inputs: ReadonlyArray<DiscoveredDocumentInput>;\n /** False once a remote trust reset cancels this listing pass. */\n isCurrent: () => Promise<boolean>;\n /** Acknowledge only after local apply; true means no pending candidates remain. */\n commit: () => Promise<boolean>;\n }>;\n cacheReferencedPrincipalPolicies?: (\n references: ReadonlyArray<ReferencedPrincipalStateResponse>,\n ) => Promise<void>;\n containerId: string;\n /** Completed unwatermarked listing, published only after local apply succeeds. */\n onFullListing?: ((documentIds: ReadonlyArray<string>) => void) | undefined;\n loadContainerDocumentWatermark: (\n containerId: string,\n ) => Promise<SyncWatermark | null>;\n listContainerDocumentLanes: ContainerDocumentLaneLister;\n /** Recheck stillCurrent under the mutation lock before persisting a listing. */\n replaceDocumentLinksBatch: (\n inputs: ReadonlyArray<DocumentLinkInput>,\n stillCurrent: () => Promise<boolean>,\n ) => Promise<void>;\n saveContainerDocumentWatermark: (\n containerId: string,\n watermark: SyncWatermark,\n stillCurrent: () => Promise<boolean>,\n ) => Promise<void>;\n upsertDiscoveredDocuments: (\n inputs: ReadonlyArray<DiscoveredDocumentInput>,\n stillCurrent: () => Promise<boolean>,\n ) => Promise<ReadonlyArray<DocumentSummary>>;\n}\n\nexport interface ListedContainerDocuments {\n isFullListing: boolean;\n items: ListedContainerDocument[];\n nextWatermark: SyncWatermark | null;\n tombstones: ContainerDocumentTombstone[];\n}\n\n/**\n * One lane's listing: complete; `null` when the server refused the lane or a\n * continuation came back without a watermark; or `\"unavailable\"` when the\n * batched request carrying it failed or timed out, which says nothing about\n * the lane and is worth retrying.\n */\nexport type ContainerDocumentLaneListing =\n | ListedContainerDocuments\n | \"unavailable\"\n | null;\n\n/**\n * Discovery could not list the container because its listing request failed\n * or timed out. Unlike a refused lane, a later attempt may succeed, so the\n * reconciler treats it as a failed reconciliation, not a skipped one.\n */\nexport class ContainerDocumentListingUnavailableError extends Error {\n constructor(readonly containerId: string) {\n super(`Document listing for container ${containerId} is unavailable`);\n this.name = \"ContainerDocumentListingUnavailableError\";\n }\n}\n"]}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Select the server's publication-based recovery contract during rollout.
|
|
2
|
+
* Continuation positions inside the token remain opaque to the device. */
|
|
3
|
+
export function isPublicationWatermark(watermark) {
|
|
4
|
+
return watermark?.id.startsWith("d1/") === true;
|
|
5
|
+
}
|
|
6
|
+
/** An initial legacy cursor can replay on the upgraded API. Once a page has
|
|
7
|
+
* been read, both sides of the continuation must use the same ordering. */
|
|
8
|
+
export function changesDiscoveryOrder(previous, next, isContinuation) {
|
|
9
|
+
if (!previous || !next)
|
|
10
|
+
return false;
|
|
11
|
+
return isPublicationWatermark(previous)
|
|
12
|
+
? !isPublicationWatermark(next)
|
|
13
|
+
: isContinuation && isPublicationWatermark(next);
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=publicationWatermark.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"publicationWatermark.js","sourceRoot":"","sources":["publicationWatermark.ts"],"names":[],"mappings":"AAEA;0EAC0E;AAC1E,MAAM,UAAU,sBAAsB,CACpC,SAA2C;IAE3C,OAAO,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC;AAClD,CAAC;AAED;2EAC2E;AAC3E,MAAM,UAAU,qBAAqB,CACnC,QAA0C,EAC1C,IAA0B,EAC1B,cAAuB;IAEvB,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACrC,OAAO,sBAAsB,CAAC,QAAQ,CAAC;QACrC,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC;QAC/B,CAAC,CAAC,cAAc,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC","sourcesContent":["import type { SyncWatermark } from \"@tearleads/validators/response\";\n\n/** Select the server's publication-based recovery contract during rollout.\n * Continuation positions inside the token remain opaque to the device. */\nexport function isPublicationWatermark(\n watermark: SyncWatermark | null | undefined,\n): boolean {\n return watermark?.id.startsWith(\"d1/\") === true;\n}\n\n/** An initial legacy cursor can replay on the upgraded API. Once a page has\n * been read, both sides of the continuation must use the same ordering. */\nexport function changesDiscoveryOrder(\n previous: SyncWatermark | null | undefined,\n next: SyncWatermark | null,\n isContinuation: boolean,\n): boolean {\n if (!previous || !next) return false;\n return isPublicationWatermark(previous)\n ? !isPublicationWatermark(next)\n : isContinuation && isPublicationWatermark(next);\n}\n"]}
|
|
@@ -9,7 +9,7 @@ async function finishRemoteHydration(input) {
|
|
|
9
9
|
if (input.changedCount > 0) {
|
|
10
10
|
input.host.updateSnapshot();
|
|
11
11
|
let insertedCount = 0;
|
|
12
|
-
for (const containerId of input.
|
|
12
|
+
for (const containerId of input.seenContainerObservations.keys()) {
|
|
13
13
|
if (!input.containerIdsBeforeHydration.has(containerId)) {
|
|
14
14
|
insertedCount++;
|
|
15
15
|
}
|
|
@@ -32,7 +32,7 @@ export async function hydrateRemoteContainers(input) {
|
|
|
32
32
|
if (!canHydrateRemoteContainers(state) || input.isCurrent?.() === false) {
|
|
33
33
|
return 0;
|
|
34
34
|
}
|
|
35
|
-
const
|
|
35
|
+
const seenContainerObservations = new Map();
|
|
36
36
|
const containerIdsBeforeHydration = new Set(state.containersById.keys());
|
|
37
37
|
const childIdsByParentId = createContainerChildIndex(state.containersById);
|
|
38
38
|
const { lanes, queueParentLane } = createContainerParentHydrationQueue({
|
|
@@ -63,7 +63,7 @@ export async function hydrateRemoteContainers(input) {
|
|
|
63
63
|
isCurrent: input.isCurrent,
|
|
64
64
|
lanes,
|
|
65
65
|
queueParentLane: queueDiscoveredParentLane,
|
|
66
|
-
|
|
66
|
+
seenContainerObservations,
|
|
67
67
|
state,
|
|
68
68
|
});
|
|
69
69
|
const { changedCount } = result;
|
|
@@ -76,7 +76,7 @@ export async function hydrateRemoteContainers(input) {
|
|
|
76
76
|
host,
|
|
77
77
|
isCurrent: input.isCurrent,
|
|
78
78
|
onFullyHydrated: input.onFullyHydrated,
|
|
79
|
-
|
|
79
|
+
seenContainerObservations,
|
|
80
80
|
state,
|
|
81
81
|
});
|
|
82
82
|
if (input.reconnect?.isCurrent() && canHydrateRemoteContainers(state)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hydrateRemoteContainers.js","sourceRoot":"","sources":["hydrateRemoteContainers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,mCAAmC,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EACL,0BAA0B,EAC1B,2BAA2B,GAC5B,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAAE,gCAAgC,EAAE,MAAM,sBAAsB,CAAC;AAQxE,KAAK,UAAU,qBAAqB,CAAC,KASpC;IACC,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK,EAAE,CAAC;QAClC,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"hydrateRemoteContainers.js","sourceRoot":"","sources":["hydrateRemoteContainers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,OAAO,EAAE,mCAAmC,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EACL,0BAA0B,EAC1B,2BAA2B,GAC5B,MAAM,iDAAiD,CAAC;AACzD,OAAO,EAAE,gCAAgC,EAAE,MAAM,sBAAsB,CAAC;AAQxE,KAAK,UAAU,qBAAqB,CAAC,KASpC;IACC,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK,EAAE,CAAC;QAClC,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,yBAAyB,CAAC,IAAI,EAAE,EAAE,CAAC;YACjE,IAAI,CAAC,KAAK,CAAC,2BAA2B,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxD,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAC1B,gCAAgC,CAAC;YAC/B,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,aAAa;SACd,CAAC,CACH,CAAC;IACJ,CAAC;IACD,IACE,KAAK,CAAC,QAAQ;QACd,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK;QAC7B,KAAK,CAAC,eAAe,EACrB,CAAC;QACD,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,kGAAkG;AAClG,IAAI,iBAAiB,GAAG,CAAC,CAAC;AAE1B,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAU7C;IACC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC9B,IAAI,CAAC,0BAA0B,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK,EAAE,CAAC;QACxE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5D,MAAM,2BAA2B,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,MAAM,kBAAkB,GAAG,yBAAyB,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAC3E,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,mCAAmC,CAAC;QACrE,YAAY,EAAE,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE;QACzC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,sBAAsB,EAAE,KAAK,CAAC,sBAAsB;QACpD,sBAAsB,EAAE,KAAK,CAAC,sBAAsB;KACrD,CAAC,CAAC;IACH,gFAAgF;IAChF,gFAAgF;IAChF,gFAAgF;IAChF,MAAM,yBAAyB,GAA6B,CAAC,WAAW,EAAE,EAAE;QAC1E,IACE,KAAK,CAAC,2BAA2B;YACjC,CAAC,WAAW,KAAK,IAAI,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,EACvE,CAAC;YACD,eAAe,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC;IACF,IAAI,WAA8B,CAAC;IACnC,MAAM,MAAM,GAAG,MAAM,2BAA2B,CAAC;QAC/C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI;YAC5B,YAAY,EAAE,IAAI,EAAE,iBAAiB,EAAE;YACvC,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,WAAW,KAAK,KAAK,CAAC;YACxB,CAAC;SACF;QACD,kBAAkB;QAClB,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK;QACL,eAAe,EAAE,yBAAyB;QAC1C,yBAAyB;QACzB,KAAK;KACN,CAAC,CAAC;IACH,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IAChC,MAAM,qBAAqB,CAAC;QAC1B,YAAY;QACZ,QAAQ,EACN,MAAM,CAAC,OAAO,KAAK,UAAU;YAC7B,0BAA0B,CAAC,KAAK,CAAC;YACjC,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK;QAC/B,2BAA2B;QAC3B,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,yBAAyB;QACzB,KAAK;KACN,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,0BAA0B,CAAC,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,eAAe,GAAG,KAAK,CAAC,sBAAsB;YAClD,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,SAAS,CAAC;QACd,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;QACxD,sFAAsF;QACtF,IAAI,eAAe;YAAE,MAAM,eAAe,CAAC;IAC7C,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC","sourcesContent":["import { createContainerChildIndex } from \"./lanes/childIndex\";\nimport { createContainerParentHydrationQueue } from \"./lanes/parentLaneQueue\";\nimport {\n canHydrateRemoteContainers,\n hydrateContainerParentLanes,\n} from \"./remoteContainers/remoteContainerPageHydration\";\nimport { describeRemoteContainerHydration } from \"./remoteHydrationLog\";\nimport type {\n QueueContainerParentLane,\n ReconnectCrawl,\n RemoteContainerHydrationHost,\n RemoteContainerHydrationState,\n} from \"./types\";\n\nasync function finishRemoteHydration(input: {\n changedCount: number;\n complete: boolean;\n host: RemoteContainerHydrationHost;\n isCurrent?: (() => boolean) | undefined;\n onFullyHydrated?: (() => Promise<void> | void) | undefined;\n seenContainerObservations: ReadonlyMap<string, string>;\n containerIdsBeforeHydration: ReadonlySet<string>;\n state: RemoteContainerHydrationState;\n}): Promise<void> {\n if (input.isCurrent?.() === false) {\n return;\n }\n if (input.changedCount > 0) {\n input.host.updateSnapshot();\n let insertedCount = 0;\n for (const containerId of input.seenContainerObservations.keys()) {\n if (!input.containerIdsBeforeHydration.has(containerId)) {\n insertedCount++;\n }\n }\n input.state.runtime.util.log(\n describeRemoteContainerHydration({\n changedCount: input.changedCount,\n insertedCount,\n }),\n );\n }\n if (\n input.complete &&\n input.isCurrent?.() !== false &&\n input.onFullyHydrated\n ) {\n await input.onFullyHydrated();\n }\n}\n\n// Distinct correlation IDs prevent an acknowledged crawl joining an older in-flight HTTP listing.\nlet reconnectSequence = 0;\n\nexport async function hydrateRemoteContainers(input: {\n followDiscoveredParentLanes?: boolean | undefined;\n host: RemoteContainerHydrationHost;\n isCurrent?: (() => boolean) | undefined;\n reconnect?: ReconnectCrawl | undefined;\n onFullyHydrated?: (() => Promise<void> | void) | undefined;\n parentIds?: ReadonlyArray<string | null> | undefined;\n resetAllLaneWatermarks?: boolean | undefined;\n resetRootLaneWatermark?: boolean | undefined;\n state: RemoteContainerHydrationState;\n}): Promise<number> {\n const { host, state } = input;\n if (!canHydrateRemoteContainers(state) || input.isCurrent?.() === false) {\n return 0;\n }\n\n const seenContainerObservations = new Map<string, string>();\n const containerIdsBeforeHydration = new Set(state.containersById.keys());\n const childIdsByParentId = createContainerChildIndex(state.containersById);\n const { lanes, queueParentLane } = createContainerParentHydrationQueue({\n containerIds: state.containersById.keys(),\n parentIds: input.parentIds,\n resetAllLaneWatermarks: input.resetAllLaneWatermarks,\n resetRootLaneWatermark: input.resetRootLaneWatermark,\n });\n // Follow child lanes during an explicit recursive crawl or for containers first\n // discovered in this pass. The latter lets a newly authorized root populate its\n // descendants immediately, while routine ticks keep known subtrees cache-first.\n const queueDiscoveredParentLane: QueueContainerParentLane = (containerId) => {\n if (\n input.followDiscoveredParentLanes ||\n (containerId !== null && !containerIdsBeforeHydration.has(containerId))\n ) {\n queueParentLane(containerId);\n }\n };\n let manualError: Error | undefined;\n const result = await hydrateContainerParentLanes({\n reconnect: input.reconnect && {\n laneIdPrefix: `r${++reconnectSequence}`,\n recordManualError: (error) => {\n manualError ??= error;\n },\n },\n childIdsByParentId,\n host,\n isCurrent: input.isCurrent,\n lanes,\n queueParentLane: queueDiscoveredParentLane,\n seenContainerObservations,\n state,\n });\n const { changedCount } = result;\n await finishRemoteHydration({\n changedCount,\n complete:\n result.outcome === \"complete\" &&\n canHydrateRemoteContainers(state) &&\n input.isCurrent?.() !== false,\n containerIdsBeforeHydration,\n host,\n isCurrent: input.isCurrent,\n onFullyHydrated: input.onFullyHydrated,\n seenContainerObservations,\n state,\n });\n if (input.reconnect?.isCurrent() && canHydrateRemoteContainers(state)) {\n const manualRejection = input.resetAllLaneWatermarks\n ? manualError\n : undefined;\n input.reconnect.record(result.outcome, manualRejection);\n // A manual refresh sharing this crawl observes the error it would receive on its own.\n if (manualRejection) throw manualRejection;\n }\n return changedCount;\n}\n"]}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { ContainerSyncWatermarkRepository, } from "../../../../data/persistence/container-contents/containerSyncWatermarkPersistence.js";
|
|
2
|
+
const lane = (containerId) => ({
|
|
3
|
+
kind: "container_restoration",
|
|
4
|
+
containerId,
|
|
5
|
+
});
|
|
6
|
+
/** The same access loss can appear on later folder publications. Read all
|
|
7
|
+
* receipts for this page together; ordinary pages need no SQLite lookup. */
|
|
8
|
+
export async function pendingFolderRestorations(input) {
|
|
9
|
+
const candidates = input.items.filter((item) => !!item.restorationId &&
|
|
10
|
+
!input.isPaused?.(item) &&
|
|
11
|
+
!!input.childIdsByParentId.get(item.id)?.size);
|
|
12
|
+
if (!candidates.length)
|
|
13
|
+
return [];
|
|
14
|
+
const receipts = await new ContainerSyncWatermarkRepository(input.execSql).loadWatermarkRecords(candidates.map((item) => lane(item.id)));
|
|
15
|
+
return candidates.filter((item, index) => receipts[index]?.watermark.id !== item.restorationId);
|
|
16
|
+
}
|
|
17
|
+
export async function recordFolderRestoration(execSql, item, settled, stillCurrent) {
|
|
18
|
+
if (!item || !settled)
|
|
19
|
+
return settled;
|
|
20
|
+
return new ContainerSyncWatermarkRepository(execSql).saveWatermark(lane(item.id), {
|
|
21
|
+
id: item.restorationId,
|
|
22
|
+
updatedAt: item.updatedAt,
|
|
23
|
+
}, { stillCurrent });
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=restorationReceipt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"restorationReceipt.js","sourceRoot":"","sources":["restorationReceipt.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gCAAgC,GACjC,MAAM,mFAAmF,CAAC;AAW3F,MAAM,IAAI,GAAG,CAAC,WAAmB,EAA8B,EAAE,CAAC,CAAC;IACjE,IAAI,EAAE,uBAAuB;IAC7B,WAAW;CACZ,CAAC,CAAC;AAEH;4EAC4E;AAC5E,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,KAK/C;IACC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CACnC,CAAC,IAAI,EAA6B,EAAE,CAClC,CAAC,CAAC,IAAI,CAAC,aAAa;QACpB,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC;QACvB,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAChD,CAAC;IACF,IAAI,CAAC,UAAU,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,IAAI,gCAAgC,CACzD,KAAK,CAAC,OAAO,CACd,CAAC,oBAAoB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,UAAU,CAAC,MAAM,CACtB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC,aAAa,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,OAAgB,EAChB,IAAmC,EACnC,OAAgB,EAChB,YAA2B;IAE3B,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;QAAE,OAAO,OAAO,CAAC;IACtC,OAAO,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,aAAa,CAChE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EACb;QACE,EAAE,EAAE,IAAI,CAAC,aAAa;QACtB,SAAS,EAAE,IAAI,CAAC,SAAS;KAC1B,EACD,EAAE,YAAY,EAAE,CACjB,CAAC;AACJ,CAAC","sourcesContent":["import {\n type ContainerSyncWatermarkLane,\n ContainerSyncWatermarkRepository,\n} from \"../../../../data/persistence/container-contents/containerSyncWatermarkPersistence\";\nimport type { ExecSql } from \"../../../../data/sqlite/sqlSchema\";\nimport type {\n ContainerChildIndex,\n ListedRemoteContainerPageItem,\n} from \"../types\";\n\ntype RestoredContainer = ListedRemoteContainerPageItem & {\n restorationId: string;\n};\n\nconst lane = (containerId: string): ContainerSyncWatermarkLane => ({\n kind: \"container_restoration\",\n containerId,\n});\n\n/** The same access loss can appear on later folder publications. Read all\n * receipts for this page together; ordinary pages need no SQLite lookup. */\nexport async function pendingFolderRestorations(input: {\n childIdsByParentId: ContainerChildIndex;\n execSql: ExecSql;\n isPaused?: ((item: ListedRemoteContainerPageItem) => boolean) | undefined;\n items: ReadonlyArray<ListedRemoteContainerPageItem>;\n}): Promise<RestoredContainer[]> {\n const candidates = input.items.filter(\n (item): item is RestoredContainer =>\n !!item.restorationId &&\n !input.isPaused?.(item) &&\n !!input.childIdsByParentId.get(item.id)?.size,\n );\n if (!candidates.length) return [];\n const receipts = await new ContainerSyncWatermarkRepository(\n input.execSql,\n ).loadWatermarkRecords(candidates.map((item) => lane(item.id)));\n return candidates.filter(\n (item, index) => receipts[index]?.watermark.id !== item.restorationId,\n );\n}\n\nexport async function recordFolderRestoration(\n execSql: ExecSql,\n item: RestoredContainer | undefined,\n settled: boolean,\n stillCurrent: () => boolean,\n) {\n if (!item || !settled) return settled;\n return new ContainerSyncWatermarkRepository(execSql).saveWatermark(\n lane(item.id),\n {\n id: item.restorationId,\n updatedAt: item.updatedAt,\n },\n { stillCurrent },\n );\n}\n"]}
|