@tearleads/client-sdk 0.1.252 → 0.1.254
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Tearleads.js +3 -3
- package/Tearleads.js.map +1 -1
- package/client/containerContents/documentDiscovery.js +0 -2
- package/client/containerContents/documentDiscovery.js.map +1 -1
- package/client/containerContents/index.d.ts +2 -0
- package/client/containerContents/index.js +4 -9
- package/client/containerContents/index.js.map +1 -1
- package/client/containerContents/storeRuntime.d.ts +17 -0
- package/client/containerContents/storeRuntime.js +31 -0
- package/client/containerContents/storeRuntime.js.map +1 -0
- package/client/deviceFirst/index.d.ts +2 -3
- package/client/deviceFirst/index.js +44 -22
- package/client/deviceFirst/index.js.map +1 -1
- package/client/runtime/workflowRuntime.d.ts +2 -0
- package/client/runtime/workflowRuntime.js +13 -6
- package/client/runtime/workflowRuntime.js.map +1 -1
- package/data/persistence/container-contents/containerContentsPersistenceTypes.d.ts +7 -10
- package/data/persistence/container-contents/containerContentsPersistenceTypes.js.map +1 -1
- package/data/persistence/container-contents/containerDeletionPersistence.js +5 -1
- package/data/persistence/container-contents/containerDeletionPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerHydrationPersistence.js +1 -1
- package/data/persistence/container-contents/containerHydrationPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerMetadataMutationPersistence.js +11 -4
- package/data/persistence/container-contents/containerMetadataMutationPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerRemovalMetadata.js +47 -0
- package/data/persistence/container-contents/containerRemovalMetadata.js.map +1 -0
- package/data/persistence/container-contents/containerRemovalPlacements.js +22 -0
- package/data/persistence/container-contents/containerRemovalPlacements.js.map +1 -0
- package/data/persistence/container-contents/containerSyncWatermarkPersistence.d.ts +2 -1
- package/data/persistence/container-contents/containerSyncWatermarkPersistence.js +10 -2
- package/data/persistence/container-contents/containerSyncWatermarkPersistence.js.map +1 -1
- package/data/persistence/documents/documentDiscoveryCandidate.d.ts +1 -0
- package/data/persistence/documents/documentDiscoveryCandidate.js +3 -1
- package/data/persistence/documents/documentDiscoveryCandidate.js.map +1 -1
- package/data/persistence/documents/documentDiscoveryEvidencePersistence.d.ts +1 -1
- package/data/persistence/documents/documentDiscoveryEvidencePersistence.js +16 -4
- package/data/persistence/documents/documentDiscoveryEvidencePersistence.js.map +1 -1
- package/data/sqlite/migrations/localSchemaMigrations.js +11 -0
- package/data/sqlite/migrations/localSchemaMigrations.js.map +1 -1
- package/data/sqlite/schema/containerSyncWatermarkSchema.js +6 -3
- package/data/sqlite/schema/containerSyncWatermarkSchema.js.map +1 -1
- package/data/sqlite/schema/index.d.ts +17 -0
- package/data/sync/syncCoordinator.js +17 -10
- package/data/sync/syncCoordinator.js.map +1 -1
- package/internal/api-client/ApiClient.d.ts +1 -0
- package/internal/api-client/client/apiClientContext.d.ts +1 -0
- package/internal/api-client/client/containerReads.d.ts +1 -0
- package/internal/api-client/requestInternals.js +14 -6
- package/internal/api-client/requestInternals.js.map +1 -1
- package/internal/validators/operation/containerReads.d.ts +29 -2
- package/internal/validators/operation/containerReads.js +2 -1
- package/internal/validators/operation/containerReads.js.map +1 -1
- package/internal/validators/request/containerDocumentLanes.d.ts +7 -0
- package/internal/validators/request/containerDocumentLanes.js +6 -0
- package/internal/validators/request/containerDocumentLanes.js.map +1 -1
- package/internal/validators/response/container.d.ts +8 -0
- package/internal/validators/response/container.js +1 -0
- package/internal/validators/response/container.js.map +1 -1
- package/internal/validators/response/containerParentLanes.d.ts +2 -0
- package/package.json +1 -1
- package/stores/container-contents/containerContentsStore.d.ts +3 -9
- package/stores/container-contents/containerContentsStore.js +24 -21
- package/stores/container-contents/containerContentsStore.js.map +1 -1
- package/stores/container-contents/containerState/storeScope.js +6 -0
- package/stores/container-contents/containerState/storeScope.js.map +1 -0
- package/stores/container-contents/hydration/backgroundHydration.js +2 -1
- package/stores/container-contents/hydration/backgroundHydration.js.map +1 -1
- package/stores/container-contents/hydration/reconnectRefresh.d.ts +10 -0
- package/stores/container-contents/hydration/reconnectRefresh.js +68 -0
- package/stores/container-contents/hydration/reconnectRefresh.js.map +1 -0
- package/stores/container-contents/hydration/reconnectRefreshMachine.js +49 -0
- package/stores/container-contents/hydration/reconnectRefreshMachine.js.map +1 -0
- package/stores/container-contents/hydration/remoteHydrationRefresh.js +4 -1
- package/stores/container-contents/hydration/remoteHydrationRefresh.js.map +1 -1
- package/stores/container-contents/hydration/remoteHydrationRequest.js +79 -10
- package/stores/container-contents/hydration/remoteHydrationRequest.js.map +1 -1
- package/stores/container-contents/hydration/startupHydration.js +2 -9
- package/stores/container-contents/hydration/startupHydration.js.map +1 -1
- package/stores/container-contents/hydration/treeResetMachine.js +61 -0
- package/stores/container-contents/hydration/treeResetMachine.js.map +1 -0
- package/stores/container-contents/localRefresh.js +28 -12
- package/stores/container-contents/localRefresh.js.map +1 -1
- package/stores/container-contents/remoteEvents/remoteContainerIngestion.js +4 -1
- package/stores/container-contents/remoteEvents/remoteContainerIngestion.js.map +1 -1
- package/stores/container-contents/remoteWrites/writeGeneration.js +5 -2
- package/stores/container-contents/remoteWrites/writeGeneration.js.map +1 -1
- package/stores/container-contents/state.d.ts +3 -7
- package/stores/container-contents/state.js +27 -16
- package/stores/container-contents/state.js.map +1 -1
- package/stores/container-contents/syncAgent/index.d.ts +7 -2
- package/stores/container-contents/syncAgent/index.js +77 -49
- package/stores/container-contents/syncAgent/index.js.map +1 -1
- package/stores/container-contents/syncAgent/syncAgentTypes.d.ts +5 -2
- package/stores/container-contents/syncAgent/syncAgentTypes.js.map +1 -1
- package/stores/container-contents/syncAgent/syncLaneIteration.js +6 -3
- package/stores/container-contents/syncAgent/syncLaneIteration.js.map +1 -1
- package/stores/local-projection/localProjectionStore.js +24 -15
- package/stores/local-projection/localProjectionStore.js.map +1 -1
- package/stores/local-projection/registry.js +2 -2
- package/stores/local-projection/registry.js.map +1 -1
- package/workflows/container-contents/container-state/create/createChild.js +3 -15
- package/workflows/container-contents/container-state/create/createChild.js.map +1 -1
- package/workflows/container-contents/containerPersistence.d.ts +4 -8
- package/workflows/container-contents/containerPersistence.js +22 -12
- package/workflows/container-contents/containerPersistence.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/containerDocumentListing.js +52 -4
- package/workflows/container-contents/documentDiscovery/containerDocumentListing.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/documentDiscoveryApply.js +4 -2
- package/workflows/container-contents/documentDiscovery/documentDiscoveryApply.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/documentDiscoveryEvidence.js +17 -12
- package/workflows/container-contents/documentDiscovery/documentDiscoveryEvidence.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/documentDiscoveryTypes.js.map +1 -1
- package/workflows/container-contents/documentDiscovery/publicationWatermark.js +15 -0
- package/workflows/container-contents/documentDiscovery/publicationWatermark.js.map +1 -0
- package/workflows/container-contents/localState.js +3 -0
- package/workflows/container-contents/localState.js.map +1 -1
- package/workflows/container-contents/remoteHydration/hydrateRemoteContainers.d.ts +12 -0
- package/workflows/container-contents/remoteHydration/hydrateRemoteContainers.js +93 -0
- package/workflows/container-contents/remoteHydration/hydrateRemoteContainers.js.map +1 -0
- package/workflows/container-contents/remoteHydration/index.d.ts +3 -11
- package/workflows/container-contents/remoteHydration/index.js +17 -66
- package/workflows/container-contents/remoteHydration/index.js.map +1 -1
- package/workflows/container-contents/remoteHydration/lanes/parentLaneFetch.js +3 -2
- package/workflows/container-contents/remoteHydration/lanes/parentLaneFetch.js.map +1 -1
- package/workflows/container-contents/remoteHydration/lanes/restorationReceipt.js +25 -0
- package/workflows/container-contents/remoteHydration/lanes/restorationReceipt.js.map +1 -0
- package/workflows/container-contents/remoteHydration/lanes/tombstoneAccess.js +107 -0
- package/workflows/container-contents/remoteHydration/lanes/tombstoneAccess.js.map +1 -0
- package/workflows/container-contents/remoteHydration/principalPolicyCache.js +57 -5
- package/workflows/container-contents/remoteHydration/principalPolicyCache.js.map +1 -1
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerPageHydration.js +129 -84
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerPageHydration.js.map +1 -1
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerState.js +32 -10
- package/workflows/container-contents/remoteHydration/remoteContainers/remoteContainerState.js.map +1 -1
- package/workflows/container-contents/remoteHydration/tombstoneApplication.js +67 -24
- package/workflows/container-contents/remoteHydration/tombstoneApplication.js.map +1 -1
- package/workflows/container-contents/remoteHydration/types.d.ts +16 -1
- package/workflows/container-contents/remoteHydration/types.js.map +1 -1
- package/workflows/container-contents/remoteReset/remoteResetCursorScope.js +3 -3
- package/workflows/container-contents/remoteReset/remoteResetCursorScope.js.map +1 -1
- package/workflows/container-contents/runtime.d.ts +3 -0
- package/workflows/container-contents/runtime.js.map +1 -1
- package/workflows/container-contents/storedContainerState.js +17 -7
- package/workflows/container-contents/storedContainerState.js.map +1 -1
- package/workflows/container-contents/syncLane.d.ts +1 -0
- package/workflows/container-contents/syncLane.js +9 -1
- package/workflows/container-contents/syncLane.js.map +1 -1
- package/workflows/container-contents/syncPause/discoveryGuard.js +6 -2
- package/workflows/container-contents/syncPause/discoveryGuard.js.map +1 -1
- package/workflows/container-contents/tombstones/documentTombstoneGate.js +6 -3
- package/workflows/container-contents/tombstones/documentTombstoneGate.js.map +1 -1
- package/workflows/shared/runtimeInput.d.ts +2 -0
- package/workflows/shared/runtimeInput.js.map +1 -1
- package/workflows/container-contents/remoteHydration/completeHydration.js +0 -25
- package/workflows/container-contents/remoteHydration/completeHydration.js.map +0 -1
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { MAX_CONTAINER_DOCUMENT_LANES, } from "../../../../internal/validators/request/index.js";
|
|
2
|
+
import { isContainerNotFoundFailure } from "../../../../data/containers/shared/mutationFailures.js";
|
|
3
|
+
import { captureRemovalPlacementHeads } from "../../../../data/persistence/container-contents/containerRemovalPlacements.js";
|
|
4
|
+
import { isPublicationWatermark } from "../../documentDiscovery/publicationWatermark.js";
|
|
5
|
+
import { collectRemovedContainers } from "../tombstoneReasons.js";
|
|
6
|
+
function classifyLane(result) {
|
|
7
|
+
if (!result)
|
|
8
|
+
return "unknown";
|
|
9
|
+
if ("page" in result)
|
|
10
|
+
return "readable";
|
|
11
|
+
return result.failure.status === 403 ||
|
|
12
|
+
isContainerNotFoundFailure(result.failure)
|
|
13
|
+
? "denied"
|
|
14
|
+
: "unknown";
|
|
15
|
+
}
|
|
16
|
+
/** A successful lane read proves the device should retain its existing row.
|
|
17
|
+
* A lane-local access denial or coded absence confirms a discovery removal.
|
|
18
|
+
* Any other outcome leaves the page retryable. */
|
|
19
|
+
async function readTombstoneCascadeAccess(api, containerIds, afterParentPage, isCurrent) {
|
|
20
|
+
const denied = new Set();
|
|
21
|
+
for (let index = 0; index < containerIds.length; index += MAX_CONTAINER_DOCUMENT_LANES) {
|
|
22
|
+
if (isCurrent?.() === false)
|
|
23
|
+
return null;
|
|
24
|
+
const ids = containerIds.slice(index, index + MAX_CONTAINER_DOCUMENT_LANES);
|
|
25
|
+
const response = await api.listContainerDocumentLanes({
|
|
26
|
+
afterParentPage,
|
|
27
|
+
lanes: ids.map((containerId, i) => ({
|
|
28
|
+
containerId,
|
|
29
|
+
laneId: String(i),
|
|
30
|
+
watermark: null,
|
|
31
|
+
limit: 1,
|
|
32
|
+
})),
|
|
33
|
+
}, { reportErrors: false });
|
|
34
|
+
if (!response || response.results.length !== ids.length)
|
|
35
|
+
return null;
|
|
36
|
+
for (const [i, containerId] of ids.entries()) {
|
|
37
|
+
const result = response.results.find((row) => row.laneId === String(i));
|
|
38
|
+
const outcome = classifyLane(result);
|
|
39
|
+
if (outcome === "unknown")
|
|
40
|
+
return null;
|
|
41
|
+
if (outcome === "denied")
|
|
42
|
+
denied.add(containerId);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return { denied };
|
|
46
|
+
}
|
|
47
|
+
export async function resolveTombstoneCascade(input) {
|
|
48
|
+
const cascade = (preservedContainerIds) => collectRemovedContainers({
|
|
49
|
+
childIdsByParentId: input.childIdsByParentId,
|
|
50
|
+
containersById: input.state.containersById,
|
|
51
|
+
preservedContainerIds,
|
|
52
|
+
tombstones: [input.tombstone],
|
|
53
|
+
});
|
|
54
|
+
const candidates = cascade(input.preservedContainerIds);
|
|
55
|
+
const clockConflict = candidates.removedContainerIds.some((id) => {
|
|
56
|
+
const held = input.expectedContainerStates.get(id)?.container.serverUpdatedAt;
|
|
57
|
+
return held != null && held > input.tombstone.updatedAt;
|
|
58
|
+
});
|
|
59
|
+
// A directly shared or moved child may be behind the saved parent cursor
|
|
60
|
+
// regardless of its entity clock. Check every cached child before an
|
|
61
|
+
// ancestor removal from a server that can fence that permission read.
|
|
62
|
+
const hasCachedDescendants = isPublicationWatermark(input.afterParentPage?.watermark) &&
|
|
63
|
+
candidates.removedContainerIds.some((id) => id !== input.tombstone.containerId);
|
|
64
|
+
if (!input.reconcileDescendants && !clockConflict && !hasCachedDescendants)
|
|
65
|
+
return { cascade: candidates, denied: null };
|
|
66
|
+
if (!input.afterParentPage)
|
|
67
|
+
return null;
|
|
68
|
+
const ids = [
|
|
69
|
+
...new Set([
|
|
70
|
+
input.tombstone.containerId,
|
|
71
|
+
...candidates.removedContainerIds,
|
|
72
|
+
]),
|
|
73
|
+
];
|
|
74
|
+
const placements = await captureRemovalPlacementHeads(input.state.runtime.infra.execSql, ids.flatMap((id) => {
|
|
75
|
+
const held = input.expectedContainerStates.get(id);
|
|
76
|
+
return held ? [held.container] : [];
|
|
77
|
+
}));
|
|
78
|
+
const access = await readTombstoneCascadeAccess(input.state.runtime.apiClient, ids, input.afterParentPage, input.isCurrent);
|
|
79
|
+
// A move before the server upgrade may have left a superseded removal in
|
|
80
|
+
// its old lane. Preserve a currently readable root and advance past it.
|
|
81
|
+
if (!access) {
|
|
82
|
+
if (input.isCurrent?.() !== false) {
|
|
83
|
+
// The request boundary reports a lagging replica as an unavailable
|
|
84
|
+
// response. Resume this exact lane on the next scheduled refresh.
|
|
85
|
+
input.state.containerListingFailures =
|
|
86
|
+
(input.state.containerListingFailures ?? 0) + 1;
|
|
87
|
+
input.state.containerParentIdsNeedingHydration?.add(input.afterParentPage.parentId);
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
// A restored row only asks to reconcile its descendants. A newer loss of
|
|
92
|
+
// the root itself has its own publication.
|
|
93
|
+
if (input.reconcileDescendants)
|
|
94
|
+
access.denied.delete(input.tombstone.containerId);
|
|
95
|
+
return {
|
|
96
|
+
// A cached descendant may have moved away from a still-readable ancestor.
|
|
97
|
+
// The batch checked every candidate independently.
|
|
98
|
+
cascade: {
|
|
99
|
+
...candidates,
|
|
100
|
+
removedContainerIds: candidates.removedContainerIds.filter((id) => access.denied.has(id)),
|
|
101
|
+
fenceOnlyContainerIds: candidates.fenceOnlyContainerIds.filter((id) => access.denied.has(id)),
|
|
102
|
+
absentDeletedContainerIds: candidates.absentDeletedContainerIds.filter((id) => access.denied.has(id)),
|
|
103
|
+
},
|
|
104
|
+
denied: new Map([...access.denied].map((id) => [id, placements.get(id) ?? null])),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=tombstoneAccess.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tombstoneAccess.js","sourceRoot":"","sources":["tombstoneAccess.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,4BAA4B,GAC7B,MAAM,+BAA+B,CAAC;AAKvC,OAAO,EAAE,0BAA0B,EAAE,MAAM,qDAAqD,CAAC;AACjG,OAAO,EAAE,4BAA4B,EAAE,MAAM,4EAA4E,CAAC;AAC1H,OAAO,EAAE,sBAAsB,EAAE,MAAM,8CAA8C,CAAC;AACtF,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAO/D,SAAS,YAAY,CACnB,MAAyE;IAEzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,IAAI,MAAM,IAAI,MAAM;QAAE,OAAO,UAAU,CAAC;IACxC,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,GAAG;QAClC,0BAA0B,CAAC,MAAM,CAAC,OAAO,CAAC;QAC1C,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;kDAEkD;AAClD,KAAK,UAAU,0BAA0B,CACvC,GAA0D,EAC1D,YAA+B,EAC/B,eAEC,EACD,SAAuC;IAEvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,KACE,IAAI,KAAK,GAAG,CAAC,EACb,KAAK,GAAG,YAAY,CAAC,MAAM,EAC3B,KAAK,IAAI,4BAA4B,EACrC,CAAC;QACD,IAAI,SAAS,EAAE,EAAE,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QACzC,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,4BAA4B,CAAC,CAAC;QAC5E,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,0BAA0B,CACnD;YACE,eAAe;YACf,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClC,WAAW;gBACX,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;gBACjB,SAAS,EAAE,IAAI;gBACf,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;SACJ,EACD,EAAE,YAAY,EAAE,KAAK,EAAE,CACxB,CAAC;QACF,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACrE,KAAK,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;YACrC,IAAI,OAAO,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YACvC,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAS7C;IAIC,MAAM,OAAO,GAAG,CAAC,qBAA0C,EAAE,EAAE,CAC7D,wBAAwB,CAAC;QACvB,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,cAAc;QAC1C,qBAAqB;QACrB,UAAU,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;KAC9B,CAAC,CAAC;IACL,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACxD,MAAM,aAAa,GAAG,UAAU,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;QAC/D,MAAM,IAAI,GACR,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,eAAe,CAAC;QACnE,OAAO,IAAI,IAAI,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC;IAC1D,CAAC,CAAC,CAAC;IACH,yEAAyE;IACzE,qEAAqE;IACrE,sEAAsE;IACtE,MAAM,oBAAoB,GACxB,sBAAsB,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,CAAC;QACxD,UAAU,CAAC,mBAAmB,CAAC,IAAI,CACjC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,KAAK,CAAC,SAAS,CAAC,WAAW,CAC3C,CAAC;IACJ,IAAI,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,aAAa,IAAI,CAAC,oBAAoB;QACxE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAE/C,IAAI,CAAC,KAAK,CAAC,eAAe;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,GAAG,GAAG;QACV,GAAG,IAAI,GAAG,CAAC;YACT,KAAK,CAAC,SAAS,CAAC,WAAW;YAC3B,GAAG,UAAU,CAAC,mBAAmB;SAClC,CAAC;KACH,CAAC;IACF,MAAM,UAAU,GAAG,MAAM,4BAA4B,CACnD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EACjC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;QACjB,MAAM,IAAI,GAAG,KAAK,CAAC,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtC,CAAC,CAAC,CACH,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAC7C,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAC7B,GAAG,EACH,KAAK,CAAC,eAAe,EACrB,KAAK,CAAC,SAAS,CAChB,CAAC;IACF,yEAAyE;IACzE,wEAAwE;IACxE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK,EAAE,CAAC;YAClC,mEAAmE;YACnE,kEAAkE;YAClE,KAAK,CAAC,KAAK,CAAC,wBAAwB;gBAClC,CAAC,KAAK,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAClD,KAAK,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CACjD,KAAK,CAAC,eAAe,CAAC,QAAQ,CAC/B,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,yEAAyE;IACzE,2CAA2C;IAC3C,IAAI,KAAK,CAAC,oBAAoB;QAC5B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACpD,OAAO;QACL,0EAA0E;QAC1E,mDAAmD;QACnD,OAAO,EAAE;YACP,GAAG,UAAU;YACb,mBAAmB,EAAE,UAAU,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAChE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CACtB;YACD,qBAAqB,EAAE,UAAU,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CACtB;YACD,yBAAyB,EAAE,UAAU,CAAC,yBAAyB,CAAC,MAAM,CACpE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAC9B;SACF;QACD,MAAM,EAAE,IAAI,GAAG,CACb,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CACjE;KACF,CAAC;AACJ,CAAC","sourcesContent":["import {\n type ListContainerDocumentLanesRequest,\n MAX_CONTAINER_DOCUMENT_LANES,\n} from \"@tearleads/validators/request\";\nimport type {\n ContainerSyncTombstone,\n ListContainerDocumentLanesResponse,\n} from \"@tearleads/validators/response\";\nimport { isContainerNotFoundFailure } from \"../../../../data/containers/shared/mutationFailures\";\nimport { captureRemovalPlacementHeads } from \"../../../../data/persistence/container-contents/containerRemovalPlacements\";\nimport { isPublicationWatermark } from \"../../documentDiscovery/publicationWatermark\";\nimport { collectRemovedContainers } from \"../tombstoneReasons\";\nimport type {\n ContainerChildIndex,\n ExpectedContainerState,\n RemoteContainerHydrationState,\n} from \"../types\";\n\nfunction classifyLane(\n result: ListContainerDocumentLanesResponse[\"results\"][number] | undefined,\n) {\n if (!result) return \"unknown\";\n if (\"page\" in result) return \"readable\";\n return result.failure.status === 403 ||\n isContainerNotFoundFailure(result.failure)\n ? \"denied\"\n : \"unknown\";\n}\n\n/** A successful lane read proves the device should retain its existing row.\n * A lane-local access denial or coded absence confirms a discovery removal.\n * Any other outcome leaves the page retryable. */\nasync function readTombstoneCascadeAccess(\n api: RemoteContainerHydrationState[\"runtime\"][\"apiClient\"],\n containerIds: readonly string[],\n afterParentPage: NonNullable<\n ListContainerDocumentLanesRequest[\"afterParentPage\"]\n >,\n isCurrent?: (() => boolean) | undefined,\n) {\n const denied = new Set<string>();\n for (\n let index = 0;\n index < containerIds.length;\n index += MAX_CONTAINER_DOCUMENT_LANES\n ) {\n if (isCurrent?.() === false) return null;\n const ids = containerIds.slice(index, index + MAX_CONTAINER_DOCUMENT_LANES);\n const response = await api.listContainerDocumentLanes(\n {\n afterParentPage,\n lanes: ids.map((containerId, i) => ({\n containerId,\n laneId: String(i),\n watermark: null,\n limit: 1,\n })),\n },\n { reportErrors: false },\n );\n if (!response || response.results.length !== ids.length) return null;\n for (const [i, containerId] of ids.entries()) {\n const result = response.results.find((row) => row.laneId === String(i));\n const outcome = classifyLane(result);\n if (outcome === \"unknown\") return null;\n if (outcome === \"denied\") denied.add(containerId);\n }\n }\n return { denied };\n}\n\nexport async function resolveTombstoneCascade(input: {\n reconcileDescendants: boolean;\n afterParentPage: ListContainerDocumentLanesRequest[\"afterParentPage\"];\n childIdsByParentId: ContainerChildIndex;\n expectedContainerStates: ReadonlyMap<string, ExpectedContainerState>;\n isCurrent?: (() => boolean) | undefined;\n preservedContainerIds: ReadonlySet<string>;\n state: RemoteContainerHydrationState;\n tombstone: ContainerSyncTombstone;\n}): Promise<{\n cascade: ReturnType<typeof collectRemovedContainers>;\n denied: ReadonlyMap<string, string | null> | null;\n} | null> {\n const cascade = (preservedContainerIds: ReadonlySet<string>) =>\n collectRemovedContainers({\n childIdsByParentId: input.childIdsByParentId,\n containersById: input.state.containersById,\n preservedContainerIds,\n tombstones: [input.tombstone],\n });\n const candidates = cascade(input.preservedContainerIds);\n const clockConflict = candidates.removedContainerIds.some((id) => {\n const held =\n input.expectedContainerStates.get(id)?.container.serverUpdatedAt;\n return held != null && held > input.tombstone.updatedAt;\n });\n // A directly shared or moved child may be behind the saved parent cursor\n // regardless of its entity clock. Check every cached child before an\n // ancestor removal from a server that can fence that permission read.\n const hasCachedDescendants =\n isPublicationWatermark(input.afterParentPage?.watermark) &&\n candidates.removedContainerIds.some(\n (id) => id !== input.tombstone.containerId,\n );\n if (!input.reconcileDescendants && !clockConflict && !hasCachedDescendants)\n return { cascade: candidates, denied: null };\n\n if (!input.afterParentPage) return null;\n const ids = [\n ...new Set([\n input.tombstone.containerId,\n ...candidates.removedContainerIds,\n ]),\n ];\n const placements = await captureRemovalPlacementHeads(\n input.state.runtime.infra.execSql,\n ids.flatMap((id) => {\n const held = input.expectedContainerStates.get(id);\n return held ? [held.container] : [];\n }),\n );\n const access = await readTombstoneCascadeAccess(\n input.state.runtime.apiClient,\n ids,\n input.afterParentPage,\n input.isCurrent,\n );\n // A move before the server upgrade may have left a superseded removal in\n // its old lane. Preserve a currently readable root and advance past it.\n if (!access) {\n if (input.isCurrent?.() !== false) {\n // The request boundary reports a lagging replica as an unavailable\n // response. Resume this exact lane on the next scheduled refresh.\n input.state.containerListingFailures =\n (input.state.containerListingFailures ?? 0) + 1;\n input.state.containerParentIdsNeedingHydration?.add(\n input.afterParentPage.parentId,\n );\n }\n return null;\n }\n // A restored row only asks to reconcile its descendants. A newer loss of\n // the root itself has its own publication.\n if (input.reconcileDescendants)\n access.denied.delete(input.tombstone.containerId);\n return {\n // A cached descendant may have moved away from a still-readable ancestor.\n // The batch checked every candidate independently.\n cascade: {\n ...candidates,\n removedContainerIds: candidates.removedContainerIds.filter((id) =>\n access.denied.has(id),\n ),\n fenceOnlyContainerIds: candidates.fenceOnlyContainerIds.filter((id) =>\n access.denied.has(id),\n ),\n absentDeletedContainerIds: candidates.absentDeletedContainerIds.filter(\n (id) => access.denied.has(id),\n ),\n },\n denied: new Map(\n [...access.denied].map((id) => [id, placements.get(id) ?? null]),\n ),\n };\n}\n"]}
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
import { referencedPrincipalPolicyKey } from "../../../data/keyingProjectionVerification/principalPolicy/principalPolicyCache.js";
|
|
2
|
+
import { isKeyingVerificationError } from "../../../data/shared/securityIncidents.js";
|
|
3
|
+
async function readPolicyRefusal(warm) {
|
|
4
|
+
try {
|
|
5
|
+
await warm();
|
|
6
|
+
return undefined;
|
|
7
|
+
}
|
|
8
|
+
catch (error) {
|
|
9
|
+
if (isKeyingVerificationError(error))
|
|
10
|
+
return error;
|
|
11
|
+
throw error;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
2
14
|
export async function cacheRemoteContainerPrincipalPolicies(input) {
|
|
3
15
|
const cacheReferencedPrincipalPolicies = input.cacheReferencedPrincipalPolicies;
|
|
4
16
|
if (!cacheReferencedPrincipalPolicies) {
|
|
@@ -17,10 +29,50 @@ export async function cacheRemoteContainerPrincipalPolicies(input) {
|
|
|
17
29
|
}
|
|
18
30
|
referencesByOrganizationId.set(organizationId, organizationReferences);
|
|
19
31
|
}
|
|
20
|
-
await Promise.all([...referencesByOrganizationId].map(([organizationId, referencesByKey]) =>
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
32
|
+
await Promise.all([...referencesByOrganizationId].map(async ([organizationId, referencesByKey]) => {
|
|
33
|
+
const references = [...referencesByKey.values()];
|
|
34
|
+
const warm = (group) => cacheReferencedPrincipalPolicies({
|
|
35
|
+
organizationId,
|
|
36
|
+
references: group,
|
|
37
|
+
stillCurrent: input.stillCurrent,
|
|
38
|
+
});
|
|
39
|
+
if (!input.onRefused)
|
|
40
|
+
return warm(references);
|
|
41
|
+
const batchError = await readPolicyRefusal(() => warm(references));
|
|
42
|
+
if (!batchError)
|
|
43
|
+
return;
|
|
44
|
+
// A failed batch identifies no individual reference. Isolate it
|
|
45
|
+
// without excluding independent containers in the same organization.
|
|
46
|
+
for (const reference of references) {
|
|
47
|
+
if (input.stillCurrent?.() === false)
|
|
48
|
+
return;
|
|
49
|
+
const error = references.length === 1
|
|
50
|
+
? batchError
|
|
51
|
+
: await readPolicyRefusal(() => warm([reference]));
|
|
52
|
+
if (error)
|
|
53
|
+
input.onRefused(organizationId, reference, error);
|
|
54
|
+
}
|
|
55
|
+
}));
|
|
56
|
+
}
|
|
57
|
+
export async function warmListedContainerPolicies(items, warmer, recordManualError, stillCurrent) {
|
|
58
|
+
const refused = new Set();
|
|
59
|
+
const key = (organizationId, reference) => JSON.stringify([organizationId, referencedPrincipalPolicyKey(reference)]);
|
|
60
|
+
await cacheRemoteContainerPrincipalPolicies({
|
|
61
|
+
cacheReferencedPrincipalPolicies: warmer,
|
|
62
|
+
remoteContainers: items,
|
|
63
|
+
stillCurrent,
|
|
64
|
+
...(recordManualError && {
|
|
65
|
+
onRefused: (org, ref, error) => {
|
|
66
|
+
recordManualError(error);
|
|
67
|
+
refused.add(key(org, ref));
|
|
68
|
+
},
|
|
69
|
+
}),
|
|
70
|
+
});
|
|
71
|
+
if (!refused.size)
|
|
72
|
+
return { items, outcome: "complete" };
|
|
73
|
+
return {
|
|
74
|
+
outcome: "refused",
|
|
75
|
+
items: items.filter((item) => !item.metadataReferencedPrincipals.some((ref) => refused.has(key(item.organizationId, ref)))),
|
|
76
|
+
};
|
|
25
77
|
}
|
|
26
78
|
//# sourceMappingURL=principalPolicyCache.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"principalPolicyCache.js","sourceRoot":"","sources":["principalPolicyCache.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,4BAA4B,EAAE,MAAM,iFAAiF,CAAC;
|
|
1
|
+
{"version":3,"file":"principalPolicyCache.js","sourceRoot":"","sources":["principalPolicyCache.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,4BAA4B,EAAE,MAAM,iFAAiF,CAAC;AAC/H,OAAO,EAAE,yBAAyB,EAAE,MAAM,wCAAwC,CAAC;AAUnF,KAAK,UAAU,iBAAiB,CAC9B,IAAyB;IAEzB,IAAI,CAAC;QACH,MAAM,IAAI,EAAE,CAAC;QACb,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,yBAAyB,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACnD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qCAAqC,CAAC,KAW3D;IACC,MAAM,gCAAgC,GACpC,KAAK,CAAC,gCAAgC,CAAC;IACzC,IAAI,CAAC,gCAAgC,EAAE,CAAC;QACtC,OAAO;IACT,CAAC;IAED,MAAM,0BAA0B,GAAG,IAAI,GAAG,EAGvC,CAAC;IAEJ,KAAK,MAAM,eAAe,IAAI,KAAK,CAAC,gBAAgB,EAAE,CAAC;QACrD,MAAM,UAAU,GAAG,eAAe,CAAC,4BAA4B,CAAC;QAChE,MAAM,EAAE,cAAc,EAAE,GAAG,eAAe,CAAC;QAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC/C,SAAS;QACX,CAAC;QAED,MAAM,sBAAsB,GAC1B,0BAA0B,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;QAC9D,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,sBAAsB,CAAC,GAAG,CACxB,4BAA4B,CAAC,SAAS,CAAC,EACvC,SAAS,CACV,CAAC;QACJ,CAAC;QACD,0BAA0B,CAAC,GAAG,CAAC,cAAc,EAAE,sBAAsB,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,0BAA0B,CAAC,CAAC,GAAG,CACjC,KAAK,EAAE,CAAC,cAAc,EAAE,eAAe,CAAC,EAAE,EAAE;QAC1C,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,CAAC,KAAoC,EAAE,EAAE,CACpD,gCAAgC,CAAC;YAC/B,cAAc;YACd,UAAU,EAAE,KAAK;YACjB,YAAY,EAAE,KAAK,CAAC,YAAY;SACjC,CAAC,CAAC;QACL,IAAI,CAAC,KAAK,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,UAAU;YAAE,OAAO;QACxB,gEAAgE;QAChE,qEAAqE;QACrE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,YAAY,EAAE,EAAE,KAAK,KAAK;gBAAE,OAAO;YAC7C,MAAM,KAAK,GACT,UAAU,CAAC,MAAM,KAAK,CAAC;gBACrB,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,MAAM,iBAAiB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACvD,IAAI,KAAK;gBAAE,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CACF,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAG/C,KAAmB,EACnB,MAAuC,EACvC,iBAA0C,EAC1C,YAA4B;IAE5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,GAAG,GAAG,CAAC,cAAsB,EAAE,SAA6B,EAAE,EAAE,CACpE,IAAI,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE,4BAA4B,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,qCAAqC,CAAC;QAC1C,gCAAgC,EAAE,MAAM;QACxC,gBAAgB,EAAE,KAAK;QACvB,YAAY;QACZ,GAAG,CAAC,iBAAiB,IAAI;YACvB,SAAS,EAAE,CAAC,GAAW,EAAE,GAAuB,EAAE,KAAY,EAAE,EAAE;gBAChE,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7B,CAAC;SACF,CAAC;KACH,CAAC,CAAC;IACH,IAAI,CAAC,OAAO,CAAC,IAAI;QAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IACzD,OAAO;QACL,OAAO,EAAE,SAAS;QAClB,KAAK,EAAE,KAAK,CAAC,MAAM,CACjB,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAC3C,CACJ;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { ReferencedPrincipalPolicyWarmer } from \"../../../data/keyingProjectionVerification\";\nimport { referencedPrincipalPolicyKey } from \"../../../data/keyingProjectionVerification/principalPolicy/principalPolicyCache\";\nimport { isKeyingVerificationError } from \"../../../data/shared/securityIncidents\";\nimport type { RemoteContainer } from \"./types\";\n\ntype RemoteContainerPrincipalReferences = Pick<\n RemoteContainer,\n \"metadataReferencedPrincipals\" | \"organizationId\"\n>;\ntype PrincipalReference =\n RemoteContainer[\"metadataReferencedPrincipals\"][number];\n\nasync function readPolicyRefusal(\n warm: () => Promise<void>,\n): Promise<Error | undefined> {\n try {\n await warm();\n return undefined;\n } catch (error) {\n if (isKeyingVerificationError(error)) return error;\n throw error;\n }\n}\n\nexport async function cacheRemoteContainerPrincipalPolicies(input: {\n readonly cacheReferencedPrincipalPolicies?:\n | ReferencedPrincipalPolicyWarmer\n | undefined;\n readonly onRefused?: (\n organizationId: string,\n reference: PrincipalReference,\n error: Error,\n ) => void;\n readonly remoteContainers: readonly RemoteContainerPrincipalReferences[];\n readonly stillCurrent?: (() => boolean) | undefined;\n}): Promise<void> {\n const cacheReferencedPrincipalPolicies =\n input.cacheReferencedPrincipalPolicies;\n if (!cacheReferencedPrincipalPolicies) {\n return;\n }\n\n const referencesByOrganizationId = new Map<\n string,\n Map<string, PrincipalReference>\n >();\n\n for (const remoteContainer of input.remoteContainers) {\n const references = remoteContainer.metadataReferencedPrincipals;\n const { organizationId } = remoteContainer;\n if (references.length === 0 || !organizationId) {\n continue;\n }\n\n const organizationReferences =\n referencesByOrganizationId.get(organizationId) ?? new Map();\n for (const reference of references) {\n organizationReferences.set(\n referencedPrincipalPolicyKey(reference),\n reference,\n );\n }\n referencesByOrganizationId.set(organizationId, organizationReferences);\n }\n\n await Promise.all(\n [...referencesByOrganizationId].map(\n async ([organizationId, referencesByKey]) => {\n const references = [...referencesByKey.values()];\n const warm = (group: readonly PrincipalReference[]) =>\n cacheReferencedPrincipalPolicies({\n organizationId,\n references: group,\n stillCurrent: input.stillCurrent,\n });\n if (!input.onRefused) return warm(references);\n const batchError = await readPolicyRefusal(() => warm(references));\n if (!batchError) return;\n // A failed batch identifies no individual reference. Isolate it\n // without excluding independent containers in the same organization.\n for (const reference of references) {\n if (input.stillCurrent?.() === false) return;\n const error =\n references.length === 1\n ? batchError\n : await readPolicyRefusal(() => warm([reference]));\n if (error) input.onRefused(organizationId, reference, error);\n }\n },\n ),\n );\n}\n\nexport async function warmListedContainerPolicies<\n T extends RemoteContainerPrincipalReferences,\n>(\n items: readonly T[],\n warmer: ReferencedPrincipalPolicyWarmer,\n recordManualError?: (error: Error) => void,\n stillCurrent?: () => boolean,\n): Promise<{ items: readonly T[]; outcome: \"complete\" | \"refused\" }> {\n const refused = new Set<string>();\n const key = (organizationId: string, reference: PrincipalReference) =>\n JSON.stringify([organizationId, referencedPrincipalPolicyKey(reference)]);\n await cacheRemoteContainerPrincipalPolicies({\n cacheReferencedPrincipalPolicies: warmer,\n remoteContainers: items,\n stillCurrent,\n ...(recordManualError && {\n onRefused: (org: string, ref: PrincipalReference, error: Error) => {\n recordManualError(error);\n refused.add(key(org, ref));\n },\n }),\n });\n if (!refused.size) return { items, outcome: \"complete\" };\n return {\n outcome: \"refused\",\n items: items.filter(\n (item) =>\n !item.metadataReferencedPrincipals.some((ref) =>\n refused.has(key(item.organizationId, ref)),\n ),\n ),\n };\n}\n"]}
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { createPrincipalPolicyOperation, } from "../../../principals/principalPolicyService.js";
|
|
2
|
+
import { changesDiscoveryOrder } from "../../documentDiscovery/publicationWatermark.js";
|
|
2
3
|
import { prefetchDestinationProjections } from "../destination/destinationPrefetch.js";
|
|
3
4
|
import { cachedDestinationRole } from "../destination/destinationRoleCache.js";
|
|
4
5
|
import { markContainerParentLaneFetched } from "../lanes/laneFetchMarkers.js";
|
|
5
6
|
import { fetchContainerParentLaneBatch } from "../lanes/parentLaneFetch.js";
|
|
6
|
-
import {
|
|
7
|
+
import { warmListedContainerPolicies } from "../principalPolicyCache.js";
|
|
7
8
|
import { applyContainerTombstones, getApplicableRemoteContainerItems, } from "../tombstoneApplication.js";
|
|
8
9
|
import { containerStateMatchesFingerprint } from "./containerStateFingerprint.js";
|
|
9
10
|
import { loadPausedContainerFilter } from "./organizationSyncPauseFilter.js";
|
|
10
11
|
import { listRemoteContainerIdsWithPendingMetadataUpdates, listRemoteContainerIdsWithPendingStructuralIntents, upsertIsolatedRemoteContainerState, } from "./remoteContainerState.js";
|
|
11
12
|
const CONTAINER_PARENT_HYDRATION_CONCURRENCY = 4;
|
|
13
|
+
/** A retryable lane takes precedence over a refused proof elsewhere in the crawl. */
|
|
14
|
+
function combineOutcomes(a, b) {
|
|
15
|
+
if (a === "retry" || b === "retry")
|
|
16
|
+
return "retry";
|
|
17
|
+
if (a === "refused" || b === "refused")
|
|
18
|
+
return "refused";
|
|
19
|
+
return "complete";
|
|
20
|
+
}
|
|
12
21
|
// Re-checked inside the commit: a pause from another device can land after
|
|
13
22
|
// the page was filtered, and its cleanup may already have run.
|
|
14
23
|
function currentUnlessPaused(input, container) {
|
|
@@ -22,18 +31,14 @@ function matchesListedState(state, expectedContainerStates, containerId) {
|
|
|
22
31
|
});
|
|
23
32
|
}
|
|
24
33
|
async function applyRemoteContainerPage(input) {
|
|
25
|
-
const {
|
|
34
|
+
const { seenContainerObservations, state } = input;
|
|
26
35
|
let hydratedCount = 0;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}),
|
|
32
|
-
remoteContainers: items,
|
|
33
|
-
stillCurrent: input.isCurrent,
|
|
34
|
-
});
|
|
36
|
+
const { items, outcome: policyOutcome } = await warmListedContainerPolicies(input.items, state.runtime.principalPolicies.createWarmer({
|
|
37
|
+
operation: input.policyOperation,
|
|
38
|
+
}), input.reconnect?.recordManualError, input.isCurrent);
|
|
39
|
+
let outcome = policyOutcome;
|
|
35
40
|
if (input.isCurrent?.() === false) {
|
|
36
|
-
return { changedCount: 0,
|
|
41
|
+
return { changedCount: 0, outcome: "retry" };
|
|
37
42
|
}
|
|
38
43
|
const [containerIdsWithPendingMetadataUpdates, containerIdsWithPendingStructuralIntents,] = await Promise.all([
|
|
39
44
|
listRemoteContainerIdsWithPendingMetadataUpdates({
|
|
@@ -46,66 +51,119 @@ async function applyRemoteContainerPage(input) {
|
|
|
46
51
|
}),
|
|
47
52
|
]);
|
|
48
53
|
if (input.isCurrent?.() === false) {
|
|
49
|
-
return { changedCount: 0,
|
|
54
|
+
return { changedCount: 0, outcome: "retry" };
|
|
50
55
|
}
|
|
51
56
|
const prefetchedProjections = await prefetchDestinationProjections({
|
|
52
|
-
containers: items.filter((item) => !
|
|
57
|
+
containers: items.filter((item) => !seenContainerObservations.has(item.id) &&
|
|
53
58
|
!state.containersById.has(item.id) &&
|
|
54
|
-
!expectedHydrationTombstones.get(item.id) &&
|
|
59
|
+
!input.expectedHydrationTombstones.get(item.id) &&
|
|
55
60
|
!cachedDestinationRole(state.runtime.infra.execSql, item)),
|
|
56
61
|
isCurrent: input.isCurrent,
|
|
57
62
|
runtime: state.runtime,
|
|
58
63
|
});
|
|
59
64
|
for (const container of items) {
|
|
60
65
|
if (input.isCurrent?.() === false) {
|
|
61
|
-
return { changedCount: hydratedCount,
|
|
66
|
+
return { changedCount: hydratedCount, outcome: "retry" };
|
|
62
67
|
}
|
|
63
|
-
|
|
64
|
-
|
|
68
|
+
const observation = JSON.stringify(container);
|
|
69
|
+
if (seenContainerObservations.get(container.id) === observation &&
|
|
70
|
+
state.containersById.has(container.id)) {
|
|
71
|
+
input.queueParentLane(container.id);
|
|
65
72
|
continue;
|
|
66
73
|
}
|
|
67
|
-
if (!matchesListedState(state, expectedContainerStates, container.id)) {
|
|
68
|
-
|
|
74
|
+
if (!matchesListedState(state, input.expectedContainerStates, container.id)) {
|
|
75
|
+
outcome = "retry";
|
|
69
76
|
continue;
|
|
70
77
|
}
|
|
78
|
+
let failure = "retry";
|
|
71
79
|
const upserted = await upsertIsolatedRemoteContainerState({
|
|
72
|
-
childIdsByParentId,
|
|
80
|
+
childIdsByParentId: input.childIdsByParentId,
|
|
73
81
|
containerIdsWithPendingMetadataUpdates,
|
|
74
82
|
containerIdsWithPendingStructuralIntents,
|
|
75
|
-
host,
|
|
83
|
+
host: input.host,
|
|
76
84
|
isCurrent: currentUnlessPaused(input, container),
|
|
77
|
-
expectedHydrationTombstone: expectedHydrationTombstones.get(container.id) ?? null,
|
|
85
|
+
expectedHydrationTombstone: input.expectedHydrationTombstones.get(container.id) ?? null,
|
|
78
86
|
prefetchedProjection: prefetchedProjections.get(container.id),
|
|
79
87
|
remoteContainer: container,
|
|
80
88
|
state,
|
|
81
|
-
})
|
|
89
|
+
}, (outcome) => {
|
|
90
|
+
failure = outcome;
|
|
91
|
+
}, input.reconnect?.recordManualError);
|
|
82
92
|
if (!upserted) {
|
|
83
|
-
|
|
93
|
+
outcome = combineOutcomes(outcome, failure);
|
|
84
94
|
continue;
|
|
85
95
|
}
|
|
86
|
-
|
|
96
|
+
seenContainerObservations.set(container.id, observation);
|
|
87
97
|
hydratedCount += 1;
|
|
88
98
|
if (input.isCurrent?.() === false) {
|
|
89
|
-
return { changedCount: hydratedCount,
|
|
99
|
+
return { changedCount: hydratedCount, outcome: "retry" };
|
|
90
100
|
}
|
|
91
|
-
queueParentLane(container.id);
|
|
101
|
+
input.queueParentLane(container.id);
|
|
92
102
|
}
|
|
93
|
-
return { changedCount: hydratedCount,
|
|
103
|
+
return { changedCount: hydratedCount, outcome };
|
|
94
104
|
}
|
|
95
105
|
export function canHydrateRemoteContainers(state) {
|
|
96
106
|
return (state.runtime.auth.isAuthenticated &&
|
|
97
107
|
state.runtime.state.online &&
|
|
98
108
|
state.runtime.infra.dbStatus === "ready");
|
|
99
109
|
}
|
|
110
|
+
function cursorKey(watermark) {
|
|
111
|
+
return JSON.stringify([watermark.updatedAt, watermark.id]);
|
|
112
|
+
}
|
|
113
|
+
function hasUnusableContinuation({ lane, response, }) {
|
|
114
|
+
const next = response.nextWatermark;
|
|
115
|
+
if (!next)
|
|
116
|
+
return response.hasMore;
|
|
117
|
+
// Opaque positions can advance with an older business timestamp.
|
|
118
|
+
// A drained lane may echo its current position, but never an earlier one.
|
|
119
|
+
if (next.id === lane.watermark?.id &&
|
|
120
|
+
next.updatedAt === lane.watermark.updatedAt)
|
|
121
|
+
return response.hasMore;
|
|
122
|
+
return lane.continuationCursors?.has(cursorKey(next)) === true;
|
|
123
|
+
}
|
|
124
|
+
function queueNextPage({ lane, response }, skipWatermarkWrite, queueContinuationLane) {
|
|
125
|
+
if (response.hasMore && response.nextWatermark) {
|
|
126
|
+
const continuationCursors = lane.continuationCursors ?? new Set();
|
|
127
|
+
if (lane.watermark)
|
|
128
|
+
continuationCursors.add(cursorKey(lane.watermark));
|
|
129
|
+
continuationCursors.add(cursorKey(response.nextWatermark));
|
|
130
|
+
queueContinuationLane({
|
|
131
|
+
parentId: lane.parentId,
|
|
132
|
+
watermark: response.nextWatermark,
|
|
133
|
+
skipWatermarkWrite,
|
|
134
|
+
continuationCursors,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
async function checkpointPage(input) {
|
|
139
|
+
const { lane, response, syncLane } = input.fetchedPage;
|
|
140
|
+
// Only a drained chain can advance a reconnect checkpoint. Otherwise a
|
|
141
|
+
// cycle can alternate the durable cursor between successive attempts.
|
|
142
|
+
if (input.reconnect && response.hasMore)
|
|
143
|
+
return true;
|
|
144
|
+
const nextWatermark = response.nextWatermark ??
|
|
145
|
+
(lane.continuationCursors ? (lane.watermark ?? null) : null);
|
|
146
|
+
return markContainerParentLaneFetched({
|
|
147
|
+
isCurrent: input.isCurrent,
|
|
148
|
+
response: { ...response, nextWatermark },
|
|
149
|
+
state: input.state,
|
|
150
|
+
syncLane,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
100
153
|
async function applyContainerParentLanePage(input) {
|
|
101
|
-
const { childIdsByParentId, fetchedPage, host,
|
|
102
|
-
const { expectedContainerStates, expectedHydrationTombstones, lane, response,
|
|
154
|
+
const { childIdsByParentId, fetchedPage, host, state } = input;
|
|
155
|
+
const { expectedContainerStates, expectedHydrationTombstones, lane, response, } = fetchedPage;
|
|
103
156
|
let changedCount = 0;
|
|
104
157
|
if (input.isCurrent?.() === false) {
|
|
105
|
-
return { changedCount,
|
|
158
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
106
159
|
}
|
|
160
|
+
if (changesDiscoveryOrder(lane.watermark, response.nextWatermark, !!lane.continuationCursors))
|
|
161
|
+
return { changedCount, outcome: "retry", shouldStop: false };
|
|
107
162
|
const remoteContainerItems = getApplicableRemoteContainerItems(response);
|
|
163
|
+
const isPaused = await loadPausedContainerFilter(state);
|
|
108
164
|
const tombstoneResult = await applyContainerTombstones({
|
|
165
|
+
isPaused,
|
|
166
|
+
parentId: lane.parentId,
|
|
109
167
|
childIdsByParentId,
|
|
110
168
|
expectedContainerStates,
|
|
111
169
|
isCurrent: input.isCurrent,
|
|
@@ -114,23 +172,19 @@ async function applyContainerParentLanePage(input) {
|
|
|
114
172
|
state,
|
|
115
173
|
});
|
|
116
174
|
if (!tombstoneResult.current) {
|
|
117
|
-
return { changedCount,
|
|
175
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
118
176
|
}
|
|
119
177
|
changedCount += tombstoneResult.changedCount;
|
|
120
178
|
if (input.isCurrent?.() === false) {
|
|
121
|
-
return { changedCount,
|
|
179
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
122
180
|
}
|
|
123
181
|
if (tombstoneResult.changedCount > 0) {
|
|
124
|
-
//
|
|
125
|
-
// document priming so their null-scoped passes run now rather than on
|
|
126
|
-
// the next startup.
|
|
182
|
+
// Prime documents orphaned by this cascade during the current pass.
|
|
127
183
|
host.requestDocumentPriming?.();
|
|
128
184
|
}
|
|
129
|
-
//
|
|
130
|
-
// Skipping them here, after tombstones saw the full page, still lets the
|
|
131
|
-
// lane complete and its watermark advance; their children are never queued.
|
|
132
|
-
const isPaused = await loadPausedContainerFilter(state);
|
|
185
|
+
// Paused folders keep their local state while the lane advances.
|
|
133
186
|
const appliedPage = await applyRemoteContainerPage({
|
|
187
|
+
reconnect: input.reconnect,
|
|
134
188
|
childIdsByParentId,
|
|
135
189
|
expectedContainerStates,
|
|
136
190
|
expectedHydrationTombstones,
|
|
@@ -139,40 +193,28 @@ async function applyContainerParentLanePage(input) {
|
|
|
139
193
|
isPaused,
|
|
140
194
|
items: remoteContainerItems.filter((item) => !isPaused(item)),
|
|
141
195
|
policyOperation: input.policyOperation,
|
|
142
|
-
queueParentLane,
|
|
143
|
-
|
|
196
|
+
queueParentLane: input.queueParentLane,
|
|
197
|
+
seenContainerObservations: input.seenContainerObservations,
|
|
144
198
|
state,
|
|
145
199
|
});
|
|
146
200
|
changedCount += appliedPage.changedCount;
|
|
147
|
-
|
|
148
|
-
// Keep this page retryable without discarding independent lanes, including
|
|
149
|
-
// children of roots first discovered here. Otherwise the root becomes known
|
|
150
|
-
// but its children are never fetched by subsequent root-only refreshes.
|
|
151
|
-
return { changedCount, complete: false, shouldStop: false };
|
|
152
|
-
}
|
|
201
|
+
const outcome = tombstoneResult.completed ? appliedPage.outcome : "retry";
|
|
153
202
|
if (input.isCurrent?.() === false) {
|
|
154
|
-
return { changedCount,
|
|
155
|
-
}
|
|
156
|
-
const didMarkFetched = await markContainerParentLaneFetched({
|
|
157
|
-
isCurrent: input.isCurrent,
|
|
158
|
-
response,
|
|
159
|
-
state,
|
|
160
|
-
syncLane,
|
|
161
|
-
});
|
|
162
|
-
if (!didMarkFetched) {
|
|
163
|
-
return { changedCount, complete: false, shouldStop: true };
|
|
203
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
164
204
|
}
|
|
165
|
-
|
|
166
|
-
|
|
205
|
+
// A bad continuation invalidates the cursor, independently verified rows still apply.
|
|
206
|
+
if (hasUnusableContinuation(fetchedPage))
|
|
207
|
+
return { changedCount, outcome: "retry", shouldStop: false };
|
|
208
|
+
// A refused item leaves a gap in this lane. Later pages can still discover
|
|
209
|
+
// independent children, but none may move its durable cursor past the gap.
|
|
210
|
+
if (outcome !== "complete" && !input.reconnect)
|
|
211
|
+
return { changedCount, outcome, shouldStop: false };
|
|
212
|
+
const skipWatermarkWrite = lane.skipWatermarkWrite || outcome !== "complete";
|
|
213
|
+
if (!skipWatermarkWrite && !(await checkpointPage(input))) {
|
|
214
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
167
215
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
}
|
|
171
|
-
queueContinuationLane({
|
|
172
|
-
parentId: lane.parentId,
|
|
173
|
-
watermark: response.nextWatermark,
|
|
174
|
-
});
|
|
175
|
-
return { changedCount, complete: true, shouldStop: false };
|
|
216
|
+
queueNextPage(fetchedPage, skipWatermarkWrite, input.queueContinuationLane);
|
|
217
|
+
return { changedCount, outcome, shouldStop: false };
|
|
176
218
|
}
|
|
177
219
|
function takeContainerParentLaneBatch(input) {
|
|
178
220
|
const { lanes, state } = input;
|
|
@@ -188,18 +230,19 @@ function takeContainerParentLaneBatch(input) {
|
|
|
188
230
|
return batch;
|
|
189
231
|
}
|
|
190
232
|
async function applyContainerParentLaneBatch(input) {
|
|
191
|
-
const { childIdsByParentId, fetchedPages, host, lanes, queueParentLane,
|
|
233
|
+
const { childIdsByParentId, fetchedPages, host, lanes, queueParentLane, seenContainerObservations, state, } = input;
|
|
192
234
|
let changedCount = 0;
|
|
193
|
-
let
|
|
235
|
+
let outcome = "complete";
|
|
194
236
|
for (const fetchedPage of fetchedPages) {
|
|
195
237
|
if (!canHydrateRemoteContainers(state) || input.isCurrent?.() === false) {
|
|
196
|
-
return { changedCount,
|
|
238
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
197
239
|
}
|
|
198
240
|
if (fetchedPage.lane.parentId !== null &&
|
|
199
241
|
!state.containersById.has(fetchedPage.lane.parentId)) {
|
|
200
242
|
continue;
|
|
201
243
|
}
|
|
202
244
|
const result = await applyContainerParentLanePage({
|
|
245
|
+
reconnect: input.reconnect,
|
|
203
246
|
childIdsByParentId,
|
|
204
247
|
fetchedPage,
|
|
205
248
|
host,
|
|
@@ -207,26 +250,26 @@ async function applyContainerParentLaneBatch(input) {
|
|
|
207
250
|
policyOperation: input.policyOperation,
|
|
208
251
|
queueContinuationLane: (lane) => lanes.push(lane),
|
|
209
252
|
queueParentLane,
|
|
210
|
-
|
|
253
|
+
seenContainerObservations,
|
|
211
254
|
state,
|
|
212
255
|
});
|
|
213
256
|
changedCount += result.changedCount;
|
|
214
|
-
|
|
257
|
+
outcome = combineOutcomes(outcome, result.outcome);
|
|
215
258
|
if (result.shouldStop) {
|
|
216
|
-
return { changedCount,
|
|
259
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
217
260
|
}
|
|
218
261
|
}
|
|
219
|
-
return { changedCount,
|
|
262
|
+
return { changedCount, outcome, shouldStop: false };
|
|
220
263
|
}
|
|
221
264
|
export async function hydrateContainerParentLanes(input) {
|
|
222
|
-
const { childIdsByParentId, host, lanes, queueParentLane,
|
|
265
|
+
const { childIdsByParentId, host, lanes, queueParentLane, seenContainerObservations, state, } = input;
|
|
223
266
|
let changedCount = 0;
|
|
224
|
-
let
|
|
267
|
+
let outcome = "complete";
|
|
225
268
|
// One hydration pass is one policy operation: its pages share a discovery.
|
|
226
269
|
const policyOperation = createPrincipalPolicyOperation();
|
|
227
270
|
while (lanes.length > 0) {
|
|
228
271
|
if (!canHydrateRemoteContainers(state) || input.isCurrent?.() === false) {
|
|
229
|
-
return { changedCount,
|
|
272
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
230
273
|
}
|
|
231
274
|
const batch = takeContainerParentLaneBatch({ lanes, state });
|
|
232
275
|
if (batch.length === 0) {
|
|
@@ -234,16 +277,18 @@ export async function hydrateContainerParentLanes(input) {
|
|
|
234
277
|
}
|
|
235
278
|
const fetchedPages = await fetchContainerParentLaneBatch({
|
|
236
279
|
batch,
|
|
280
|
+
laneIdPrefix: input.reconnect?.laneIdPrefix,
|
|
237
281
|
isCurrent: input.isCurrent,
|
|
238
282
|
state,
|
|
239
283
|
});
|
|
240
284
|
if (!fetchedPages) {
|
|
241
|
-
return { changedCount,
|
|
285
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
242
286
|
}
|
|
243
287
|
if (input.isCurrent?.() === false) {
|
|
244
|
-
return { changedCount,
|
|
288
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
245
289
|
}
|
|
246
290
|
const result = await applyContainerParentLaneBatch({
|
|
291
|
+
reconnect: input.reconnect,
|
|
247
292
|
childIdsByParentId,
|
|
248
293
|
fetchedPages,
|
|
249
294
|
host,
|
|
@@ -251,15 +296,15 @@ export async function hydrateContainerParentLanes(input) {
|
|
|
251
296
|
lanes,
|
|
252
297
|
policyOperation,
|
|
253
298
|
queueParentLane,
|
|
254
|
-
|
|
299
|
+
seenContainerObservations,
|
|
255
300
|
state,
|
|
256
301
|
});
|
|
257
302
|
changedCount += result.changedCount;
|
|
258
|
-
|
|
303
|
+
outcome = combineOutcomes(outcome, result.outcome);
|
|
259
304
|
if (result.shouldStop) {
|
|
260
|
-
return { changedCount,
|
|
305
|
+
return { changedCount, outcome: "retry", shouldStop: true };
|
|
261
306
|
}
|
|
262
307
|
}
|
|
263
|
-
return { changedCount,
|
|
308
|
+
return { changedCount, outcome, shouldStop: false };
|
|
264
309
|
}
|
|
265
310
|
//# sourceMappingURL=remoteContainerPageHydration.js.map
|