@tearleads/client-sdk 0.1.42 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/containerDocumentDiscovery.js +71 -29
- package/client/containerDocumentDiscovery.js.map +1 -1
- package/client/deviceFirst.js +4 -5
- package/client/deviceFirst.js.map +1 -1
- package/data/blobs/transferPool.js +35 -8
- package/data/blobs/transferPool.js.map +1 -1
- package/data/containers/shared/projection.js +12 -0
- package/data/containers/shared/projection.js.map +1 -1
- package/data/documents/blob/shared/types.d.ts +2 -3
- package/data/documents/blob/shared/types.js.map +1 -1
- package/data/documents/shared/events.js +0 -1
- package/data/documents/shared/events.js.map +1 -1
- package/data/persistence/container-contents/containerContentsPersistence.js +6 -22
- package/data/persistence/container-contents/containerContentsPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerMetadataMutationPersistence.js +5 -15
- package/data/persistence/container-contents/containerMetadataMutationPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerPendingUpdatePersistence.js +6 -15
- package/data/persistence/container-contents/containerPendingUpdatePersistence.js.map +1 -1
- package/data/persistence/container-contents/containerRecoveryPersistence.js +3 -4
- package/data/persistence/container-contents/containerRecoveryPersistence.js.map +1 -1
- package/data/persistence/container-contents/containerRemovalWork.js +3 -5
- package/data/persistence/container-contents/containerRemovalWork.js.map +1 -1
- package/data/persistence/container-contents/dormantContainerMetadata.js +1 -5
- package/data/persistence/container-contents/dormantContainerMetadata.js.map +1 -1
- package/data/persistence/containers/containerSyncWatermarkPersistence.d.ts +11 -3
- package/data/persistence/containers/containerSyncWatermarkPersistence.js +46 -58
- package/data/persistence/containers/containerSyncWatermarkPersistence.js.map +1 -1
- package/data/persistence/documents/documentsPersistence.js +4 -9
- package/data/persistence/documents/documentsPersistence.js.map +1 -1
- package/data/persistence/documents/internal/discardDocument.js +1 -2
- package/data/persistence/documents/internal/discardDocument.js.map +1 -1
- package/data/persistence/documents/internal/documentDeletionPersistence.js +1 -2
- package/data/persistence/documents/internal/documentDeletionPersistence.js.map +1 -1
- package/data/persistence/documents/internal/documentMutationPersistence.js +17 -25
- package/data/persistence/documents/internal/documentMutationPersistence.js.map +1 -1
- package/data/persistence/documents/internal/orphanSideRows.js +2 -14
- package/data/persistence/documents/internal/orphanSideRows.js.map +1 -1
- package/data/persistence/documents/internal/syncQueuePersistence.js +3 -3
- package/data/persistence/documents/internal/syncQueuePersistence.js.map +1 -1
- package/data/sqlite/documentHistoryPersistence.d.ts +9 -3
- package/data/sqlite/documentHistoryPersistence.js +31 -8
- package/data/sqlite/documentHistoryPersistence.js.map +1 -1
- package/data/sqlite/documentPendingUpdatePersistence.d.ts +58 -11
- package/data/sqlite/documentPendingUpdatePersistence.js +137 -63
- package/data/sqlite/documentPendingUpdatePersistence.js.map +1 -1
- package/data/sqlite/documentPersistence.d.ts +1 -1
- package/data/sqlite/documentPersistence.js +1 -1
- package/data/sqlite/documentPersistence.js.map +1 -1
- package/data/sqlite/documentPersistenceTypes.d.ts +2 -2
- package/data/sqlite/documentPersistenceTypes.js.map +1 -1
- package/data/sqlite/documentRecordPersistence.js +5 -4
- package/data/sqlite/documentRecordPersistence.js.map +1 -1
- package/data/sqlite/schema.d.ts +167 -517
- package/data/sqlite/schema.js +68 -82
- package/data/sqlite/schema.js.map +1 -1
- package/internal/api-client/ApiClient.d.ts +18 -41
- package/internal/api-client/ApiClient.js +9 -14
- package/internal/api-client/ApiClient.js.map +1 -1
- package/internal/api-client/apiRequestRuntime.d.ts +2 -0
- package/internal/api-client/apiRequestRuntime.js +4 -0
- package/internal/api-client/apiRequestRuntime.js.map +1 -1
- package/internal/api-client/binaryRequestOperationTransport.d.ts +18 -0
- package/internal/api-client/index.d.ts +1 -0
- package/internal/api-client/index.js +1 -0
- package/internal/api-client/index.js.map +1 -1
- package/internal/api-client/operationTransportFactory.d.ts +6 -0
- package/internal/api-client/requestInternals.js +7 -7
- package/internal/api-client/requestInternals.js.map +1 -1
- package/internal/api-client/routes/blobs/get.d.ts +7 -0
- package/internal/api-client/routes/blobs/get.js.map +1 -1
- package/internal/api-client/routes/blobs/multipart.js +1 -11
- package/internal/api-client/routes/blobs/multipart.js.map +1 -1
- package/internal/api-client/routes/blobs/multipartUpload.d.ts +14 -0
- package/internal/api-client/routes/blobs/multipartUpload.js +94 -43
- package/internal/api-client/routes/blobs/multipartUpload.js.map +1 -1
- package/internal/api-client/routes/containers/reads.d.ts +13 -7
- package/internal/api-client/routes/containers/reads.js +36 -15
- package/internal/api-client/routes/containers/reads.js.map +1 -1
- package/internal/api-client/types.d.ts +1 -5
- package/internal/api-client/types.js.map +1 -1
- package/internal/validators/containerReadRefinements.js +15 -5
- package/internal/validators/containerReadRefinements.js.map +1 -1
- package/internal/validators/containerReadSchemas.d.ts +36 -0
- package/internal/validators/containerReadSchemas.js +64 -1
- package/internal/validators/containerReadSchemas.js.map +1 -1
- package/internal/validators/operation/attachments.d.ts +80 -160
- package/internal/validators/operation/auth.d.ts +760 -1664
- package/internal/validators/operation/blobDownloads.js +1 -1
- package/internal/validators/operation/blobDownloads.js.map +1 -1
- package/internal/validators/operation/containerMutations.d.ts +416 -856
- package/internal/validators/operation/containerReads.d.ts +118 -182
- package/internal/validators/operation/containerReads.js +20 -54
- package/internal/validators/operation/containerReads.js.map +1 -1
- package/internal/validators/operation/documentMutations.d.ts +120 -264
- package/internal/validators/operation/documentSync.d.ts +40 -80
- package/internal/validators/operation/index.d.ts +2 -2
- package/internal/validators/operation/index.js +2 -2
- package/internal/validators/operation/index.js.map +1 -1
- package/internal/validators/operation/multipartBlobs.d.ts +70 -190
- package/internal/validators/operation/multipartBlobs.js +2 -24
- package/internal/validators/operation/multipartBlobs.js.map +1 -1
- package/internal/validators/operation/multipartPartUploads.d.ts +5 -5
- package/internal/validators/operation/multipartPartUploads.js +6 -20
- package/internal/validators/operation/multipartPartUploads.js.map +1 -1
- package/internal/validators/operation/organizations.d.ts +882 -1906
- package/internal/validators/operation/principals.d.ts +120 -240
- package/internal/validators/operation/registry.d.ts +3 -4
- package/internal/validators/operation/registry.js +3 -4
- package/internal/validators/operation/registry.js.map +1 -1
- package/internal/validators/request/blob.d.ts +100 -201
- package/internal/validators/request/blob.js +6 -13
- package/internal/validators/request/blob.js.map +1 -1
- package/internal/validators/request/container.d.ts +136 -272
- package/internal/validators/request/container.js +16 -10
- package/internal/validators/request/container.js.map +1 -1
- package/internal/validators/request/containerDocumentLanes.d.ts +23 -0
- package/internal/validators/request/containerDocumentLanes.js +23 -0
- package/internal/validators/request/containerDocumentLanes.js.map +1 -0
- package/internal/validators/request/containerManifestRef.d.ts +54 -0
- package/internal/validators/request/containerManifestRef.js +28 -0
- package/internal/validators/request/containerManifestRef.js.map +1 -0
- package/internal/validators/request/containerMetadata.d.ts +160 -368
- package/internal/validators/request/containerParentLanes.js +5 -35
- package/internal/validators/request/containerParentLanes.js.map +1 -1
- package/internal/validators/request/document.d.ts +100 -245
- package/internal/validators/request/document.js +1 -2
- package/internal/validators/request/document.js.map +1 -1
- package/internal/validators/request/documentSyncSchema.d.ts +42 -109
- package/internal/validators/request/documentSyncSchema.js +8 -25
- package/internal/validators/request/documentSyncSchema.js.map +1 -1
- package/internal/validators/request/index.d.ts +2 -1
- package/internal/validators/request/index.js +2 -1
- package/internal/validators/request/index.js.map +1 -1
- package/internal/validators/request/organization.d.ts +160 -320
- package/internal/validators/request/organizationProvisioning.d.ts +2424 -5312
- package/internal/validators/request/principal.d.ts +160 -320
- package/internal/validators/request/registration.d.ts +760 -1664
- package/internal/validators/response/blob.d.ts +48 -27
- package/internal/validators/response/blob.js +3 -10
- package/internal/validators/response/blob.js.map +1 -1
- package/internal/validators/response/containerDocumentLanes.d.ts +82 -0
- package/internal/validators/response/containerDocumentLanes.js +35 -0
- package/internal/validators/response/containerDocumentLanes.js.map +1 -0
- package/internal/validators/response/containerParentLanes.js +3 -15
- package/internal/validators/response/containerParentLanes.js.map +1 -1
- package/internal/validators/response/index.d.ts +3 -2
- package/internal/validators/response/index.js +2 -2
- package/internal/validators/response/index.js.map +1 -1
- package/internal/validators/util/blob.d.ts +7 -0
- package/internal/validators/util/blob.js +7 -0
- package/internal/validators/util/blob.js.map +1 -1
- package/internal/validators/util/blobTransferSchemas.d.ts +41 -0
- package/internal/validators/util/blobTransferSchemas.js +39 -0
- package/internal/validators/util/blobTransferSchemas.js.map +1 -0
- package/internal/validators/util/containerLimits.d.ts +7 -0
- package/internal/validators/util/containerLimits.js +7 -0
- package/internal/validators/util/containerLimits.js.map +1 -1
- package/internal/validators/util/index.d.ts +2 -1
- package/internal/validators/util/index.js +2 -1
- package/internal/validators/util/index.js.map +1 -1
- package/package.json +1 -1
- package/sync/reconciliation/knownContainerSweep.js +57 -54
- package/sync/reconciliation/knownContainerSweep.js.map +1 -1
- package/sync/reconciliation/reconcileContainer.js +49 -17
- package/sync/reconciliation/reconcileContainer.js.map +1 -1
- package/sync/reconciliation/service.js +44 -16
- package/sync/reconciliation/service.js.map +1 -1
- package/sync/reconciliation/serviceTypes.d.ts +23 -2
- package/sync/reconciliation/serviceTypes.js.map +1 -1
- package/workflows/blobs/multipartFirstParts.js +82 -0
- package/workflows/blobs/multipartFirstParts.js.map +1 -0
- package/workflows/blobs/multipartStageResolution.js +83 -0
- package/workflows/blobs/multipartStageResolution.js.map +1 -0
- package/workflows/blobs/multipartUpload.js +45 -89
- package/workflows/blobs/multipartUpload.js.map +1 -1
- package/workflows/container-contents/containerDocumentListing.js +92 -38
- package/workflows/container-contents/containerDocumentListing.js.map +1 -1
- package/workflows/container-contents/containerPersistence.d.ts +8 -2
- package/workflows/container-contents/containerPersistence.js +6 -7
- package/workflows/container-contents/containerPersistence.js.map +1 -1
- package/workflows/container-contents/documentDiscovery.d.ts +3 -3
- package/workflows/container-contents/documentDiscovery.js +24 -96
- package/workflows/container-contents/documentDiscovery.js.map +1 -1
- package/workflows/container-contents/documentDiscoveryApply.js +65 -0
- package/workflows/container-contents/documentDiscoveryApply.js.map +1 -0
- package/workflows/container-contents/documentDiscoveryLanes.js +53 -0
- package/workflows/container-contents/documentDiscoveryLanes.js.map +1 -0
- package/workflows/container-contents/documentDiscoveryTypes.d.ts +25 -19
- package/workflows/container-contents/documentDiscoveryTypes.js.map +1 -1
- package/workflows/container-contents/documentPriming.js +2 -1
- package/workflows/container-contents/documentPriming.js.map +1 -1
- package/workflows/container-contents/documentQueries/sql.js +4 -4
- package/workflows/container-contents/documentQueries/sql.js.map +1 -1
- package/workflows/container-contents/pendingWrites/deferredTails.js +6 -3
- package/workflows/container-contents/pendingWrites/deferredTails.js.map +1 -1
- package/workflows/container-contents/pendingWrites/sources.js +2 -1
- package/workflows/container-contents/pendingWrites/sources.js.map +1 -1
- package/workflows/container-contents/pendingWrites.js +2 -1
- package/workflows/container-contents/pendingWrites.js.map +1 -1
- package/workflows/container-contents/remoteHydration/laneFetchMarkers.js +11 -17
- package/workflows/container-contents/remoteHydration/laneFetchMarkers.js.map +1 -1
- package/workflows/containers/child/create.js +3 -3
- package/workflows/containers/child/create.js.map +1 -1
- package/workflows/containers/child/move.js +2 -2
- package/workflows/containers/child/move.js.map +1 -1
- package/workflows/containers/child/mutationRequestCore.js +5 -4
- package/workflows/containers/child/mutationRequestCore.js.map +1 -1
- package/workflows/containers/child/recitePlan.js +3 -2
- package/workflows/containers/child/recitePlan.js.map +1 -1
- package/workflows/containers/child/sharePlanCore.js +2 -2
- package/workflows/containers/child/sharePlanCore.js.map +1 -1
- package/workflows/containers/root/create.js +2 -2
- package/workflows/containers/root/create.js.map +1 -1
- package/workflows/registration/provisioningBootstrapBuilders.d.ts +20 -46
- package/workflows/sync/remoteReset.js +9 -9
- package/workflows/sync/remoteReset.js.map +1 -1
- package/workflows/sync/remoteResetCursorScope.js +10 -18
- package/workflows/sync/remoteResetCursorScope.js.map +1 -1
- package/workflows/sync/remoteResetPlans.js +6 -3
- package/workflows/sync/remoteResetPlans.js.map +1 -1
- package/workflows/sync/remoteResetScope.js +4 -11
- package/workflows/sync/remoteResetScope.js.map +1 -1
- package/internal/validators/operation/blobTransferSchemas.js +0 -11
- package/internal/validators/operation/blobTransferSchemas.js.map +0 -1
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const signedBlobUrlSchema: z.ZodCustom<string, string>;
|
|
3
|
+
export declare const signedBlobUrlExpirySchema: z.ZodCustom<string, string>;
|
|
4
|
+
export declare const blobPartContentMd5Schema: z.ZodCustom<string, string>;
|
|
5
|
+
/** S3/R2 multipart part numbers run from 1 through 10,000. */
|
|
6
|
+
export declare const MAX_MULTIPART_BLOB_PART_NUMBER = 10000;
|
|
7
|
+
/**
|
|
8
|
+
* One encrypted part as the client will send it: its number, exact length, and
|
|
9
|
+
* the transport Content-MD5 a signed upload URL binds.
|
|
10
|
+
*/
|
|
11
|
+
export declare const MultipartBlobPartDescriptorSchema: z.ZodObject<{
|
|
12
|
+
byteLength: z.ZodCustom<number, number>;
|
|
13
|
+
contentMd5: z.ZodCustom<string, string>;
|
|
14
|
+
partNumber: z.ZodCustom<number, number>;
|
|
15
|
+
}, z.core.$strict>;
|
|
16
|
+
export type MultipartBlobPartDescriptor = z.infer<typeof MultipartBlobPartDescriptorSchema>;
|
|
17
|
+
/**
|
|
18
|
+
* Where a part's bytes go: the API byte route for stores that cannot sign, or
|
|
19
|
+
* a short-lived URL that signs exactly one part of one upload.
|
|
20
|
+
*/
|
|
21
|
+
export declare const MultipartBlobPartUploadSourceSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
22
|
+
kind: z.ZodLiteral<"api">;
|
|
23
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
24
|
+
kind: z.ZodLiteral<"signed-url">;
|
|
25
|
+
url: z.ZodCustom<string, string>;
|
|
26
|
+
expiresAt: z.ZodCustom<string, string>;
|
|
27
|
+
}, z.core.$strict>], "kind">;
|
|
28
|
+
export type MultipartBlobPartUploadSource = z.infer<typeof MultipartBlobPartUploadSourceSchema>;
|
|
29
|
+
export declare const MultipartBlobPartUploadSchema: z.ZodObject<{
|
|
30
|
+
byteLength: z.ZodCustom<number, number>;
|
|
31
|
+
contentMd5: z.ZodCustom<string, string>;
|
|
32
|
+
partNumber: z.ZodCustom<number, number>;
|
|
33
|
+
source: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
34
|
+
kind: z.ZodLiteral<"api">;
|
|
35
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
36
|
+
kind: z.ZodLiteral<"signed-url">;
|
|
37
|
+
url: z.ZodCustom<string, string>;
|
|
38
|
+
expiresAt: z.ZodCustom<string, string>;
|
|
39
|
+
}, z.core.$strict>], "kind">;
|
|
40
|
+
}, z.core.$strict>;
|
|
41
|
+
export type MultipartBlobPartUpload = z.infer<typeof MultipartBlobPartUploadSchema>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { registerJsonSchemaFragment } from "../jsonSchema.js";
|
|
3
|
+
import { boundedPositiveIntegerSchema } from "../schema.js";
|
|
4
|
+
import { MAX_MULTIPART_BLOB_PART_BYTES } from "./blob.js";
|
|
5
|
+
function stringSchemaFragment(schema) {
|
|
6
|
+
const { $schema: _dialect, ...fragment } = z.toJSONSchema(schema);
|
|
7
|
+
return registerJsonSchemaFragment(z.custom((value) => schema.safeParse(value).success), fragment);
|
|
8
|
+
}
|
|
9
|
+
export const signedBlobUrlSchema = stringSchemaFragment(z.url().regex(/^https?:\/\//u));
|
|
10
|
+
export const signedBlobUrlExpirySchema = stringSchemaFragment(z.iso.datetime());
|
|
11
|
+
// R2 transport checksum; assembled SHA-256 remains the application authority.
|
|
12
|
+
export const blobPartContentMd5Schema = stringSchemaFragment(z.string().regex(/^[A-Za-z0-9+/]{21}[AQgw]==$/u));
|
|
13
|
+
/** S3/R2 multipart part numbers run from 1 through 10,000. */
|
|
14
|
+
export const MAX_MULTIPART_BLOB_PART_NUMBER = 10_000;
|
|
15
|
+
/**
|
|
16
|
+
* One encrypted part as the client will send it: its number, exact length, and
|
|
17
|
+
* the transport Content-MD5 a signed upload URL binds.
|
|
18
|
+
*/
|
|
19
|
+
export const MultipartBlobPartDescriptorSchema = z.strictObject({
|
|
20
|
+
byteLength: boundedPositiveIntegerSchema(MAX_MULTIPART_BLOB_PART_BYTES),
|
|
21
|
+
contentMd5: blobPartContentMd5Schema,
|
|
22
|
+
partNumber: boundedPositiveIntegerSchema(MAX_MULTIPART_BLOB_PART_NUMBER),
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* Where a part's bytes go: the API byte route for stores that cannot sign, or
|
|
26
|
+
* a short-lived URL that signs exactly one part of one upload.
|
|
27
|
+
*/
|
|
28
|
+
export const MultipartBlobPartUploadSourceSchema = z.discriminatedUnion("kind", [
|
|
29
|
+
z.strictObject({ kind: z.literal("api") }),
|
|
30
|
+
z.strictObject({
|
|
31
|
+
kind: z.literal("signed-url"),
|
|
32
|
+
url: signedBlobUrlSchema,
|
|
33
|
+
expiresAt: signedBlobUrlExpirySchema,
|
|
34
|
+
}),
|
|
35
|
+
]);
|
|
36
|
+
export const MultipartBlobPartUploadSchema = MultipartBlobPartDescriptorSchema.extend({
|
|
37
|
+
source: MultipartBlobPartUploadSourceSchema,
|
|
38
|
+
});
|
|
39
|
+
//# sourceMappingURL=blobTransferSchemas.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"blobTransferSchemas.js","sourceRoot":"","sources":["blobTransferSchemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,0BAA0B,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,6BAA6B,EAAE,MAAM,QAAQ,CAAC;AAEvD,SAAS,oBAAoB,CAAC,MAAyB;IACrD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAClE,OAAO,0BAA0B,CAC/B,CAAC,CAAC,MAAM,CAAS,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAC5D,QAAQ,CACT,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,mBAAmB,GAAG,oBAAoB,CACrD,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAC/B,CAAC;AACF,MAAM,CAAC,MAAM,yBAAyB,GAAG,oBAAoB,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;AAEhF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,wBAAwB,GAAG,oBAAoB,CAC1D,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC,CACjD,CAAC;AAEF,8DAA8D;AAC9D,MAAM,CAAC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAErD;;;GAGG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,CAAC,YAAY,CAAC;IAC9D,UAAU,EAAE,4BAA4B,CAAC,6BAA6B,CAAC;IACvE,UAAU,EAAE,wBAAwB;IACpC,UAAU,EAAE,4BAA4B,CAAC,8BAA8B,CAAC;CACzE,CAAC,CAAC;AAMH;;;GAGG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC,kBAAkB,CACrE,MAAM,EACN;IACE,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;IAC1C,CAAC,CAAC,YAAY,CAAC;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;QAC7B,GAAG,EAAE,mBAAmB;QACxB,SAAS,EAAE,yBAAyB;KACrC,CAAC;CACH,CACF,CAAC;AAMF,MAAM,CAAC,MAAM,6BAA6B,GACxC,iCAAiC,CAAC,MAAM,CAAC;IACvC,MAAM,EAAE,mCAAmC;CAC5C,CAAC,CAAC","sourcesContent":["import { z } from \"zod\";\nimport { registerJsonSchemaFragment } from \"../jsonSchema\";\nimport { boundedPositiveIntegerSchema } from \"../schema\";\nimport { MAX_MULTIPART_BLOB_PART_BYTES } from \"./blob\";\n\nfunction stringSchemaFragment(schema: z.ZodType<string>) {\n const { $schema: _dialect, ...fragment } = z.toJSONSchema(schema);\n return registerJsonSchemaFragment(\n z.custom<string>((value) => schema.safeParse(value).success),\n fragment,\n );\n}\n\nexport const signedBlobUrlSchema = stringSchemaFragment(\n z.url().regex(/^https?:\\/\\//u),\n);\nexport const signedBlobUrlExpirySchema = stringSchemaFragment(z.iso.datetime());\n\n// R2 transport checksum; assembled SHA-256 remains the application authority.\nexport const blobPartContentMd5Schema = stringSchemaFragment(\n z.string().regex(/^[A-Za-z0-9+/]{21}[AQgw]==$/u),\n);\n\n/** S3/R2 multipart part numbers run from 1 through 10,000. */\nexport const MAX_MULTIPART_BLOB_PART_NUMBER = 10_000;\n\n/**\n * One encrypted part as the client will send it: its number, exact length, and\n * the transport Content-MD5 a signed upload URL binds.\n */\nexport const MultipartBlobPartDescriptorSchema = z.strictObject({\n byteLength: boundedPositiveIntegerSchema(MAX_MULTIPART_BLOB_PART_BYTES),\n contentMd5: blobPartContentMd5Schema,\n partNumber: boundedPositiveIntegerSchema(MAX_MULTIPART_BLOB_PART_NUMBER),\n});\n\nexport type MultipartBlobPartDescriptor = z.infer<\n typeof MultipartBlobPartDescriptorSchema\n>;\n\n/**\n * Where a part's bytes go: the API byte route for stores that cannot sign, or\n * a short-lived URL that signs exactly one part of one upload.\n */\nexport const MultipartBlobPartUploadSourceSchema = z.discriminatedUnion(\n \"kind\",\n [\n z.strictObject({ kind: z.literal(\"api\") }),\n z.strictObject({\n kind: z.literal(\"signed-url\"),\n url: signedBlobUrlSchema,\n expiresAt: signedBlobUrlExpirySchema,\n }),\n ],\n);\n\nexport type MultipartBlobPartUploadSource = z.infer<\n typeof MultipartBlobPartUploadSourceSchema\n>;\n\nexport const MultipartBlobPartUploadSchema =\n MultipartBlobPartDescriptorSchema.extend({\n source: MultipartBlobPartUploadSourceSchema,\n });\n\nexport type MultipartBlobPartUpload = z.infer<\n typeof MultipartBlobPartUploadSchema\n>;\n"]}
|
|
@@ -1,2 +1,9 @@
|
|
|
1
1
|
/** Maximum number of containers in a root-to-leaf path, including both ends. */
|
|
2
2
|
export declare const MAX_CONTAINER_PATH_LENGTH = 100;
|
|
3
|
+
/**
|
|
4
|
+
* Maximum number of stored manifests one container mutation may cite as the
|
|
5
|
+
* mutated container's history. Inline bundles (8-14 KiB each) fit about this
|
|
6
|
+
* many under the 32 MiB request ceiling; a reference is ~150 bytes, so the
|
|
7
|
+
* bound keeps that capacity instead of letting a small body cite ~50x more.
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAX_CONTAINER_MANIFEST_HISTORY_REFS = 4096;
|
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
/** Maximum number of containers in a root-to-leaf path, including both ends. */
|
|
2
2
|
export const MAX_CONTAINER_PATH_LENGTH = 100;
|
|
3
|
+
/**
|
|
4
|
+
* Maximum number of stored manifests one container mutation may cite as the
|
|
5
|
+
* mutated container's history. Inline bundles (8-14 KiB each) fit about this
|
|
6
|
+
* many under the 32 MiB request ceiling; a reference is ~150 bytes, so the
|
|
7
|
+
* bound keeps that capacity instead of letting a small body cite ~50x more.
|
|
8
|
+
*/
|
|
9
|
+
export const MAX_CONTAINER_MANIFEST_HISTORY_REFS = 4096;
|
|
3
10
|
//# sourceMappingURL=containerLimits.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"containerLimits.js","sourceRoot":"","sources":["containerLimits.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC","sourcesContent":["/** Maximum number of containers in a root-to-leaf path, including both ends. */\nexport const MAX_CONTAINER_PATH_LENGTH = 100;\n"]}
|
|
1
|
+
{"version":3,"file":"containerLimits.js","sourceRoot":"","sources":["containerLimits.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mCAAmC,GAAG,IAAI,CAAC","sourcesContent":["/** Maximum number of containers in a root-to-leaf path, including both ends. */\nexport const MAX_CONTAINER_PATH_LENGTH = 100;\n\n/**\n * Maximum number of stored manifests one container mutation may cite as the\n * mutated container's history. Inline bundles (8-14 KiB each) fit about this\n * many under the 32 MiB request ceiling; a reference is ~150 bytes, so the\n * bound keeps that capacity instead of letting a small body cite ~50x more.\n */\nexport const MAX_CONTAINER_MANIFEST_HISTORY_REFS = 4096;\n"]}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { type AccessEventBundleWireResponse, AccessEventBundleWireResponseSchema, type AccessManifestBundleWire, type AccessManifestBundleWireResponse, AccessManifestBundleWireResponseSchema, AccessManifestBundleWireSchema, isAccessEventBundleWireResponse, isAccessManifestBundleWire, isAccessManifestBundleWireArray, isAccessManifestBundleWireResponse, isOptionalAccessManifestBundleWireArray, } from "./accessManifestBundle.js";
|
|
2
|
-
export { MAX_MULTIPART_BLOB_PART_BYTES } from "./blob.js";
|
|
2
|
+
export { MAX_INITIATE_PART_UPLOADS, MAX_MULTIPART_BLOB_PART_BYTES, } from "./blob.js";
|
|
3
|
+
export { blobPartContentMd5Schema, MAX_MULTIPART_BLOB_PART_NUMBER, type MultipartBlobPartDescriptor, MultipartBlobPartDescriptorSchema, type MultipartBlobPartUpload, MultipartBlobPartUploadSchema, type MultipartBlobPartUploadSource, MultipartBlobPartUploadSourceSchema, signedBlobUrlExpirySchema, signedBlobUrlSchema, } from "./blobTransferSchemas.js";
|
|
3
4
|
export { CONTAINER_KEK_LOG_PAGE_LIMIT, CONTAINER_KEK_LOG_PRINCIPAL_SCOPE_LIMIT, CONTAINER_KEK_WRAPS_PER_EPOCH_LIMIT, ContainerKekKeyringWireRecordSchema, isContainerKekKeyringWireRecord, MAX_CONTAINER_KEY_EPOCH, MAX_INLINE_CONTAINER_REKEYS, MAX_ROTATION_CONTAINER_REKEYS, MAX_SEALED_CONTAINER_KEK_KEYRING_BASE64_LENGTH, SEALED_CONTAINER_KEK_KEYRING_AEAD_OVERHEAD_BYTES, SEALED_CONTAINER_KEK_KEYRING_ENTRY_BYTES, SEALED_CONTAINER_KEK_KEYRING_HEADER_BYTES, sealedContainerKekKeyringBytes, } from "./containerKekKeyringWire.js";
|
|
4
5
|
export { MAX_CONTAINER_PATH_LENGTH } from "./containerLimits.js";
|
|
5
6
|
export { MAX_DOCUMENT_SYNC_AUTHORIZATION_PATH_REFS, MAX_DOCUMENT_SYNC_AUTHORIZATION_PATHS, MAX_DOCUMENT_SYNC_CONTENT_KEY_TARGETS, MAX_DOCUMENT_SYNC_OUTGOING_UPDATES, MAX_DOCUMENT_SYNC_PULL_CURSOR_LENGTH, MAX_DOCUMENT_SYNC_REQUEST_BYTES, MAX_DOCUMENT_SYNC_RESPONSE_ENVELOPE_BYTES, MAX_DOCUMENT_SYNC_RESPONSE_PAGE_BYTES, MAX_DOCUMENT_SYNC_RESPONSE_PAGE_UPDATES, MAX_DOCUMENT_SYNC_RESPONSE_UPDATE_PAGE_BYTES, } from "./documentSyncLimits.js";
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { AccessEventBundleWireResponseSchema, AccessManifestBundleWireResponseSchema, AccessManifestBundleWireSchema, isAccessEventBundleWireResponse, isAccessManifestBundleWire, isAccessManifestBundleWireArray, isAccessManifestBundleWireResponse, isOptionalAccessManifestBundleWireArray, } from "./accessManifestBundle.js";
|
|
2
|
-
export { MAX_MULTIPART_BLOB_PART_BYTES } from "./blob.js";
|
|
2
|
+
export { MAX_INITIATE_PART_UPLOADS, MAX_MULTIPART_BLOB_PART_BYTES, } from "./blob.js";
|
|
3
|
+
export { blobPartContentMd5Schema, MAX_MULTIPART_BLOB_PART_NUMBER, MultipartBlobPartDescriptorSchema, MultipartBlobPartUploadSchema, MultipartBlobPartUploadSourceSchema, signedBlobUrlExpirySchema, signedBlobUrlSchema, } from "./blobTransferSchemas.js";
|
|
3
4
|
export { CONTAINER_KEK_LOG_PAGE_LIMIT, CONTAINER_KEK_LOG_PRINCIPAL_SCOPE_LIMIT, CONTAINER_KEK_WRAPS_PER_EPOCH_LIMIT, ContainerKekKeyringWireRecordSchema, isContainerKekKeyringWireRecord, MAX_CONTAINER_KEY_EPOCH, MAX_INLINE_CONTAINER_REKEYS, MAX_ROTATION_CONTAINER_REKEYS, MAX_SEALED_CONTAINER_KEK_KEYRING_BASE64_LENGTH, SEALED_CONTAINER_KEK_KEYRING_AEAD_OVERHEAD_BYTES, SEALED_CONTAINER_KEK_KEYRING_ENTRY_BYTES, SEALED_CONTAINER_KEK_KEYRING_HEADER_BYTES, sealedContainerKekKeyringBytes, } from "./containerKekKeyringWire.js";
|
|
4
5
|
export { MAX_CONTAINER_PATH_LENGTH } from "./containerLimits.js";
|
|
5
6
|
export { MAX_DOCUMENT_SYNC_AUTHORIZATION_PATH_REFS, MAX_DOCUMENT_SYNC_AUTHORIZATION_PATHS, MAX_DOCUMENT_SYNC_CONTENT_KEY_TARGETS, MAX_DOCUMENT_SYNC_OUTGOING_UPDATES, MAX_DOCUMENT_SYNC_PULL_CURSOR_LENGTH, MAX_DOCUMENT_SYNC_REQUEST_BYTES, MAX_DOCUMENT_SYNC_RESPONSE_ENVELOPE_BYTES, MAX_DOCUMENT_SYNC_RESPONSE_PAGE_BYTES, MAX_DOCUMENT_SYNC_RESPONSE_PAGE_UPDATES, MAX_DOCUMENT_SYNC_RESPONSE_UPDATE_PAGE_BYTES, } from "./documentSyncLimits.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,mCAAmC,EAGnC,sCAAsC,EACtC,8BAA8B,EAC9B,+BAA+B,EAC/B,0BAA0B,EAC1B,+BAA+B,EAC/B,kCAAkC,EAClC,uCAAuC,GACxC,MAAM,wBAAwB,CAAC;AAChC,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,mCAAmC,EAGnC,sCAAsC,EACtC,8BAA8B,EAC9B,+BAA+B,EAC/B,0BAA0B,EAC1B,+BAA+B,EAC/B,kCAAkC,EAClC,uCAAuC,GACxC,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,yBAAyB,EACzB,6BAA6B,GAC9B,MAAM,QAAQ,CAAC;AAChB,OAAO,EACL,wBAAwB,EACxB,8BAA8B,EAE9B,iCAAiC,EAEjC,6BAA6B,EAE7B,mCAAmC,EACnC,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,4BAA4B,EAC5B,uCAAuC,EACvC,mCAAmC,EACnC,mCAAmC,EACnC,+BAA+B,EAC/B,uBAAuB,EACvB,2BAA2B,EAC3B,6BAA6B,EAC7B,8CAA8C,EAC9C,gDAAgD,EAChD,wCAAwC,EACxC,yCAAyC,EACzC,8BAA8B,GAC/B,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EACL,yCAAyC,EACzC,qCAAqC,EACrC,qCAAqC,EACrC,kCAAkC,EAClC,oCAAoC,EACpC,+BAA+B,EAC/B,yCAAyC,EACzC,qCAAqC,EACrC,uCAAuC,EACvC,4CAA4C,GAC7C,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,uBAAuB,EACvB,4BAA4B,GAE7B,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,yBAAyB,EACzB,2BAA2B,EAC3B,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,sCAAsC,EAEtC,0CAA0C,GAC3C,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,mCAAmC,EACnC,kCAAkC,GACnC,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,yBAAyB,EACzB,yBAAyB,EACzB,yBAAyB,EACzB,iBAAiB,EACjB,iBAAiB,EACjB,yBAAyB,EACzB,0BAA0B,EAC1B,gBAAgB,EAChB,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,0BAA0B,EAC1B,aAAa,EACb,kBAAkB,EAClB,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,yBAAyB,EACzB,wBAAwB,EACxB,WAAW,EACX,mBAAmB,EACnB,iBAAiB,EACjB,yBAAyB,EACzB,wBAAwB,EACxB,2BAA2B,EAC3B,iBAAiB,GAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,2BAA2B,EAC3B,mBAAmB,EACnB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC","sourcesContent":["export {\n type AccessEventBundleWireResponse,\n AccessEventBundleWireResponseSchema,\n type AccessManifestBundleWire,\n type AccessManifestBundleWireResponse,\n AccessManifestBundleWireResponseSchema,\n AccessManifestBundleWireSchema,\n isAccessEventBundleWireResponse,\n isAccessManifestBundleWire,\n isAccessManifestBundleWireArray,\n isAccessManifestBundleWireResponse,\n isOptionalAccessManifestBundleWireArray,\n} from \"./accessManifestBundle\";\nexport {\n MAX_INITIATE_PART_UPLOADS,\n MAX_MULTIPART_BLOB_PART_BYTES,\n} from \"./blob\";\nexport {\n blobPartContentMd5Schema,\n MAX_MULTIPART_BLOB_PART_NUMBER,\n type MultipartBlobPartDescriptor,\n MultipartBlobPartDescriptorSchema,\n type MultipartBlobPartUpload,\n MultipartBlobPartUploadSchema,\n type MultipartBlobPartUploadSource,\n MultipartBlobPartUploadSourceSchema,\n signedBlobUrlExpirySchema,\n signedBlobUrlSchema,\n} from \"./blobTransferSchemas\";\nexport {\n CONTAINER_KEK_LOG_PAGE_LIMIT,\n CONTAINER_KEK_LOG_PRINCIPAL_SCOPE_LIMIT,\n CONTAINER_KEK_WRAPS_PER_EPOCH_LIMIT,\n ContainerKekKeyringWireRecordSchema,\n isContainerKekKeyringWireRecord,\n MAX_CONTAINER_KEY_EPOCH,\n MAX_INLINE_CONTAINER_REKEYS,\n MAX_ROTATION_CONTAINER_REKEYS,\n MAX_SEALED_CONTAINER_KEK_KEYRING_BASE64_LENGTH,\n SEALED_CONTAINER_KEK_KEYRING_AEAD_OVERHEAD_BYTES,\n SEALED_CONTAINER_KEK_KEYRING_ENTRY_BYTES,\n SEALED_CONTAINER_KEK_KEYRING_HEADER_BYTES,\n sealedContainerKekKeyringBytes,\n} from \"./containerKekKeyringWire\";\nexport { MAX_CONTAINER_PATH_LENGTH } from \"./containerLimits\";\nexport {\n MAX_DOCUMENT_SYNC_AUTHORIZATION_PATH_REFS,\n MAX_DOCUMENT_SYNC_AUTHORIZATION_PATHS,\n MAX_DOCUMENT_SYNC_CONTENT_KEY_TARGETS,\n MAX_DOCUMENT_SYNC_OUTGOING_UPDATES,\n MAX_DOCUMENT_SYNC_PULL_CURSOR_LENGTH,\n MAX_DOCUMENT_SYNC_REQUEST_BYTES,\n MAX_DOCUMENT_SYNC_RESPONSE_ENVELOPE_BYTES,\n MAX_DOCUMENT_SYNC_RESPONSE_PAGE_BYTES,\n MAX_DOCUMENT_SYNC_RESPONSE_PAGE_UPDATES,\n MAX_DOCUMENT_SYNC_RESPONSE_UPDATE_PAGE_BYTES,\n} from \"./documentSyncLimits\";\nexport {\n isSerializedKeyEnvelope,\n isSerializedKeyEnvelopeArray,\n type SerializedKeyEnvelope,\n} from \"./keyEnvelope\";\nexport {\n KNOWN_MANIFEST_HINT_LIMIT,\n KnownManifestsHeadersSchema,\n parseKnownManifestHints,\n} from \"./knownManifestWire\";\nexport {\n isOrganizationReplacementAuthorization,\n type OrganizationReplacementAuthorization,\n OrganizationReplacementAuthorizationSchema,\n} from \"./organizationReplacement\";\nexport {\n PRINCIPAL_POLICY_HISTORY_PAGE_LIMIT,\n PRINCIPAL_POLICY_REPAIR_HEAD_LIMIT,\n} from \"./principalHistoryWire\";\nexport {\n hasArrayProperty,\n hasBooleanProperty,\n hasNonEmptyStringProperty,\n hasNullableNumberProperty,\n hasNullableStringProperty,\n hasNumberProperty,\n hasObjectProperty,\n hasOptionalStringProperty,\n hasPositiveIntegerProperty,\n hasPropertyValue,\n hasStringProperty,\n isNonEmptyStringArray,\n isOptionalRecordArray,\n isOptionalRecordArrayArray,\n isRecordArray,\n isRecordArrayArray,\n isStringArray,\n} from \"./properties\";\nexport {\n AUTH_CHALLENGE_HEX_LENGTH,\n isAuthChallengeHexString,\n isByteArray,\n isByteArrayOfLength,\n isSha256HexString,\n ML_DSA87_PUBLIC_KEY_BYTES,\n ML_DSA87_SIGNATURE_BYTES,\n ML_KEM1024_PUBLIC_KEY_BYTES,\n SHA256_HEX_LENGTH,\n} from \"./protocol\";\nexport {\n CANONICAL_SIGNED_AT_PATTERN,\n isCanonicalSignedAt,\n MAX_SIGNED_AT_YEAR,\n MIN_SIGNED_AT_YEAR,\n SIGNED_AT_CONTRACT,\n} from \"./signedAt\";\nexport { isUuidV4String } from \"./uuid\";\nexport { isWalLsnString, parseWalLsn } from \"./walLsn\";\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { MAX_CONTAINER_DOCUMENT_LANES } from "../../internal/validators/request/index.js";
|
|
1
2
|
import { acknowledgeContainerForce, } from "./idleBackfill.js";
|
|
2
3
|
import { rearmFailedSweepContainers, } from "./laneFailure.js";
|
|
3
|
-
import {
|
|
4
|
+
import { reconcileContainers, } from "./reconcileContainer.js";
|
|
4
5
|
const pendingRetries = new WeakMap();
|
|
5
6
|
export function clearPendingDiscoveryRetries(state) {
|
|
6
7
|
for (const { timer } of pendingRetries.get(state)?.values() ?? [])
|
|
@@ -33,51 +34,62 @@ export function isCurrentReconciliationLifecycle(state, generation) {
|
|
|
33
34
|
return (state.lifecycleGeneration === generation &&
|
|
34
35
|
(state.active || generation === 0));
|
|
35
36
|
}
|
|
36
|
-
// Reconcile
|
|
37
|
-
// back when
|
|
37
|
+
// Reconcile containers whose discovered marks are already set, rolling a mark
|
|
38
|
+
// back when its container is skipped or fails so a transient error (or a
|
|
38
39
|
// container that later becomes eligible) can be retried.
|
|
39
|
-
export async function
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
40
|
+
export async function reconcileMarkedContainers(host, state, containers, fullRevalidation = false) {
|
|
41
|
+
const results = await reconcileContainers(host, containers.map(({ containerId, forceDocumentContentPull }) => ({
|
|
42
|
+
containerId,
|
|
43
|
+
documentIds: fullRevalidation
|
|
44
|
+
? undefined
|
|
45
|
+
: state.forcedDocumentIds.get(containerId),
|
|
46
|
+
forceDocumentContentPull,
|
|
47
|
+
onFullListing: state.initialDocumentProbe.captureListing(containerId),
|
|
48
|
+
onPendingDiscovery: (delayMs) => schedulePendingDiscovery(state, containerId, delayMs),
|
|
49
|
+
})));
|
|
50
|
+
for (const result of results) {
|
|
51
|
+
if (result.status !== "reconciled") {
|
|
52
|
+
state.discoveredContainerIds.delete(result.containerId);
|
|
51
53
|
}
|
|
52
|
-
return reconciled;
|
|
53
|
-
}
|
|
54
|
-
catch (error) {
|
|
55
|
-
state.discoveredContainerIds.delete(containerId);
|
|
56
|
-
throw error;
|
|
57
54
|
}
|
|
55
|
+
return results;
|
|
58
56
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Reconciles one batch of a sweep. Force generations are read before the
|
|
59
|
+
* batched listing starts, so a hint arriving while the batch is in flight
|
|
60
|
+
* keeps its newer generation and runs again.
|
|
61
|
+
*/
|
|
62
|
+
async function sweepContainerBatch(input) {
|
|
63
|
+
const { containerIds, forceAllDocumentContentPulls, host, state } = input;
|
|
64
|
+
const forceGenerations = new Map(containerIds.map((containerId) => [
|
|
65
|
+
containerId,
|
|
66
|
+
state.forcedContainerGenerations.get(containerId),
|
|
67
|
+
]));
|
|
68
|
+
const results = await reconcileMarkedContainers(host, state, containerIds.map((containerId) => ({
|
|
69
|
+
containerId,
|
|
70
|
+
forceDocumentContentPull: forceAllDocumentContentPulls ||
|
|
71
|
+
forceGenerations.get(containerId) !== undefined,
|
|
72
|
+
})), forceAllDocumentContentPulls);
|
|
73
|
+
if (!isCurrentReconciliationLifecycle(state, input.lifecycleGeneration)) {
|
|
74
|
+
return null;
|
|
63
75
|
}
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (reconciled) {
|
|
71
|
-
acknowledgeContainerForce(state, containerId, forceGeneration);
|
|
76
|
+
const failedForces = [];
|
|
77
|
+
let firstError;
|
|
78
|
+
for (const result of results) {
|
|
79
|
+
const forceGeneration = forceGenerations.get(result.containerId);
|
|
80
|
+
if (result.status === "reconciled") {
|
|
81
|
+
acknowledgeContainerForce(state, result.containerId, forceGeneration);
|
|
72
82
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
83
|
+
else if (result.status === "failed") {
|
|
84
|
+
if (forceGeneration !== undefined) {
|
|
85
|
+
failedForces.push({ containerId: result.containerId, forceGeneration });
|
|
86
|
+
}
|
|
87
|
+
if (!host.isIgnorableError(result.error) && firstError === undefined) {
|
|
88
|
+
firstError = result.error;
|
|
89
|
+
}
|
|
78
90
|
}
|
|
79
|
-
return { error, forceGeneration, status: "failed" };
|
|
80
91
|
}
|
|
92
|
+
return { failedForces, firstError };
|
|
81
93
|
}
|
|
82
94
|
export async function sweepKnownContainers(input) {
|
|
83
95
|
const { forceAllDocumentContentPulls, host, knownIds, lifecycleGeneration, state, } = input;
|
|
@@ -98,28 +110,19 @@ export async function sweepKnownContainers(input) {
|
|
|
98
110
|
// block refreshing the rest. Surface the first real error after the sweep.
|
|
99
111
|
let firstError;
|
|
100
112
|
const failedForces = [];
|
|
101
|
-
for (
|
|
102
|
-
const
|
|
103
|
-
|
|
113
|
+
for (let offset = 0; offset < containerIds.length; offset += MAX_CONTAINER_DOCUMENT_LANES) {
|
|
114
|
+
const batch = await sweepContainerBatch({
|
|
115
|
+
containerIds: containerIds.slice(offset, offset + MAX_CONTAINER_DOCUMENT_LANES),
|
|
104
116
|
forceAllDocumentContentPulls,
|
|
105
117
|
host,
|
|
106
118
|
lifecycleGeneration,
|
|
107
119
|
state,
|
|
108
120
|
});
|
|
109
|
-
if (
|
|
121
|
+
if (!batch)
|
|
110
122
|
return;
|
|
111
|
-
|
|
112
|
-
if (
|
|
113
|
-
|
|
114
|
-
failedForces.push({
|
|
115
|
-
containerId,
|
|
116
|
-
forceGeneration: result.forceGeneration,
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
if (!host.isIgnorableError(result.error) && firstError === undefined) {
|
|
120
|
-
firstError = result.error;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
+
failedForces.push(...batch.failedForces);
|
|
124
|
+
if (firstError === undefined)
|
|
125
|
+
firstError = batch.firstError;
|
|
123
126
|
}
|
|
124
127
|
rearmFailedSweepContainers(state, failedForces, lifecycleGeneration);
|
|
125
128
|
if (firstError !== undefined) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knownContainerSweep.js","sourceRoot":"","sources":["knownContainerSweep.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yBAAyB,GAE1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEL,0BAA0B,GAC3B,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAY7D,MAAM,cAAc,GAAG,IAAI,OAAO,EAG/B,CAAC;AAEJ,MAAM,UAAU,4BAA4B,CAC1C,KAA+B;IAE/B,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;QAC/D,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,wBAAwB,CAC/B,KAA+B,EAC/B,WAAmB,EACnB,OAAe;IAEf,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO;IAC1B,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;IACvD,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG,IAAI,GAAG;QAAE,OAAO;IAC5C,IAAI,QAAQ;QAAE,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,mBAAmB,CAAC;IAC7C,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE;QACH,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,UAAU;YAAE,OAAO;QACtE,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACjD,KAAK,CAAC,KAAK,CAAC,OAAO,CACjB,WAAW,EACX,KAAK,CAAC,iBAAiB,KAAK,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAC5D,CAAC;QACF,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC5B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CACrB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,KAA+B,EAC/B,UAAkB;IAElB,OAAO,CACL,KAAK,CAAC,mBAAmB,KAAK,UAAU;QACxC,CAAC,KAAK,CAAC,MAAM,IAAI,UAAU,KAAK,CAAC,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,yEAAyE;AACzE,yDAAyD;AACzD,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,IAAwB,EACxB,KAA+B,EAC/B,WAAmB,EACnB,wBAAiC,EACjC,gBAAgB,GAAG,KAAK;IAExB,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,WAAW,EAAE;YAChE,wBAAwB;YACxB,WAAW,EAAE,gBAAgB;gBAC3B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC;YAC5C,aAAa,EAAE,KAAK,CAAC,oBAAoB,CAAC,cAAc,CAAC,WAAW,CAAC;YACrE,kBAAkB,EAAE,CAAC,OAAO,EAAE,EAAE,CAC9B,wBAAwB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC;SACxD,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAOD,KAAK,UAAU,cAAc,CAAC,KAM7B;IACC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAChE,IAAI,CAAC,gCAAgC,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IACD,MAAM,eAAe,GAAG,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC1E,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAC/C,IAAI,EACJ,KAAK,EACL,WAAW,EACX,KAAK,CAAC,4BAA4B,IAAI,eAAe,KAAK,SAAS,EACnE,KAAK,CAAC,4BAA4B,CACnC,CAAC;QACF,IAAI,CAAC,gCAAgC,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,IAAI,UAAU,EAAE,CAAC;YACf,yBAAyB,CAAC,KAAK,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,gCAAgC,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAM1C;IACC,MAAM,EACJ,4BAA4B,EAC5B,IAAI,EACJ,QAAQ,EACR,mBAAmB,EACnB,KAAK,GACN,GAAG,KAAK,CAAC;IACV,IAAI,CAAC,gCAAgC,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO;IACT,CAAC;IACD,MAAM,YAAY,GAAG,4BAA4B;QAC/C,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,QAAQ,CAAC,MAAM,CACb,CAAC,WAAW,EAAE,EAAE,CACd,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC;YACjD,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,CACjD,CAAC;IACN,2EAA2E;IAC3E,4EAA4E;IAC5E,4EAA4E;IAC5E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IACD,0EAA0E;IAC1E,2EAA2E;IAC3E,IAAI,UAAmB,CAAC;IACxB,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC;YAClC,WAAW;YACX,4BAA4B;YAC5B,IAAI;YACJ,mBAAmB;YACnB,KAAK;SACN,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;gBACzC,YAAY,CAAC,IAAI,CAAC;oBAChB,WAAW;oBACX,eAAe,EAAE,MAAM,CAAC,eAAe;iBACxC,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBACrE,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IACD,0BAA0B,CAAC,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACrE,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,UAAU,CAAC;IACnB,CAAC;AACH,CAAC","sourcesContent":["import {\n acknowledgeContainerForce,\n type IdleBackfillState,\n} from \"./idleBackfill\";\nimport {\n type FailedForcedContainer,\n rearmFailedSweepContainers,\n} from \"./laneFailure\";\nimport { reconcileOneContainer } from \"./reconcileContainer\";\nimport type { ReconciliationHost } from \"./serviceTypes\";\n\ninterface KnownContainerSweepState extends IdleBackfillState {\n active: boolean;\n lane: {\n requestSync: () => void;\n requestSyncAfter: (delayMs: number) => void;\n } | null;\n lifecycleGeneration: number;\n}\n\nconst pendingRetries = new WeakMap<\n KnownContainerSweepState,\n Map<string, { due: number; timer: ReturnType<typeof setTimeout> }>\n>();\n\nexport function clearPendingDiscoveryRetries(\n state: KnownContainerSweepState,\n): void {\n for (const { timer } of pendingRetries.get(state)?.values() ?? [])\n clearTimeout(timer);\n pendingRetries.delete(state);\n}\n\nfunction schedulePendingDiscovery(\n state: KnownContainerSweepState,\n containerId: string,\n delayMs: number,\n): void {\n if (!state.active) return;\n const retries = pendingRetries.get(state) ?? new Map();\n pendingRetries.set(state, retries);\n const due = Date.now() + delayMs;\n const existing = retries.get(containerId);\n if (existing && existing.due <= due) return;\n if (existing) clearTimeout(existing.timer);\n const generation = state.lifecycleGeneration;\n const timer = setTimeout(\n () => {\n retries.delete(containerId);\n if (!state.active || state.lifecycleGeneration !== generation) return;\n state.discoveredContainerIds.delete(containerId);\n state.queue.enqueue(\n containerId,\n state.activeContainerId === containerId ? \"active\" : \"idle\",\n );\n state.lane?.requestSync();\n },\n Math.max(1, delayMs),\n );\n retries.set(containerId, { due, timer });\n}\n\nexport function isCurrentReconciliationLifecycle(\n state: KnownContainerSweepState,\n generation: number,\n): boolean {\n return (\n state.lifecycleGeneration === generation &&\n (state.active || generation === 0)\n );\n}\n\n// Reconcile a container whose discovered mark is already set, rolling the mark\n// back when the container is skipped or fails so a transient error (or a\n// container that later becomes eligible) can be retried.\nexport async function reconcileMarkedContainer(\n host: ReconciliationHost,\n state: KnownContainerSweepState,\n containerId: string,\n forceDocumentContentPull: boolean,\n fullRevalidation = false,\n): Promise<boolean> {\n try {\n const reconciled = await reconcileOneContainer(host, containerId, {\n forceDocumentContentPull,\n documentIds: fullRevalidation\n ? undefined\n : state.forcedDocumentIds.get(containerId),\n onFullListing: state.initialDocumentProbe.captureListing(containerId),\n onPendingDiscovery: (delayMs) =>\n schedulePendingDiscovery(state, containerId, delayMs),\n });\n if (!reconciled) {\n state.discoveredContainerIds.delete(containerId);\n }\n return reconciled;\n } catch (error) {\n state.discoveredContainerIds.delete(containerId);\n throw error;\n }\n}\n\ntype SweepContainerResult =\n | { status: \"failed\"; error: unknown; forceGeneration: number | undefined }\n | { status: \"stale\" }\n | { status: \"succeeded\" };\n\nasync function sweepContainer(input: {\n containerId: string;\n forceAllDocumentContentPulls: boolean;\n host: ReconciliationHost;\n lifecycleGeneration: number;\n state: KnownContainerSweepState;\n}): Promise<SweepContainerResult> {\n const { containerId, host, lifecycleGeneration, state } = input;\n if (!isCurrentReconciliationLifecycle(state, lifecycleGeneration)) {\n return { status: \"stale\" };\n }\n const forceGeneration = state.forcedContainerGenerations.get(containerId);\n try {\n const reconciled = await reconcileMarkedContainer(\n host,\n state,\n containerId,\n input.forceAllDocumentContentPulls || forceGeneration !== undefined,\n input.forceAllDocumentContentPulls,\n );\n if (!isCurrentReconciliationLifecycle(state, lifecycleGeneration)) {\n return { status: \"stale\" };\n }\n if (reconciled) {\n acknowledgeContainerForce(state, containerId, forceGeneration);\n }\n return { status: \"succeeded\" };\n } catch (error) {\n if (!isCurrentReconciliationLifecycle(state, lifecycleGeneration)) {\n return { status: \"stale\" };\n }\n return { error, forceGeneration, status: \"failed\" };\n }\n}\n\nexport async function sweepKnownContainers(input: {\n forceAllDocumentContentPulls: boolean;\n host: ReconciliationHost;\n knownIds: ReadonlyArray<string>;\n lifecycleGeneration: number;\n state: KnownContainerSweepState;\n}): Promise<void> {\n const {\n forceAllDocumentContentPulls,\n host,\n knownIds,\n lifecycleGeneration,\n state,\n } = input;\n if (!isCurrentReconciliationLifecycle(state, lifecycleGeneration)) {\n return;\n }\n const containerIds = forceAllDocumentContentPulls\n ? knownIds\n : knownIds.filter(\n (containerId) =>\n state.forcedContainerGenerations.has(containerId) ||\n !state.discoveredContainerIds.has(containerId),\n );\n // Mark only the containers this sweep will fetch. Automatic root hints are\n // discovery signals, so already-reconciled containers stay settled unless a\n // targeted event forced them; explicit full refreshes still fetch every id.\n for (const containerId of containerIds) {\n state.discoveredContainerIds.add(containerId);\n }\n // Reconcile every container independently: one failing container must not\n // block refreshing the rest. Surface the first real error after the sweep.\n let firstError: unknown;\n const failedForces: FailedForcedContainer[] = [];\n for (const containerId of containerIds) {\n const result = await sweepContainer({\n containerId,\n forceAllDocumentContentPulls,\n host,\n lifecycleGeneration,\n state,\n });\n if (result.status === \"stale\") {\n return;\n }\n if (result.status === \"failed\") {\n if (result.forceGeneration !== undefined) {\n failedForces.push({\n containerId,\n forceGeneration: result.forceGeneration,\n });\n }\n if (!host.isIgnorableError(result.error) && firstError === undefined) {\n firstError = result.error;\n }\n }\n }\n rearmFailedSweepContainers(state, failedForces, lifecycleGeneration);\n if (firstError !== undefined) {\n throw firstError;\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"knownContainerSweep.js","sourceRoot":"","sources":["knownContainerSweep.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAC7E,OAAO,EACL,yBAAyB,GAE1B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEL,0BAA0B,GAC3B,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAY9B,MAAM,cAAc,GAAG,IAAI,OAAO,EAG/B,CAAC;AAEJ,MAAM,UAAU,4BAA4B,CAC1C,KAA+B;IAE/B,KAAK,MAAM,EAAE,KAAK,EAAE,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;QAC/D,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,wBAAwB,CAC/B,KAA+B,EAC/B,WAAmB,EACnB,OAAe;IAEf,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO;IAC1B,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;IACvD,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IACjC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG,IAAI,GAAG;QAAE,OAAO;IAC5C,IAAI,QAAQ;QAAE,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,mBAAmB,CAAC;IAC7C,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE;QACH,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,UAAU;YAAE,OAAO;QACtE,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACjD,KAAK,CAAC,KAAK,CAAC,OAAO,CACjB,WAAW,EACX,KAAK,CAAC,iBAAiB,KAAK,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAC5D,CAAC;QACF,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;IAC5B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CACrB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gCAAgC,CAC9C,KAA+B,EAC/B,UAAkB;IAElB,OAAO,CACL,KAAK,CAAC,mBAAmB,KAAK,UAAU;QACxC,CAAC,KAAK,CAAC,MAAM,IAAI,UAAU,KAAK,CAAC,CAAC,CACnC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,yEAAyE;AACzE,yDAAyD;AACzD,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAwB,EACxB,KAA+B,EAC/B,UAGE,EACF,gBAAgB,GAAG,KAAK;IAExB,MAAM,OAAO,GAAG,MAAM,mBAAmB,CACvC,IAAI,EACJ,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7D,WAAW;QACX,WAAW,EAAE,gBAAgB;YAC3B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC;QAC5C,wBAAwB;QACxB,aAAa,EAAE,KAAK,CAAC,oBAAoB,CAAC,cAAc,CAAC,WAAW,CAAC;QACrE,kBAAkB,EAAE,CAAC,OAAe,EAAE,EAAE,CACtC,wBAAwB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC;KACxD,CAAC,CAAC,CACJ,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACnC,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,mBAAmB,CAAC,KAMlC;IAIC,MAAM,EAAE,YAAY,EAAE,4BAA4B,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC;IAC1E,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAChC,WAAW;QACX,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC;KAClD,CAAC,CACH,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,yBAAyB,CAC7C,IAAI,EACJ,KAAK,EACL,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACjC,WAAW;QACX,wBAAwB,EACtB,4BAA4B;YAC5B,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,SAAS;KAClD,CAAC,CAAC,EACH,4BAA4B,CAC7B,CAAC;IACF,IAAI,CAAC,gCAAgC,CAAC,KAAK,EAAE,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,IAAI,UAAmB,CAAC;IACxB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACnC,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;QACxE,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;gBAClC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,eAAe,EAAE,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;gBACrE,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC;YAC5B,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAM1C;IACC,MAAM,EACJ,4BAA4B,EAC5B,IAAI,EACJ,QAAQ,EACR,mBAAmB,EACnB,KAAK,GACN,GAAG,KAAK,CAAC;IACV,IAAI,CAAC,gCAAgC,CAAC,KAAK,EAAE,mBAAmB,CAAC,EAAE,CAAC;QAClE,OAAO;IACT,CAAC;IACD,MAAM,YAAY,GAAG,4BAA4B;QAC/C,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,QAAQ,CAAC,MAAM,CACb,CAAC,WAAW,EAAE,EAAE,CACd,KAAK,CAAC,0BAA0B,CAAC,GAAG,CAAC,WAAW,CAAC;YACjD,CAAC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,CACjD,CAAC;IACN,2EAA2E;IAC3E,4EAA4E;IAC5E,4EAA4E;IAC5E,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;QACvC,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IACD,0EAA0E;IAC1E,2EAA2E;IAC3E,IAAI,UAAmB,CAAC;IACxB,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,KACE,IAAI,MAAM,GAAG,CAAC,EACd,MAAM,GAAG,YAAY,CAAC,MAAM,EAC5B,MAAM,IAAI,4BAA4B,EACtC,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC;YACtC,YAAY,EAAE,YAAY,CAAC,KAAK,CAC9B,MAAM,EACN,MAAM,GAAG,4BAA4B,CACtC;YACD,4BAA4B;YAC5B,IAAI;YACJ,mBAAmB;YACnB,KAAK;SACN,CAAC,CAAC;QACH,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QACzC,IAAI,UAAU,KAAK,SAAS;YAAE,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IAC9D,CAAC;IACD,0BAA0B,CAAC,KAAK,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACrE,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,UAAU,CAAC;IACnB,CAAC;AACH,CAAC","sourcesContent":["import { MAX_CONTAINER_DOCUMENT_LANES } from \"@tearleads/validators/request\";\nimport {\n acknowledgeContainerForce,\n type IdleBackfillState,\n} from \"./idleBackfill\";\nimport {\n type FailedForcedContainer,\n rearmFailedSweepContainers,\n} from \"./laneFailure\";\nimport {\n type ContainerReconcileResult,\n reconcileContainers,\n} from \"./reconcileContainer\";\nimport type { ReconciliationHost } from \"./serviceTypes\";\n\ninterface KnownContainerSweepState extends IdleBackfillState {\n active: boolean;\n lane: {\n requestSync: () => void;\n requestSyncAfter: (delayMs: number) => void;\n } | null;\n lifecycleGeneration: number;\n}\n\nconst pendingRetries = new WeakMap<\n KnownContainerSweepState,\n Map<string, { due: number; timer: ReturnType<typeof setTimeout> }>\n>();\n\nexport function clearPendingDiscoveryRetries(\n state: KnownContainerSweepState,\n): void {\n for (const { timer } of pendingRetries.get(state)?.values() ?? [])\n clearTimeout(timer);\n pendingRetries.delete(state);\n}\n\nfunction schedulePendingDiscovery(\n state: KnownContainerSweepState,\n containerId: string,\n delayMs: number,\n): void {\n if (!state.active) return;\n const retries = pendingRetries.get(state) ?? new Map();\n pendingRetries.set(state, retries);\n const due = Date.now() + delayMs;\n const existing = retries.get(containerId);\n if (existing && existing.due <= due) return;\n if (existing) clearTimeout(existing.timer);\n const generation = state.lifecycleGeneration;\n const timer = setTimeout(\n () => {\n retries.delete(containerId);\n if (!state.active || state.lifecycleGeneration !== generation) return;\n state.discoveredContainerIds.delete(containerId);\n state.queue.enqueue(\n containerId,\n state.activeContainerId === containerId ? \"active\" : \"idle\",\n );\n state.lane?.requestSync();\n },\n Math.max(1, delayMs),\n );\n retries.set(containerId, { due, timer });\n}\n\nexport function isCurrentReconciliationLifecycle(\n state: KnownContainerSweepState,\n generation: number,\n): boolean {\n return (\n state.lifecycleGeneration === generation &&\n (state.active || generation === 0)\n );\n}\n\n// Reconcile containers whose discovered marks are already set, rolling a mark\n// back when its container is skipped or fails so a transient error (or a\n// container that later becomes eligible) can be retried.\nexport async function reconcileMarkedContainers(\n host: ReconciliationHost,\n state: KnownContainerSweepState,\n containers: ReadonlyArray<{\n readonly containerId: string;\n readonly forceDocumentContentPull: boolean;\n }>,\n fullRevalidation = false,\n): Promise<ReadonlyArray<ContainerReconcileResult>> {\n const results = await reconcileContainers(\n host,\n containers.map(({ containerId, forceDocumentContentPull }) => ({\n containerId,\n documentIds: fullRevalidation\n ? undefined\n : state.forcedDocumentIds.get(containerId),\n forceDocumentContentPull,\n onFullListing: state.initialDocumentProbe.captureListing(containerId),\n onPendingDiscovery: (delayMs: number) =>\n schedulePendingDiscovery(state, containerId, delayMs),\n })),\n );\n for (const result of results) {\n if (result.status !== \"reconciled\") {\n state.discoveredContainerIds.delete(result.containerId);\n }\n }\n return results;\n}\n\n/**\n * Reconciles one batch of a sweep. Force generations are read before the\n * batched listing starts, so a hint arriving while the batch is in flight\n * keeps its newer generation and runs again.\n */\nasync function sweepContainerBatch(input: {\n containerIds: ReadonlyArray<string>;\n forceAllDocumentContentPulls: boolean;\n host: ReconciliationHost;\n lifecycleGeneration: number;\n state: KnownContainerSweepState;\n}): Promise<{\n readonly failedForces: ReadonlyArray<FailedForcedContainer>;\n readonly firstError: unknown;\n} | null> {\n const { containerIds, forceAllDocumentContentPulls, host, state } = input;\n const forceGenerations = new Map(\n containerIds.map((containerId) => [\n containerId,\n state.forcedContainerGenerations.get(containerId),\n ]),\n );\n const results = await reconcileMarkedContainers(\n host,\n state,\n containerIds.map((containerId) => ({\n containerId,\n forceDocumentContentPull:\n forceAllDocumentContentPulls ||\n forceGenerations.get(containerId) !== undefined,\n })),\n forceAllDocumentContentPulls,\n );\n if (!isCurrentReconciliationLifecycle(state, input.lifecycleGeneration)) {\n return null;\n }\n const failedForces: FailedForcedContainer[] = [];\n let firstError: unknown;\n for (const result of results) {\n const forceGeneration = forceGenerations.get(result.containerId);\n if (result.status === \"reconciled\") {\n acknowledgeContainerForce(state, result.containerId, forceGeneration);\n } else if (result.status === \"failed\") {\n if (forceGeneration !== undefined) {\n failedForces.push({ containerId: result.containerId, forceGeneration });\n }\n if (!host.isIgnorableError(result.error) && firstError === undefined) {\n firstError = result.error;\n }\n }\n }\n return { failedForces, firstError };\n}\n\nexport async function sweepKnownContainers(input: {\n forceAllDocumentContentPulls: boolean;\n host: ReconciliationHost;\n knownIds: ReadonlyArray<string>;\n lifecycleGeneration: number;\n state: KnownContainerSweepState;\n}): Promise<void> {\n const {\n forceAllDocumentContentPulls,\n host,\n knownIds,\n lifecycleGeneration,\n state,\n } = input;\n if (!isCurrentReconciliationLifecycle(state, lifecycleGeneration)) {\n return;\n }\n const containerIds = forceAllDocumentContentPulls\n ? knownIds\n : knownIds.filter(\n (containerId) =>\n state.forcedContainerGenerations.has(containerId) ||\n !state.discoveredContainerIds.has(containerId),\n );\n // Mark only the containers this sweep will fetch. Automatic root hints are\n // discovery signals, so already-reconciled containers stay settled unless a\n // targeted event forced them; explicit full refreshes still fetch every id.\n for (const containerId of containerIds) {\n state.discoveredContainerIds.add(containerId);\n }\n // Reconcile every container independently: one failing container must not\n // block refreshing the rest. Surface the first real error after the sweep.\n let firstError: unknown;\n const failedForces: FailedForcedContainer[] = [];\n for (\n let offset = 0;\n offset < containerIds.length;\n offset += MAX_CONTAINER_DOCUMENT_LANES\n ) {\n const batch = await sweepContainerBatch({\n containerIds: containerIds.slice(\n offset,\n offset + MAX_CONTAINER_DOCUMENT_LANES,\n ),\n forceAllDocumentContentPulls,\n host,\n lifecycleGeneration,\n state,\n });\n if (!batch) return;\n failedForces.push(...batch.failedForces);\n if (firstError === undefined) firstError = batch.firstError;\n }\n rearmFailedSweepContainers(state, failedForces, lifecycleGeneration);\n if (firstError !== undefined) {\n throw firstError;\n }\n}\n"]}
|
|
@@ -1,31 +1,63 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
if (!host.canDiscoverContainerDocuments(containerId)) {
|
|
4
|
-
return false;
|
|
5
|
-
}
|
|
1
|
+
async function settleDiscoveredContainer(host, entry, outcome, pending) {
|
|
2
|
+
const { containerId } = entry;
|
|
6
3
|
try {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
});
|
|
12
|
-
if (discovered === null && !pending) {
|
|
13
|
-
return false;
|
|
4
|
+
if (outcome.status === "rejected")
|
|
5
|
+
throw outcome.reason;
|
|
6
|
+
if (outcome.value === null && !pending) {
|
|
7
|
+
return { containerId, status: "skipped" };
|
|
14
8
|
}
|
|
15
9
|
const delta = await host.loadContainerDelta(containerId);
|
|
16
10
|
host.applyReconciled(delta);
|
|
17
11
|
// Hint forces target named documents. Full refreshes revalidate every
|
|
18
12
|
// registered store; passive pulls only open system projections.
|
|
19
|
-
host.requestDocumentContentPull(containerId, delta.documentSummaries,
|
|
13
|
+
host.requestDocumentContentPull(containerId, delta.documentSummaries, entry.forceDocumentContentPull, entry.documentIds);
|
|
20
14
|
// A durable, scheduled retry is settled for this sweep and force request.
|
|
21
15
|
// Its own timer will reopen the lane when evidence is due.
|
|
22
|
-
return
|
|
16
|
+
return { containerId, status: "reconciled" };
|
|
23
17
|
}
|
|
24
18
|
catch (error) {
|
|
25
|
-
|
|
26
|
-
|
|
19
|
+
return host.isIgnorableError(error)
|
|
20
|
+
? { containerId, status: "skipped" }
|
|
21
|
+
: { containerId, error, status: "failed" };
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Lists every entry's documents in one batched discovery, then reconciles each
|
|
26
|
+
* container as soon as its lane applies. Results follow entry order; one
|
|
27
|
+
* container's failure never hides another's result.
|
|
28
|
+
*/
|
|
29
|
+
export async function reconcileContainers(host, entries) {
|
|
30
|
+
// A queued local root/system id can become stale before the document phase.
|
|
31
|
+
const eligible = new Map(entries
|
|
32
|
+
.filter((entry) => host.canDiscoverContainerDocuments(entry.containerId))
|
|
33
|
+
.map((entry) => [entry.containerId, entry]));
|
|
34
|
+
const pending = new Set();
|
|
35
|
+
const results = new Map();
|
|
36
|
+
const settle = async (outcome) => {
|
|
37
|
+
const entry = eligible.get(outcome.containerId);
|
|
38
|
+
if (!entry || results.has(entry.containerId))
|
|
39
|
+
return;
|
|
40
|
+
results.set(entry.containerId, await settleDiscoveredContainer(host, entry, outcome, pending.has(entry.containerId)));
|
|
41
|
+
};
|
|
42
|
+
if (eligible.size > 0) {
|
|
43
|
+
try {
|
|
44
|
+
const outcomes = await host.discoverContainerDocuments([...eligible.values()].map((entry) => ({
|
|
45
|
+
containerId: entry.containerId,
|
|
46
|
+
onFullListing: entry.onFullListing,
|
|
47
|
+
onPendingDiscovery: (delayMs) => {
|
|
48
|
+
pending.add(entry.containerId);
|
|
49
|
+
entry.onPendingDiscovery?.(delayMs);
|
|
50
|
+
},
|
|
51
|
+
})), settle);
|
|
52
|
+
for (const outcome of outcomes)
|
|
53
|
+
await settle(outcome);
|
|
54
|
+
}
|
|
55
|
+
catch (reason) {
|
|
56
|
+
for (const containerId of eligible.keys()) {
|
|
57
|
+
await settle({ containerId, reason, status: "rejected" });
|
|
58
|
+
}
|
|
27
59
|
}
|
|
28
|
-
throw error;
|
|
29
60
|
}
|
|
61
|
+
return entries.map(({ containerId }) => results.get(containerId) ?? { containerId, status: "skipped" });
|
|
30
62
|
}
|
|
31
63
|
//# sourceMappingURL=reconcileContainer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reconcileContainer.js","sourceRoot":"","sources":["reconcileContainer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"reconcileContainer.js","sourceRoot":"","sources":["reconcileContainer.ts"],"names":[],"mappings":"AAuBA,KAAK,UAAU,yBAAyB,CACtC,IAAwB,EACxB,KAA8B,EAC9B,OAAkC,EAClC,OAAgB;IAEhB,MAAM,EAAE,WAAW,EAAE,GAAG,KAAK,CAAC;IAC9B,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC;QACxD,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACvC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;QACzD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC5B,sEAAsE;QACtE,gEAAgE;QAChE,IAAI,CAAC,0BAA0B,CAC7B,WAAW,EACX,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,wBAAwB,EAC9B,KAAK,CAAC,WAAW,CAClB,CAAC;QACF,0EAA0E;QAC1E,2DAA2D;QAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;YACjC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE;YACpC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAC/C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAwB,EACxB,OAA+C;IAE/C,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,OAAO;SACJ,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;SACxE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAC9C,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoC,CAAC;IAC5D,MAAM,MAAM,GAAG,KAAK,EAAE,OAAkC,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAChD,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC;YAAE,OAAO;QACrD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,WAAW,EACjB,MAAM,yBAAyB,CAC7B,IAAI,EACJ,KAAK,EACL,OAAO,EACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAC/B,CACF,CAAC;IACJ,CAAC,CAAC;IACF,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,0BAA0B,CACpD,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACrC,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,kBAAkB,EAAE,CAAC,OAAe,EAAE,EAAE;oBACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;oBAC/B,KAAK,CAAC,kBAAkB,EAAE,CAAC,OAAO,CAAC,CAAC;gBACtC,CAAC;aACF,CAAC,CAAC,EACH,MAAM,CACP,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,QAAQ;gBAAE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,MAAM,EAAE,CAAC;YAChB,KAAK,MAAM,WAAW,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC1C,MAAM,MAAM,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAChB,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAClB,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,CACjE,CAAC;AACJ,CAAC","sourcesContent":["import type {\n ContainerDiscoveryOutcome,\n ReconciliationHost,\n} from \"./serviceTypes\";\n\nexport interface ContainerReconcileEntry {\n readonly containerId: string;\n readonly documentIds?: ReadonlySet<string> | undefined;\n readonly forceDocumentContentPull: boolean;\n readonly onFullListing?:\n | ((documentIds: ReadonlyArray<string>) => void)\n | undefined;\n readonly onPendingDiscovery?: ((delayMs: number) => void) | undefined;\n}\n\nexport type ContainerReconcileResult =\n | { readonly containerId: string; readonly status: \"reconciled\" | \"skipped\" }\n | {\n readonly containerId: string;\n readonly error: unknown;\n readonly status: \"failed\";\n };\n\nasync function settleDiscoveredContainer(\n host: ReconciliationHost,\n entry: ContainerReconcileEntry,\n outcome: ContainerDiscoveryOutcome,\n pending: boolean,\n): Promise<ContainerReconcileResult> {\n const { containerId } = entry;\n try {\n if (outcome.status === \"rejected\") throw outcome.reason;\n if (outcome.value === null && !pending) {\n return { containerId, status: \"skipped\" };\n }\n const delta = await host.loadContainerDelta(containerId);\n host.applyReconciled(delta);\n // Hint forces target named documents. Full refreshes revalidate every\n // registered store; passive pulls only open system projections.\n host.requestDocumentContentPull(\n containerId,\n delta.documentSummaries,\n entry.forceDocumentContentPull,\n entry.documentIds,\n );\n // A durable, scheduled retry is settled for this sweep and force request.\n // Its own timer will reopen the lane when evidence is due.\n return { containerId, status: \"reconciled\" };\n } catch (error) {\n return host.isIgnorableError(error)\n ? { containerId, status: \"skipped\" }\n : { containerId, error, status: \"failed\" };\n }\n}\n\n/**\n * Lists every entry's documents in one batched discovery, then reconciles each\n * container as soon as its lane applies. Results follow entry order; one\n * container's failure never hides another's result.\n */\nexport async function reconcileContainers(\n host: ReconciliationHost,\n entries: ReadonlyArray<ContainerReconcileEntry>,\n): Promise<ReadonlyArray<ContainerReconcileResult>> {\n // A queued local root/system id can become stale before the document phase.\n const eligible = new Map(\n entries\n .filter((entry) => host.canDiscoverContainerDocuments(entry.containerId))\n .map((entry) => [entry.containerId, entry]),\n );\n const pending = new Set<string>();\n const results = new Map<string, ContainerReconcileResult>();\n const settle = async (outcome: ContainerDiscoveryOutcome) => {\n const entry = eligible.get(outcome.containerId);\n if (!entry || results.has(entry.containerId)) return;\n results.set(\n entry.containerId,\n await settleDiscoveredContainer(\n host,\n entry,\n outcome,\n pending.has(entry.containerId),\n ),\n );\n };\n if (eligible.size > 0) {\n try {\n const outcomes = await host.discoverContainerDocuments(\n [...eligible.values()].map((entry) => ({\n containerId: entry.containerId,\n onFullListing: entry.onFullListing,\n onPendingDiscovery: (delayMs: number) => {\n pending.add(entry.containerId);\n entry.onPendingDiscovery?.(delayMs);\n },\n })),\n settle,\n );\n for (const outcome of outcomes) await settle(outcome);\n } catch (reason) {\n for (const containerId of eligible.keys()) {\n await settle({ containerId, reason, status: \"rejected\" });\n }\n }\n }\n return entries.map(\n ({ containerId }) =>\n results.get(containerId) ?? { containerId, status: \"skipped\" },\n );\n}\n"]}
|
|
@@ -1,20 +1,36 @@
|
|
|
1
1
|
import { getOrCreateDomainSyncCoordinator, } from "../../data/sync/syncCoordinator.js";
|
|
2
2
|
import { acknowledgeContainerForce, activateContainer, enqueueKnownContainersForIdleBackfill, markContainerForced, } from "./idleBackfill.js";
|
|
3
3
|
import { createInitialDocumentProbe } from "./initialDocumentProbe.js";
|
|
4
|
-
import { clearPendingDiscoveryRetries, isCurrentReconciliationLifecycle,
|
|
4
|
+
import { clearPendingDiscoveryRetries, isCurrentReconciliationLifecycle, reconcileMarkedContainers, sweepKnownContainers, } from "./knownContainerSweep.js";
|
|
5
5
|
import { rearmFailedContainer } from "./laneFailure.js";
|
|
6
6
|
import { clearOriginatedDocuments } from "./originatedDocuments.js";
|
|
7
7
|
import { createReconcileQueue } from "./queue.js";
|
|
8
8
|
function canReconcile(status) {
|
|
9
9
|
return status.dbStatus === "ready" && status.isAuthenticated && status.online;
|
|
10
10
|
}
|
|
11
|
+
/**
|
|
12
|
+
* Queued containers one lane run lists together. Each still verifies and
|
|
13
|
+
* applies on its own; the bound keeps one run's head verification well
|
|
14
|
+
* inside the lane watchdog.
|
|
15
|
+
*/
|
|
16
|
+
const RECONCILE_LANE_BATCH_SIZE = 4;
|
|
17
|
+
function dequeueContainerBatch(state) {
|
|
18
|
+
const containerIds = [];
|
|
19
|
+
while (containerIds.length < RECONCILE_LANE_BATCH_SIZE) {
|
|
20
|
+
const containerId = state.queue.dequeue();
|
|
21
|
+
if (!containerId)
|
|
22
|
+
break;
|
|
23
|
+
containerIds.push(containerId);
|
|
24
|
+
}
|
|
25
|
+
return containerIds;
|
|
26
|
+
}
|
|
11
27
|
async function runReconcileLane(host, state) {
|
|
12
28
|
if (!state.active || !canReconcile(host.getRuntimeStatus())) {
|
|
13
29
|
return;
|
|
14
30
|
}
|
|
15
31
|
const lifecycleGeneration = state.lifecycleGeneration;
|
|
16
|
-
const
|
|
17
|
-
if (
|
|
32
|
+
const containerIds = dequeueContainerBatch(state);
|
|
33
|
+
if (containerIds.length === 0) {
|
|
18
34
|
await state.initialDocumentProbe.run();
|
|
19
35
|
scheduleProbeContinuation(host, state);
|
|
20
36
|
return;
|
|
@@ -22,22 +38,34 @@ async function runReconcileLane(host, state) {
|
|
|
22
38
|
// Double-check at run time: a container may have been discovered (by an
|
|
23
39
|
// explicit refresh or an earlier lane pass) after it was queued. Mark it
|
|
24
40
|
// discovered up front to collapse concurrent re-enqueues into one fetch, but
|
|
25
|
-
// roll the mark back on failure so a transient error can be retried
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
41
|
+
// roll the mark back on failure so a transient error can be retried. Force
|
|
42
|
+
// generations are read before the batched listing starts.
|
|
43
|
+
const batch = containerIds.flatMap((containerId) => {
|
|
44
|
+
const forceGeneration = state.forcedContainerGenerations.get(containerId);
|
|
45
|
+
if (forceGeneration === undefined &&
|
|
46
|
+
state.discoveredContainerIds.has(containerId)) {
|
|
47
|
+
return [];
|
|
48
|
+
}
|
|
29
49
|
state.discoveredContainerIds.add(containerId);
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
50
|
+
return [{ containerId, forceGeneration }];
|
|
51
|
+
});
|
|
52
|
+
const results = await reconcileMarkedContainers(host, state, batch.map(({ containerId, forceGeneration }) => ({
|
|
53
|
+
containerId,
|
|
54
|
+
forceDocumentContentPull: forceGeneration !== undefined,
|
|
55
|
+
})));
|
|
56
|
+
let firstError;
|
|
57
|
+
results.forEach((result, index) => {
|
|
58
|
+
const forceGeneration = batch[index]?.forceGeneration;
|
|
59
|
+
if (result.status === "reconciled") {
|
|
60
|
+
acknowledgeContainerForce(state, result.containerId, forceGeneration);
|
|
35
61
|
}
|
|
36
|
-
|
|
37
|
-
rearmFailedContainer(state, containerId, forceGeneration, lifecycleGeneration);
|
|
38
|
-
|
|
62
|
+
else if (result.status === "failed") {
|
|
63
|
+
rearmFailedContainer(state, result.containerId, forceGeneration, lifecycleGeneration);
|
|
64
|
+
firstError ??= result.error;
|
|
39
65
|
}
|
|
40
|
-
}
|
|
66
|
+
});
|
|
67
|
+
if (firstError !== undefined)
|
|
68
|
+
throw firstError;
|
|
41
69
|
// Keep draining: schedule another pass while the queue holds work.
|
|
42
70
|
if (state.queue.size > 0) {
|
|
43
71
|
state.lane?.requestSync();
|