@opendatalabs/personal-server-ts-core 0.0.1-canary.01ca694 → 0.0.1-canary.052d47e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/index.d.ts +282 -9
- package/dist/api/index.d.ts.map +1 -1
- package/dist/api/index.js +867 -163
- package/dist/api/index.js.map +1 -1
- package/dist/auth/request.d.ts.map +1 -1
- package/dist/auth/request.js.map +1 -1
- package/dist/client.d.ts +8 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +21 -3
- package/dist/client.js.map +1 -1
- package/dist/contracts/binary.d.ts +12 -0
- package/dist/contracts/binary.d.ts.map +1 -1
- package/dist/contracts/binary.js +31 -0
- package/dist/contracts/binary.js.map +1 -1
- package/dist/contracts/data.d.ts +51 -5
- package/dist/contracts/data.d.ts.map +1 -1
- package/dist/contracts/data.js +166 -30
- package/dist/contracts/data.js.map +1 -1
- package/dist/contracts/grants.d.ts +7 -5
- package/dist/contracts/grants.d.ts.map +1 -1
- package/dist/contracts/grants.js +60 -37
- package/dist/contracts/grants.js.map +1 -1
- package/dist/contracts/index.d.ts +1 -1
- package/dist/contracts/index.d.ts.map +1 -1
- package/dist/contracts/index.js +1 -1
- package/dist/contracts/index.js.map +1 -1
- package/dist/derivatives/api.d.ts +48 -0
- package/dist/derivatives/api.d.ts.map +1 -0
- package/dist/derivatives/api.js +366 -0
- package/dist/derivatives/api.js.map +1 -0
- package/dist/derivatives/compute.d.ts +106 -0
- package/dist/derivatives/compute.d.ts.map +1 -0
- package/dist/derivatives/compute.js +414 -0
- package/dist/derivatives/compute.js.map +1 -0
- package/dist/derivatives/e2ee/aad.d.ts +22 -0
- package/dist/derivatives/e2ee/aad.d.ts.map +1 -0
- package/dist/derivatives/e2ee/aad.js +32 -0
- package/dist/derivatives/e2ee/aad.js.map +1 -0
- package/dist/derivatives/e2ee/attestation.d.ts +134 -0
- package/dist/derivatives/e2ee/attestation.d.ts.map +1 -0
- package/dist/derivatives/e2ee/attestation.js +248 -0
- package/dist/derivatives/e2ee/attestation.js.map +1 -0
- package/dist/derivatives/e2ee/index.d.ts +6 -0
- package/dist/derivatives/e2ee/index.d.ts.map +1 -0
- package/dist/derivatives/e2ee/index.js +6 -0
- package/dist/derivatives/e2ee/index.js.map +1 -0
- package/dist/derivatives/e2ee/jcs.d.ts +17 -0
- package/dist/derivatives/e2ee/jcs.d.ts.map +1 -0
- package/dist/derivatives/e2ee/jcs.js +55 -0
- package/dist/derivatives/e2ee/jcs.js.map +1 -0
- package/dist/derivatives/e2ee/phala.d.ts +62 -0
- package/dist/derivatives/e2ee/phala.d.ts.map +1 -0
- package/dist/derivatives/e2ee/phala.js +168 -0
- package/dist/derivatives/e2ee/phala.js.map +1 -0
- package/dist/derivatives/e2ee/suite.d.ts +56 -0
- package/dist/derivatives/e2ee/suite.d.ts.map +1 -0
- package/dist/derivatives/e2ee/suite.js +164 -0
- package/dist/derivatives/e2ee/suite.js.map +1 -0
- package/dist/derivatives/index.d.ts +10 -0
- package/dist/derivatives/index.d.ts.map +1 -0
- package/dist/derivatives/index.js +10 -0
- package/dist/derivatives/index.js.map +1 -0
- package/dist/derivatives/inference.d.ts +151 -0
- package/dist/derivatives/inference.d.ts.map +1 -0
- package/dist/derivatives/inference.js +262 -0
- package/dist/derivatives/inference.js.map +1 -0
- package/dist/derivatives/prompt.d.ts +64 -0
- package/dist/derivatives/prompt.d.ts.map +1 -0
- package/dist/derivatives/prompt.js +193 -0
- package/dist/derivatives/prompt.js.map +1 -0
- package/dist/derivatives/registration.d.ts +54 -0
- package/dist/derivatives/registration.d.ts.map +1 -0
- package/dist/derivatives/registration.js +180 -0
- package/dist/derivatives/registration.js.map +1 -0
- package/dist/derivatives/scheduler.d.ts +84 -0
- package/dist/derivatives/scheduler.d.ts.map +1 -0
- package/dist/derivatives/scheduler.js +257 -0
- package/dist/derivatives/scheduler.js.map +1 -0
- package/dist/derivatives/store.d.ts +14 -0
- package/dist/derivatives/store.d.ts.map +1 -0
- package/dist/derivatives/store.js +77 -0
- package/dist/derivatives/store.js.map +1 -0
- package/dist/derivatives/types.d.ts +92 -0
- package/dist/derivatives/types.d.ts.map +1 -0
- package/dist/derivatives/types.js +27 -0
- package/dist/derivatives/types.js.map +1 -0
- package/dist/errors/catalog.d.ts +75 -0
- package/dist/errors/catalog.d.ts.map +1 -1
- package/dist/errors/catalog.js +104 -0
- package/dist/errors/catalog.js.map +1 -1
- package/dist/grants/index.d.ts +2 -1
- package/dist/grants/index.d.ts.map +1 -1
- package/dist/grants/index.js +1 -1
- package/dist/grants/index.js.map +1 -1
- package/dist/keys/index.d.ts +1 -0
- package/dist/keys/index.d.ts.map +1 -1
- package/dist/keys/index.js +1 -1
- package/dist/keys/index.js.map +1 -1
- package/dist/keys/public-only-account.d.ts +8 -0
- package/dist/keys/public-only-account.d.ts.map +1 -0
- package/dist/keys/public-only-account.js +21 -0
- package/dist/keys/public-only-account.js.map +1 -0
- package/dist/keys/server-account.d.ts +2 -2
- package/dist/keys/server-account.d.ts.map +1 -1
- package/dist/lineage/attestation.d.ts +42 -0
- package/dist/lineage/attestation.d.ts.map +1 -0
- package/dist/lineage/attestation.js +25 -0
- package/dist/lineage/attestation.js.map +1 -0
- package/dist/lineage/gateway.d.ts +105 -0
- package/dist/lineage/gateway.d.ts.map +1 -0
- package/dist/lineage/gateway.js +242 -0
- package/dist/lineage/gateway.js.map +1 -0
- package/dist/lineage/index.d.ts +5 -0
- package/dist/lineage/index.d.ts.map +1 -0
- package/dist/lineage/index.js +5 -0
- package/dist/lineage/index.js.map +1 -0
- package/dist/lineage/lineage.d.ts +141 -0
- package/dist/lineage/lineage.d.ts.map +1 -0
- package/dist/lineage/lineage.js +255 -0
- package/dist/lineage/lineage.js.map +1 -0
- package/dist/logging/access-log.d.ts +10 -1
- package/dist/logging/access-log.d.ts.map +1 -1
- package/dist/mcp/activity.d.ts.map +1 -1
- package/dist/mcp/activity.js.map +1 -1
- package/dist/mcp/connection-api.d.ts.map +1 -1
- package/dist/mcp/connection-api.js +0 -1
- package/dist/mcp/connection-api.js.map +1 -1
- package/dist/mcp/index.d.ts +1 -0
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +1 -0
- package/dist/mcp/index.js.map +1 -1
- package/dist/mcp/read-client.d.ts +61 -1
- package/dist/mcp/read-client.d.ts.map +1 -1
- package/dist/mcp/read-client.js +397 -42
- package/dist/mcp/read-client.js.map +1 -1
- package/dist/mcp/resources.d.ts +37 -0
- package/dist/mcp/resources.d.ts.map +1 -0
- package/dist/mcp/resources.js +114 -0
- package/dist/mcp/resources.js.map +1 -0
- package/dist/mcp/search/index.d.ts.map +1 -1
- package/dist/mcp/search/index.js +108 -21
- package/dist/mcp/search/index.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +27 -9
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/session.d.ts +160 -0
- package/dist/mcp/session.d.ts.map +1 -0
- package/dist/mcp/session.js +243 -0
- package/dist/mcp/session.js.map +1 -0
- package/dist/mcp/tools.d.ts +29 -3
- package/dist/mcp/tools.d.ts.map +1 -1
- package/dist/mcp/tools.js +515 -48
- package/dist/mcp/tools.js.map +1 -1
- package/dist/mcp/types.d.ts.map +1 -1
- package/dist/payment/index.d.ts +2 -0
- package/dist/payment/index.d.ts.map +1 -0
- package/dist/payment/index.js +2 -0
- package/dist/payment/index.js.map +1 -0
- package/dist/payment/x402.d.ts +155 -0
- package/dist/payment/x402.d.ts.map +1 -0
- package/dist/payment/x402.js +402 -0
- package/dist/payment/x402.js.map +1 -0
- package/dist/policy/data-read.d.ts +18 -2
- package/dist/policy/data-read.d.ts.map +1 -1
- package/dist/policy/data-read.js +80 -49
- package/dist/policy/data-read.js.map +1 -1
- package/dist/policy/data-write.d.ts +69 -0
- package/dist/policy/data-write.d.ts.map +1 -0
- package/dist/policy/data-write.js +131 -0
- package/dist/policy/data-write.js.map +1 -0
- package/dist/policy/index.d.ts +3 -1
- package/dist/policy/index.d.ts.map +1 -1
- package/dist/policy/index.js +3 -1
- package/dist/policy/index.js.map +1 -1
- package/dist/policy/signed-artifacts.d.ts +17 -0
- package/dist/policy/signed-artifacts.d.ts.map +1 -0
- package/dist/policy/signed-artifacts.js +110 -0
- package/dist/policy/signed-artifacts.js.map +1 -0
- package/dist/ports/index.d.ts +159 -32
- package/dist/ports/index.d.ts.map +1 -1
- package/dist/ports/index.js +6 -5
- package/dist/ports/index.js.map +1 -1
- package/dist/schemas/server-config.d.ts +49 -4
- package/dist/schemas/server-config.d.ts.map +1 -1
- package/dist/schemas/server-config.js +95 -13
- package/dist/schemas/server-config.js.map +1 -1
- package/dist/signing/index.d.ts +2 -2
- package/dist/signing/index.d.ts.map +1 -1
- package/dist/signing/index.js +1 -1
- package/dist/signing/index.js.map +1 -1
- package/dist/signing/request-signer.d.ts +2 -0
- package/dist/signing/request-signer.d.ts.map +1 -1
- package/dist/signing/request-signer.js +1 -0
- package/dist/signing/request-signer.js.map +1 -1
- package/dist/signing/signer.d.ts +27 -25
- package/dist/signing/signer.d.ts.map +1 -1
- package/dist/signing/signer.js +27 -51
- package/dist/signing/signer.js.map +1 -1
- package/dist/storage/adapters/index.d.ts +1 -1
- package/dist/storage/adapters/index.d.ts.map +1 -1
- package/dist/storage/adapters/index.js +1 -1
- package/dist/storage/adapters/index.js.map +1 -1
- package/dist/storage/adapters/interface.d.ts +10 -0
- package/dist/storage/adapters/interface.d.ts.map +1 -1
- package/dist/storage/adapters/sdk.d.ts +11 -1
- package/dist/storage/adapters/sdk.d.ts.map +1 -1
- package/dist/storage/adapters/sdk.js +38 -3
- package/dist/storage/adapters/sdk.js.map +1 -1
- package/dist/storage/adapters/vana.d.ts +3 -0
- package/dist/storage/adapters/vana.d.ts.map +1 -1
- package/dist/storage/adapters/vana.js +95 -2
- package/dist/storage/adapters/vana.js.map +1 -1
- package/dist/storage/blocks/errors.d.ts.map +1 -1
- package/dist/storage/blocks/errors.js.map +1 -1
- package/dist/storage/blocks/index.d.ts +2 -0
- package/dist/storage/blocks/index.d.ts.map +1 -1
- package/dist/storage/blocks/index.js +2 -0
- package/dist/storage/blocks/index.js.map +1 -1
- package/dist/storage/blocks/select.d.ts +29 -0
- package/dist/storage/blocks/select.d.ts.map +1 -0
- package/dist/storage/blocks/select.js +120 -0
- package/dist/storage/blocks/select.js.map +1 -0
- package/dist/storage/blocks/types.d.ts +7 -0
- package/dist/storage/blocks/types.d.ts.map +1 -1
- package/dist/storage/index/manager.d.ts +30 -2
- package/dist/storage/index/manager.d.ts.map +1 -1
- package/dist/storage/index/schema.d.ts +1 -1
- package/dist/storage/index/schema.js +1 -1
- package/dist/storage/index/types.d.ts +6 -1
- package/dist/storage/index/types.d.ts.map +1 -1
- package/dist/sync/data-point-feed.d.ts +35 -0
- package/dist/sync/data-point-feed.d.ts.map +1 -0
- package/dist/sync/data-point-feed.js +122 -0
- package/dist/sync/data-point-feed.js.map +1 -0
- package/dist/sync/data-point-id.d.ts +7 -0
- package/dist/sync/data-point-id.d.ts.map +1 -0
- package/dist/sync/data-point-id.js +16 -0
- package/dist/sync/data-point-id.js.map +1 -0
- package/dist/sync/delete-data-port.d.ts +50 -0
- package/dist/sync/delete-data-port.d.ts.map +1 -0
- package/dist/sync/delete-data-port.js +244 -0
- package/dist/sync/delete-data-port.js.map +1 -0
- package/dist/sync/engine/sync-manager.d.ts +16 -4
- package/dist/sync/engine/sync-manager.d.ts.map +1 -1
- package/dist/sync/engine/sync-manager.js +93 -32
- package/dist/sync/engine/sync-manager.js.map +1 -1
- package/dist/sync/index.d.ts +7 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +7 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/issues.d.ts.map +1 -1
- package/dist/sync/issues.js +21 -3
- package/dist/sync/issues.js.map +1 -1
- package/dist/sync/pending-blob-deletions.d.ts +29 -0
- package/dist/sync/pending-blob-deletions.d.ts.map +1 -0
- package/dist/sync/pending-blob-deletions.js +116 -0
- package/dist/sync/pending-blob-deletions.js.map +1 -0
- package/dist/sync/retry-memory.d.ts +33 -0
- package/dist/sync/retry-memory.d.ts.map +1 -0
- package/dist/sync/retry-memory.js +68 -0
- package/dist/sync/retry-memory.js.map +1 -0
- package/dist/sync/scope-deletions.d.ts +155 -0
- package/dist/sync/scope-deletions.d.ts.map +1 -0
- package/dist/sync/scope-deletions.js +225 -0
- package/dist/sync/scope-deletions.js.map +1 -0
- package/dist/sync/tombstone.d.ts +27 -0
- package/dist/sync/tombstone.d.ts.map +1 -0
- package/dist/sync/tombstone.js +28 -0
- package/dist/sync/tombstone.js.map +1 -0
- package/dist/sync/workers/delete.d.ts +152 -31
- package/dist/sync/workers/delete.d.ts.map +1 -1
- package/dist/sync/workers/delete.js +457 -55
- package/dist/sync/workers/delete.js.map +1 -1
- package/dist/sync/workers/download.d.ts +108 -27
- package/dist/sync/workers/download.d.ts.map +1 -1
- package/dist/sync/workers/download.js +321 -126
- package/dist/sync/workers/download.js.map +1 -1
- package/dist/sync/workers/upload.d.ts +65 -10
- package/dist/sync/workers/upload.d.ts.map +1 -1
- package/dist/sync/workers/upload.js +371 -42
- package/dist/sync/workers/upload.js.map +1 -1
- package/dist/test-utils/e2ee-fixtures.d.ts +94 -0
- package/dist/test-utils/e2ee-fixtures.d.ts.map +1 -0
- package/dist/test-utils/e2ee-fixtures.js +116 -0
- package/dist/test-utils/e2ee-fixtures.js.map +1 -0
- package/dist/test-utils/e2ee-gateway.d.ts +61 -0
- package/dist/test-utils/e2ee-gateway.d.ts.map +1 -0
- package/dist/test-utils/e2ee-gateway.js +248 -0
- package/dist/test-utils/e2ee-gateway.js.map +1 -0
- package/dist/test-utils/index.d.ts +3 -0
- package/dist/test-utils/index.d.ts.map +1 -1
- package/dist/test-utils/index.js +3 -0
- package/dist/test-utils/index.js.map +1 -1
- package/dist/test-utils/memory-storage.d.ts +12 -0
- package/dist/test-utils/memory-storage.d.ts.map +1 -0
- package/dist/test-utils/memory-storage.js +146 -0
- package/dist/test-utils/memory-storage.js.map +1 -0
- package/dist/test-utils/wallet.d.ts +6 -0
- package/dist/test-utils/wallet.d.ts.map +1 -1
- package/dist/test-utils/wallet.js +46 -0
- package/dist/test-utils/wallet.js.map +1 -1
- package/dist/write/attribution.d.ts +185 -0
- package/dist/write/attribution.d.ts.map +1 -0
- package/dist/write/attribution.js +450 -0
- package/dist/write/attribution.js.map +1 -0
- package/dist/write/authorize.d.ts +77 -0
- package/dist/write/authorize.d.ts.map +1 -0
- package/dist/write/authorize.js +109 -0
- package/dist/write/authorize.js.map +1 -0
- package/dist/write/index.d.ts +5 -0
- package/dist/write/index.d.ts.map +1 -0
- package/dist/write/index.js +5 -0
- package/dist/write/index.js.map +1 -0
- package/dist/write/route.d.ts +61 -0
- package/dist/write/route.d.ts.map +1 -0
- package/dist/write/route.js +132 -0
- package/dist/write/route.js.map +1 -0
- package/dist/write/session.d.ts +96 -0
- package/dist/write/session.d.ts.map +1 -0
- package/dist/write/session.js +136 -0
- package/dist/write/session.js.map +1 -0
- package/package.json +19 -3
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { uploadAll } from "../workers/upload.js";
|
|
2
2
|
import { downloadAll } from "../workers/download.js";
|
|
3
|
-
import {
|
|
3
|
+
import { deleteScope, retryPendingBlobDeletions, } from "../workers/delete.js";
|
|
4
|
+
import { createDownloadRetryMemory } from "../retry-memory.js";
|
|
4
5
|
const MAX_ERRORS = 10;
|
|
5
6
|
export function createSyncManager(uploadDeps, downloadDeps, options) {
|
|
6
7
|
const pollInterval = options?.pollInterval ?? 60_000;
|
|
7
8
|
const uploadBatchSize = options?.uploadBatchSize ?? 50;
|
|
8
9
|
const notifyDebounceMs = options?.notifyDebounceMs ?? 500;
|
|
10
|
+
const transferMode = options?.transferMode ?? "upload-download";
|
|
9
11
|
let intervalId = null;
|
|
10
12
|
let notifyTimerId = null;
|
|
11
13
|
let isRunning = false;
|
|
@@ -15,13 +17,41 @@ export function createSyncManager(uploadDeps, downloadDeps, options) {
|
|
|
15
17
|
let errors = [];
|
|
16
18
|
let cycleInFlight = null;
|
|
17
19
|
let rerunRequested = false;
|
|
20
|
+
let needsFullReconcile = true;
|
|
21
|
+
// One retry memory per manager (= per boot session): a blob that 404s is
|
|
22
|
+
// attempted once per session, not once per cycle; transient storage errors
|
|
23
|
+
// back off exponentially instead of re-firing on every cycle.
|
|
24
|
+
const downloadRetryMemory = createDownloadRetryMemory();
|
|
25
|
+
const dataPointFeed = downloadDeps.dataPointFeed ?? uploadDeps.dataPointFeed;
|
|
26
|
+
const scopeDeletions = uploadDeps.scopeDeletions ?? downloadDeps.scopeDeletions;
|
|
27
|
+
// The upload worker queues guarded cleanup for ciphertext it had to
|
|
28
|
+
// abandon mid-race; it shares the manager's marker store.
|
|
29
|
+
const workerUploadDeps = {
|
|
30
|
+
...uploadDeps,
|
|
31
|
+
pendingBlobDeletions: uploadDeps.pendingBlobDeletions ?? options?.pendingBlobDeletions,
|
|
32
|
+
};
|
|
33
|
+
const workerDownloadDeps = {
|
|
34
|
+
...downloadDeps,
|
|
35
|
+
pendingBlobDeletions: downloadDeps.pendingBlobDeletions ?? options?.pendingBlobDeletions,
|
|
36
|
+
};
|
|
37
|
+
// Sync cycles and durable deletes mutate the same local index and the same
|
|
38
|
+
// registry rows. Run them one at a time (FIFO): a delete cannot start while
|
|
39
|
+
// an upload of the same scope is between "blob uploaded" and "row marked
|
|
40
|
+
// synced", and a cycle cannot start until a delete has finished all three
|
|
41
|
+
// steps. A rejected operation never blocks the ones queued behind it.
|
|
42
|
+
let mutationQueue = Promise.resolve();
|
|
43
|
+
function exclusive(operation) {
|
|
44
|
+
const run = mutationQueue.then(operation, operation);
|
|
45
|
+
mutationQueue = run.catch(() => undefined);
|
|
46
|
+
return run;
|
|
47
|
+
}
|
|
18
48
|
async function runCycle() {
|
|
19
49
|
// Prevent concurrent cycles
|
|
20
50
|
if (cycleInFlight) {
|
|
21
51
|
rerunRequested = true;
|
|
22
52
|
return cycleInFlight;
|
|
23
53
|
}
|
|
24
|
-
cycleInFlight = (async () => {
|
|
54
|
+
cycleInFlight = exclusive(async () => {
|
|
25
55
|
do {
|
|
26
56
|
rerunRequested = false;
|
|
27
57
|
const canRun = await (options?.canSync?.() ?? { ok: true });
|
|
@@ -31,35 +61,20 @@ export function createSyncManager(uploadDeps, downloadDeps, options) {
|
|
|
31
61
|
continue;
|
|
32
62
|
}
|
|
33
63
|
blocked = null;
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
onError(entry, error) {
|
|
39
|
-
pushError({
|
|
40
|
-
fileId: entry.fileId,
|
|
41
|
-
scope: entry.scope,
|
|
42
|
-
message: `Upload failed for ${entry.path}: ${error.message}`,
|
|
43
|
-
timestamp: new Date().toISOString(),
|
|
44
|
-
});
|
|
45
|
-
},
|
|
46
|
-
});
|
|
47
|
-
uploadDeps.logger.debug({ uploaded: uploadResults.length }, "Upload cycle complete");
|
|
48
|
-
}
|
|
49
|
-
catch (err) {
|
|
50
|
-
const syncError = {
|
|
51
|
-
fileId: null,
|
|
52
|
-
scope: null,
|
|
53
|
-
message: `Upload cycle failed: ${err.message}`,
|
|
54
|
-
timestamp: new Date().toISOString(),
|
|
55
|
-
};
|
|
56
|
-
pushError(syncError);
|
|
57
|
-
uploadDeps.logger.error({ error: err.message }, "Upload cycle failed");
|
|
64
|
+
// Finish blob deletions whose tombstone landed but whose storage
|
|
65
|
+
// DELETE did not. Cheap when nothing is pending; never blocks sync.
|
|
66
|
+
if (transferMode === "upload-download") {
|
|
67
|
+
await runUploadCycle();
|
|
58
68
|
}
|
|
59
69
|
try {
|
|
60
70
|
// Download new remote files
|
|
61
|
-
const
|
|
62
|
-
|
|
71
|
+
const fullReconcile = needsFullReconcile;
|
|
72
|
+
const downloadResults = await downloadAll(workerDownloadDeps, {
|
|
73
|
+
fullReconcile,
|
|
74
|
+
retryMemory: downloadRetryMemory,
|
|
75
|
+
});
|
|
76
|
+
needsFullReconcile = false;
|
|
77
|
+
downloadDeps.logger.debug({ downloaded: downloadResults.length, fullReconcile }, "Download cycle complete");
|
|
63
78
|
}
|
|
64
79
|
catch (err) {
|
|
65
80
|
const syncError = {
|
|
@@ -80,7 +95,7 @@ export function createSyncManager(uploadDeps, downloadDeps, options) {
|
|
|
80
95
|
}
|
|
81
96
|
lastSync = new Date().toISOString();
|
|
82
97
|
} while (rerunRequested && isRunning);
|
|
83
|
-
})
|
|
98
|
+
});
|
|
84
99
|
try {
|
|
85
100
|
await cycleInFlight;
|
|
86
101
|
}
|
|
@@ -88,6 +103,45 @@ export function createSyncManager(uploadDeps, downloadDeps, options) {
|
|
|
88
103
|
cycleInFlight = null;
|
|
89
104
|
}
|
|
90
105
|
}
|
|
106
|
+
async function runUploadCycle() {
|
|
107
|
+
try {
|
|
108
|
+
await retryPendingBlobDeletions({
|
|
109
|
+
deleteData: options?.deleteData,
|
|
110
|
+
pendingBlobDeletions: options?.pendingBlobDeletions,
|
|
111
|
+
dataPointFeed,
|
|
112
|
+
serverOwner: uploadDeps.serverOwner,
|
|
113
|
+
storage: uploadDeps.storage,
|
|
114
|
+
logger: uploadDeps.logger,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
uploadDeps.logger.warn({ error: err.message }, "Pending blob deletion retry failed");
|
|
119
|
+
}
|
|
120
|
+
try {
|
|
121
|
+
const uploadResults = await uploadAll(workerUploadDeps, {
|
|
122
|
+
batchSize: uploadBatchSize,
|
|
123
|
+
onError(entry, error) {
|
|
124
|
+
pushError({
|
|
125
|
+
fileId: entry.fileId,
|
|
126
|
+
scope: entry.scope,
|
|
127
|
+
message: `Upload failed for ${entry.path}: ${error.message}`,
|
|
128
|
+
timestamp: new Date().toISOString(),
|
|
129
|
+
});
|
|
130
|
+
},
|
|
131
|
+
});
|
|
132
|
+
uploadDeps.logger.debug({ uploaded: uploadResults.length }, "Upload cycle complete");
|
|
133
|
+
}
|
|
134
|
+
catch (err) {
|
|
135
|
+
const syncError = {
|
|
136
|
+
fileId: null,
|
|
137
|
+
scope: null,
|
|
138
|
+
message: `Upload cycle failed: ${err.message}`,
|
|
139
|
+
timestamp: new Date().toISOString(),
|
|
140
|
+
};
|
|
141
|
+
pushError(syncError);
|
|
142
|
+
uploadDeps.logger.error({ error: err.message }, "Upload cycle failed");
|
|
143
|
+
}
|
|
144
|
+
}
|
|
91
145
|
function scheduleNotifiedCycle() {
|
|
92
146
|
if (!isRunning || notifyTimerId !== null)
|
|
93
147
|
return;
|
|
@@ -176,9 +230,16 @@ export function createSyncManager(uploadDeps, downloadDeps, options) {
|
|
|
176
230
|
uploadDeps.logger.debug("New data notification received");
|
|
177
231
|
scheduleNotifiedCycle();
|
|
178
232
|
},
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
233
|
+
deleteScope(scope) {
|
|
234
|
+
return exclusive(() => deleteScope({
|
|
235
|
+
storage: uploadDeps.storage,
|
|
236
|
+
serverOwner: uploadDeps.serverOwner,
|
|
237
|
+
deleteData: options?.deleteData,
|
|
238
|
+
pendingBlobDeletions: options?.pendingBlobDeletions,
|
|
239
|
+
scopeDeletions,
|
|
240
|
+
dataPointFeed,
|
|
241
|
+
logger: uploadDeps.logger,
|
|
242
|
+
}, scope));
|
|
182
243
|
},
|
|
183
244
|
};
|
|
184
245
|
return manager;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync-manager.js","sourceRoot":"","sources":["../../../src/sync/engine/sync-manager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"sync-manager.js","sourceRoot":"","sources":["../../../src/sync/engine/sync-manager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,WAAW,EACX,yBAAyB,GAE1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAyD/D,MAAM,UAAU,GAAG,EAAE,CAAC;AAEtB,MAAM,UAAU,iBAAiB,CAC/B,UAA4B,EAC5B,YAAgC,EAChC,OAA4B;IAE5B,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,MAAM,CAAC;IACrD,MAAM,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,EAAE,CAAC;IACvD,MAAM,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,GAAG,CAAC;IAC1D,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,iBAAiB,CAAC;IAEhE,IAAI,UAAU,GAA0C,IAAI,CAAC;IAC7D,IAAI,aAAa,GAAyC,IAAI,CAAC;IAC/D,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,sBAAsB,GAAkB,IAAI,CAAC;IACjD,IAAI,OAAO,GAA6B,IAAI,CAAC;IAC7C,IAAI,MAAM,GAAgB,EAAE,CAAC;IAC7B,IAAI,aAAa,GAAyB,IAAI,CAAC;IAC/C,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAC9B,yEAAyE;IACzE,2EAA2E;IAC3E,8DAA8D;IAC9D,MAAM,mBAAmB,GAAG,yBAAyB,EAAE,CAAC;IACxD,MAAM,aAAa,GAAG,YAAY,CAAC,aAAa,IAAI,UAAU,CAAC,aAAa,CAAC;IAC7E,MAAM,cAAc,GAClB,UAAU,CAAC,cAAc,IAAI,YAAY,CAAC,cAAc,CAAC;IAC3D,oEAAoE;IACpE,0DAA0D;IAC1D,MAAM,gBAAgB,GAAqB;QACzC,GAAG,UAAU;QACb,oBAAoB,EAClB,UAAU,CAAC,oBAAoB,IAAI,OAAO,EAAE,oBAAoB;KACnE,CAAC;IACF,MAAM,kBAAkB,GAAuB;QAC7C,GAAG,YAAY;QACf,oBAAoB,EAClB,YAAY,CAAC,oBAAoB,IAAI,OAAO,EAAE,oBAAoB;KACrE,CAAC;IAEF,2EAA2E;IAC3E,4EAA4E;IAC5E,yEAAyE;IACzE,0EAA0E;IAC1E,sEAAsE;IACtE,IAAI,aAAa,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IACxD,SAAS,SAAS,CAAI,SAA2B;QAC/C,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACrD,aAAa,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3C,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,UAAU,QAAQ;QACrB,4BAA4B;QAC5B,IAAI,aAAa,EAAE,CAAC;YAClB,cAAc,GAAG,IAAI,CAAC;YACtB,OAAO,aAAa,CAAC;QACvB,CAAC;QAED,aAAa,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE;YACnC,GAAG,CAAC;gBACF,cAAc,GAAG,KAAK,CAAC;gBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5D,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;oBACf,OAAO,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7D,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;oBACtD,SAAS;gBACX,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBAEf,iEAAiE;gBACjE,oEAAoE;gBACpE,IAAI,YAAY,KAAK,iBAAiB,EAAE,CAAC;oBACvC,MAAM,cAAc,EAAE,CAAC;gBACzB,CAAC;gBAED,IAAI,CAAC;oBACH,4BAA4B;oBAC5B,MAAM,aAAa,GAAG,kBAAkB,CAAC;oBACzC,MAAM,eAAe,GAAG,MAAM,WAAW,CAAC,kBAAkB,EAAE;wBAC5D,aAAa;wBACb,WAAW,EAAE,mBAAmB;qBACjC,CAAC,CAAC;oBACH,kBAAkB,GAAG,KAAK,CAAC;oBAC3B,YAAY,CAAC,MAAM,CAAC,KAAK,CACvB,EAAE,UAAU,EAAE,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,EACrD,yBAAyB,CAC1B,CAAC;gBACJ,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,SAAS,GAAc;wBAC3B,MAAM,EAAE,IAAI;wBACZ,KAAK,EAAE,IAAI;wBACX,OAAO,EAAE,0BAA2B,GAAa,CAAC,OAAO,EAAE;wBAC3D,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC,CAAC;oBACF,SAAS,CAAC,SAAS,CAAC,CAAC;oBACrB,YAAY,CAAC,MAAM,CAAC,KAAK,CACvB,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,EACjC,uBAAuB,CACxB,CAAC;gBACJ,CAAC;gBAED,6BAA6B;gBAC7B,IAAI,CAAC;oBACH,sBAAsB,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5D,CAAC;gBAAC,MAAM,CAAC;oBACP,6CAA6C;gBAC/C,CAAC;gBAED,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACtC,CAAC,QAAQ,cAAc,IAAI,SAAS,EAAE;QACxC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,aAAa,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,KAAK,UAAU,cAAc;QAC3B,IAAI,CAAC;YACH,MAAM,yBAAyB,CAAC;gBAC9B,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,oBAAoB,EAAE,OAAO,EAAE,oBAAoB;gBACnD,aAAa;gBACb,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,UAAU,CAAC,MAAM,CAAC,IAAI,CACpB,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,EACjC,oCAAoC,CACrC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,SAAS,CAAC,gBAAgB,EAAE;gBACtD,SAAS,EAAE,eAAe;gBAC1B,OAAO,CAAC,KAAK,EAAE,KAAK;oBAClB,SAAS,CAAC;wBACR,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,OAAO,EAAE,qBAAqB,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE;wBAC5D,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;YACH,UAAU,CAAC,MAAM,CAAC,KAAK,CACrB,EAAE,QAAQ,EAAE,aAAa,CAAC,MAAM,EAAE,EAClC,uBAAuB,CACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,SAAS,GAAc;gBAC3B,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,wBAAyB,GAAa,CAAC,OAAO,EAAE;gBACzD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;YACF,SAAS,CAAC,SAAS,CAAC,CAAC;YACrB,UAAU,CAAC,MAAM,CAAC,KAAK,CACrB,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,EACjC,qBAAqB,CACtB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,SAAS,qBAAqB;QAC5B,IAAI,CAAC,SAAS,IAAI,aAAa,KAAK,IAAI;YAAE,OAAO;QACjD,aAAa,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,aAAa,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,SAAS;gBAAE,OAAO;YACvB,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBACpB,yCAAyC;YAC3C,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACvB,CAAC;IAED,SAAS,SAAS,CAAC,KAAgB;QACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,IAAI,MAAM,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;YAC/B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,SAAS,aAAa;QACpB,IAAI,UAAU,KAAK,IAAI;YAAE,OAAO;QAChC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBACpB,yCAAyC;YAC3C,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,YAAY,CAAC,CAAC;IACnB,CAAC;IAED,SAAS,kBAAkB;QACzB,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,aAAa,CAAC,UAAU,CAAC,CAAC;YAC1B,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,SAAS,gBAAgB;QACvB,IAAI,aAAa,KAAK,IAAI,EAAE,CAAC;YAC3B,YAAY,CAAC,aAAa,CAAC,CAAC;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAgB;QAC3B,IAAI,OAAO;YACT,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,KAAK;YACH,IAAI,SAAS;gBAAE,OAAO,CAAC,aAAa;YACpC,SAAS,GAAG,IAAI,CAAC;YAEjB,qDAAqD;YACrD,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBACpB,yCAAyC;YAC3C,CAAC,CAAC,CAAC;YAEH,aAAa,EAAE,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACR,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,SAAS,GAAG,KAAK,CAAC;YAElB,2CAA2C;YAC3C,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC;YACtB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,OAAO;YACX,2DAA2D;YAC3D,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,MAAM,QAAQ,EAAE,CAAC;YACjB,IAAI,SAAS,EAAE,CAAC;gBACd,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;QAED,SAAS;YACP,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC;YAE9D,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,SAAS;gBAClB,OAAO,EAAE,aAAa,KAAK,IAAI;gBAC/B,OAAO;gBACP,QAAQ;gBACR,sBAAsB;gBACtB,YAAY;gBACZ,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;aACpB,CAAC;QACJ,CAAC;QAED,aAAa;YACX,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC1D,qBAAqB,EAAE,CAAC;QAC1B,CAAC;QAED,WAAW,CAAC,KAAa;YACvB,OAAO,SAAS,CAAC,GAAG,EAAE,CACpB,WAAW,CACT;gBACE,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,WAAW,EAAE,UAAU,CAAC,WAAW;gBACnC,UAAU,EAAE,OAAO,EAAE,UAAU;gBAC/B,oBAAoB,EAAE,OAAO,EAAE,oBAAoB;gBACnD,cAAc;gBACd,aAAa;gBACb,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,EACD,KAAK,CACN,CACF,CAAC;QACJ,CAAC;KACF,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/sync/index.d.ts
CHANGED
|
@@ -3,4 +3,11 @@ export type { SyncCursor } from "./cursor.js";
|
|
|
3
3
|
export { classifySyncFailure, inferPayloadKind, type ClassifiedSyncFailure, type ClassifySyncFailureInput, type SyncDownloadFailureTelemetryEvent, type SyncFailureDisposition, type SyncFailureStage, type SyncFileIssue, type SyncPayloadKind, } from "./issues.js";
|
|
4
4
|
export { createSyncManager, type SyncManager, type SyncManagerOptions, } from "./engine/sync-manager.js";
|
|
5
5
|
export type { DownloadDiagnosticsHook, DownloadWorkerDeps, } from "./workers/download.js";
|
|
6
|
+
export { BLOB_DELETE_BATCH_SIZE, deleteScope, planBlobDeletions, retryPendingBlobDeletions, type DeleteScopeDeps, type DeleteScopeResult, type DeleteScopeStep, type DeleteStepStatus, type RetryPendingBlobDeletionsResult, } from "./workers/delete.js";
|
|
7
|
+
export { createGatewayDataPointFeed, feedFromGatewayClient, type GatewayDataPointFeedOptions, } from "./data-point-feed.js";
|
|
8
|
+
export { createGatewayDeleteDataPort, type GatewayDeleteDataPortOptions, } from "./delete-data-port.js";
|
|
9
|
+
export { createMemoryPendingBlobDeletionStore, createPendingBlobDeletionStore, normalizePendingBlobDeletions, type PendingBlobDeletionKv, } from "./pending-blob-deletions.js";
|
|
10
|
+
export { computeDataPointId } from "./data-point-id.js";
|
|
11
|
+
export { createScopeDeletionTracker, deletionTimestamp, isEntryCoveredByTombstone, tombstoneVersion, DEFAULT_SCOPE_DELETION_GATEWAY_RETRY_MS, DEFAULT_SCOPE_DELETION_MAX_STALENESS_MS, type ScopeDeletionResolveOptions, type ScopeDeletionSource, type ScopeDeletionTracker, type ScopeDeletionTrackerOptions, type ScopeDeletionVerdict, type ScopeTombstone, } from "./scope-deletions.js";
|
|
12
|
+
export { TOMBSTONE_DATA_HASH, TOMBSTONE_METADATA_HASH, isTombstoneRecord, } from "./tombstone.js";
|
|
6
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/sync/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sync/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,iCAAiC,EACtC,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,kBAAkB,GACxB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/sync/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxD,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EACL,mBAAmB,EACnB,gBAAgB,EAChB,KAAK,qBAAqB,EAC1B,KAAK,wBAAwB,EAC7B,KAAK,iCAAiC,EACtC,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,kBAAkB,GACxB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,iBAAiB,EACjB,yBAAyB,EACzB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,+BAA+B,GACrC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,0BAA0B,EAC1B,qBAAqB,EACrB,KAAK,2BAA2B,GACjC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,2BAA2B,EAC3B,KAAK,4BAA4B,GAClC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,oCAAoC,EACpC,8BAA8B,EAC9B,6BAA6B,EAC7B,KAAK,qBAAqB,GAC3B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,0BAA0B,EAC1B,iBAAiB,EACjB,yBAAyB,EACzB,gBAAgB,EAChB,uCAAuC,EACvC,uCAAuC,EACvC,KAAK,2BAA2B,EAChC,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,EAChC,KAAK,oBAAoB,EACzB,KAAK,cAAc,GACpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC"}
|
package/dist/sync/index.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
export { classifySyncFailure, inferPayloadKind, } from "./issues.js";
|
|
2
2
|
export { createSyncManager, } from "./engine/sync-manager.js";
|
|
3
|
+
export { BLOB_DELETE_BATCH_SIZE, deleteScope, planBlobDeletions, retryPendingBlobDeletions, } from "./workers/delete.js";
|
|
4
|
+
export { createGatewayDataPointFeed, feedFromGatewayClient, } from "./data-point-feed.js";
|
|
5
|
+
export { createGatewayDeleteDataPort, } from "./delete-data-port.js";
|
|
6
|
+
export { createMemoryPendingBlobDeletionStore, createPendingBlobDeletionStore, normalizePendingBlobDeletions, } from "./pending-blob-deletions.js";
|
|
7
|
+
export { computeDataPointId } from "./data-point-id.js";
|
|
8
|
+
export { createScopeDeletionTracker, deletionTimestamp, isEntryCoveredByTombstone, tombstoneVersion, DEFAULT_SCOPE_DELETION_GATEWAY_RETRY_MS, DEFAULT_SCOPE_DELETION_MAX_STALENESS_MS, } from "./scope-deletions.js";
|
|
9
|
+
export { TOMBSTONE_DATA_HASH, TOMBSTONE_METADATA_HASH, isTombstoneRecord, } from "./tombstone.js";
|
|
3
10
|
//# sourceMappingURL=index.js.map
|
package/dist/sync/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sync/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,mBAAmB,EACnB,gBAAgB,GAQjB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,GAGlB,MAAM,0BAA0B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sync/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,mBAAmB,EACnB,gBAAgB,GAQjB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,GAGlB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,EACL,sBAAsB,EACtB,WAAW,EACX,iBAAiB,EACjB,yBAAyB,GAM1B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,0BAA0B,EAC1B,qBAAqB,GAEtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,2BAA2B,GAE5B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,oCAAoC,EACpC,8BAA8B,EAC9B,6BAA6B,GAE9B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,0BAA0B,EAC1B,iBAAiB,EACjB,yBAAyB,EACzB,gBAAgB,EAChB,uCAAuC,EACvC,uCAAuC,GAOxC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,gBAAgB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/sync/issues.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,gBAAgB,GACxB,UAAU,GACV,eAAe,GACf,SAAS,GACT,YAAY,GACZ,mBAAmB,GACnB,aAAa,GACb,SAAS,CAAC;AAEd,MAAM,MAAM,eAAe,
|
|
1
|
+
{"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/sync/issues.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,gBAAgB,GACxB,UAAU,GACV,eAAe,GACf,SAAS,GACT,YAAY,GACZ,mBAAmB,GACnB,aAAa,GACb,SAAS,CAAC;AAEd,MAAM,MAAM,eAAe,GACzB,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE/D,MAAM,MAAM,sBAAsB,GAAG,eAAe,GAAG,WAAW,CAAC;AAEnE,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,gBAAgB,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,sBAAsB,CAAC;IACpC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,iCAAiC;IAChD,KAAK,EAAE,uBAAuB,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,gBAAgB,CAAC;IACxB,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,sBAAsB,CAAC;IACpC,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,aAAa,CAAC;IACrB,SAAS,EAAE,iCAAiC,CAAC;CAC9C;AAiBD,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,wBAAwB,GAC9B,qBAAqB,CA0CvB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,GAAG,eAAe,CAgB5E"}
|
package/dist/sync/issues.js
CHANGED
|
@@ -5,7 +5,12 @@ const DETERMINISTIC_STAGES = new Set([
|
|
|
5
5
|
"envelope_validate",
|
|
6
6
|
"block_build",
|
|
7
7
|
]);
|
|
8
|
-
|
|
8
|
+
// Only statuses that PROVE the blob cannot appear by retrying are
|
|
9
|
+
// deterministic. Everything else — 5xx, Cloudflare 52x, 401/403 auth blips
|
|
10
|
+
// (downloads are Web3Signed off the local clock), 429 — is treated as
|
|
11
|
+
// transient and bounded by the retry memory's attempt cap instead of being
|
|
12
|
+
// permanently quarantined on first sight.
|
|
13
|
+
const DETERMINISTIC_DOWNLOAD_STATUSES = new Set([404, 410]);
|
|
9
14
|
export function classifySyncFailure(input) {
|
|
10
15
|
const stage = input.stage ?? inferStage(input.error);
|
|
11
16
|
const disposition = classifyDisposition(input.error, stage);
|
|
@@ -67,9 +72,12 @@ export function inferPayloadKind(bytes) {
|
|
|
67
72
|
function classifyDisposition(error, stage) {
|
|
68
73
|
if (stage === "download") {
|
|
69
74
|
const status = getNumericProperty(error, "status") ??
|
|
70
|
-
getNumericProperty(error, "statusCode")
|
|
75
|
+
getNumericProperty(error, "statusCode") ??
|
|
76
|
+
getDownloadStatusFromMessage(error);
|
|
71
77
|
if (status !== undefined) {
|
|
72
|
-
return
|
|
78
|
+
return DETERMINISTIC_DOWNLOAD_STATUSES.has(status)
|
|
79
|
+
? "deterministic"
|
|
80
|
+
: "transient";
|
|
73
81
|
}
|
|
74
82
|
return "transient";
|
|
75
83
|
}
|
|
@@ -115,6 +123,16 @@ function getStringProperty(value, key) {
|
|
|
115
123
|
const property = value[key];
|
|
116
124
|
return typeof property === "string" ? property : "";
|
|
117
125
|
}
|
|
126
|
+
// The SDK's StorageError carries the HTTP status only in its message —
|
|
127
|
+
// "vana-storage download failed: 404 Not Found" — with no numeric property.
|
|
128
|
+
// Without this fallback a 404'd blob classifies transient and is retried
|
|
129
|
+
// every sync cycle forever. The anchored "download failed: " prefix mirrors
|
|
130
|
+
// the provider's stable message shape and avoids misreading digits in
|
|
131
|
+
// network-error causes (IPs, ports), which stay transient.
|
|
132
|
+
function getDownloadStatusFromMessage(error) {
|
|
133
|
+
const match = /download failed: ([1-5]\d{2})\b/.exec(getRawMessage(error));
|
|
134
|
+
return match ? Number(match[1]) : undefined;
|
|
135
|
+
}
|
|
118
136
|
function getNumericProperty(value, key) {
|
|
119
137
|
if (typeof value !== "object" || value === null || !(key in value)) {
|
|
120
138
|
return undefined;
|
package/dist/sync/issues.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"issues.js","sourceRoot":"","sources":["../../src/sync/issues.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"issues.js","sourceRoot":"","sources":["../../src/sync/issues.ts"],"names":[],"mappings":"AAmEA,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAmB;IACrD,eAAe;IACf,SAAS;IACT,YAAY;IACZ,mBAAmB;IACnB,aAAa;CACd,CAAC,CAAC;AAEH,kEAAkE;AAClE,2EAA2E;AAC3E,sEAAsE;AACtE,2EAA2E;AAC3E,0CAA0C;AAC1C,MAAM,+BAA+B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE5D,MAAM,UAAU,mBAAmB,CACjC,KAA+B;IAE/B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,WAAW,KAAK,WAAW,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;IAExD,MAAM,KAAK,GAAkB;QAC3B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK;QACL,UAAU;QACV,OAAO;QACP,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,WAAW,EAAE,SAAS;QACtB,UAAU,EAAE,SAAS;QACrB,WAAW;QACX,SAAS;KACV,CAAC;IAEF,MAAM,SAAS,GAAsC;QACnD,KAAK,EAAE,uBAAuB;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK;QACL,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,UAAU;QACV,WAAW;QACX,SAAS;QACT,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;QAClD,cAAc,EAAE,KAAK,CAAC,cAAc;KACrC,CAAC;IAEF,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAA0B;IACzD,MAAM,IAAI,GACR,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IAEjC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,IAAI,OAAO,CAAC,UAAU,CAAC,6BAA6B,CAAC;QAAE,OAAO,SAAS,CAAC;IACxE,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC;IACtE,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACvE,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,kBAAkB,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAE/C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAc,EACd,KAAuB;IAEvB,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QACzB,MAAM,MAAM,GACV,kBAAkB,CAAC,KAAK,EAAE,QAAQ,CAAC;YACnC,kBAAkB,CAAC,KAAK,EAAE,YAAY,CAAC;YACvC,4BAA4B,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,+BAA+B,CAAC,GAAG,CAAC,MAAM,CAAC;gBAChD,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,WAAW,CAAC;QAClB,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,IAAI,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,YAAY,CAAC;IAEtD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5D,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IAEnD,IAAI,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;QAChD,OAAO,mBAAmB,CAAC;IAC7B,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACnE,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC;IAChC,CAAC;IACD,OAAO,OAAO,KAAK,CAAC;AACtB,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,YAAY,KAAK;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IACjD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC;AAC/D,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,GAAW;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC9E,MAAM,QAAQ,GAAI,KAAiC,CAAC,GAAG,CAAC,CAAC;IACzD,OAAO,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;AACtD,CAAC;AAED,uEAAuE;AACvE,4EAA4E;AAC5E,yEAAyE;AACzE,4EAA4E;AAC5E,sEAAsE;AACtE,2DAA2D;AAC3D,SAAS,4BAA4B,CAAC,KAAc;IAClD,MAAM,KAAK,GAAG,iCAAiC,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc,EAAE,GAAW;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;QACnE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,QAAQ,GAAI,KAAiC,CAAC,GAAG,CAAC,CAAC;IACzD,OAAO,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7D,CAAC;AAED,SAAS,kBAAkB,CAAC,KAA0B;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAuB,EACvB,WAAmC;IAEnC,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,UAAU;YACb,OAAO,WAAW,KAAK,WAAW;gBAChC,CAAC,CAAC,2DAA2D;gBAC7D,CAAC,CAAC,uDAAuD,CAAC;QAC9D,KAAK,eAAe;YAClB,OAAO,6DAA6D,CAAC;QACvE,KAAK,SAAS;YACZ,OAAO,0CAA0C,CAAC;QACpD,KAAK,YAAY;YACf,OAAO,+CAA+C,CAAC;QACzD,KAAK,mBAAmB;YACtB,OAAO,0DAA0D,CAAC;QACpE,KAAK,aAAa;YAChB,OAAO,qDAAqD,CAAC;QAC/D,KAAK,SAAS;YACZ,OAAO,sCAAsC,CAAC;IAClD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { PendingBlobDeletion, PendingBlobDeletionStore } from "../ports/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Minimal persistence seam for the pending-blob-deletion markers. Hosts
|
|
4
|
+
* supply the storage (a JSON file on Node, IndexedDB state in the browser);
|
|
5
|
+
* the store keeps the set semantics, de-duplication and mutation ordering in
|
|
6
|
+
* one place. `write` must replace the whole value atomically (tmp + rename on
|
|
7
|
+
* disk, a single IndexedDB put in the browser) so a crash mid-write cannot
|
|
8
|
+
* leave a torn marker file.
|
|
9
|
+
*/
|
|
10
|
+
export interface PendingBlobDeletionKv {
|
|
11
|
+
read(): Promise<PendingBlobDeletion[] | null>;
|
|
12
|
+
write(keys: PendingBlobDeletion[]): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Accept the stored value from either marker generation: exact keys
|
|
16
|
+
* (`{ scope, version }`) or the pre-key format that recorded whole scopes,
|
|
17
|
+
* which maps to a version-less marker the retry expands into exact keys.
|
|
18
|
+
*/
|
|
19
|
+
export declare function normalizePendingBlobDeletions(stored: unknown): PendingBlobDeletion[];
|
|
20
|
+
/**
|
|
21
|
+
* Every operation is a read-modify-write of the whole marker set, so the
|
|
22
|
+
* store serialises them: a delete recording keys while a sync cycle's retry
|
|
23
|
+
* clears others must not lose either update. Ordering is FIFO, and a failed
|
|
24
|
+
* operation never blocks the ones queued behind it.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createPendingBlobDeletionStore(kv: PendingBlobDeletionKv): PendingBlobDeletionStore;
|
|
27
|
+
/** Non-durable store: markers survive only for the process lifetime. */
|
|
28
|
+
export declare function createMemoryPendingBlobDeletionStore(): PendingBlobDeletionStore;
|
|
29
|
+
//# sourceMappingURL=pending-blob-deletions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pending-blob-deletions.d.ts","sourceRoot":"","sources":["../../src/sync/pending-blob-deletions.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EACnB,wBAAwB,EACzB,MAAM,mBAAmB,CAAC;AAE3B;;;;;;;GAOG;AACH,MAAM,WAAW,qBAAqB;IACpC,IAAI,IAAI,OAAO,CAAC,mBAAmB,EAAE,GAAG,IAAI,CAAC,CAAC;IAC9C,KAAK,CAAC,IAAI,EAAE,mBAAmB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnD;AAwBD;;;;GAIG;AACH,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,OAAO,GACd,mBAAmB,EAAE,CAwBvB;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAC5C,EAAE,EAAE,qBAAqB,GACxB,wBAAwB,CAgD1B;AAED,wEAAwE;AACxE,wBAAgB,oCAAoC,IAAI,wBAAwB,CAU/E"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
function markerId(key) {
|
|
2
|
+
const range = key.range ? `${key.range.from}-${key.range.to}` : "";
|
|
3
|
+
return `${key.scope}\u0000${key.version ?? ""}\u0000${range}`;
|
|
4
|
+
}
|
|
5
|
+
function normalizeRange(value) {
|
|
6
|
+
if (typeof value !== "object" || value === null)
|
|
7
|
+
return undefined;
|
|
8
|
+
const { from, to } = value;
|
|
9
|
+
if (typeof from !== "string" ||
|
|
10
|
+
typeof to !== "string" ||
|
|
11
|
+
!/^\d+$/.test(from) ||
|
|
12
|
+
!/^\d+$/.test(to) ||
|
|
13
|
+
BigInt(from) > BigInt(to)) {
|
|
14
|
+
return undefined;
|
|
15
|
+
}
|
|
16
|
+
return { from, to };
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Accept the stored value from either marker generation: exact keys
|
|
20
|
+
* (`{ scope, version }`) or the pre-key format that recorded whole scopes,
|
|
21
|
+
* which maps to a version-less marker the retry expands into exact keys.
|
|
22
|
+
*/
|
|
23
|
+
export function normalizePendingBlobDeletions(stored) {
|
|
24
|
+
if (!Array.isArray(stored))
|
|
25
|
+
return [];
|
|
26
|
+
const keys = [];
|
|
27
|
+
for (const item of stored) {
|
|
28
|
+
if (typeof item === "string") {
|
|
29
|
+
keys.push({ scope: item, version: null });
|
|
30
|
+
}
|
|
31
|
+
else if (typeof item === "object" &&
|
|
32
|
+
item !== null &&
|
|
33
|
+
typeof item.scope === "string") {
|
|
34
|
+
const version = item.version;
|
|
35
|
+
const range = typeof version === "string"
|
|
36
|
+
? undefined
|
|
37
|
+
: normalizeRange(item.range);
|
|
38
|
+
keys.push({
|
|
39
|
+
scope: item.scope,
|
|
40
|
+
version: typeof version === "string" ? version : null,
|
|
41
|
+
...(range ? { range } : {}),
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return keys;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Every operation is a read-modify-write of the whole marker set, so the
|
|
49
|
+
* store serialises them: a delete recording keys while a sync cycle's retry
|
|
50
|
+
* clears others must not lose either update. Ordering is FIFO, and a failed
|
|
51
|
+
* operation never blocks the ones queued behind it.
|
|
52
|
+
*/
|
|
53
|
+
export function createPendingBlobDeletionStore(kv) {
|
|
54
|
+
let queue = Promise.resolve();
|
|
55
|
+
function serialized(operation) {
|
|
56
|
+
const run = queue.then(operation, operation);
|
|
57
|
+
queue = run.catch(() => undefined);
|
|
58
|
+
return run;
|
|
59
|
+
}
|
|
60
|
+
async function current() {
|
|
61
|
+
return normalizePendingBlobDeletions(await kv.read());
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
list() {
|
|
65
|
+
return serialized(current);
|
|
66
|
+
},
|
|
67
|
+
add(keys) {
|
|
68
|
+
return serialized(async () => {
|
|
69
|
+
if (keys.length === 0)
|
|
70
|
+
return;
|
|
71
|
+
const existing = await current();
|
|
72
|
+
const known = new Set(existing.map(markerId));
|
|
73
|
+
const next = [...existing];
|
|
74
|
+
for (const key of keys) {
|
|
75
|
+
const id = markerId(key);
|
|
76
|
+
if (known.has(id))
|
|
77
|
+
continue;
|
|
78
|
+
known.add(id);
|
|
79
|
+
next.push({
|
|
80
|
+
scope: key.scope,
|
|
81
|
+
version: key.version,
|
|
82
|
+
...(key.range ? { range: { ...key.range } } : {}),
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
if (next.length === existing.length)
|
|
86
|
+
return;
|
|
87
|
+
await kv.write(next);
|
|
88
|
+
});
|
|
89
|
+
},
|
|
90
|
+
remove(keys) {
|
|
91
|
+
return serialized(async () => {
|
|
92
|
+
if (keys.length === 0)
|
|
93
|
+
return;
|
|
94
|
+
const existing = await current();
|
|
95
|
+
const gone = new Set(keys.map(markerId));
|
|
96
|
+
const next = existing.filter((key) => !gone.has(markerId(key)));
|
|
97
|
+
if (next.length === existing.length)
|
|
98
|
+
return;
|
|
99
|
+
await kv.write(next);
|
|
100
|
+
});
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** Non-durable store: markers survive only for the process lifetime. */
|
|
105
|
+
export function createMemoryPendingBlobDeletionStore() {
|
|
106
|
+
let keys = [];
|
|
107
|
+
return createPendingBlobDeletionStore({
|
|
108
|
+
async read() {
|
|
109
|
+
return keys;
|
|
110
|
+
},
|
|
111
|
+
async write(next) {
|
|
112
|
+
keys = next.map((key) => ({ ...key }));
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=pending-blob-deletions.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pending-blob-deletions.js","sourceRoot":"","sources":["../../src/sync/pending-blob-deletions.ts"],"names":[],"mappings":"AAkBA,SAAS,QAAQ,CAAC,GAAwB;IACxC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO,GAAG,GAAG,CAAC,KAAK,SAAS,GAAG,CAAC,OAAO,IAAI,EAAE,SAAS,KAAK,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,cAAc,CACrB,KAAc;IAEd,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,KAAyC,CAAC;IAC/D,IACE,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,EAAE,KAAK,QAAQ;QACtB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;QACnB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,EACzB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,6BAA6B,CAC3C,MAAe;IAEf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,IAAI,GAA0B,EAAE,CAAC;IACvC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;aAAM,IACL,OAAO,IAAI,KAAK,QAAQ;YACxB,IAAI,KAAK,IAAI;YACb,OAAQ,IAA4B,CAAC,KAAK,KAAK,QAAQ,EACvD,CAAC;YACD,MAAM,OAAO,GAAI,IAA8B,CAAC,OAAO,CAAC;YACxD,MAAM,KAAK,GACT,OAAO,OAAO,KAAK,QAAQ;gBACzB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,cAAc,CAAE,IAA4B,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,CAAC,IAAI,CAAC;gBACR,KAAK,EAAG,IAA0B,CAAC,KAAK;gBACxC,OAAO,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;gBACrD,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAC5C,EAAyB;IAEzB,IAAI,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,SAAS,UAAU,CAAI,SAA2B;QAChD,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC7C,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,UAAU,OAAO;QACpB,OAAO,6BAA6B,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,OAAO;QACL,IAAI;YACF,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QACD,GAAG,CAAC,IAAI;YACN,OAAO,UAAU,CAAC,KAAK,IAAI,EAAE;gBAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO;gBAC9B,MAAM,QAAQ,GAAG,MAAM,OAAO,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9C,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACzB,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;wBAAE,SAAS;oBAC5B,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACd,IAAI,CAAC,IAAI,CAAC;wBACR,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,GAAG,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAClD,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;oBAAE,OAAO;gBAC5C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,CAAC,IAAI;YACT,OAAO,UAAU,CAAC,KAAK,IAAI,EAAE;gBAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO;gBAC9B,MAAM,QAAQ,GAAG,MAAM,OAAO,EAAE,CAAC;gBACjC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChE,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;oBAAE,OAAO;gBAC5C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,oCAAoC;IAClD,IAAI,IAAI,GAA0B,EAAE,CAAC;IACrC,OAAO,8BAA8B,CAAC;QACpC,KAAK,CAAC,IAAI;YACR,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,IAAI;YACd,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export type DownloadAttemptDecision = "attempt" | "backoff" | "give-up";
|
|
2
|
+
export interface DownloadRetryMemoryOptions {
|
|
3
|
+
/** Attempt cap for retryable failures (default 5). */
|
|
4
|
+
maxTransientAttempts?: number;
|
|
5
|
+
/** Base backoff, doubled per failed attempt (default 30s). */
|
|
6
|
+
backoffBaseMs?: number;
|
|
7
|
+
/** Clock override for tests. */
|
|
8
|
+
now?: () => number;
|
|
9
|
+
}
|
|
10
|
+
export declare const DEFAULT_MAX_TRANSIENT_ATTEMPTS = 5;
|
|
11
|
+
export declare const DEFAULT_BACKOFF_BASE_MS = 30000;
|
|
12
|
+
export interface DownloadRetryMemory {
|
|
13
|
+
/** Whether the next download of `key` should run, wait, or never happen. */
|
|
14
|
+
decide(key: string): DownloadAttemptDecision;
|
|
15
|
+
/** Record a failed attempt; `retryable` follows the issue classification. */
|
|
16
|
+
recordFailure(key: string, retryable: boolean): void;
|
|
17
|
+
/** Clear the failure history after a successful download. */
|
|
18
|
+
recordSuccess(key: string): void;
|
|
19
|
+
/**
|
|
20
|
+
* The listing restarted from the beginning (repair pass / full
|
|
21
|
+
* reconcile) — give transient failures a fresh attempt budget so the
|
|
22
|
+
* reset can actually re-fetch what it re-lists. Deterministic (404)
|
|
23
|
+
* entries stay dead: re-listing doesn't make missing bytes appear.
|
|
24
|
+
*/
|
|
25
|
+
onListingReset(): void;
|
|
26
|
+
}
|
|
27
|
+
/** Retry-memory key for a data-point record: id + version. */
|
|
28
|
+
export declare function downloadRetryKey(record: {
|
|
29
|
+
id: string;
|
|
30
|
+
expectedVersion: string;
|
|
31
|
+
}): string;
|
|
32
|
+
export declare function createDownloadRetryMemory(options?: DownloadRetryMemoryOptions): DownloadRetryMemory;
|
|
33
|
+
//# sourceMappingURL=retry-memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-memory.d.ts","sourceRoot":"","sources":["../../src/sync/retry-memory.ts"],"names":[],"mappings":"AAqBA,MAAM,MAAM,uBAAuB,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;AAExE,MAAM,WAAW,0BAA0B;IACzC,sDAAsD;IACtD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,8DAA8D;IAC9D,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gCAAgC;IAChC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,8BAA8B,IAAI,CAAC;AAChD,eAAO,MAAM,uBAAuB,QAAS,CAAC;AAE9C,MAAM,WAAW,mBAAmB;IAClC,4EAA4E;IAC5E,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,uBAAuB,CAAC;IAC7C,6EAA6E;IAC7E,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI,CAAC;IACrD,6DAA6D;IAC7D,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC;;;;;OAKG;IACH,cAAc,IAAI,IAAI,CAAC;CACxB;AAED,8DAA8D;AAC9D,wBAAgB,gBAAgB,CAAC,MAAM,EAAE;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,MAAM,CAET;AAQD,wBAAgB,yBAAyB,CACvC,OAAO,GAAE,0BAA+B,GACvC,mBAAmB,CA4CrB"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Cross-cycle download retry memory.
|
|
2
|
+
//
|
|
3
|
+
// downloadAll re-lists the owner's data points every sync cycle (the cursor
|
|
4
|
+
// only advances past fully-successful pages, and a single-page listing has no
|
|
5
|
+
// nextCursor at all), so without memory a record whose blob fails to download
|
|
6
|
+
// is re-fetched on every cycle. With the DCR page triggering a sync per
|
|
7
|
+
// readiness poll, one missing blob produced a 700+ request 404 storm against
|
|
8
|
+
// storage (2026-07-08 prod incident).
|
|
9
|
+
//
|
|
10
|
+
// Policy:
|
|
11
|
+
// - non-retryable failures (deterministic — e.g. HTTP 404: the blob is not
|
|
12
|
+
// there and waiting cannot heal it): never attempt again for the lifetime
|
|
13
|
+
// of this memory (one attempt per boot session).
|
|
14
|
+
// - retryable failures (transient — 5xx, network): exponential backoff,
|
|
15
|
+
// capped at DEFAULT_MAX_TRANSIENT_ATTEMPTS attempts, then give up for the
|
|
16
|
+
// session.
|
|
17
|
+
//
|
|
18
|
+
// Keys pair the record id with its expectedVersion (`downloadRetryKey`), so a
|
|
19
|
+
// re-registered version — new bytes under a new blob key — always gets a
|
|
20
|
+
// fresh set of attempts.
|
|
21
|
+
export const DEFAULT_MAX_TRANSIENT_ATTEMPTS = 5;
|
|
22
|
+
export const DEFAULT_BACKOFF_BASE_MS = 30_000;
|
|
23
|
+
/** Retry-memory key for a data-point record: id + version. */
|
|
24
|
+
export function downloadRetryKey(record) {
|
|
25
|
+
return `${record.id}@${record.expectedVersion}`;
|
|
26
|
+
}
|
|
27
|
+
export function createDownloadRetryMemory(options = {}) {
|
|
28
|
+
const maxTransientAttempts = options.maxTransientAttempts ?? DEFAULT_MAX_TRANSIENT_ATTEMPTS;
|
|
29
|
+
const backoffBaseMs = options.backoffBaseMs ?? DEFAULT_BACKOFF_BASE_MS;
|
|
30
|
+
const now = options.now ?? Date.now;
|
|
31
|
+
// DPv2 keys one record per (owner, scope), so the map stays small — no
|
|
32
|
+
// eviction needed.
|
|
33
|
+
const entries = new Map();
|
|
34
|
+
return {
|
|
35
|
+
decide(key) {
|
|
36
|
+
const entry = entries.get(key);
|
|
37
|
+
if (!entry)
|
|
38
|
+
return "attempt";
|
|
39
|
+
if (entry.permanent || entry.attempts >= maxTransientAttempts) {
|
|
40
|
+
return "give-up";
|
|
41
|
+
}
|
|
42
|
+
return now() < entry.nextRetryAtMs ? "backoff" : "attempt";
|
|
43
|
+
},
|
|
44
|
+
recordFailure(key, retryable) {
|
|
45
|
+
if (!retryable) {
|
|
46
|
+
entries.set(key, { attempts: 0, nextRetryAtMs: 0, permanent: true });
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const attempts = (entries.get(key)?.attempts ?? 0) + 1;
|
|
50
|
+
entries.set(key, {
|
|
51
|
+
attempts,
|
|
52
|
+
nextRetryAtMs: now() + backoffBaseMs * 2 ** (attempts - 1),
|
|
53
|
+
permanent: false,
|
|
54
|
+
});
|
|
55
|
+
},
|
|
56
|
+
recordSuccess(key) {
|
|
57
|
+
entries.delete(key);
|
|
58
|
+
},
|
|
59
|
+
onListingReset() {
|
|
60
|
+
for (const [key, entry] of entries) {
|
|
61
|
+
if (!entry.permanent) {
|
|
62
|
+
entries.delete(key);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=retry-memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-memory.js","sourceRoot":"","sources":["../../src/sync/retry-memory.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,8EAA8E;AAC9E,wEAAwE;AACxE,6EAA6E;AAC7E,sCAAsC;AACtC,EAAE;AACF,UAAU;AACV,6EAA6E;AAC7E,8EAA8E;AAC9E,qDAAqD;AACrD,0EAA0E;AAC1E,8EAA8E;AAC9E,eAAe;AACf,EAAE;AACF,8EAA8E;AAC9E,yEAAyE;AACzE,yBAAyB;AAazB,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAChD,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AAkB9C,8DAA8D;AAC9D,MAAM,UAAU,gBAAgB,CAAC,MAGhC;IACC,OAAO,GAAG,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;AAClD,CAAC;AAQD,MAAM,UAAU,yBAAyB,CACvC,UAAsC,EAAE;IAExC,MAAM,oBAAoB,GACxB,OAAO,CAAC,oBAAoB,IAAI,8BAA8B,CAAC;IACjE,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,uBAAuB,CAAC;IACvE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,uEAAuE;IACvE,mBAAmB;IACnB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE9C,OAAO;QACL,MAAM,CAAC,GAAG;YACR,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK;gBAAE,OAAO,SAAS,CAAC;YAC7B,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,IAAI,oBAAoB,EAAE,CAAC;gBAC9D,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,OAAO,GAAG,EAAE,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7D,CAAC;QAED,aAAa,CAAC,GAAG,EAAE,SAAS;YAC1B,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACrE,OAAO;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;gBACf,QAAQ;gBACR,aAAa,EAAE,GAAG,EAAE,GAAG,aAAa,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAC1D,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,aAAa,CAAC,GAAG;YACf,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QAED,cAAc;YACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;gBACnC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;oBACrB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|