@narumitw/pi-sync-daemon 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +21 -0
- package/README.md +89 -2
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +128 -0
- package/dist/client.d.ts +55 -0
- package/dist/client.js +484 -0
- package/dist/consent.d.ts +24 -0
- package/dist/consent.js +205 -0
- package/dist/document.d.ts +10 -0
- package/dist/document.js +6 -0
- package/dist/endpoint.d.ts +8 -0
- package/dist/endpoint.js +91 -0
- package/dist/engine/backends/backend-factory.d.ts +6 -0
- package/dist/engine/backends/backend-factory.js +16 -0
- package/dist/engine/backends/backend-identity.d.ts +3 -0
- package/dist/engine/backends/backend-identity.js +38 -0
- package/dist/engine/backends/backend-types.d.ts +70 -0
- package/dist/engine/backends/backend-types.js +1 -0
- package/dist/engine/backends/git/git-backend.d.ts +55 -0
- package/dist/engine/backends/git/git-backend.js +700 -0
- package/dist/engine/backends/git/git-config.d.ts +5 -0
- package/dist/engine/backends/git/git-config.js +118 -0
- package/dist/engine/backends/git/git-runner.d.ts +25 -0
- package/dist/engine/backends/git/git-runner.js +232 -0
- package/dist/engine/backends/git/git-storage.d.ts +39 -0
- package/dist/engine/backends/git/git-storage.js +226 -0
- package/dist/engine/backends/s3/s3-backend.d.ts +32 -0
- package/dist/engine/backends/s3/s3-backend.js +436 -0
- package/dist/engine/backends/s3/s3-client.d.ts +36 -0
- package/dist/engine/backends/s3/s3-client.js +352 -0
- package/dist/engine/backends/s3/s3-conditions.d.ts +4 -0
- package/dist/engine/backends/s3/s3-conditions.js +66 -0
- package/dist/engine/backends/snapshot-pointer.d.ts +6 -0
- package/dist/engine/backends/snapshot-pointer.js +22 -0
- package/dist/engine/backends/sync-backend.d.ts +71 -0
- package/dist/engine/backends/sync-backend.js +23 -0
- package/dist/engine/backends/webdav/webdav-backend.d.ts +34 -0
- package/dist/engine/backends/webdav/webdav-backend.js +534 -0
- package/dist/engine/backends/webdav/webdav-client.d.ts +36 -0
- package/dist/engine/backends/webdav/webdav-client.js +326 -0
- package/dist/engine/backends/webdav/webdav-config.d.ts +5 -0
- package/dist/engine/backends/webdav/webdav-config.js +73 -0
- package/dist/engine/commands/command-types.d.ts +31 -0
- package/dist/engine/commands/command-types.js +1 -0
- package/dist/engine/host.d.ts +70 -0
- package/dist/engine/host.js +16 -0
- package/dist/engine/paths.d.ts +9 -0
- package/dist/engine/paths.js +71 -0
- package/dist/engine/settings/config-errors.d.ts +1 -0
- package/dist/engine/settings/config-errors.js +4 -0
- package/dist/engine/settings/config-file.d.ts +33 -0
- package/dist/engine/settings/config-file.js +630 -0
- package/dist/engine/settings/config.d.ts +24 -0
- package/dist/engine/settings/config.js +200 -0
- package/dist/engine/settings/settings-management.d.ts +24 -0
- package/dist/engine/settings/settings-management.js +191 -0
- package/dist/engine/settings/settings-store.d.ts +8 -0
- package/dist/engine/settings/settings-store.js +41 -0
- package/dist/engine/settings/settings-types.d.ts +134 -0
- package/dist/engine/settings/settings-types.js +1 -0
- package/dist/engine/settings/settings-validation.d.ts +15 -0
- package/dist/engine/settings/settings-validation.js +356 -0
- package/dist/engine/snapshot/session-paths.d.ts +12 -0
- package/dist/engine/snapshot/session-paths.js +91 -0
- package/dist/engine/snapshot/snapshot-apply.d.ts +22 -0
- package/dist/engine/snapshot/snapshot-apply.js +262 -0
- package/dist/engine/snapshot/snapshot-case-replacement.d.ts +15 -0
- package/dist/engine/snapshot/snapshot-case-replacement.js +82 -0
- package/dist/engine/snapshot/snapshot-codec.d.ts +6 -0
- package/dist/engine/snapshot/snapshot-codec.js +53 -0
- package/dist/engine/snapshot/snapshot-exclude-safety.d.ts +7 -0
- package/dist/engine/snapshot/snapshot-exclude-safety.js +47 -0
- package/dist/engine/snapshot/snapshot-mutation-queues.d.ts +9 -0
- package/dist/engine/snapshot/snapshot-mutation-queues.js +133 -0
- package/dist/engine/snapshot/snapshot-paths.d.ts +1 -0
- package/dist/engine/snapshot/snapshot-paths.js +11 -0
- package/dist/engine/snapshot/snapshot-root-transition.d.ts +25 -0
- package/dist/engine/snapshot/snapshot-root-transition.js +122 -0
- package/dist/engine/snapshot/snapshot-transaction-directory.d.ts +2 -0
- package/dist/engine/snapshot/snapshot-transaction-directory.js +16 -0
- package/dist/engine/snapshot/snapshot-transaction-plan.d.ts +12 -0
- package/dist/engine/snapshot/snapshot-transaction-plan.js +68 -0
- package/dist/engine/snapshot/snapshot-transaction.d.ts +15 -0
- package/dist/engine/snapshot/snapshot-transaction.js +787 -0
- package/dist/engine/snapshot/snapshot-types.d.ts +48 -0
- package/dist/engine/snapshot/snapshot-types.js +1 -0
- package/dist/engine/snapshot/snapshot.d.ts +30 -0
- package/dist/engine/snapshot/snapshot.js +364 -0
- package/dist/engine/state/json-file.d.ts +6 -0
- package/dist/engine/state/json-file.js +49 -0
- package/dist/engine/state/lock-policy.d.ts +2 -0
- package/dist/engine/state/lock-policy.js +2 -0
- package/dist/engine/state/lock.d.ts +20 -0
- package/dist/engine/state/lock.js +259 -0
- package/dist/engine/state/lockfile-fs.d.ts +13 -0
- package/dist/engine/state/lockfile-fs.js +13 -0
- package/dist/engine/state/merge-baseline-store.d.ts +11 -0
- package/dist/engine/state/merge-baseline-store.js +187 -0
- package/dist/engine/state/mutation-directory-sync.d.ts +5 -0
- package/dist/engine/state/mutation-directory-sync.js +34 -0
- package/dist/engine/state/operation-availability.d.ts +49 -0
- package/dist/engine/state/operation-availability.js +40 -0
- package/dist/engine/state/state-directory.d.ts +14 -0
- package/dist/engine/state/state-directory.js +225 -0
- package/dist/engine/state/state-types.d.ts +27 -0
- package/dist/engine/state/state-types.js +1 -0
- package/dist/engine/state/sync-state-store.d.ts +11 -0
- package/dist/engine/state/sync-state-store.js +54 -0
- package/dist/engine/sync/agent-candidate.d.ts +44 -0
- package/dist/engine/sync/agent-candidate.js +207 -0
- package/dist/engine/sync/agent-plan.d.ts +8 -0
- package/dist/engine/sync/agent-plan.js +84 -0
- package/dist/engine/sync/agent-policy.d.ts +9 -0
- package/dist/engine/sync/agent-policy.js +10 -0
- package/dist/engine/sync/agent-refusal-retry.d.ts +4 -0
- package/dist/engine/sync/agent-refusal-retry.js +39 -0
- package/dist/engine/sync/agent-request.d.ts +14 -0
- package/dist/engine/sync/agent-request.js +13 -0
- package/dist/engine/sync/agent-runner.d.ts +6 -0
- package/dist/engine/sync/agent-runner.js +183 -0
- package/dist/engine/sync/agent-store.d.ts +7 -0
- package/dist/engine/sync/agent-store.js +177 -0
- package/dist/engine/sync/agent-sync.d.ts +4 -0
- package/dist/engine/sync/agent-sync.js +132 -0
- package/dist/engine/sync/apply-review.d.ts +8 -0
- package/dist/engine/sync/apply-review.js +51 -0
- package/dist/engine/sync/conflict-artifact-ownership.d.ts +4 -0
- package/dist/engine/sync/conflict-artifact-ownership.js +53 -0
- package/dist/engine/sync/conflict-artifacts.d.ts +32 -0
- package/dist/engine/sync/conflict-artifacts.js +158 -0
- package/dist/engine/sync/conflict-groups.d.ts +9 -0
- package/dist/engine/sync/conflict-groups.js +76 -0
- package/dist/engine/sync/conflict-preview.d.ts +4 -0
- package/dist/engine/sync/conflict-preview.js +53 -0
- package/dist/engine/sync/conflict-resolution.d.ts +13 -0
- package/dist/engine/sync/conflict-resolution.js +53 -0
- package/dist/engine/sync/conflict-retention.d.ts +5 -0
- package/dist/engine/sync/conflict-retention.js +61 -0
- package/dist/engine/sync/conflict-review.d.ts +4 -0
- package/dist/engine/sync/conflict-review.js +73 -0
- package/dist/engine/sync/content-conflicts.d.ts +10 -0
- package/dist/engine/sync/content-conflicts.js +70 -0
- package/dist/engine/sync/continuous-errors.d.ts +14 -0
- package/dist/engine/sync/continuous-errors.js +66 -0
- package/dist/engine/sync/continuous-lease.d.ts +9 -0
- package/dist/engine/sync/continuous-lease.js +206 -0
- package/dist/engine/sync/continuous-record.d.ts +31 -0
- package/dist/engine/sync/continuous-record.js +199 -0
- package/dist/engine/sync/continuous-storage.d.ts +14 -0
- package/dist/engine/sync/continuous-storage.js +59 -0
- package/dist/engine/sync/continuous-sync.d.ts +15 -0
- package/dist/engine/sync/continuous-sync.js +496 -0
- package/dist/engine/sync/continuous-watch.d.ts +3 -0
- package/dist/engine/sync/continuous-watch.js +65 -0
- package/dist/engine/sync/exclude-policy.d.ts +5 -0
- package/dist/engine/sync/exclude-policy.js +79 -0
- package/dist/engine/sync/field-migration.d.ts +5 -0
- package/dist/engine/sync/field-migration.js +20 -0
- package/dist/engine/sync/file-merge-planner.d.ts +36 -0
- package/dist/engine/sync/file-merge-planner.js +123 -0
- package/dist/engine/sync/initialization.d.ts +12 -0
- package/dist/engine/sync/initialization.js +75 -0
- package/dist/engine/sync/json-document.d.ts +29 -0
- package/dist/engine/sync/json-document.js +172 -0
- package/dist/engine/sync/local-field-policy.d.ts +2 -0
- package/dist/engine/sync/local-field-policy.js +26 -0
- package/dist/engine/sync/local-fields.d.ts +9 -0
- package/dist/engine/sync/local-fields.js +116 -0
- package/dist/engine/sync/merge-apply.d.ts +5 -0
- package/dist/engine/sync/merge-apply.js +386 -0
- package/dist/engine/sync/merge-journal.d.ts +31 -0
- package/dist/engine/sync/merge-journal.js +179 -0
- package/dist/engine/sync/merge-transfer-error.d.ts +3 -0
- package/dist/engine/sync/merge-transfer-error.js +64 -0
- package/dist/engine/sync/merged-sync.d.ts +6 -0
- package/dist/engine/sync/merged-sync.js +618 -0
- package/dist/engine/sync/mutation-owner.d.ts +2 -0
- package/dist/engine/sync/mutation-owner.js +18 -0
- package/dist/engine/sync/package-merge.d.ts +3 -0
- package/dist/engine/sync/package-merge.js +82 -0
- package/dist/engine/sync/partial-progress.d.ts +15 -0
- package/dist/engine/sync/partial-progress.js +39 -0
- package/dist/engine/sync/remote-snapshot.d.ts +28 -0
- package/dist/engine/sync/remote-snapshot.js +62 -0
- package/dist/engine/sync/resolver-stream.d.ts +4 -0
- package/dist/engine/sync/resolver-stream.js +34 -0
- package/dist/engine/sync/session-merge.d.ts +3 -0
- package/dist/engine/sync/session-merge.js +178 -0
- package/dist/engine/sync/session-push.d.ts +5 -0
- package/dist/engine/sync/session-push.js +32 -0
- package/dist/engine/sync/settings-conflicts.d.ts +8 -0
- package/dist/engine/sync/settings-conflicts.js +56 -0
- package/dist/engine/sync/settings-merge.d.ts +13 -0
- package/dist/engine/sync/settings-merge.js +93 -0
- package/dist/engine/sync/signals.d.ts +2 -0
- package/dist/engine/sync/signals.js +8 -0
- package/dist/engine/sync/startup-timings.d.ts +11 -0
- package/dist/engine/sync/startup-timings.js +44 -0
- package/dist/engine/sync/sync-decision.d.ts +35 -0
- package/dist/engine/sync/sync-decision.js +66 -0
- package/dist/engine/sync/sync-error-guidance.d.ts +2 -0
- package/dist/engine/sync/sync-error-guidance.js +55 -0
- package/dist/engine/sync/sync-errors.d.ts +9 -0
- package/dist/engine/sync/sync-errors.js +18 -0
- package/dist/engine/sync/sync-inspection.d.ts +26 -0
- package/dist/engine/sync/sync-inspection.js +40 -0
- package/dist/engine/sync/sync-loaders.d.ts +7 -0
- package/dist/engine/sync/sync-loaders.js +8 -0
- package/dist/engine/sync/sync-local.d.ts +5 -0
- package/dist/engine/sync/sync-local.js +40 -0
- package/dist/engine/sync/sync-mutations.d.ts +30 -0
- package/dist/engine/sync/sync-mutations.js +589 -0
- package/dist/engine/sync/sync-operations.d.ts +2 -0
- package/dist/engine/sync/sync-operations.js +3 -0
- package/dist/engine/sync/sync-policy.d.ts +85 -0
- package/dist/engine/sync/sync-policy.js +356 -0
- package/dist/engine/sync/sync-queries.d.ts +7 -0
- package/dist/engine/sync/sync-queries.js +195 -0
- package/dist/engine/sync/sync-state.d.ts +31 -0
- package/dist/engine/sync/sync-state.js +92 -0
- package/dist/engine/sync/text-merge.d.ts +19 -0
- package/dist/engine/sync/text-merge.js +124 -0
- package/dist/engine/types.d.ts +5 -0
- package/dist/engine/types.js +2 -0
- package/dist/engine/ui/merge-review.d.ts +2 -0
- package/dist/engine/ui/merge-review.js +7 -0
- package/dist/engine/ui/merged-sync-summary.d.ts +25 -0
- package/dist/engine/ui/merged-sync-summary.js +86 -0
- package/dist/engine/ui/sync-format.d.ts +13 -0
- package/dist/engine/ui/sync-format.js +150 -0
- package/dist/engine/ui/sync-status.d.ts +9 -0
- package/dist/engine/ui/sync-status.js +30 -0
- package/dist/engine/ui/terminal-text.d.ts +3 -0
- package/dist/engine/ui/terminal-text.js +9 -0
- package/dist/engine/ui/transfer-changes.d.ts +12 -0
- package/dist/engine/ui/transfer-changes.js +24 -0
- package/dist/engine/ui/transfer-review.d.ts +3 -0
- package/dist/engine/ui/transfer-review.js +19 -0
- package/dist/engine-service.d.ts +11 -0
- package/dist/engine-service.js +315 -0
- package/dist/identity.d.ts +20 -0
- package/dist/identity.js +133 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +4 -0
- package/dist/lifecycle-log.d.ts +7 -0
- package/dist/lifecycle-log.js +53 -0
- package/dist/lifecycle.d.ts +2 -0
- package/dist/lifecycle.js +21 -0
- package/dist/model-signature.d.ts +2 -0
- package/dist/model-signature.js +7 -0
- package/dist/monitor.d.ts +11 -0
- package/dist/monitor.js +245 -0
- package/dist/participant-admission.d.ts +5 -0
- package/dist/participant-admission.js +30 -0
- package/dist/protocol.d.ts +97 -0
- package/dist/protocol.js +90 -0
- package/dist/resolver-host.d.ts +11 -0
- package/dist/resolver-host.js +78 -0
- package/dist/server.d.ts +18 -0
- package/dist/server.js +724 -0
- package/dist/service-host.d.ts +7 -0
- package/dist/service-host.js +174 -0
- package/docs/architecture.md +133 -0
- package/package.json +54 -4
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { encodeKey, posixJoin } from "../../paths.js";
|
|
3
|
+
import { sessionTokenWarnings } from "../../settings/config.js";
|
|
4
|
+
import { decodeSnapshot, encodeSnapshot } from "../../snapshot/snapshot-codec.js";
|
|
5
|
+
import { validatePortableSnapshot } from "../../sync/local-fields.js";
|
|
6
|
+
import { throwIfAborted } from "../../sync/signals.js";
|
|
7
|
+
import { syncErrorGuidance } from "../../sync/sync-error-guidance.js";
|
|
8
|
+
import { portableSnapshotSelection } from "../../sync/sync-policy.js";
|
|
9
|
+
import { createLatestPointer, remoteHistoryEntry } from "../snapshot-pointer.js";
|
|
10
|
+
import { SyncBackendConflictError, SyncBackendPublicationOutcomeUnknownError, } from "../sync-backend.js";
|
|
11
|
+
import { S3Client, S3HttpError, S3ObjectAlreadyExistsError } from "./s3-client.js";
|
|
12
|
+
import { requireS3Etag, verifyS3Conditions } from "./s3-conditions.js";
|
|
13
|
+
const VERSION = 1;
|
|
14
|
+
const POST_COMMIT_TIMEOUT_MS = 30_000;
|
|
15
|
+
export class S3SyncBackend {
|
|
16
|
+
config;
|
|
17
|
+
postCommitTimeoutMs;
|
|
18
|
+
identity;
|
|
19
|
+
destination;
|
|
20
|
+
capability = "conditional-required";
|
|
21
|
+
checksums = new Map();
|
|
22
|
+
recoveryHead;
|
|
23
|
+
constructor(config, postCommitTimeoutMs = POST_COMMIT_TIMEOUT_MS) {
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.postCommitTimeoutMs = postCommitTimeoutMs;
|
|
26
|
+
assertSafeDestination(config);
|
|
27
|
+
this.identity = s3BackendIdentity(config);
|
|
28
|
+
this.destination = s3Destination(config);
|
|
29
|
+
}
|
|
30
|
+
sameRevision(left, right) {
|
|
31
|
+
return left === right;
|
|
32
|
+
}
|
|
33
|
+
matchesUncommittedRecoveryHead(current, recorded) {
|
|
34
|
+
return (this.sameRevision(current.revision, recorded.revision) ||
|
|
35
|
+
(current.snapshotId === recorded.snapshotId &&
|
|
36
|
+
current.snapshotRef === recorded.snapshotRef &&
|
|
37
|
+
current.revision === this.recoveryHead?.current &&
|
|
38
|
+
recorded.revision === this.recoveryHead?.legacy));
|
|
39
|
+
}
|
|
40
|
+
async readHead(signal) {
|
|
41
|
+
this.recoveryHead = undefined;
|
|
42
|
+
const object = await new S3Client(this.config, signal).getJson(latestKey(this.config));
|
|
43
|
+
throwIfAborted(signal);
|
|
44
|
+
if (object.missing)
|
|
45
|
+
return undefined;
|
|
46
|
+
const pointer = requirePointer(object.value, "Remote latest pointer is malformed.", this.config.destination.namespace);
|
|
47
|
+
this.registerChecksum(pointer.snapshot, pointer.sha256);
|
|
48
|
+
const head = remoteHead(pointer, this.identity, object.etag);
|
|
49
|
+
if (object.etag && /^"[^"\r\n]+"$/u.test(object.etag)) {
|
|
50
|
+
// Old clients hashed R2's compression-weakened ETag. Bind the alias to the
|
|
51
|
+
// exact current pointer and strong ETag, solely for inactive-journal retirement.
|
|
52
|
+
this.recoveryHead = {
|
|
53
|
+
current: head.revision,
|
|
54
|
+
legacy: remoteHead(pointer, this.identity, `W/${object.etag}`).revision,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return head;
|
|
58
|
+
}
|
|
59
|
+
async readSnapshot(reference, signal) {
|
|
60
|
+
const expectedChecksum = this.checksums.get(reference) ?? (await this.resolveChecksum(reference, signal));
|
|
61
|
+
const object = await new S3Client(this.config, signal).getBuffer(snapshotKey(this.config, reference));
|
|
62
|
+
throwIfAborted(signal);
|
|
63
|
+
if (!object.value)
|
|
64
|
+
throw new Error(`Snapshot not found: ${reference}`);
|
|
65
|
+
if (expectedChecksum && sha256(object.value) !== expectedChecksum) {
|
|
66
|
+
throw new Error("Remote snapshot checksum mismatch.");
|
|
67
|
+
}
|
|
68
|
+
const snapshot = await decodeSnapshot(object.value, { signal });
|
|
69
|
+
if (snapshot.id !== reference || snapshot.profile !== this.config.destination.namespace) {
|
|
70
|
+
throw new Error("Remote snapshot identity mismatch.");
|
|
71
|
+
}
|
|
72
|
+
validateSnapshotBundle(snapshot);
|
|
73
|
+
return snapshot;
|
|
74
|
+
}
|
|
75
|
+
async publishSnapshot(snapshot, expected, options = {}) {
|
|
76
|
+
throwIfAborted(options.signal);
|
|
77
|
+
assertSnapshotIdentity(snapshot, this.config.destination.namespace);
|
|
78
|
+
await verifyS3Conditions(this.config, options.signal);
|
|
79
|
+
throwIfAborted(options.signal);
|
|
80
|
+
const stagedKey = snapshotKey(this.config, snapshot.id);
|
|
81
|
+
const encoded = await encodeSnapshot(snapshot);
|
|
82
|
+
throwIfAborted(options.signal);
|
|
83
|
+
const pointer = pointerFor(this.config, snapshot, sha256(encoded));
|
|
84
|
+
const cancellableClient = new S3Client(this.config, options.signal);
|
|
85
|
+
try {
|
|
86
|
+
await cancellableClient.putBuffer(stagedKey, encoded, "application/gzip", {
|
|
87
|
+
ifAbsent: true,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
if (!(error instanceof S3ObjectAlreadyExistsError))
|
|
92
|
+
throw error;
|
|
93
|
+
const existing = await cancellableClient.getBuffer(stagedKey);
|
|
94
|
+
if (!existing.value || sha256(existing.value) !== pointer.sha256) {
|
|
95
|
+
throw new SyncBackendConflictError(`Immutable snapshot id already exists with different content: ${snapshot.id}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const currentObject = await cancellableClient.getJson(latestKey(this.config));
|
|
99
|
+
throwIfAborted(options.signal);
|
|
100
|
+
let current;
|
|
101
|
+
if (!currentObject.missing) {
|
|
102
|
+
const currentPointer = requirePointer(currentObject.value, "Remote latest pointer is malformed.", this.config.destination.namespace);
|
|
103
|
+
this.registerChecksum(currentPointer.snapshot, currentPointer.sha256);
|
|
104
|
+
current = remoteHead(currentPointer, this.identity, currentObject.etag);
|
|
105
|
+
}
|
|
106
|
+
if (!matchesExpected(current, expected)) {
|
|
107
|
+
throw new SyncBackendConflictError("Remote changed while pushing. Run /sync pull first, then retry.", {
|
|
108
|
+
currentHead: current,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
const condition = currentObject.missing ? { ifAbsent: true } : { ifMatch: requireS3Etag(currentObject.etag) };
|
|
112
|
+
throwIfAborted(options.signal);
|
|
113
|
+
options.onCommit?.();
|
|
114
|
+
// The active-head PUT is the publication boundary. Do not bind it or its
|
|
115
|
+
// verification to a user-cancellation signal after the boundary begins.
|
|
116
|
+
const commitClient = new S3Client(this.config, AbortSignal.timeout(this.postCommitTimeoutMs));
|
|
117
|
+
try {
|
|
118
|
+
await commitClient.putJson(latestKey(this.config), pointer, condition);
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (error instanceof S3ObjectAlreadyExistsError ||
|
|
122
|
+
(error instanceof S3HttpError && (error.status === 412 || error.status === 409))) {
|
|
123
|
+
throw new SyncBackendConflictError("Remote conditional publication was rejected; reread and retry.", {
|
|
124
|
+
cause: error,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
throw new SyncBackendPublicationOutcomeUnknownError(`Remote publication outcome is unknown: ${errorMessage(error)}`, { cause: error });
|
|
128
|
+
}
|
|
129
|
+
let verifiedObject;
|
|
130
|
+
try {
|
|
131
|
+
verifiedObject = await commitClient.getJson(latestKey(this.config));
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
throw new SyncBackendPublicationOutcomeUnknownError(`Remote snapshot may be active, but publication could not be verified: ${errorMessage(error)}`, { cause: error });
|
|
135
|
+
}
|
|
136
|
+
let verifiedPointer;
|
|
137
|
+
try {
|
|
138
|
+
verifiedPointer = requirePointer(verifiedObject.value, "Remote latest pointer is malformed after publication.", this.config.destination.namespace);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
throw new SyncBackendPublicationOutcomeUnknownError(`Remote snapshot may be active, but publication verification was malformed: ${errorMessage(error)}`, { cause: error });
|
|
142
|
+
}
|
|
143
|
+
if (!samePointer(verifiedPointer, pointer)) {
|
|
144
|
+
throw new SyncBackendConflictError("Remote latest changed immediately after push. Run /sync status before continuing.", {
|
|
145
|
+
phase: "after-commit",
|
|
146
|
+
currentHead: remoteHead(verifiedPointer, this.identity, verifiedObject.etag),
|
|
147
|
+
candidateMayHaveBeenActive: true,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
this.registerChecksum(verifiedPointer.snapshot, verifiedPointer.sha256);
|
|
151
|
+
const head = remoteHead(verifiedPointer, this.identity, verifiedObject.etag);
|
|
152
|
+
const warning = await this.updateHistorySafely(commitClient, pointer);
|
|
153
|
+
return { head, warnings: warning ? [warning] : [] };
|
|
154
|
+
}
|
|
155
|
+
async listHistory(signal) {
|
|
156
|
+
const object = await new S3Client(this.config, signal).getJson(historyKey(this.config));
|
|
157
|
+
throwIfAborted(signal);
|
|
158
|
+
if (object.missing)
|
|
159
|
+
return [];
|
|
160
|
+
const pointers = requireHistory(object.value, this.config.destination.namespace);
|
|
161
|
+
for (const pointer of pointers) {
|
|
162
|
+
const known = this.checksums.get(pointer.snapshot);
|
|
163
|
+
if (known && known !== pointer.sha256) {
|
|
164
|
+
throw new Error("Remote history conflicts with an already observed snapshot checksum.");
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
for (const pointer of pointers)
|
|
168
|
+
this.registerChecksum(pointer.snapshot, pointer.sha256);
|
|
169
|
+
return pointers.map(remoteHistoryEntry);
|
|
170
|
+
}
|
|
171
|
+
async diagnose(signal) {
|
|
172
|
+
throwIfAborted(signal);
|
|
173
|
+
const diagnostics = [
|
|
174
|
+
{
|
|
175
|
+
key: "s3-config",
|
|
176
|
+
level: "info",
|
|
177
|
+
message: `s3 config: ok (${this.config.destination.bucket}/${storageRoot(this.config)})`,
|
|
178
|
+
},
|
|
179
|
+
...sessionTokenWarnings(this.config.profile).map((message) => ({
|
|
180
|
+
key: "s3-session-token",
|
|
181
|
+
level: "warning",
|
|
182
|
+
message,
|
|
183
|
+
})),
|
|
184
|
+
];
|
|
185
|
+
const deadline = AbortSignal.timeout(10_000);
|
|
186
|
+
const probeSignal = signal ? AbortSignal.any([signal, deadline]) : deadline;
|
|
187
|
+
try {
|
|
188
|
+
const result = await new S3Client(this.config, probeSignal).diagnoseRead(latestKey(this.config));
|
|
189
|
+
throwIfAborted(probeSignal);
|
|
190
|
+
const messages = {
|
|
191
|
+
readable: "s3 remote read: HTTP success. Snapshot contents and write access are not tested.",
|
|
192
|
+
"missing-key": "s3 remote read: no snapshot at this path (NoSuchKey). Write access is not tested; verify the bucket and path before first sync.",
|
|
193
|
+
"missing-bucket": "s3 remote read: bucket not found (NoSuchBucket). Create the bucket with your provider or edit the setup's bucket name.",
|
|
194
|
+
"unknown-404": "s3 remote read: HTTP 404; cannot confirm whether the bucket or snapshot is missing. Verify the endpoint, bucket, and path with your provider. Write access is not tested.",
|
|
195
|
+
};
|
|
196
|
+
diagnostics.push({
|
|
197
|
+
key: "s3-read",
|
|
198
|
+
level: result === "readable" ? "info" : "warning",
|
|
199
|
+
message: messages[result],
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
throwIfAborted(signal);
|
|
204
|
+
diagnostics.push({
|
|
205
|
+
key: "s3-read",
|
|
206
|
+
level: "warning",
|
|
207
|
+
message: syncErrorGuidance(deadline.aborted ? new Error("S3 diagnostic request timed out.") : error),
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
return diagnostics;
|
|
211
|
+
}
|
|
212
|
+
async resolveChecksum(reference, signal) {
|
|
213
|
+
await this.readHead(signal);
|
|
214
|
+
const headChecksum = this.checksums.get(reference);
|
|
215
|
+
if (headChecksum)
|
|
216
|
+
return headChecksum;
|
|
217
|
+
await this.listHistory(signal);
|
|
218
|
+
return this.checksums.get(reference);
|
|
219
|
+
}
|
|
220
|
+
registerChecksum(reference, checksum) {
|
|
221
|
+
const known = this.checksums.get(reference);
|
|
222
|
+
if (known && known !== checksum) {
|
|
223
|
+
throw new Error("Remote snapshot reference was rebound to a different checksum.");
|
|
224
|
+
}
|
|
225
|
+
this.checksums.set(reference, checksum);
|
|
226
|
+
}
|
|
227
|
+
async updateHistorySafely(client, pointer) {
|
|
228
|
+
try {
|
|
229
|
+
await this.updateHistory(client, pointer);
|
|
230
|
+
return undefined;
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
return `Remote snapshot is active, but history could not be updated: ${errorMessage(error)}. Run /sync doctor before relying on history.`;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
async updateHistory(client, pointer) {
|
|
237
|
+
const object = await client.getJson(historyKey(this.config));
|
|
238
|
+
const snapshots = object.missing ? [] : requireHistory(object.value, this.config.destination.namespace);
|
|
239
|
+
const existing = snapshots.find((snapshot) => snapshot.snapshot === pointer.snapshot);
|
|
240
|
+
if (existing) {
|
|
241
|
+
if (!samePointer(existing, pointer))
|
|
242
|
+
throw new Error("Remote history rebound an immutable snapshot reference.");
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
const next = [...snapshots, pointer].slice(-100);
|
|
246
|
+
const condition = object.missing ? { ifAbsent: true } : { ifMatch: requireS3Etag(object.etag) };
|
|
247
|
+
await client.putJson(historyKey(this.config), { version: VERSION, snapshots: next }, condition);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
export function latestKey(config) {
|
|
251
|
+
return posixJoin(storageRoot(config), "latest.json");
|
|
252
|
+
}
|
|
253
|
+
export function historyKey(config) {
|
|
254
|
+
return posixJoin(storageRoot(config), "history.json");
|
|
255
|
+
}
|
|
256
|
+
export function snapshotKey(config, id) {
|
|
257
|
+
requireSnapshotReference(id);
|
|
258
|
+
return posixJoin(storageRoot(config), "snapshots", `${id}.json.gz`);
|
|
259
|
+
}
|
|
260
|
+
export function storageRoot(config) {
|
|
261
|
+
return config.destination.prefix === "./" ? "" : config.destination.prefix;
|
|
262
|
+
}
|
|
263
|
+
export function pointerFor(config, snapshot, checksum) {
|
|
264
|
+
return createLatestPointer(config.destination.namespace, snapshot, checksum);
|
|
265
|
+
}
|
|
266
|
+
export function s3BackendIdentity(config) {
|
|
267
|
+
const destination = JSON.stringify([
|
|
268
|
+
secretFreeEndpoint(config.profile.endpoint),
|
|
269
|
+
trimSlashes(config.destination.bucket),
|
|
270
|
+
trimSlashes(config.destination.prefix),
|
|
271
|
+
]);
|
|
272
|
+
return `s3:${sha256(Buffer.from(destination))}`;
|
|
273
|
+
}
|
|
274
|
+
function s3Destination(config) {
|
|
275
|
+
let host = secretFreeEndpoint(config.profile.endpoint);
|
|
276
|
+
try {
|
|
277
|
+
host = new URL(host).hostname;
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
host = "invalid S3 endpoint";
|
|
281
|
+
}
|
|
282
|
+
return `${host} · ${config.destination.bucket}/${storageRoot(config)}`;
|
|
283
|
+
}
|
|
284
|
+
function secretFreeEndpoint(value) {
|
|
285
|
+
const normalized = value.trim();
|
|
286
|
+
try {
|
|
287
|
+
const url = new URL(normalized);
|
|
288
|
+
url.username = "";
|
|
289
|
+
url.password = "";
|
|
290
|
+
return url.toString();
|
|
291
|
+
}
|
|
292
|
+
catch {
|
|
293
|
+
return normalized.replace(/\/\/[^/@\s]+@/u, "//");
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
function trimSlashes(value) {
|
|
297
|
+
return value.replace(/^\/+|\/+$/g, "");
|
|
298
|
+
}
|
|
299
|
+
function remoteHead(pointer, identity, etag) {
|
|
300
|
+
return {
|
|
301
|
+
snapshotRef: pointer.snapshot,
|
|
302
|
+
snapshotId: pointer.snapshot,
|
|
303
|
+
revision: `s3:${sha256(Buffer.from(canonicalJson([identity, etag ?? null, pointer])))}`,
|
|
304
|
+
createdAt: pointer.createdAt,
|
|
305
|
+
machine: pointer.machine,
|
|
306
|
+
syncSessions: pointer.syncSessions === true,
|
|
307
|
+
...(pointer.selection === undefined ? {} : { selection: pointer.selection }),
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
function samePointer(left, right) {
|
|
311
|
+
return canonicalJson(left) === canonicalJson(right);
|
|
312
|
+
}
|
|
313
|
+
function canonicalJson(value) {
|
|
314
|
+
return JSON.stringify(canonicalValue(value));
|
|
315
|
+
}
|
|
316
|
+
function canonicalValue(value) {
|
|
317
|
+
if (Array.isArray(value))
|
|
318
|
+
return value.map(canonicalValue);
|
|
319
|
+
if (!value || typeof value !== "object")
|
|
320
|
+
return value;
|
|
321
|
+
return Object.fromEntries(Object.keys(value)
|
|
322
|
+
.sort()
|
|
323
|
+
.map((key) => [key, canonicalValue(value[key])]));
|
|
324
|
+
}
|
|
325
|
+
function matchesExpected(head, expected) {
|
|
326
|
+
if (expected.kind === "missing")
|
|
327
|
+
return head === undefined;
|
|
328
|
+
return head?.revision === expected.revision;
|
|
329
|
+
}
|
|
330
|
+
function requirePointer(value, message, expectedProfile) {
|
|
331
|
+
if (!value ||
|
|
332
|
+
value.version !== VERSION ||
|
|
333
|
+
value.profile !== expectedProfile ||
|
|
334
|
+
typeof value.snapshot !== "string" ||
|
|
335
|
+
!isSafeSnapshotReference(value.snapshot) ||
|
|
336
|
+
typeof value.sha256 !== "string" ||
|
|
337
|
+
!/^[0-9a-f]{64}$/.test(value.sha256) ||
|
|
338
|
+
typeof value.createdAt !== "string" ||
|
|
339
|
+
!isSafeMetadata(value.createdAt, 64) ||
|
|
340
|
+
typeof value.machine !== "string" ||
|
|
341
|
+
!isSafeMetadata(value.machine, 256) ||
|
|
342
|
+
(value.syncSessions !== undefined && typeof value.syncSessions !== "boolean")) {
|
|
343
|
+
throw new Error(message);
|
|
344
|
+
}
|
|
345
|
+
if (value.selection !== undefined)
|
|
346
|
+
portableSnapshotSelection(value.selection);
|
|
347
|
+
return value;
|
|
348
|
+
}
|
|
349
|
+
function requireHistory(value, expectedProfile) {
|
|
350
|
+
if (!value || value.version !== VERSION || !Array.isArray(value.snapshots)) {
|
|
351
|
+
throw new Error("Remote history is malformed.");
|
|
352
|
+
}
|
|
353
|
+
const pointers = value.snapshots.map((pointer) => requirePointer(pointer, "Remote history entry is malformed.", expectedProfile));
|
|
354
|
+
const references = new Set();
|
|
355
|
+
for (const pointer of pointers) {
|
|
356
|
+
if (references.has(pointer.snapshot)) {
|
|
357
|
+
throw new Error("Remote history contains duplicate snapshot references.");
|
|
358
|
+
}
|
|
359
|
+
references.add(pointer.snapshot);
|
|
360
|
+
}
|
|
361
|
+
return pointers;
|
|
362
|
+
}
|
|
363
|
+
function assertSnapshotIdentity(snapshot, expectedProfile) {
|
|
364
|
+
validatePortableSnapshot(snapshot);
|
|
365
|
+
if (snapshot.profile !== expectedProfile ||
|
|
366
|
+
!isSafeSnapshotReference(snapshot.id) ||
|
|
367
|
+
!isSafeMetadata(snapshot.createdAt, 64) ||
|
|
368
|
+
!isSafeMetadata(snapshot.machine, 256) ||
|
|
369
|
+
!Array.isArray(snapshot.files)) {
|
|
370
|
+
throw new Error("Invalid snapshot identity for S3 publication.");
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
function validateSnapshotBundle(snapshot) {
|
|
374
|
+
const paths = new Set();
|
|
375
|
+
for (const file of snapshot.files) {
|
|
376
|
+
if (!file ||
|
|
377
|
+
typeof file.path !== "string" ||
|
|
378
|
+
typeof file.contentBase64 !== "string" ||
|
|
379
|
+
typeof file.sha256 !== "string" ||
|
|
380
|
+
!/^[0-9a-f]{64}$/.test(file.sha256) ||
|
|
381
|
+
paths.has(file.path)) {
|
|
382
|
+
throw new Error("Remote snapshot bundle is malformed.");
|
|
383
|
+
}
|
|
384
|
+
const content = Buffer.from(file.contentBase64, "base64");
|
|
385
|
+
if (content.toString("base64") !== file.contentBase64 || sha256(content) !== file.sha256) {
|
|
386
|
+
throw new Error("Remote snapshot file checksum mismatch.");
|
|
387
|
+
}
|
|
388
|
+
paths.add(file.path);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
function assertSafeDestination(config) {
|
|
392
|
+
for (const [label, value, allowEmpty] of [
|
|
393
|
+
["bucket", config.destination.bucket, false],
|
|
394
|
+
["prefix", config.destination.prefix, true],
|
|
395
|
+
["namespace", config.destination.namespace, false],
|
|
396
|
+
]) {
|
|
397
|
+
if (label === "prefix" && value === "./")
|
|
398
|
+
continue;
|
|
399
|
+
if ((!allowEmpty && value.length === 0) ||
|
|
400
|
+
value.includes("\\") ||
|
|
401
|
+
hasControlCharacter(value) ||
|
|
402
|
+
value.split("/").some((segment) => segment === "." || segment === "..")) {
|
|
403
|
+
throw new Error(`Invalid S3 storage location ${label}.`);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
function requireSnapshotReference(reference) {
|
|
408
|
+
if (!isSafeSnapshotReference(reference)) {
|
|
409
|
+
throw new Error("Invalid S3 snapshot reference.");
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
function isSafeSnapshotReference(reference) {
|
|
413
|
+
return (reference.length > 0 &&
|
|
414
|
+
reference.length <= 512 &&
|
|
415
|
+
reference !== "." &&
|
|
416
|
+
reference !== ".." &&
|
|
417
|
+
!reference.includes("/") &&
|
|
418
|
+
!reference.includes("\\") &&
|
|
419
|
+
!hasControlCharacter(reference));
|
|
420
|
+
}
|
|
421
|
+
function isSafeMetadata(value, maxLength) {
|
|
422
|
+
return value.length > 0 && value.length <= maxLength && !hasControlCharacter(value);
|
|
423
|
+
}
|
|
424
|
+
function hasControlCharacter(value) {
|
|
425
|
+
return [...value].some((character) => {
|
|
426
|
+
const code = character.codePointAt(0) ?? 0;
|
|
427
|
+
return code < 0x20 || code === 0x7f;
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
function sha256(value) {
|
|
431
|
+
return createHash("sha256").update(value).digest("hex");
|
|
432
|
+
}
|
|
433
|
+
function errorMessage(error) {
|
|
434
|
+
return error instanceof Error ? error.message : String(error);
|
|
435
|
+
}
|
|
436
|
+
export { encodeKey };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { RemoteObject, ResolvedS3Backend } from "../backend-types.js";
|
|
2
|
+
export declare class S3ObjectAlreadyExistsError extends Error {
|
|
3
|
+
readonly key: string;
|
|
4
|
+
constructor(key: string);
|
|
5
|
+
}
|
|
6
|
+
export declare class S3HttpError extends Error {
|
|
7
|
+
readonly status: number;
|
|
8
|
+
readonly retryAfterMs?: number | undefined;
|
|
9
|
+
constructor(message: string, status: number, retryAfterMs?: number | undefined);
|
|
10
|
+
}
|
|
11
|
+
export declare class S3Client {
|
|
12
|
+
private config;
|
|
13
|
+
private endpoint;
|
|
14
|
+
private signal?;
|
|
15
|
+
private requestTimeoutMs;
|
|
16
|
+
private omitSessionTokenAfterRejection;
|
|
17
|
+
constructor(config: ResolvedS3Backend, signal?: AbortSignal, requestTimeoutMs?: number);
|
|
18
|
+
/** Read-only access probe. A bare 404 cannot distinguish a missing key from a missing bucket. */
|
|
19
|
+
diagnoseRead(key: string): Promise<"readable" | "missing-key" | "missing-bucket" | "unknown-404">;
|
|
20
|
+
getJson<T>(key: string): Promise<RemoteObject<T>>;
|
|
21
|
+
getBuffer(key: string): Promise<RemoteObject<Buffer>>;
|
|
22
|
+
putJson(key: string, value: unknown, options?: {
|
|
23
|
+
ifAbsent?: boolean;
|
|
24
|
+
ifMatch?: string;
|
|
25
|
+
}): Promise<void>;
|
|
26
|
+
putBuffer(key: string, body: Buffer, contentType: string, options?: {
|
|
27
|
+
ifAbsent?: boolean;
|
|
28
|
+
ifMatch?: string;
|
|
29
|
+
}): Promise<void>;
|
|
30
|
+
delete(key: string): Promise<void>;
|
|
31
|
+
private request;
|
|
32
|
+
private transportSignal;
|
|
33
|
+
private shouldRetryWithoutSessionToken;
|
|
34
|
+
private readErrorText;
|
|
35
|
+
private redact;
|
|
36
|
+
}
|