@narumitw/pi-sync-daemon 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +11 -1
- package/dist/cli.js +2 -0
- package/dist/client-transport.d.ts +14 -0
- package/dist/client-transport.js +66 -0
- package/dist/client.d.ts +15 -8
- package/dist/client.js +116 -86
- package/dist/consent.js +33 -11
- package/dist/daemon-debug.d.ts +3 -0
- package/dist/daemon-debug.js +33 -0
- package/dist/daemon-recovery.d.ts +4 -0
- package/dist/daemon-recovery.js +75 -0
- package/dist/engine/backends/git/git-backend.d.ts +8 -0
- package/dist/engine/backends/git/git-backend.js +39 -12
- package/dist/engine/backends/git/git-runner.js +6 -1
- package/dist/engine/backends/git/git-storage.js +5 -9
- package/dist/engine/backends/publication-encoding.d.ts +3 -0
- package/dist/engine/backends/publication-encoding.js +19 -0
- package/dist/engine/backends/s3/s3-backend.js +10 -12
- package/dist/engine/backends/sync-backend.d.ts +4 -0
- package/dist/engine/backends/webdav/webdav-backend.d.ts +0 -2
- package/dist/engine/backends/webdav/webdav-backend.js +15 -26
- package/dist/engine/backends/webdav/webdav-client.js +10 -0
- package/dist/engine/commands/command-types.d.ts +1 -0
- package/dist/engine/debug-log.d.ts +27 -0
- package/dist/engine/debug-log.js +134 -0
- package/dist/engine/paths.d.ts +2 -0
- package/dist/engine/paths.js +7 -1
- package/dist/engine/settings/config-file.d.ts +10 -1
- package/dist/engine/settings/config-file.js +62 -17
- package/dist/engine/settings/config.d.ts +7 -0
- package/dist/engine/settings/config.js +49 -14
- package/dist/engine/snapshot/snapshot-apply.d.ts +2 -1
- package/dist/engine/snapshot/snapshot-apply.js +30 -8
- package/dist/engine/snapshot/snapshot-codec.js +3 -1
- package/dist/engine/snapshot/snapshot-payload.d.ts +4 -0
- package/dist/engine/snapshot/snapshot-payload.js +32 -0
- package/dist/engine/snapshot/snapshot-transaction-images.d.ts +2 -0
- package/dist/engine/snapshot/snapshot-transaction-images.js +34 -0
- package/dist/engine/snapshot/snapshot-transaction.d.ts +4 -1
- package/dist/engine/snapshot/snapshot-transaction.js +50 -7
- package/dist/engine/snapshot/snapshot-types.d.ts +10 -2
- package/dist/engine/snapshot/snapshot.d.ts +3 -1
- package/dist/engine/snapshot/snapshot.js +64 -25
- package/dist/engine/state/lock.d.ts +2 -1
- package/dist/engine/state/lock.js +157 -17
- package/dist/engine/sync/continuous-sync.d.ts +10 -1
- package/dist/engine/sync/continuous-sync.js +35 -11
- package/dist/engine/sync/initialization.d.ts +1 -1
- package/dist/engine/sync/initialization.js +2 -2
- package/dist/engine/sync/local-fields.d.ts +6 -1
- package/dist/engine/sync/local-fields.js +42 -19
- package/dist/engine/sync/merged-sync.js +13 -10
- package/dist/engine/sync/remote-snapshot.js +18 -3
- package/dist/engine/sync/session-push.js +13 -4
- package/dist/engine/sync/startup-bootstrap.d.ts +1 -0
- package/dist/engine/sync/startup-bootstrap.js +105 -0
- package/dist/engine/sync/startup-recovery-evidence.d.ts +1 -0
- package/dist/engine/sync/startup-recovery-evidence.js +37 -0
- package/dist/engine/sync/startup-timings.d.ts +9 -2
- package/dist/engine/sync/startup-timings.js +80 -8
- package/dist/engine/sync/sync-error-guidance.js +6 -1
- package/dist/engine/sync/sync-inspection.js +4 -7
- package/dist/engine/sync/sync-local.d.ts +2 -1
- package/dist/engine/sync/sync-local.js +3 -1
- package/dist/engine/sync/sync-mutations.js +9 -3
- package/dist/engine/sync/sync-policy.d.ts +2 -0
- package/dist/engine/sync/sync-policy.js +9 -1
- package/dist/engine/sync/sync-state.d.ts +8 -3
- package/dist/engine/sync/sync-state.js +12 -4
- package/dist/engine-service.d.ts +6 -0
- package/dist/engine-service.js +54 -10
- package/dist/identity.d.ts +5 -2
- package/dist/identity.js +36 -6
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/monitor.js +20 -1
- package/dist/paged-review.d.ts +35 -0
- package/dist/paged-review.js +177 -0
- package/dist/participant-ledger.d.ts +16 -0
- package/dist/participant-ledger.js +149 -0
- package/dist/protocol.d.ts +15 -0
- package/dist/protocol.js +4 -0
- package/dist/server.d.ts +7 -0
- package/dist/server.js +268 -120
- package/dist/service-host.d.ts +2 -1
- package/dist/service-host.js +29 -9
- package/dist/state-wait-jobs.d.ts +8 -0
- package/dist/state-wait-jobs.js +23 -0
- package/dist/state-wait.d.ts +7 -0
- package/dist/state-wait.js +34 -0
- package/docs/architecture.md +22 -0
- package/package.json +1 -1
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { appendDebugEvent } from "./engine/debug-log.js";
|
|
2
|
+
import { readLocalConfigForDiagnostics } from "./engine/settings/config-file.js";
|
|
3
|
+
import { validateSettingsDocument } from "./engine/settings/settings-validation.js";
|
|
4
|
+
export function createDaemonDebug(root) {
|
|
5
|
+
let waiting = [];
|
|
6
|
+
let flushing = false;
|
|
7
|
+
async function flush() {
|
|
8
|
+
while (waiting.length) {
|
|
9
|
+
const batch = waiting;
|
|
10
|
+
waiting = [];
|
|
11
|
+
try {
|
|
12
|
+
const snapshot = await readLocalConfigForDiagnostics(root);
|
|
13
|
+
if (!snapshot || validateSettingsDocument(snapshot.document.parsed).debug !== true)
|
|
14
|
+
continue;
|
|
15
|
+
for (const { event, current } of batch) {
|
|
16
|
+
if ((current?.() ?? true) && snapshot.stillCurrent())
|
|
17
|
+
appendDebugEvent(root, "pi-sync-daemon", event);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
catch { }
|
|
21
|
+
}
|
|
22
|
+
flushing = false;
|
|
23
|
+
}
|
|
24
|
+
return (event, current) => {
|
|
25
|
+
if (waiting.length >= 32)
|
|
26
|
+
return;
|
|
27
|
+
waiting.push({ event, current });
|
|
28
|
+
if (flushing)
|
|
29
|
+
return;
|
|
30
|
+
flushing = true;
|
|
31
|
+
queueMicrotask(() => void flush());
|
|
32
|
+
};
|
|
33
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type Identity } from "./protocol.js";
|
|
2
|
+
export type DaemonRecoveryTarget = Pick<Identity, "root" | "pid" | "instance">;
|
|
3
|
+
export declare function inspectDaemonRecovery(root: string, signal?: AbortSignal): Promise<DaemonRecoveryTarget | undefined>;
|
|
4
|
+
export declare function stopDaemonForRecovery(target: DaemonRecoveryTarget, signal?: AbortSignal): Promise<void>;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { acquireStartupLock, checkUpgradeConsent, finishUpgrade, hello, requestDaemon } from "./client.js";
|
|
2
|
+
import { canonicalRootGuard, processExists, processExited, readIdentity, readProcessBirth, readUpgradeOwner, recordUpgradeOwner, runtimePaths, upgradeRepairStarted, } from "./identity.js";
|
|
3
|
+
import { withCleanup } from "./lifecycle.js";
|
|
4
|
+
import { record } from "./protocol.js";
|
|
5
|
+
async function recoveryOwner(paths) {
|
|
6
|
+
const pending = await readUpgradeOwner(paths);
|
|
7
|
+
if (pending && (pending.purpose !== "cleanup" || (await upgradeRepairStarted(paths, pending))))
|
|
8
|
+
throw new Error("A daemon upgrade or repair is pending. Follow the daemon recovery guide.");
|
|
9
|
+
const identity = await readIdentity(paths);
|
|
10
|
+
if (pending && identity && !sameOwner(pending, identity))
|
|
11
|
+
throw new Error("Daemon owner changed during recovery. Keep the recorded evidence.");
|
|
12
|
+
return { identity, pending };
|
|
13
|
+
}
|
|
14
|
+
function sameOwner(left, right) {
|
|
15
|
+
return left.root === right.root && left.pid === right.pid && left.instance === right.instance;
|
|
16
|
+
}
|
|
17
|
+
export async function inspectDaemonRecovery(root, signal) {
|
|
18
|
+
const paths = await runtimePaths(root);
|
|
19
|
+
const assertRoot = canonicalRootGuard(paths.root);
|
|
20
|
+
signal?.throwIfAborted();
|
|
21
|
+
const { identity, pending } = await recoveryOwner(paths);
|
|
22
|
+
if (!pending && identity)
|
|
23
|
+
await hello(paths, identity, signal);
|
|
24
|
+
signal?.throwIfAborted();
|
|
25
|
+
assertRoot();
|
|
26
|
+
const owner = pending ?? identity;
|
|
27
|
+
if (owner)
|
|
28
|
+
return { root: owner.root, pid: owner.pid, instance: owner.instance };
|
|
29
|
+
}
|
|
30
|
+
export async function stopDaemonForRecovery(target, signal) {
|
|
31
|
+
const paths = await runtimePaths(target.root);
|
|
32
|
+
const assertRoot = canonicalRootGuard(paths.root);
|
|
33
|
+
signal?.throwIfAborted();
|
|
34
|
+
const release = await acquireStartupLock(paths, signal);
|
|
35
|
+
await withCleanup(async () => {
|
|
36
|
+
signal?.throwIfAborted();
|
|
37
|
+
assertRoot();
|
|
38
|
+
const { identity, pending } = await recoveryOwner(paths);
|
|
39
|
+
const owner = pending ?? identity;
|
|
40
|
+
if (!owner || !sameOwner(owner, target))
|
|
41
|
+
throw new Error("Daemon owner changed. Review the current owner before stopping it.");
|
|
42
|
+
let barrier = pending;
|
|
43
|
+
if (!barrier && identity) {
|
|
44
|
+
const before = await readProcessBirth(identity.pid);
|
|
45
|
+
await hello(paths, identity, signal);
|
|
46
|
+
const birth = before === undefined ? undefined : await readProcessBirth(identity.pid);
|
|
47
|
+
if (!processExists(identity.pid) || (before !== undefined && before !== birth))
|
|
48
|
+
throw new Error("Daemon process changed during authentication. Review the current owner.");
|
|
49
|
+
await checkUpgradeConsent(paths, assertRoot, signal);
|
|
50
|
+
const current = await readIdentity(paths);
|
|
51
|
+
if (!current || !sameOwner(identity, current))
|
|
52
|
+
throw new Error("Daemon owner changed. Review the current owner before stopping it.");
|
|
53
|
+
signal?.throwIfAborted();
|
|
54
|
+
assertRoot();
|
|
55
|
+
barrier = { ...target, version: identity.version, birth, purpose: "cleanup" };
|
|
56
|
+
await recordUpgradeOwner(paths, barrier);
|
|
57
|
+
}
|
|
58
|
+
if (!barrier)
|
|
59
|
+
throw new Error("Daemon recovery has no recorded owner.");
|
|
60
|
+
if (identity && !(await processExited(barrier))) {
|
|
61
|
+
await checkUpgradeConsent(paths, assertRoot, signal);
|
|
62
|
+
try {
|
|
63
|
+
const reply = await requestDaemon(paths, identity, "stop", {}, signal);
|
|
64
|
+
if (!record(reply) || reply.stopping !== true)
|
|
65
|
+
throw new Error("Daemon did not acknowledge stop.");
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
signal?.throwIfAborted();
|
|
69
|
+
if (await readIdentity(paths))
|
|
70
|
+
throw new Error("Daemon stop could not be confirmed. Keep recovery evidence and retry.", { cause: error });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
await finishUpgrade(paths, barrier, assertRoot, signal);
|
|
74
|
+
}, release);
|
|
75
|
+
}
|
|
@@ -25,15 +25,23 @@ export declare class GitSyncBackend implements SyncBackend {
|
|
|
25
25
|
private readonly afterLsRemoteForTest?;
|
|
26
26
|
private readonly afterPayloadWriteForTest?;
|
|
27
27
|
private cacheReady?;
|
|
28
|
+
private cachedPublication?;
|
|
28
29
|
constructor(config: ResolvedGitBackend, options?: GitBackendOptions);
|
|
29
30
|
sameRevision(left: string, right: string): boolean;
|
|
30
31
|
readHead(signal?: AbortSignal): Promise<RemoteHead | undefined>;
|
|
32
|
+
readHeadAndSnapshot(signal?: AbortSignal): Promise<{
|
|
33
|
+
head: RemoteHead;
|
|
34
|
+
snapshot: Snapshot;
|
|
35
|
+
} | undefined>;
|
|
31
36
|
readSnapshot(reference: string, signal?: AbortSignal): Promise<Snapshot>;
|
|
37
|
+
private requireReachablePublication;
|
|
38
|
+
private readPublicationSnapshot;
|
|
32
39
|
publishSnapshot(snapshot: Snapshot, expected: ExpectedRemoteHead, options?: PublishSnapshotOptions): Promise<PublishSnapshotResult>;
|
|
33
40
|
listHistory(signal?: AbortSignal): Promise<RemoteHistoryEntry[]>;
|
|
34
41
|
diagnose(signal?: AbortSignal): Promise<BackendDiagnostic[]>;
|
|
35
42
|
private resolveSnapshotReference;
|
|
36
43
|
private headForSha;
|
|
44
|
+
private readCachedPublication;
|
|
37
45
|
private fetchRemoteHead;
|
|
38
46
|
private tryReadHistoryManifestBatch;
|
|
39
47
|
private createCommit;
|
|
@@ -3,6 +3,7 @@ import { createHash, randomUUID } from "node:crypto";
|
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { posixJoin } from "../../paths.js";
|
|
6
|
+
import { snapshotFileFromBlob } from "../../snapshot/snapshot-payload.js";
|
|
6
7
|
import { IGNORE_SNAPSHOT_VERSION } from "../../snapshot/snapshot-types.js";
|
|
7
8
|
import { stateDir } from "../../state/state-directory.js";
|
|
8
9
|
import { throwIfAborted } from "../../sync/signals.js";
|
|
@@ -30,9 +31,11 @@ export class GitSyncBackend {
|
|
|
30
31
|
afterLsRemoteForTest;
|
|
31
32
|
afterPayloadWriteForTest;
|
|
32
33
|
cacheReady;
|
|
34
|
+
cachedPublication;
|
|
33
35
|
constructor(config, options = {}) {
|
|
34
36
|
this.config = config;
|
|
35
37
|
assertGitDestination(config);
|
|
38
|
+
this.config = { ...config, profile: { ...config.profile }, destination: { ...config.destination } };
|
|
36
39
|
this.allowLocalRemotes = options.allowLocalRemotes === true;
|
|
37
40
|
if (!this.allowLocalRemotes)
|
|
38
41
|
assertProductionRemote(config.profile.remote);
|
|
@@ -53,22 +56,37 @@ export class GitSyncBackend {
|
|
|
53
56
|
const sha = await this.fetchRemoteHead(signal);
|
|
54
57
|
if (!sha)
|
|
55
58
|
return undefined;
|
|
56
|
-
|
|
57
|
-
|
|
59
|
+
return this.headForSha(sha, signal);
|
|
60
|
+
}
|
|
61
|
+
async readHeadAndSnapshot(signal) {
|
|
62
|
+
const commit = await this.fetchRemoteHead(signal);
|
|
63
|
+
if (!commit)
|
|
64
|
+
return undefined;
|
|
65
|
+
const publication = await this.readCachedPublication(commit, signal);
|
|
66
|
+
await this.requireReachablePublication(commit, commit, undefined, signal);
|
|
67
|
+
const snapshot = await this.readPublicationSnapshot(publication, signal);
|
|
68
|
+
return { head: remoteHead(commit, publication.manifest, this.identity), snapshot };
|
|
58
69
|
}
|
|
59
70
|
async readSnapshot(reference, signal) {
|
|
60
71
|
const head = await this.fetchRemoteHead(signal);
|
|
61
72
|
if (!head)
|
|
62
73
|
throw new Error(`Git snapshot publication was not found: ${reference}`);
|
|
63
74
|
const commit = await this.resolveSnapshotReference(reference, head, signal);
|
|
75
|
+
await this.requireReachablePublication(reference, commit, head, signal);
|
|
76
|
+
return this.readPublicationSnapshot(await this.readCachedPublication(commit, signal), signal);
|
|
77
|
+
}
|
|
78
|
+
async requireReachablePublication(reference, commit, head, signal) {
|
|
64
79
|
try {
|
|
65
80
|
await this.git(["cat-file", "-e", `${commit}^{commit}`], { signal });
|
|
66
|
-
|
|
81
|
+
if (head !== undefined)
|
|
82
|
+
await this.git(["merge-base", "--is-ancestor", commit, head], { signal });
|
|
67
83
|
}
|
|
68
84
|
catch (error) {
|
|
85
|
+
throwIfAborted(signal);
|
|
69
86
|
throw new Error(`Git snapshot publication was not found: ${reference}`, { cause: error });
|
|
70
87
|
}
|
|
71
|
-
|
|
88
|
+
}
|
|
89
|
+
async readPublicationSnapshot({ manifest, payloadEntries }, signal) {
|
|
72
90
|
let blobs;
|
|
73
91
|
try {
|
|
74
92
|
blobs = await readGitBlobs(payloadEntries.map((entry) => entry.object), {
|
|
@@ -88,10 +106,10 @@ export class GitSyncBackend {
|
|
|
88
106
|
throwIfAborted(signal);
|
|
89
107
|
const files = manifest.files.map((file, index) => {
|
|
90
108
|
const content = blobs[index];
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
return
|
|
109
|
+
const mismatch = `Git snapshot file checksum or size mismatch: ${file.path}`;
|
|
110
|
+
if (!content || content.byteLength !== file.size)
|
|
111
|
+
throw new Error(mismatch);
|
|
112
|
+
return snapshotFileFromBlob(file.path, content, file.sha256, mismatch);
|
|
95
113
|
});
|
|
96
114
|
const snapshot = {
|
|
97
115
|
version: manifest.snapshotVersion,
|
|
@@ -100,8 +118,8 @@ export class GitSyncBackend {
|
|
|
100
118
|
machine: manifest.machine,
|
|
101
119
|
profile: manifest.profile,
|
|
102
120
|
...(manifest.snapshotSyncSessions === undefined ? {} : { syncSessions: manifest.snapshotSyncSessions }),
|
|
103
|
-
...(manifest.selection === undefined ? {} : { selection: manifest.selection }),
|
|
104
|
-
...(manifest.localFields === undefined ? {} : { localFields: manifest.localFields }),
|
|
121
|
+
...(manifest.selection === undefined ? {} : { selection: structuredClone(manifest.selection) }),
|
|
122
|
+
...(manifest.localFields === undefined ? {} : { localFields: [...manifest.localFields] }),
|
|
105
123
|
files,
|
|
106
124
|
};
|
|
107
125
|
validateGitSnapshot(snapshot, manifest, this.config.destination.namespace);
|
|
@@ -294,9 +312,18 @@ export class GitSyncBackend {
|
|
|
294
312
|
return matches[0];
|
|
295
313
|
}
|
|
296
314
|
async headForSha(sha, signal) {
|
|
297
|
-
const { manifest } = await this.
|
|
315
|
+
const { manifest } = await this.readCachedPublication(sha, signal);
|
|
298
316
|
return remoteHead(sha, manifest, this.identity);
|
|
299
317
|
}
|
|
318
|
+
async readCachedPublication(commit, signal) {
|
|
319
|
+
throwIfAborted(signal);
|
|
320
|
+
if (this.cachedPublication?.commit === commit)
|
|
321
|
+
return this.cachedPublication.publication;
|
|
322
|
+
const publication = await this.readPublication(commit, signal);
|
|
323
|
+
throwIfAborted(signal);
|
|
324
|
+
this.cachedPublication = { commit, publication };
|
|
325
|
+
return publication;
|
|
326
|
+
}
|
|
300
327
|
async fetchRemoteHead(signal) {
|
|
301
328
|
await this.ensureCache(signal);
|
|
302
329
|
const result = await this.git(["ls-remote", "--refs", this.config.profile.remote, this.remoteRef()], { signal });
|
|
@@ -611,7 +638,7 @@ function remoteHead(sha, manifest, identity) {
|
|
|
611
638
|
createdAt: manifest.createdAt,
|
|
612
639
|
machine: manifest.machine,
|
|
613
640
|
syncSessions: manifest.syncSessions,
|
|
614
|
-
...(manifest.selection === undefined ? {} : { selection: manifest.selection }),
|
|
641
|
+
...(manifest.selection === undefined ? {} : { selection: structuredClone(manifest.selection) }),
|
|
615
642
|
};
|
|
616
643
|
}
|
|
617
644
|
function matchesExpected(current, expected, identity) {
|
|
@@ -33,7 +33,12 @@ export async function runGit(args, options = {}) {
|
|
|
33
33
|
];
|
|
34
34
|
if (options.allowFileProtocol)
|
|
35
35
|
protocolArgs.push("-c", "protocol.file.allow=always");
|
|
36
|
-
const commandArgs = [
|
|
36
|
+
const commandArgs = [
|
|
37
|
+
"--no-replace-objects",
|
|
38
|
+
...(options.gitDir ? [`--git-dir=${options.gitDir}`] : []),
|
|
39
|
+
...protocolArgs,
|
|
40
|
+
...args,
|
|
41
|
+
];
|
|
37
42
|
const inheritedEnvironment = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("GIT_") &&
|
|
38
43
|
key !== "PAGER" &&
|
|
39
44
|
key !== "EDITOR" &&
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
1
|
import { assertAgentDirectorySettingsSize, MAX_AGENT_DIRECTORY_BYTES } from "../../snapshot/snapshot-bounds.js";
|
|
2
|
+
import { materializeSnapshotFilePayload } from "../../snapshot/snapshot-payload.js";
|
|
3
3
|
import { IGNORE_SNAPSHOT_VERSION, isSupportedSnapshotVersion, } from "../../snapshot/snapshot-types.js";
|
|
4
4
|
import { sameLocalFields, validateIgnoreSnapshotIdentity, validateIgnoreSnapshotSessionFlag, validatePortableSnapshot, validateSnapshotFieldPolicy, } from "../../sync/local-fields.js";
|
|
5
|
-
import { assertAgentDirectoryPathLayout, configuredSnapshotPathMatcher, portableSnapshotSelection, sameSnapshotSelection, snapshotSelectionInclude, validateSnapshotSelection, } from "../../sync/sync-policy.js";
|
|
5
|
+
import { assertAgentDirectoryPathLayout, assertSnapshotExcludesDebugLog, assertSnapshotPathExcludesDebugLog, configuredSnapshotPathMatcher, portableSnapshotSelection, sameSnapshotSelection, snapshotSelectionInclude, validateSnapshotSelection, } from "../../sync/sync-policy.js";
|
|
6
6
|
export const GIT_MANIFEST_VERSION = 2;
|
|
7
7
|
export const MAX_GIT_MANIFEST_BYTES = 1024 * 1024;
|
|
8
8
|
export const MAX_GIT_TREE_OUTPUT_BYTES = 16 * 1024 * 1024;
|
|
@@ -76,6 +76,7 @@ export function requireGitManifest(value) {
|
|
|
76
76
|
throw new Error("Git publication manifest file is malformed.");
|
|
77
77
|
}
|
|
78
78
|
const file = rawFile;
|
|
79
|
+
assertSnapshotPathExcludesDebugLog(file.path);
|
|
79
80
|
if (!hasExactKeys(file, ["path", "sha256", "size"]) ||
|
|
80
81
|
!isSafeSnapshotPath(file.path) ||
|
|
81
82
|
(selected !== undefined && !selected(file.path)) ||
|
|
@@ -129,6 +130,7 @@ export function validateGitSnapshot(snapshot, manifest, namespace) {
|
|
|
129
130
|
}
|
|
130
131
|
}
|
|
131
132
|
export function prepareGitSnapshot(snapshot, namespace) {
|
|
133
|
+
assertSnapshotExcludesDebugLog(snapshot);
|
|
132
134
|
if (snapshot.version !== IGNORE_SNAPSHOT_VERSION)
|
|
133
135
|
snapshotSelectionInclude(snapshot);
|
|
134
136
|
validatePortableSnapshot(snapshot);
|
|
@@ -159,10 +161,7 @@ export function prepareGitSnapshot(snapshot, namespace) {
|
|
|
159
161
|
paths.has(file.path)) {
|
|
160
162
|
throw new Error("Invalid Git snapshot file.");
|
|
161
163
|
}
|
|
162
|
-
const content =
|
|
163
|
-
if (content.toString("base64") !== file.contentBase64 || sha256(content) !== file.sha256) {
|
|
164
|
-
throw new Error("Git snapshot file checksum mismatch.");
|
|
165
|
-
}
|
|
164
|
+
const content = materializeSnapshotFilePayload(file, "Git snapshot file checksum mismatch.");
|
|
166
165
|
if (!isGitPayloadSizeAllowed(content.byteLength)) {
|
|
167
166
|
throw new Error(`Git snapshot file exceeds GitHub's ${MAX_GIT_PAYLOAD_BYTES}-byte regular-Git limit: ${file.path}`);
|
|
168
167
|
}
|
|
@@ -243,9 +242,6 @@ function assertNoPathConflicts(paths) {
|
|
|
243
242
|
}
|
|
244
243
|
}
|
|
245
244
|
}
|
|
246
|
-
function sha256(value) {
|
|
247
|
-
return createHash("sha256").update(value).digest("hex");
|
|
248
|
-
}
|
|
249
245
|
function hasControlCharacter(value) {
|
|
250
246
|
// biome-ignore lint/suspicious/noControlCharactersInRegex: Reject untrusted terminal/ref controls.
|
|
251
247
|
return /[\u0000-\u001f\u007f-\u009f]/u.test(value);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { encodeSnapshot } from "../snapshot/snapshot-codec.js";
|
|
2
|
+
export function capturePublicationImage(snapshot) {
|
|
3
|
+
if (!Array.isArray(snapshot.files))
|
|
4
|
+
throw new Error("Invalid snapshot publication files.");
|
|
5
|
+
return {
|
|
6
|
+
...snapshot,
|
|
7
|
+
selection: structuredClone(snapshot.selection),
|
|
8
|
+
localFields: snapshot.localFields && [...snapshot.localFields],
|
|
9
|
+
files: snapshot.files.map((file) => ({ ...file })),
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export async function encodeForPublication(image, prepareRemote) {
|
|
13
|
+
const [prepared, encoding] = await Promise.allSettled([prepareRemote(), encodeSnapshot(image)]);
|
|
14
|
+
if (prepared.status === "rejected")
|
|
15
|
+
throw prepared.reason;
|
|
16
|
+
if (encoding.status === "rejected")
|
|
17
|
+
throw encoding.reason;
|
|
18
|
+
return encoding.value;
|
|
19
|
+
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { encodeKey, posixJoin } from "../../paths.js";
|
|
3
3
|
import { sessionTokenWarnings } from "../../settings/config.js";
|
|
4
|
-
import { decodeSnapshot
|
|
4
|
+
import { decodeSnapshot } from "../../snapshot/snapshot-codec.js";
|
|
5
|
+
import { assertSnapshotFilePayload } from "../../snapshot/snapshot-payload.js";
|
|
5
6
|
import { validatePortableSnapshot } from "../../sync/local-fields.js";
|
|
6
7
|
import { throwIfAborted } from "../../sync/signals.js";
|
|
7
8
|
import { syncErrorGuidance } from "../../sync/sync-error-guidance.js";
|
|
8
9
|
import { portableSnapshotSelection } from "../../sync/sync-policy.js";
|
|
10
|
+
import { capturePublicationImage, encodeForPublication } from "../publication-encoding.js";
|
|
9
11
|
import { createLatestPointer, remoteHistoryEntry } from "../snapshot-pointer.js";
|
|
10
12
|
import { SyncBackendConflictError, SyncBackendPublicationOutcomeUnknownError, } from "../sync-backend.js";
|
|
11
13
|
import { S3Client, S3HttpError, S3ObjectAlreadyExistsError } from "./s3-client.js";
|
|
@@ -74,16 +76,15 @@ export class S3SyncBackend {
|
|
|
74
76
|
}
|
|
75
77
|
async publishSnapshot(snapshot, expected, options = {}) {
|
|
76
78
|
throwIfAborted(options.signal);
|
|
77
|
-
|
|
79
|
+
const image = capturePublicationImage(snapshot);
|
|
80
|
+
assertSnapshotIdentity(image, this.config.destination.namespace);
|
|
78
81
|
if (options.validateIgnorePolicy)
|
|
79
82
|
await options.validateIgnorePolicy();
|
|
80
83
|
throwIfAborted(options.signal);
|
|
81
|
-
await verifyS3Conditions(this.config, options.signal);
|
|
84
|
+
const encoded = await encodeForPublication(image, () => verifyS3Conditions(this.config, options.signal));
|
|
82
85
|
throwIfAborted(options.signal);
|
|
83
|
-
const stagedKey = snapshotKey(this.config,
|
|
84
|
-
const
|
|
85
|
-
throwIfAborted(options.signal);
|
|
86
|
-
const pointer = pointerFor(this.config, snapshot, sha256(encoded));
|
|
86
|
+
const stagedKey = snapshotKey(this.config, image.id);
|
|
87
|
+
const pointer = pointerFor(this.config, image, sha256(encoded));
|
|
87
88
|
const cancellableClient = new S3Client(this.config, options.signal);
|
|
88
89
|
if (options.validateIgnorePolicy)
|
|
89
90
|
await options.validateIgnorePolicy();
|
|
@@ -98,7 +99,7 @@ export class S3SyncBackend {
|
|
|
98
99
|
throw error;
|
|
99
100
|
const existing = await cancellableClient.getBuffer(stagedKey);
|
|
100
101
|
if (!existing.value || sha256(existing.value) !== pointer.sha256) {
|
|
101
|
-
throw new SyncBackendConflictError(`Immutable snapshot id already exists with different content: ${
|
|
102
|
+
throw new SyncBackendConflictError(`Immutable snapshot id already exists with different content: ${image.id}`);
|
|
102
103
|
}
|
|
103
104
|
}
|
|
104
105
|
const currentObject = await cancellableClient.getJson(latestKey(this.config));
|
|
@@ -389,10 +390,7 @@ function validateSnapshotBundle(snapshot) {
|
|
|
389
390
|
paths.has(file.path)) {
|
|
390
391
|
throw new Error("Remote snapshot bundle is malformed.");
|
|
391
392
|
}
|
|
392
|
-
|
|
393
|
-
if (content.toString("base64") !== file.contentBase64 || sha256(content) !== file.sha256) {
|
|
394
|
-
throw new Error("Remote snapshot file checksum mismatch.");
|
|
395
|
-
}
|
|
393
|
+
assertSnapshotFilePayload(file, "Remote snapshot file checksum mismatch.");
|
|
396
394
|
paths.add(file.path);
|
|
397
395
|
}
|
|
398
396
|
}
|
|
@@ -49,6 +49,10 @@ export interface SyncBackend {
|
|
|
49
49
|
/** Recovery-only equivalence for retiring an inactive candidate; never authorizes publication or apply. */
|
|
50
50
|
matchesUncommittedRecoveryHead?(current: RemoteHead, recorded: RemoteHead): boolean;
|
|
51
51
|
readHead(signal?: AbortSignal): Promise<RemoteHead | undefined>;
|
|
52
|
+
readHeadAndSnapshot?(signal?: AbortSignal): Promise<{
|
|
53
|
+
head: RemoteHead;
|
|
54
|
+
snapshot: Snapshot;
|
|
55
|
+
} | undefined>;
|
|
52
56
|
readSnapshot(reference: string, signal?: AbortSignal): Promise<Snapshot>;
|
|
53
57
|
publishSnapshot(snapshot: Snapshot, expected: ExpectedRemoteHead, options?: PublishSnapshotOptions): Promise<PublishSnapshotResult>;
|
|
54
58
|
listHistory(signal?: AbortSignal): Promise<RemoteHistoryEntry[]>;
|
|
@@ -8,7 +8,6 @@ export declare class WebDavSyncBackend implements SyncBackend {
|
|
|
8
8
|
readonly destination: string;
|
|
9
9
|
private conditionalVerified;
|
|
10
10
|
private readonly checksums;
|
|
11
|
-
private capabilityCheck?;
|
|
12
11
|
constructor(config: ResolvedWebDavBackend, postCommitTimeoutMs?: number);
|
|
13
12
|
get capability(): "atomic-conditional" | "conditional-required";
|
|
14
13
|
sameRevision(left: string, right: string): boolean;
|
|
@@ -17,7 +16,6 @@ export declare class WebDavSyncBackend implements SyncBackend {
|
|
|
17
16
|
publishSnapshot(snapshot: Snapshot, expected: ExpectedRemoteHead, options?: PublishSnapshotOptions): Promise<PublishSnapshotResult>;
|
|
18
17
|
listHistory(signal?: AbortSignal): Promise<RemoteHistoryEntry[]>;
|
|
19
18
|
diagnose(signal?: AbortSignal): Promise<BackendDiagnostic[]>;
|
|
20
|
-
private ensureAtomicConditions;
|
|
21
19
|
private runCapabilityProbe;
|
|
22
20
|
private resolveChecksum;
|
|
23
21
|
private registerChecksum;
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { decodeSnapshot
|
|
2
|
+
import { decodeSnapshot } from "../../snapshot/snapshot-codec.js";
|
|
3
|
+
import { assertSnapshotFilePayload } from "../../snapshot/snapshot-payload.js";
|
|
3
4
|
import { validatePortableSnapshot } from "../../sync/local-fields.js";
|
|
4
5
|
import { throwIfAborted } from "../../sync/signals.js";
|
|
5
6
|
import { syncErrorGuidance } from "../../sync/sync-error-guidance.js";
|
|
6
7
|
import { portableSnapshotSelection } from "../../sync/sync-policy.js";
|
|
8
|
+
import { capturePublicationImage, encodeForPublication } from "../publication-encoding.js";
|
|
7
9
|
import { createLatestPointer, remoteHistoryEntry } from "../snapshot-pointer.js";
|
|
8
10
|
import { SyncBackendConflictError, SyncBackendPublicationOutcomeUnknownError, } from "../sync-backend.js";
|
|
9
11
|
import { WebDavClient, WebDavHttpError, WebDavPreconditionError } from "./webdav-client.js";
|
|
@@ -16,7 +18,6 @@ export class WebDavSyncBackend {
|
|
|
16
18
|
destination;
|
|
17
19
|
conditionalVerified = false;
|
|
18
20
|
checksums = new Map();
|
|
19
|
-
capabilityCheck;
|
|
20
21
|
constructor(config, postCommitTimeoutMs = POST_COMMIT_TIMEOUT_MS) {
|
|
21
22
|
this.config = config;
|
|
22
23
|
this.postCommitTimeoutMs = postCommitTimeoutMs;
|
|
@@ -58,18 +59,19 @@ export class WebDavSyncBackend {
|
|
|
58
59
|
}
|
|
59
60
|
async publishSnapshot(snapshot, expected, options = {}) {
|
|
60
61
|
throwIfAborted(options.signal);
|
|
61
|
-
|
|
62
|
+
const image = capturePublicationImage(snapshot);
|
|
63
|
+
assertSnapshotIdentity(image, this.config.destination.namespace);
|
|
62
64
|
if (options.validateIgnorePolicy)
|
|
63
65
|
await options.validateIgnorePolicy();
|
|
64
66
|
throwIfAborted(options.signal);
|
|
65
|
-
await this.ensureAtomicConditions(options.signal);
|
|
66
|
-
throwIfAborted(options.signal);
|
|
67
67
|
const client = new WebDavClient(this.config, options.signal);
|
|
68
|
-
await
|
|
69
|
-
|
|
68
|
+
const encoded = await encodeForPublication(image, async () => {
|
|
69
|
+
await this.runCapabilityProbe(options.signal);
|
|
70
|
+
await client.ensureCollection(snapshotsPath(this.config));
|
|
71
|
+
});
|
|
70
72
|
throwIfAborted(options.signal);
|
|
71
|
-
const pointer = createLatestPointer(this.config.destination.namespace,
|
|
72
|
-
const stagedPath = snapshotPath(this.config,
|
|
73
|
+
const pointer = createLatestPointer(this.config.destination.namespace, image, sha256(encoded));
|
|
74
|
+
const stagedPath = snapshotPath(this.config, image.id);
|
|
73
75
|
if (options.validateIgnorePolicy)
|
|
74
76
|
await options.validateIgnorePolicy();
|
|
75
77
|
throwIfAborted(options.signal);
|
|
@@ -81,7 +83,7 @@ export class WebDavSyncBackend {
|
|
|
81
83
|
throw error;
|
|
82
84
|
const existing = await client.getBuffer(stagedPath);
|
|
83
85
|
if (!existing.value || sha256(existing.value) !== pointer.sha256) {
|
|
84
|
-
throw new SyncBackendConflictError(`Immutable snapshot id already exists with different content: ${
|
|
86
|
+
throw new SyncBackendConflictError(`Immutable snapshot id already exists with different content: ${image.id}`);
|
|
85
87
|
}
|
|
86
88
|
}
|
|
87
89
|
const currentObject = await client.getJson(latestPath(this.config));
|
|
@@ -173,7 +175,6 @@ export class WebDavSyncBackend {
|
|
|
173
175
|
}, { key: "webdav-cleanup", level: "info", message: "webdav probe cleanup: ok" });
|
|
174
176
|
}
|
|
175
177
|
catch (error) {
|
|
176
|
-
this.capabilityCheck = undefined;
|
|
177
178
|
this.conditionalVerified = false;
|
|
178
179
|
if (error instanceof Error && error.name === "AbortError")
|
|
179
180
|
throw error;
|
|
@@ -202,18 +203,8 @@ export class WebDavSyncBackend {
|
|
|
202
203
|
}
|
|
203
204
|
return diagnostics;
|
|
204
205
|
}
|
|
205
|
-
ensureAtomicConditions(signal) {
|
|
206
|
-
if (this.capabilityCheck)
|
|
207
|
-
return this.capabilityCheck;
|
|
208
|
-
this.conditionalVerified = false;
|
|
209
|
-
const check = this.runCapabilityProbe(signal).finally(() => {
|
|
210
|
-
if (this.capabilityCheck === check)
|
|
211
|
-
this.capabilityCheck = undefined;
|
|
212
|
-
});
|
|
213
|
-
this.capabilityCheck = check;
|
|
214
|
-
return check;
|
|
215
|
-
}
|
|
216
206
|
async runCapabilityProbe(signal) {
|
|
207
|
+
this.conditionalVerified = false;
|
|
217
208
|
throwIfAborted(signal);
|
|
218
209
|
const probeCollection = joinRemote(rootPath(this.config), ".pi-sync-probes", randomUUID());
|
|
219
210
|
const probe = `${probeCollection}/conditional.txt`;
|
|
@@ -259,6 +250,7 @@ export class WebDavSyncBackend {
|
|
|
259
250
|
}
|
|
260
251
|
if (operationError)
|
|
261
252
|
throw operationError;
|
|
253
|
+
throwIfAborted(signal);
|
|
262
254
|
this.conditionalVerified = true;
|
|
263
255
|
}
|
|
264
256
|
async resolveChecksum(reference, signal) {
|
|
@@ -442,10 +434,7 @@ function validateSnapshotBundle(snapshot) {
|
|
|
442
434
|
typeof file.sha256 !== "string") {
|
|
443
435
|
throw new Error("Remote snapshot bundle is malformed.");
|
|
444
436
|
}
|
|
445
|
-
|
|
446
|
-
if (content.toString("base64") !== file.contentBase64 || sha256(content) !== file.sha256) {
|
|
447
|
-
throw new Error("Remote snapshot bundle is malformed.");
|
|
448
|
-
}
|
|
437
|
+
assertSnapshotFilePayload(file, "Remote snapshot bundle is malformed.");
|
|
449
438
|
paths.add(file.path);
|
|
450
439
|
}
|
|
451
440
|
}
|
|
@@ -110,6 +110,16 @@ export class WebDavClient {
|
|
|
110
110
|
}
|
|
111
111
|
async ensureCollection(remotePath) {
|
|
112
112
|
const segments = safeSegments(remotePath);
|
|
113
|
+
if (segments.length === 0)
|
|
114
|
+
return;
|
|
115
|
+
try {
|
|
116
|
+
await this.makeCollection(segments.join("/"));
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (segments.length < 2 || !(error instanceof WebDavHttpError) || (error.status !== 409 && error.status !== 404))
|
|
121
|
+
throw error;
|
|
122
|
+
}
|
|
113
123
|
for (let index = 1; index <= segments.length; index += 1) {
|
|
114
124
|
await this.makeCollection(segments.slice(0, index).join("/"));
|
|
115
125
|
}
|
|
@@ -12,6 +12,7 @@ export interface CommandOptions {
|
|
|
12
12
|
automaticFingerprint?: string;
|
|
13
13
|
signal?: AbortSignal;
|
|
14
14
|
onCommit?: () => void;
|
|
15
|
+
noteDebugOutcome?: (outcome: "blocked" | "cancelled" | "skipped") => void;
|
|
15
16
|
/** In-memory startup diagnostics only; never persisted or sent to a backend. */
|
|
16
17
|
startupTimings?: StartupTimings;
|
|
17
18
|
/** Internal validated resolver result; never parsed from command arguments. */
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
declare const OPERATIONS: readonly ["initialize", "inspect", "status", "diff", "doctor", "history", "push", "pull", "sync", "startup-transfer", "shutdown", "conflicts", "rollback", "unlock", "migrate-state", "daemon.authorize", "daemon.authorize-connected", "daemon.handover"];
|
|
2
|
+
declare const OUTCOMES: readonly ["started", "completed", "failed", "cancelled", "blocked", "skipped"];
|
|
3
|
+
type DebugSource = "pi-sync" | "pi-sync-daemon";
|
|
4
|
+
type DebugOperation = (typeof OPERATIONS)[number];
|
|
5
|
+
type Outcome = (typeof OUTCOMES)[number];
|
|
6
|
+
export type DebugEvent = {
|
|
7
|
+
event: "startup-timings";
|
|
8
|
+
report: string;
|
|
9
|
+
} | {
|
|
10
|
+
event: "connection";
|
|
11
|
+
outcome: Exclude<Outcome, "started" | "blocked">;
|
|
12
|
+
} | {
|
|
13
|
+
event: "lifecycle";
|
|
14
|
+
outcome: "starting" | "ready" | "drained" | "failed";
|
|
15
|
+
} | {
|
|
16
|
+
event: "job";
|
|
17
|
+
operation: DebugOperation;
|
|
18
|
+
outcome: Outcome;
|
|
19
|
+
operationId?: string;
|
|
20
|
+
} | {
|
|
21
|
+
event: "continuous-cycle";
|
|
22
|
+
outcome: Exclude<Outcome, "started">;
|
|
23
|
+
reason?: string;
|
|
24
|
+
};
|
|
25
|
+
export declare function debugOperation(operation: string): DebugOperation | undefined;
|
|
26
|
+
export declare function appendDebugEvent(root: string, source: DebugSource, event: DebugEvent): boolean;
|
|
27
|
+
export {};
|