@narumitw/pi-sync-daemon 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +11 -1
- package/dist/cli.js +2 -0
- package/dist/client-transport.d.ts +14 -0
- package/dist/client-transport.js +66 -0
- package/dist/client.d.ts +15 -8
- package/dist/client.js +116 -86
- package/dist/consent.js +33 -11
- package/dist/daemon-debug.d.ts +3 -0
- package/dist/daemon-debug.js +33 -0
- package/dist/daemon-recovery.d.ts +4 -0
- package/dist/daemon-recovery.js +75 -0
- package/dist/engine/backends/git/git-backend.d.ts +8 -0
- package/dist/engine/backends/git/git-backend.js +39 -12
- package/dist/engine/backends/git/git-runner.js +6 -1
- package/dist/engine/backends/git/git-storage.js +5 -9
- package/dist/engine/backends/publication-encoding.d.ts +3 -0
- package/dist/engine/backends/publication-encoding.js +19 -0
- package/dist/engine/backends/s3/s3-backend.js +10 -12
- package/dist/engine/backends/sync-backend.d.ts +4 -0
- package/dist/engine/backends/webdav/webdav-backend.d.ts +0 -2
- package/dist/engine/backends/webdav/webdav-backend.js +15 -26
- package/dist/engine/backends/webdav/webdav-client.js +10 -0
- package/dist/engine/commands/command-types.d.ts +1 -0
- package/dist/engine/debug-log.d.ts +27 -0
- package/dist/engine/debug-log.js +134 -0
- package/dist/engine/ignore/ignore-initialization.d.ts +5 -1
- package/dist/engine/ignore/ignore-initialization.js +50 -14
- package/dist/engine/ignore/ignore-settings.js +11 -14
- package/dist/engine/ignore/ignore-source.d.ts +1 -1
- package/dist/engine/ignore/ignore-source.js +1 -1
- package/dist/engine/paths.d.ts +2 -0
- package/dist/engine/paths.js +7 -1
- package/dist/engine/settings/config-file.d.ts +10 -1
- package/dist/engine/settings/config-file.js +62 -17
- package/dist/engine/settings/config.d.ts +7 -0
- package/dist/engine/settings/config.js +49 -14
- package/dist/engine/snapshot/snapshot-apply.d.ts +2 -1
- package/dist/engine/snapshot/snapshot-apply.js +30 -8
- package/dist/engine/snapshot/snapshot-codec.js +3 -1
- package/dist/engine/snapshot/snapshot-payload.d.ts +4 -0
- package/dist/engine/snapshot/snapshot-payload.js +32 -0
- package/dist/engine/snapshot/snapshot-transaction-images.d.ts +2 -0
- package/dist/engine/snapshot/snapshot-transaction-images.js +34 -0
- package/dist/engine/snapshot/snapshot-transaction.d.ts +4 -1
- package/dist/engine/snapshot/snapshot-transaction.js +50 -7
- package/dist/engine/snapshot/snapshot-types.d.ts +10 -2
- package/dist/engine/snapshot/snapshot.d.ts +3 -1
- package/dist/engine/snapshot/snapshot.js +64 -25
- package/dist/engine/state/lock.d.ts +2 -1
- package/dist/engine/state/lock.js +157 -17
- package/dist/engine/sync/continuous-sync.d.ts +10 -1
- package/dist/engine/sync/continuous-sync.js +35 -11
- package/dist/engine/sync/initialization.d.ts +1 -1
- package/dist/engine/sync/initialization.js +2 -2
- package/dist/engine/sync/local-fields.d.ts +6 -1
- package/dist/engine/sync/local-fields.js +42 -19
- package/dist/engine/sync/merged-sync.js +13 -10
- package/dist/engine/sync/remote-snapshot.js +18 -3
- package/dist/engine/sync/session-push.js +13 -4
- package/dist/engine/sync/startup-bootstrap.d.ts +1 -0
- package/dist/engine/sync/startup-bootstrap.js +105 -0
- package/dist/engine/sync/startup-recovery-evidence.d.ts +1 -0
- package/dist/engine/sync/startup-recovery-evidence.js +37 -0
- package/dist/engine/sync/startup-timings.d.ts +9 -2
- package/dist/engine/sync/startup-timings.js +80 -8
- package/dist/engine/sync/sync-error-guidance.js +22 -4
- package/dist/engine/sync/sync-inspection.js +4 -7
- package/dist/engine/sync/sync-local.d.ts +2 -1
- package/dist/engine/sync/sync-local.js +3 -1
- package/dist/engine/sync/sync-mutations.js +13 -7
- package/dist/engine/sync/sync-policy.d.ts +2 -0
- package/dist/engine/sync/sync-policy.js +9 -1
- package/dist/engine/sync/sync-state.d.ts +8 -3
- package/dist/engine/sync/sync-state.js +12 -4
- package/dist/engine/ui/sync-format.d.ts +16 -5
- package/dist/engine/ui/sync-format.js +23 -55
- package/dist/engine/ui/transfer-review.d.ts +2 -1
- package/dist/engine/ui/transfer-review.js +3 -3
- package/dist/engine-service.d.ts +6 -0
- package/dist/engine-service.js +54 -10
- package/dist/identity.d.ts +5 -2
- package/dist/identity.js +36 -6
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/monitor.js +20 -1
- package/dist/paged-review.d.ts +35 -0
- package/dist/paged-review.js +177 -0
- package/dist/participant-ledger.d.ts +16 -0
- package/dist/participant-ledger.js +149 -0
- package/dist/protocol.d.ts +15 -0
- package/dist/protocol.js +4 -0
- package/dist/server.d.ts +7 -0
- package/dist/server.js +268 -120
- package/dist/service-host.d.ts +2 -1
- package/dist/service-host.js +29 -9
- package/dist/state-wait-jobs.d.ts +8 -0
- package/dist/state-wait-jobs.js +23 -0
- package/dist/state-wait.d.ts +7 -0
- package/dist/state-wait.js +34 -0
- package/docs/architecture.md +23 -1
- package/package.json +1 -1
|
@@ -33,7 +33,7 @@ async function operationalConfig(document, setupName, signal) {
|
|
|
33
33
|
return config;
|
|
34
34
|
if (engineHost().ignoreFileOperationsAvailable === false)
|
|
35
35
|
return requireLegacyOperationConfig(config);
|
|
36
|
-
return resolveOperationPolicy(config, setupName, signal, document);
|
|
36
|
+
return resolveOperationPolicy(config, setupName, signal, document && { kind: "checked", document });
|
|
37
37
|
}
|
|
38
38
|
export async function loadSettingsConfig(setupName) {
|
|
39
39
|
return configFromSettings(await readSettingsForCheck(), setupName);
|
|
@@ -56,10 +56,31 @@ export async function resolveIgnoreConfig(settings, setupName, signal) {
|
|
|
56
56
|
signal?.throwIfAborted();
|
|
57
57
|
return resolveOperationPolicy(configFromSettings(validateSettingsDocument(settings), setupName), setupName, signal);
|
|
58
58
|
}
|
|
59
|
+
export function createSettingsConfigView(document) {
|
|
60
|
+
const settings = validateSettingsDocument(document.parsed);
|
|
61
|
+
const ignoreNames = reservedIgnoreFilenames(settings);
|
|
62
|
+
const metadata = (setupName) => projectSettingsConfig(settings, setupName, () => ignoreNames);
|
|
63
|
+
let fingerprint;
|
|
64
|
+
return {
|
|
65
|
+
activeSetupName: settings.activeSyncSetup,
|
|
66
|
+
setupNames: Object.keys(settings.syncSetups),
|
|
67
|
+
metadata,
|
|
68
|
+
async observe(setupName, signal) {
|
|
69
|
+
signal?.throwIfAborted();
|
|
70
|
+
const config = metadata(setupName);
|
|
71
|
+
if (!config.reservedIgnoreNames)
|
|
72
|
+
return config;
|
|
73
|
+
if (engineHost().ignoreFileOperationsAvailable === false)
|
|
74
|
+
return requireLegacyOperationConfig(config);
|
|
75
|
+
fingerprint ??= settingsDocumentFingerprint(document);
|
|
76
|
+
return resolveOperationPolicy(config, setupName, signal, { kind: "view", fingerprint });
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
}
|
|
59
80
|
function settingsDocumentFingerprint(document) {
|
|
60
81
|
return documentFingerprint(document && { bytes: document.bytes.toString("base64"), identity: document.identity });
|
|
61
82
|
}
|
|
62
|
-
async function resolveOperationPolicy(config, setupName, signal,
|
|
83
|
+
async function resolveOperationPolicy(config, setupName, signal, settingsSource) {
|
|
63
84
|
if (!config.reservedIgnoreNames)
|
|
64
85
|
return config;
|
|
65
86
|
const root = getEngineRoot();
|
|
@@ -68,8 +89,9 @@ async function resolveOperationPolicy(config, setupName, signal, original) {
|
|
|
68
89
|
if (root !== getEngineRoot())
|
|
69
90
|
throw new Error("The Pi agent directory changed; reopen the review before continuing.");
|
|
70
91
|
};
|
|
71
|
-
const
|
|
72
|
-
const
|
|
92
|
+
const original = settingsSource?.kind === "checked" ? settingsSource.document : undefined;
|
|
93
|
+
const configDocumentFingerprint = settingsSource?.kind === "view" ? settingsSource.fingerprint : original && settingsDocumentFingerprint(original);
|
|
94
|
+
const settingsFingerprint = settingsSource?.kind === "view" ? undefined : syncCheckConfigFingerprint(config);
|
|
73
95
|
const validateSettings = (document) => {
|
|
74
96
|
validateRoot();
|
|
75
97
|
if ((original &&
|
|
@@ -81,16 +103,21 @@ async function resolveOperationPolicy(config, setupName, signal, original) {
|
|
|
81
103
|
};
|
|
82
104
|
const source = config.pathPolicy === "ignore-file" ? await readIgnoreSource(root, config.ignoreFile, signal) : undefined;
|
|
83
105
|
validateRoot();
|
|
84
|
-
if (
|
|
106
|
+
if (settingsSource && source && !source.present) {
|
|
85
107
|
throw new Error(`Ignore file is missing: ${safeTerminalText(path.join(root, source.filename))}. Create it and reopen Pi Sync; starter rules are only a preview.`);
|
|
86
108
|
}
|
|
87
109
|
if (original && source)
|
|
88
110
|
validateSettings(await readActiveLocalConfigDocumentForRepair());
|
|
89
|
-
|
|
111
|
+
const observed = {
|
|
90
112
|
...config,
|
|
91
113
|
...(configDocumentFingerprint ? { configDocumentFingerprint } : {}),
|
|
92
114
|
...(source ? { ignorePolicy: compileEffectiveIgnorePolicy(source, ignorePolicyInput(config)) } : {}),
|
|
93
115
|
...(source?.identity ? { ignoreSourceIdentity: source.identity } : {}),
|
|
116
|
+
};
|
|
117
|
+
if (settingsSource?.kind === "view")
|
|
118
|
+
return observed;
|
|
119
|
+
return {
|
|
120
|
+
...observed,
|
|
94
121
|
validateIgnorePolicy: async () => {
|
|
95
122
|
validateRoot();
|
|
96
123
|
await withLockedLocalConfigDocument(async (document, recheck) => {
|
|
@@ -121,7 +148,21 @@ function ignorePolicyInput(config) {
|
|
|
121
148
|
configuredFilenames: config.reservedIgnoreNames ?? [],
|
|
122
149
|
};
|
|
123
150
|
}
|
|
151
|
+
function reservedIgnoreFilenames(settings) {
|
|
152
|
+
const names = new Set();
|
|
153
|
+
for (const { sync } of Object.values(settings.syncSetups)) {
|
|
154
|
+
if (sync.pathPolicy === "ignore-file")
|
|
155
|
+
names.add(validateIgnoreFilename(sync.ignoreFile ?? DEFAULT_IGNORE_FILENAME).toLowerCase());
|
|
156
|
+
}
|
|
157
|
+
if (names.size === 0)
|
|
158
|
+
return undefined;
|
|
159
|
+
names.add(DEFAULT_IGNORE_FILENAME);
|
|
160
|
+
return [...names].sort();
|
|
161
|
+
}
|
|
124
162
|
export function configFromSettings(settings, setupName) {
|
|
163
|
+
return projectSettingsConfig(settings, setupName, () => reservedIgnoreFilenames(settings));
|
|
164
|
+
}
|
|
165
|
+
function projectSettingsConfig(settings, setupName, getReservedIgnoreNames) {
|
|
125
166
|
const selectedName = setupName ?? settings.activeSyncSetup;
|
|
126
167
|
if (!selectedName)
|
|
127
168
|
throw new Error("No sync setups are configured.");
|
|
@@ -134,17 +175,11 @@ export function configFromSettings(settings, setupName) {
|
|
|
134
175
|
if (!connection) {
|
|
135
176
|
throw new Error(`Invalid pi-sync settings: sync setup “${selectedName}” references missing storage connection “${connectionName}”.`);
|
|
136
177
|
}
|
|
137
|
-
const
|
|
138
|
-
for (const { sync } of Object.values(settings.syncSetups)) {
|
|
139
|
-
if (sync.pathPolicy === "ignore-file")
|
|
140
|
-
ignoreNames.add(validateIgnoreFilename(sync.ignoreFile ?? DEFAULT_IGNORE_FILENAME).toLowerCase());
|
|
141
|
-
}
|
|
142
|
-
if (ignoreNames.size > 0)
|
|
143
|
-
ignoreNames.add(DEFAULT_IGNORE_FILENAME);
|
|
178
|
+
const reservedIgnoreNames = getReservedIgnoreNames();
|
|
144
179
|
return {
|
|
145
180
|
...resolveSyncConfig(selectedName, setup, connectionName, connection, settings.onSwitch, settings.skipSecretScan ?? false, settings.showStatus ?? true),
|
|
146
181
|
debug: settings.debug ?? false,
|
|
147
|
-
...(
|
|
182
|
+
...(reservedIgnoreNames ? { reservedIgnoreNames } : {}),
|
|
148
183
|
};
|
|
149
184
|
}
|
|
150
185
|
/** A validated setup-facing projection used by manager and settings UI. */
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type SkippedSnapshotWrite } from "./snapshot-transaction.js";
|
|
1
2
|
import { type Snapshot, type SnapshotApplyPlan, type SnapshotOptions } from "./snapshot-types.js";
|
|
2
3
|
export declare function applySnapshot(snapshot: Snapshot, protectedRelativePaths?: Set<string>, options?: Pick<SnapshotOptions, "include" | "exclude" | "scope" | "reinclude" | "ignorePolicy" | "reservedIgnoreNames" | "setupName" | "recoverySetupKey" | "ignoreSourceIdentity" | "sessionDir" | "syncFiles" | "syncSessions" | "extraFiles" | "signal" | "validateMutation" | "validateIgnorePolicy" | "validateApplyReview" | "expectedFileHashes"> & {
|
|
3
4
|
reviewedLocal?: Pick<Snapshot, "files">;
|
|
@@ -17,5 +18,5 @@ export declare function preflightSnapshotMutations(root: string, plan: {
|
|
|
17
18
|
target: string;
|
|
18
19
|
content: Buffer;
|
|
19
20
|
}>;
|
|
20
|
-
}, sessionDir?: string, options?: MutationGuards): Promise<
|
|
21
|
+
}, sessionDir?: string, options?: MutationGuards, unchangedHashes?: ReadonlyMap<string, string>): Promise<SkippedSnapshotWrite[]>;
|
|
21
22
|
export {};
|
|
@@ -4,7 +4,8 @@ import path from "node:path";
|
|
|
4
4
|
import { assertWithinRoot, isDeniedPath, isPathInside, parentPaths, safeJoin, toPosix } from "../paths.js";
|
|
5
5
|
import { syncDirectory } from "../state/json-file.js";
|
|
6
6
|
import { stateDir } from "../state/state-directory.js";
|
|
7
|
-
import {
|
|
7
|
+
import { collidingPaths } from "../sync/file-merge-planner.js";
|
|
8
|
+
import { assertSnapshotExcludesDebugLog, compileSyncPolicyExclude, requireLegacyTransferPolicy, sameSnapshotSelection, selectionForSnapshot, } from "../sync/sync-policy.js";
|
|
8
9
|
import { agentDir } from "./session-paths.js";
|
|
9
10
|
import { agentDirectoryConfiguredSessionDir, agentDirectoryInventory, createSnapshot, isSessionFilePath, isSessionPath, sessionStorageRoot, snapshotIncludesSessions, snapshotTarget, } from "./snapshot.js";
|
|
10
11
|
import { assertAgentDirectorySnapshotBound } from "./snapshot-bounds.js";
|
|
@@ -12,7 +13,7 @@ import { assertProjectedAgentDirectoryBudget } from "./snapshot-entry-projection
|
|
|
12
13
|
import { agentDirectoryPhysicalSessionPath, assertUnexcludedTargets } from "./snapshot-exclude-safety.js";
|
|
13
14
|
import { decodeBase64Strict } from "./snapshot-payload.js";
|
|
14
15
|
import { reviewedSnapshotPreimages } from "./snapshot-reviewed-preimages.js";
|
|
15
|
-
import { applySnapshotTransaction, recoverPendingSnapshotTransactions } from "./snapshot-transaction.js";
|
|
16
|
+
import { applySnapshotTransaction, recoverPendingSnapshotTransactions, } from "./snapshot-transaction.js";
|
|
16
17
|
import { ignorePolicyValidator } from "./snapshot-transaction-policy.js";
|
|
17
18
|
import { IGNORE_SNAPSHOT_VERSION, } from "./snapshot-types.js";
|
|
18
19
|
function sha256(value) {
|
|
@@ -22,6 +23,7 @@ function fileHashMap(snapshot) {
|
|
|
22
23
|
return Object.fromEntries(snapshot.files.map((file) => [file.path, file.sha256]));
|
|
23
24
|
}
|
|
24
25
|
function assertSnapshotApplyPolicy(snapshot, options) {
|
|
26
|
+
assertSnapshotExcludesDebugLog(snapshot);
|
|
25
27
|
if (snapshot.version !== IGNORE_SNAPSHOT_VERSION) {
|
|
26
28
|
requireLegacyTransferPolicy(snapshot);
|
|
27
29
|
if (options.ignorePolicy)
|
|
@@ -77,6 +79,16 @@ export async function applySnapshot(snapshot, protectedRelativePaths = new Set()
|
|
|
77
79
|
]);
|
|
78
80
|
}
|
|
79
81
|
const plan = await addTopLevelCaseVariantDeletes(root, protectSnapshotApplyPlan(root, preflightSnapshotApply(root, snapshot, current, options), protectedRelativePaths, sessionDir), snapshot, options);
|
|
82
|
+
const currentHashes = new Map(current.files.map(({ path, sha256 }) => [path, sha256]));
|
|
83
|
+
const dependencies = collidingPaths([...plan.deletes, ...plan.writes.map(({ target }) => target)].map(toPosix));
|
|
84
|
+
const unchangedHashes = new Map();
|
|
85
|
+
for (const file of snapshot.files) {
|
|
86
|
+
if (currentHashes.get(file.path) !== file.sha256)
|
|
87
|
+
continue;
|
|
88
|
+
const target = snapshotTarget(root, file.path, sessionDir);
|
|
89
|
+
if (!dependencies.has(toPosix(target)))
|
|
90
|
+
unchangedHashes.set(target, file.sha256);
|
|
91
|
+
}
|
|
80
92
|
if (inventory) {
|
|
81
93
|
assertProjectedAgentDirectoryBudget(inventory, {
|
|
82
94
|
deletes: plan.deletes,
|
|
@@ -85,8 +97,10 @@ export async function applySnapshot(snapshot, protectedRelativePaths = new Set()
|
|
|
85
97
|
{ target: stateDir(root) },
|
|
86
98
|
],
|
|
87
99
|
}, options);
|
|
88
|
-
await preflightSnapshotMutations(root, plan, sessionDir, options);
|
|
89
100
|
}
|
|
101
|
+
const skipped = inventory
|
|
102
|
+
? await preflightSnapshotMutations(root, plan, sessionDir, options, unchangedHashes)
|
|
103
|
+
: undefined;
|
|
90
104
|
if (options.reviewedLocal || inventory) {
|
|
91
105
|
const images = reviewedSnapshotPreimages(root, sessionDir, current.files, plan, inventory);
|
|
92
106
|
const reviewed = options.reviewedLocal;
|
|
@@ -102,9 +116,7 @@ export async function applySnapshot(snapshot, protectedRelativePaths = new Set()
|
|
|
102
116
|
transactionOptions.expectedPreimages.set(target, image);
|
|
103
117
|
}
|
|
104
118
|
}
|
|
105
|
-
|
|
106
|
-
await preflightSnapshotMutations(root, plan, sessionDir, options);
|
|
107
|
-
await applySnapshotTransaction(plan, transactionOptions);
|
|
119
|
+
await applySnapshotTransaction(plan, transactionOptions, skipped ?? (await preflightSnapshotMutations(root, plan, sessionDir, options, unchangedHashes)));
|
|
108
120
|
return appliedFileHashMap(snapshot, current, protectedRelativePaths);
|
|
109
121
|
}
|
|
110
122
|
function sameReviewedFiles(scanned, reviewed) {
|
|
@@ -242,7 +254,8 @@ function excludedApplyPath(root, options) {
|
|
|
242
254
|
return (relative) => excluded(relative) ||
|
|
243
255
|
(/^sessions(?:\/|$)/iu.test(relative) ? !isSessionFilePath(relative) : physicalSessionPath(relative));
|
|
244
256
|
}
|
|
245
|
-
export async function preflightSnapshotMutations(root, plan, sessionDir, options = {}) {
|
|
257
|
+
export async function preflightSnapshotMutations(root, plan, sessionDir, options = {}, unchangedHashes) {
|
|
258
|
+
const skipped = [];
|
|
246
259
|
assertMutationCurrent(options);
|
|
247
260
|
await assertUnexcludedTargets(root, [...plan.deletes, ...plan.writes.map((item) => item.target)], {
|
|
248
261
|
...options,
|
|
@@ -255,9 +268,17 @@ export async function preflightSnapshotMutations(root, plan, sessionDir, options
|
|
|
255
268
|
assertMutationCurrent(options);
|
|
256
269
|
}
|
|
257
270
|
for (const item of plan.writes) {
|
|
258
|
-
await prepareSnapshotWrite(rootForTarget(root, item.target, sessionDir), item.target, deletePaths, options);
|
|
271
|
+
const stat = await prepareSnapshotWrite(rootForTarget(root, item.target, sessionDir), item.target, deletePaths, options);
|
|
259
272
|
assertMutationCurrent(options);
|
|
273
|
+
const hash = unchangedHashes?.get(item.target);
|
|
274
|
+
if (hash !== undefined && stat?.isFile() && stat.nlink === 1) {
|
|
275
|
+
const canonical = await fs.realpath(item.target);
|
|
276
|
+
assertMutationCurrent(options);
|
|
277
|
+
if (canonical === item.target)
|
|
278
|
+
skipped.push({ ...item, preimage: `file:${hash}` });
|
|
279
|
+
}
|
|
260
280
|
}
|
|
281
|
+
return skipped;
|
|
261
282
|
}
|
|
262
283
|
function assertMutationCurrent(options) {
|
|
263
284
|
options.validateMutation?.();
|
|
@@ -296,6 +317,7 @@ async function prepareSnapshotWrite(root, target, deletePaths, options) {
|
|
|
296
317
|
if (stat?.isDirectory() && !deletePaths.has(target)) {
|
|
297
318
|
throw new Error(`Refusing to overwrite directory during snapshot apply: ${target}`);
|
|
298
319
|
}
|
|
320
|
+
return stat;
|
|
299
321
|
}
|
|
300
322
|
async function ensureSafeDirectory(root, directory, deletePaths, options) {
|
|
301
323
|
assertMutationCurrent(options);
|
|
@@ -4,7 +4,7 @@ import { pipeline } from "node:stream/promises";
|
|
|
4
4
|
import { promisify } from "node:util";
|
|
5
5
|
import { createGunzip, gzip } from "node:zlib";
|
|
6
6
|
import { throwIfAborted } from "../sync/signals.js";
|
|
7
|
-
import { snapshotSelectionInclude, validateSnapshotSelection } from "../sync/sync-policy.js";
|
|
7
|
+
import { assertSnapshotExcludesDebugLog, snapshotSelectionInclude, validateSnapshotSelection, } from "../sync/sync-policy.js";
|
|
8
8
|
import { IGNORE_SNAPSHOT_VERSION, isSupportedSnapshotVersion } from "./snapshot-types.js";
|
|
9
9
|
const VERSION = 1;
|
|
10
10
|
const MAX_DECOMPRESSED_SNAPSHOT_BYTES = 512 * 1024 * 1024;
|
|
@@ -12,6 +12,7 @@ const gzipAsync = promisify(gzip);
|
|
|
12
12
|
export async function encodeSnapshot(snapshot) {
|
|
13
13
|
if (!isSupportedSnapshotVersion(snapshot?.version))
|
|
14
14
|
throw new Error("Unsupported snapshot format.");
|
|
15
|
+
assertSnapshotExcludesDebugLog(snapshot);
|
|
15
16
|
if (snapshot.version !== IGNORE_SNAPSHOT_VERSION)
|
|
16
17
|
snapshotSelectionInclude(snapshot);
|
|
17
18
|
validateSnapshotSelection(snapshot);
|
|
@@ -46,6 +47,7 @@ export async function decodeSnapshot(buffer, options = {}) {
|
|
|
46
47
|
}
|
|
47
48
|
if (parsed.version === IGNORE_SNAPSHOT_VERSION && !isUtf8(payload))
|
|
48
49
|
throw new Error("Invalid snapshot UTF-8.");
|
|
50
|
+
assertSnapshotExcludesDebugLog(parsed);
|
|
49
51
|
validateSnapshotSelection(parsed);
|
|
50
52
|
if (parsed.version !== VERSION) {
|
|
51
53
|
const { normalizeLocalFields, validatePortableSnapshot } = await import("../sync/local-fields.js");
|
|
@@ -1,2 +1,6 @@
|
|
|
1
|
+
import type { SnapshotFile } from "./snapshot-types.js";
|
|
2
|
+
export declare function assertSnapshotFilePayload(file: SnapshotFile, message: string): void;
|
|
3
|
+
export declare function materializeSnapshotFilePayload(file: SnapshotFile, message: string): Buffer;
|
|
4
|
+
export declare function snapshotFileFromBlob(path: string, content: Buffer, sha256: string, message: string): SnapshotFile;
|
|
1
5
|
export declare function snapshotBase64Size(value: string, filePath: string): number;
|
|
2
6
|
export declare function decodeBase64Strict(value: string, filePath: string): Buffer<ArrayBuffer>;
|
|
@@ -1,3 +1,35 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
const checked = new WeakMap();
|
|
3
|
+
function checkPayload(file, message, materialize = false) {
|
|
4
|
+
const { contentBase64, sha256 } = file;
|
|
5
|
+
if (typeof contentBase64 !== "string" || typeof sha256 !== "string")
|
|
6
|
+
throw new Error(message);
|
|
7
|
+
const cacheable = Object.getOwnPropertyDescriptor(file, "contentBase64")?.value === contentBase64 &&
|
|
8
|
+
Object.getOwnPropertyDescriptor(file, "sha256")?.value === sha256;
|
|
9
|
+
const previous = cacheable ? checked.get(file) : undefined;
|
|
10
|
+
if (previous?.contentBase64 === contentBase64 && previous.sha256 === sha256)
|
|
11
|
+
return materialize ? Buffer.from(contentBase64, "base64") : undefined;
|
|
12
|
+
const content = Buffer.from(contentBase64, "base64");
|
|
13
|
+
if (content.toString("base64") !== contentBase64 || createHash("sha256").update(content).digest("hex") !== sha256)
|
|
14
|
+
throw new Error(message);
|
|
15
|
+
if (cacheable)
|
|
16
|
+
checked.set(file, { contentBase64, sha256 });
|
|
17
|
+
return content;
|
|
18
|
+
}
|
|
19
|
+
export function assertSnapshotFilePayload(file, message) {
|
|
20
|
+
checkPayload(file, message);
|
|
21
|
+
}
|
|
22
|
+
export function materializeSnapshotFilePayload(file, message) {
|
|
23
|
+
return checkPayload(file, message, true);
|
|
24
|
+
}
|
|
25
|
+
export function snapshotFileFromBlob(path, content, sha256, message) {
|
|
26
|
+
if (createHash("sha256").update(content).digest("hex") !== sha256)
|
|
27
|
+
throw new Error(message);
|
|
28
|
+
const contentBase64 = content.toString("base64");
|
|
29
|
+
const file = { path, contentBase64, sha256 };
|
|
30
|
+
checked.set(file, { contentBase64, sha256 });
|
|
31
|
+
return file;
|
|
32
|
+
}
|
|
1
33
|
export function snapshotBase64Size(value, filePath) {
|
|
2
34
|
if (typeof value !== "string" || value.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}(?![\s\S])/u.test(value))
|
|
3
35
|
throw new Error(`Invalid base64 content in snapshot file: ${filePath}`);
|
|
@@ -25,6 +25,8 @@ export declare function scopedPathIdentity(target: string, guards: Guards): Prom
|
|
|
25
25
|
export declare function scopedIdentity(stat: BigIntStats): string;
|
|
26
26
|
export declare function installedVacancyIdentityMatches(image: ScopedImage, expected: string): boolean;
|
|
27
27
|
export declare function stableScopedIdentity(target: string, stat: BigIntStats): string;
|
|
28
|
+
export declare function verifyUnchangedFile(target: string, content: Buffer, guards: Guards): Promise<BigIntStats>;
|
|
29
|
+
export declare function recheckUnchangedFile(target: string, stat: BigIntStats, guards: Guards): Promise<void>;
|
|
28
30
|
export declare function readTransactionJournal(file: string, guards: Guards, maxBytes?: number): Promise<string | undefined>;
|
|
29
31
|
export declare function scopedTransactionImage(target: string, guards: Guards, copyTo?: string, checkPath?: (source: string) => Promise<void>, beforeCopyFile?: (handle: FileHandle) => Promise<void>): Promise<ScopedImage>;
|
|
30
32
|
export declare function ownedScopedSubset(image: ScopedImage, expected: ScopedImage): boolean;
|
|
@@ -125,6 +125,40 @@ async function readStableFile(target, original, guards, consume) {
|
|
|
125
125
|
}
|
|
126
126
|
await recheck(target, original, guards);
|
|
127
127
|
}
|
|
128
|
+
export async function verifyUnchangedFile(target, content, guards) {
|
|
129
|
+
const stat = await lstat(target, guards);
|
|
130
|
+
if (!stat?.isFile() || stat.nlink !== 1n || stat.size !== BigInt(content.length))
|
|
131
|
+
skippedPreimageChanged();
|
|
132
|
+
await recheckUnchangedFile(target, stat, guards);
|
|
133
|
+
await readStableFile(target, stat, guards, (chunk, offset) => {
|
|
134
|
+
if (!chunk.equals(content.subarray(offset, offset + chunk.length)))
|
|
135
|
+
skippedPreimageChanged();
|
|
136
|
+
});
|
|
137
|
+
return stat;
|
|
138
|
+
}
|
|
139
|
+
export async function recheckUnchangedFile(target, stat, guards) {
|
|
140
|
+
validate(guards);
|
|
141
|
+
let canonical;
|
|
142
|
+
try {
|
|
143
|
+
canonical = await fs.realpath(target);
|
|
144
|
+
}
|
|
145
|
+
catch (error) {
|
|
146
|
+
validate(guards);
|
|
147
|
+
const code = error.code;
|
|
148
|
+
if (code === "ENOENT" || code === "ENOTDIR")
|
|
149
|
+
skippedPreimageChanged();
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
validate(guards);
|
|
153
|
+
if (canonical !== target)
|
|
154
|
+
skippedPreimageChanged();
|
|
155
|
+
const current = await lstat(target, guards);
|
|
156
|
+
if (!current || !sameObservation(stat, current))
|
|
157
|
+
skippedPreimageChanged();
|
|
158
|
+
}
|
|
159
|
+
function skippedPreimageChanged() {
|
|
160
|
+
throw new Error("Skipped file preimage changed before apply; snapshot installation refused.");
|
|
161
|
+
}
|
|
128
162
|
export async function readTransactionJournal(file, guards, maxBytes = MAX_TRANSACTION_JOURNAL_BYTES) {
|
|
129
163
|
const stat = await lstat(file, guards);
|
|
130
164
|
if (!stat)
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { type TransactionOptions } from "./snapshot-transaction-policy.js";
|
|
2
2
|
import type { SnapshotApplyPlan } from "./snapshot-types.js";
|
|
3
|
-
export
|
|
3
|
+
export type SkippedSnapshotWrite = SnapshotApplyPlan["writes"][number] & {
|
|
4
|
+
preimage: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function applySnapshotTransaction(plan: SnapshotApplyPlan, options?: TransactionOptions, skipped?: readonly SkippedSnapshotWrite[]): Promise<void>;
|
|
4
7
|
export declare function recoverSnapshotTransactionsOnStartup(options?: TransactionOptions): Promise<void>;
|
|
5
8
|
export declare function recoverPendingSnapshotTransactions(options?: TransactionOptions): Promise<void>;
|
|
@@ -9,15 +9,15 @@ import { normalizeSyncExclude } from "../sync/exclude-policy.js";
|
|
|
9
9
|
import { mergePathIdentity } from "../sync/file-merge-planner.js";
|
|
10
10
|
import { includeFromSelectionConfig, selectionForSnapshot } from "../sync/sync-policy.js";
|
|
11
11
|
import { agentDir, configuredSessionDir } from "./session-paths.js";
|
|
12
|
-
import { agentDirectoryConfiguredSessionDir } from "./snapshot.js";
|
|
12
|
+
import { agentDirectoryConfiguredSessionDir, scanAgentDirectoryInventory } from "./snapshot.js";
|
|
13
13
|
import { caseReplacementSpelling, coalesceCaseReplacements } from "./snapshot-case-replacement.js";
|
|
14
|
-
import { recheckAgentDirectoryBudget } from "./snapshot-entry-projection.js";
|
|
14
|
+
import { assertProjectedAgentDirectoryBudget, recheckAgentDirectoryBudget } from "./snapshot-entry-projection.js";
|
|
15
15
|
import { assertUnexcludedTargets } from "./snapshot-exclude-safety.js";
|
|
16
16
|
import { withMissingTargetQueues, withTargetQueues } from "./snapshot-mutation-queues.js";
|
|
17
17
|
import { sessionStorageRoot } from "./snapshot-paths.js";
|
|
18
18
|
import { prepareSessionRootTransition, resolveTransitionSessionRoot, stableAgentDirectorySettingsImage, } from "./snapshot-root-transition.js";
|
|
19
19
|
import { pendingTransactionEntries, transactionRoot } from "./snapshot-transaction-directory.js";
|
|
20
|
-
import { legacyImage, MAX_TRANSACTION_JOURNAL_BYTES, ownedScopedPostTree, readTransactionJournal, recheckScopedImage, removeScopedImage, scopedDescendant, scopedIdentity, scopedPathIdentity, scopedTransactionImage, stableScopedIdentity, } from "./snapshot-transaction-images.js";
|
|
20
|
+
import { legacyImage, MAX_TRANSACTION_JOURNAL_BYTES, ownedScopedPostTree, readTransactionJournal, recheckScopedImage, recheckUnchangedFile, removeScopedImage, scopedDescendant, scopedIdentity, scopedPathIdentity, scopedTransactionImage, stableScopedIdentity, verifyUnchangedFile, } from "./snapshot-transaction-images.js";
|
|
21
21
|
import { indexTransactionPlan } from "./snapshot-transaction-plan.js";
|
|
22
22
|
import { AGENT_DIRECTORY_JOURNAL_VERSION, assertAllowedTarget, assertRecoveryPolicy, assertSafeParents, BOUND_DIRECTORY_JOURNAL_VERSION, BOUND_INCLUDED_PATHS_JOURNAL_VERSION, IGNORE_POLICY_JOURNAL_VERSION, ignorePolicyValidator, ignoreSourceBinding, isScopedJournal, isStrictlyInside, recoveryPolicyKey, StaleIgnorePolicyError, touchesProtectedTarget, transactionHasSessionTargets, validateJournal, validateJournalPolicy, } from "./snapshot-transaction-policy.js";
|
|
23
23
|
import { assertNoUnclaimedLegacyStage, buildScopedRecoveryStage, cleanScopedLegacyStage, cleanScopedWorkspace, inspectScopedRecoveryStage, prepareLegacyRecoveryStage, recoveryStagePath, } from "./snapshot-transaction-stage.js";
|
|
@@ -25,7 +25,7 @@ import { verifyTarget } from "./snapshot-transaction-verification.js";
|
|
|
25
25
|
const JOURNAL_VERSION = 4;
|
|
26
26
|
class ScopedRemovalError extends Error {
|
|
27
27
|
}
|
|
28
|
-
export async function applySnapshotTransaction(plan, options = {}) {
|
|
28
|
+
export async function applySnapshotTransaction(plan, options = {}, skipped = []) {
|
|
29
29
|
options.validateMutation?.();
|
|
30
30
|
const root = path.resolve(agentDir());
|
|
31
31
|
if (options.scope === "agent-directory") {
|
|
@@ -90,7 +90,10 @@ export async function applySnapshotTransaction(plan, options = {}) {
|
|
|
90
90
|
expectedPreimages.set(target, evidence.image);
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
|
+
const skippedPreimages = new Map(skipped.map(({ target, preimage }) => [target, preimage]));
|
|
93
94
|
for (const [target, expected] of expectedPreimages) {
|
|
95
|
+
if (skippedPreimages.get(target) === expected)
|
|
96
|
+
continue;
|
|
94
97
|
if (scoped || options.reservedIgnoreNames?.length)
|
|
95
98
|
await assertUnexcludedTargets(root, [target], options);
|
|
96
99
|
const current = scoped ? (await scopedTransactionImage(target, options)).value : await legacyImage(target);
|
|
@@ -101,7 +104,29 @@ export async function applySnapshotTransaction(plan, options = {}) {
|
|
|
101
104
|
}
|
|
102
105
|
if (validateIgnorePolicy)
|
|
103
106
|
await validateIgnorePolicy();
|
|
104
|
-
const
|
|
107
|
+
const mutationTargets = skipped.length ? targets.filter((target) => !skippedPreimages.has(target)) : targets;
|
|
108
|
+
const writes = skipped.length ? plan.writes.filter(({ target }) => !skippedPreimages.has(target)) : plan.writes;
|
|
109
|
+
if (mutationTargets.length === 0) {
|
|
110
|
+
if (options.validateApplyReview)
|
|
111
|
+
await options.validateApplyReview();
|
|
112
|
+
if (scoped) {
|
|
113
|
+
const pendingStateEntry = stateDir(root);
|
|
114
|
+
const inventory = await scanAgentDirectoryInventory(root, options, pendingStateEntry);
|
|
115
|
+
options.validateMutation?.();
|
|
116
|
+
options.signal?.throwIfAborted();
|
|
117
|
+
assertProjectedAgentDirectoryBudget(inventory, { deletes: [], writes: [{ target: pendingStateEntry }] }, options);
|
|
118
|
+
}
|
|
119
|
+
if (validateIgnorePolicy)
|
|
120
|
+
await validateIgnorePolicy();
|
|
121
|
+
await verifySkippedWrites(root, skipped, heldKeys, options);
|
|
122
|
+
options.validateMutation?.();
|
|
123
|
+
options.signal?.throwIfAborted();
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
const mutationPlan = skipped.length
|
|
127
|
+
? { ...coalesced.plan, writes: coalesced.plan.writes.filter(({ target }) => !skippedPreimages.has(target)) }
|
|
128
|
+
: coalesced.plan;
|
|
129
|
+
const transaction = await prepareTransaction(mutationPlan, mutationTargets, { ...options, expectedPreimages }, coalesced.replacements);
|
|
105
130
|
const entriesByTarget = new Map(transaction.journal.entries.flatMap((entry) => [
|
|
106
131
|
[entry.target, entry],
|
|
107
132
|
...(entry.afterTarget ? [[entry.afterTarget, entry]] : []),
|
|
@@ -135,7 +160,7 @@ export async function applySnapshotTransaction(plan, options = {}) {
|
|
|
135
160
|
await retireRemovalIntent(transaction.directory, transaction.journal, transaction.journal.entries.filter((entry) => entry.removalPending && (entry.afterImage !== "missing" || (entry.postFiles?.length ?? 0) > 0)), options);
|
|
136
161
|
// Only this uninterrupted call knows which deleted writes have not attempted installation yet.
|
|
137
162
|
const uninstalledWrites = new Set(transaction.deletedWrites);
|
|
138
|
-
for (const item of
|
|
163
|
+
for (const item of writes) {
|
|
139
164
|
options.signal?.throwIfAborted();
|
|
140
165
|
const entry = entriesByTarget.get(item.target);
|
|
141
166
|
if (!entry)
|
|
@@ -194,7 +219,7 @@ export async function applySnapshotTransaction(plan, options = {}) {
|
|
|
194
219
|
}
|
|
195
220
|
options.validateMutation?.();
|
|
196
221
|
options.signal?.throwIfAborted();
|
|
197
|
-
await syncMutationParents(
|
|
222
|
+
await syncMutationParents(mutationTargets, [transaction.journal.root, ...(transaction.journal.sessionRoot ? [transaction.journal.sessionRoot] : [])], options);
|
|
198
223
|
options.validateMutation?.();
|
|
199
224
|
options.signal?.throwIfAborted();
|
|
200
225
|
if (validateIgnorePolicy)
|
|
@@ -202,6 +227,9 @@ export async function applySnapshotTransaction(plan, options = {}) {
|
|
|
202
227
|
});
|
|
203
228
|
if (validateIgnorePolicy)
|
|
204
229
|
await validateIgnorePolicy();
|
|
230
|
+
await verifySkippedWrites(root, skipped, heldKeys, options);
|
|
231
|
+
options.validateMutation?.();
|
|
232
|
+
options.signal?.throwIfAborted();
|
|
205
233
|
}
|
|
206
234
|
catch (error) {
|
|
207
235
|
// Aborted owners cannot roll back files after another session has replaced them.
|
|
@@ -226,6 +254,21 @@ export async function applySnapshotTransaction(plan, options = {}) {
|
|
|
226
254
|
await completeTransaction(transaction.directory, transaction.journal);
|
|
227
255
|
});
|
|
228
256
|
}
|
|
257
|
+
async function verifySkippedWrites(root, skipped, heldKeys, options) {
|
|
258
|
+
if (!skipped.length)
|
|
259
|
+
return;
|
|
260
|
+
await assertUnexcludedTargets(root, skipped.map(({ target }) => target), options, { recursive: false });
|
|
261
|
+
const sessionRoot = options.sessionDir ? sessionStorageRoot(root, options.sessionDir) : undefined;
|
|
262
|
+
const verified = [];
|
|
263
|
+
for (const { target, content } of skipped) {
|
|
264
|
+
if (!heldKeys.has(target) || touchesProtectedTarget(target, options.protectedTargets))
|
|
265
|
+
throw new Error("Skipped file lost its transaction queue or session protection changed; review is required.");
|
|
266
|
+
await assertSafeParents(root, sessionRoot, target);
|
|
267
|
+
verified.push({ target, stat: await verifyUnchangedFile(target, content, options) });
|
|
268
|
+
}
|
|
269
|
+
for (const { target, stat } of verified)
|
|
270
|
+
await recheckUnchangedFile(target, stat, options);
|
|
271
|
+
}
|
|
229
272
|
export async function recoverSnapshotTransactionsOnStartup(options = {}) {
|
|
230
273
|
if (!(await pendingTransactionEntries()).some((entry) => entry.isDirectory()))
|
|
231
274
|
return;
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
-
export interface
|
|
1
|
+
export interface SnapshotFileHash {
|
|
2
2
|
path: string;
|
|
3
|
-
contentBase64: string;
|
|
4
3
|
sha256: string;
|
|
5
4
|
}
|
|
5
|
+
export interface SnapshotFile extends SnapshotFileHash {
|
|
6
|
+
contentBase64: string;
|
|
7
|
+
}
|
|
8
|
+
export interface SnapshotObservation {
|
|
9
|
+
scope: DirectoryScope["scope"];
|
|
10
|
+
files: Array<SnapshotFileHash & {
|
|
11
|
+
settingsContent?: Buffer;
|
|
12
|
+
}>;
|
|
13
|
+
}
|
|
6
14
|
export interface EffectiveIgnorePolicy {
|
|
7
15
|
filename: string;
|
|
8
16
|
sourceDigest: string;
|
|
@@ -2,10 +2,12 @@ import { reservedIgnoreTargetGuard } from "./snapshot-exclude-safety.js";
|
|
|
2
2
|
export { isDeniedPath } from "../paths.js";
|
|
3
3
|
export { sessionStorageRoot } from "./snapshot-paths.js";
|
|
4
4
|
import { type SyncSelectionConfig } from "../sync/sync-policy.js";
|
|
5
|
-
import { type AgentDirectoryInventory, type Snapshot, type SnapshotFile, type SnapshotOptions } from "./snapshot-types.js";
|
|
5
|
+
import { type AgentDirectoryInventory, type Snapshot, type SnapshotFile, type SnapshotObservation, type SnapshotOptions } from "./snapshot-types.js";
|
|
6
6
|
export { canonicalSnapshotPathForConfig, isConfiguredSnapshotPath } from "../sync/sync-policy.js";
|
|
7
7
|
export declare function agentDirectoryInventory(snapshot: Pick<Snapshot, "files">): AgentDirectoryInventory | undefined;
|
|
8
8
|
export declare function regenerateSnapshotIdentity(snapshot: Snapshot): Snapshot;
|
|
9
|
+
export declare function observeSnapshot(options?: SnapshotOptions, retainSettings?: boolean): Promise<SnapshotObservation>;
|
|
10
|
+
export declare function snapshotWithObservedInventory(snapshot: Snapshot, observation: SnapshotObservation): Snapshot | undefined;
|
|
9
11
|
export declare function createSnapshot(profile: string, options?: SnapshotOptions): Promise<Snapshot>;
|
|
10
12
|
export declare function collectFiles(root: string, options?: SnapshotOptions, excluded?: (relativePath: string, directory?: boolean) => boolean): Promise<SnapshotFile[]>;
|
|
11
13
|
export declare function agentDirectoryConfiguredSessionDir(signal?: AbortSignal, root?: string, options?: Pick<SnapshotOptions, "reservedIgnoreNames">): Promise<string | undefined>;
|