@narumitw/pi-sync 0.30.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +159 -236
- package/package.json +14 -6
- package/src/backend-factory.ts +20 -0
- package/src/command.ts +39 -39
- package/src/config-errors.ts +7 -0
- package/src/config-file.ts +64 -8
- package/src/config.ts +605 -476
- package/src/file-selection.ts +121 -140
- package/src/git-backend.ts +832 -0
- package/src/git-config.ts +136 -0
- package/src/git-runner.ts +276 -0
- package/src/git-storage.ts +288 -0
- package/src/git-ui.ts +307 -0
- package/src/manager-helpers.ts +64 -0
- package/src/manager-ui.ts +537 -542
- package/src/paths.ts +4 -1
- package/src/s3-backend.ts +498 -0
- package/src/s3-client.ts +147 -64
- package/src/s3-credentials-ui.ts +92 -0
- package/src/secret-input.ts +199 -0
- package/src/settings-management.ts +191 -277
- package/src/settings-ui.ts +94 -115
- package/src/setup-switch.ts +150 -0
- package/src/snapshot-apply.ts +13 -7
- package/src/snapshot-codec.ts +48 -0
- package/src/snapshot-transaction.ts +3 -1
- package/src/snapshot.ts +63 -32
- package/src/storage-connections-ui.ts +308 -0
- package/src/sync-backend.ts +97 -0
- package/src/sync-format.ts +31 -33
- package/src/sync-operations.ts +912 -0
- package/src/sync-policy.ts +137 -61
- package/src/sync-setups-ui.ts +177 -0
- package/src/sync-state.ts +48 -48
- package/src/sync.ts +180 -739
- package/src/types.ts +177 -53
- package/src/webdav-backend.ts +678 -0
- package/src/webdav-client.ts +390 -0
- package/src/webdav-config.ts +92 -0
- package/src/webdav-ui.ts +421 -0
- package/src/target-switch.ts +0 -119
|
@@ -0,0 +1,832 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
import {
|
|
6
|
+
normalizeGitBranch,
|
|
7
|
+
normalizeGitDirectory,
|
|
8
|
+
normalizeGitRemote,
|
|
9
|
+
normalizeGitRemoteIdentity,
|
|
10
|
+
validateGitNamespace,
|
|
11
|
+
} from "./git-config.js";
|
|
12
|
+
import { GitCommandError, readGitBlobs, runGit } from "./git-runner.js";
|
|
13
|
+
import {
|
|
14
|
+
GIT_MANIFEST_VERSION,
|
|
15
|
+
type GitManifest,
|
|
16
|
+
MAX_GIT_MANIFEST_BYTES,
|
|
17
|
+
MAX_GIT_TREE_OUTPUT_BYTES,
|
|
18
|
+
type PreparedGitFile,
|
|
19
|
+
parseGitTree,
|
|
20
|
+
prepareGitSnapshot,
|
|
21
|
+
requireGitManifest,
|
|
22
|
+
validateGitPublicationTree,
|
|
23
|
+
validateGitSnapshot,
|
|
24
|
+
} from "./git-storage.js";
|
|
25
|
+
import { posixJoin } from "./paths.js";
|
|
26
|
+
import {
|
|
27
|
+
type BackendDiagnostic,
|
|
28
|
+
type ExpectedRemoteHead,
|
|
29
|
+
type PublishSnapshotOptions,
|
|
30
|
+
type PublishSnapshotResult,
|
|
31
|
+
type RemoteHead,
|
|
32
|
+
type RemoteHistoryEntry,
|
|
33
|
+
type SyncBackend,
|
|
34
|
+
SyncBackendConflictError,
|
|
35
|
+
SyncBackendPublicationOutcomeUnknownError,
|
|
36
|
+
} from "./sync-backend.js";
|
|
37
|
+
import type { ResolvedGitBackend, Snapshot, SnapshotFile } from "./types.js";
|
|
38
|
+
|
|
39
|
+
const COMMAND_TIMEOUT_MS = 30_000;
|
|
40
|
+
const POST_COMMIT_TIMEOUT_MS = 45_000;
|
|
41
|
+
const gitCacheMutationQueues = new Map<string, Promise<void>>();
|
|
42
|
+
|
|
43
|
+
export interface GitBackendOptions {
|
|
44
|
+
cacheRoot?: string;
|
|
45
|
+
allowLocalRemotes?: boolean;
|
|
46
|
+
commandTimeoutMs?: number;
|
|
47
|
+
postCommitTimeoutMs?: number;
|
|
48
|
+
/** Deterministic fault injection used only by the local backend test suite. */
|
|
49
|
+
afterPushForTest?: () => void | Promise<void>;
|
|
50
|
+
afterLsRemoteForTest?: () => void | Promise<void>;
|
|
51
|
+
afterPayloadWriteForTest?: () => void | Promise<void>;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export class GitSyncBackend implements SyncBackend {
|
|
55
|
+
readonly identity: string;
|
|
56
|
+
readonly destination: string;
|
|
57
|
+
readonly capability = "lease-protected" as const;
|
|
58
|
+
private readonly cacheRoot: string;
|
|
59
|
+
private readonly cacheDir: string;
|
|
60
|
+
private readonly allowLocalRemotes: boolean;
|
|
61
|
+
private readonly commandTimeoutMs: number;
|
|
62
|
+
private readonly postCommitTimeoutMs: number;
|
|
63
|
+
private readonly afterPushForTest?: () => void | Promise<void>;
|
|
64
|
+
private readonly afterLsRemoteForTest?: () => void | Promise<void>;
|
|
65
|
+
private readonly afterPayloadWriteForTest?: () => void | Promise<void>;
|
|
66
|
+
private cacheReady?: Promise<void>;
|
|
67
|
+
|
|
68
|
+
constructor(
|
|
69
|
+
private readonly config: ResolvedGitBackend,
|
|
70
|
+
options: GitBackendOptions = {},
|
|
71
|
+
) {
|
|
72
|
+
assertGitDestination(config);
|
|
73
|
+
this.allowLocalRemotes = options.allowLocalRemotes === true;
|
|
74
|
+
if (!this.allowLocalRemotes) assertProductionRemote(config.profile.remote);
|
|
75
|
+
this.identity = gitBackendIdentity(config);
|
|
76
|
+
this.destination = gitDestination(config);
|
|
77
|
+
this.cacheRoot = options.cacheRoot ?? path.join(getAgentDir(), ".pisync", "git");
|
|
78
|
+
this.cacheDir = path.join(this.cacheRoot, this.identity.slice("git:".length), "repository.git");
|
|
79
|
+
this.commandTimeoutMs = options.commandTimeoutMs ?? COMMAND_TIMEOUT_MS;
|
|
80
|
+
this.postCommitTimeoutMs = options.postCommitTimeoutMs ?? POST_COMMIT_TIMEOUT_MS;
|
|
81
|
+
this.afterPushForTest = options.afterPushForTest;
|
|
82
|
+
this.afterLsRemoteForTest = options.afterLsRemoteForTest;
|
|
83
|
+
this.afterPayloadWriteForTest = options.afterPayloadWriteForTest;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
sameRevision(left: string, right: string) {
|
|
87
|
+
return decodeRevision(left, this.identity) === decodeRevision(right, this.identity);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async readHead(signal?: AbortSignal): Promise<RemoteHead | undefined> {
|
|
91
|
+
const sha = await this.fetchRemoteHead(signal);
|
|
92
|
+
if (!sha) return undefined;
|
|
93
|
+
const { manifest } = await this.readPublication(sha, signal);
|
|
94
|
+
return remoteHead(sha, manifest, this.identity);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async readSnapshot(reference: string, signal?: AbortSignal): Promise<Snapshot> {
|
|
98
|
+
const head = await this.fetchRemoteHead(signal);
|
|
99
|
+
if (!head) throw new Error(`Git snapshot publication was not found: ${reference}`);
|
|
100
|
+
const commit = await this.resolveSnapshotReference(reference, head, signal);
|
|
101
|
+
try {
|
|
102
|
+
await this.git(["cat-file", "-e", `${commit}^{commit}`], { signal });
|
|
103
|
+
await this.git(["merge-base", "--is-ancestor", commit, head], { signal });
|
|
104
|
+
} catch (error) {
|
|
105
|
+
throw new Error(`Git snapshot publication was not found: ${reference}`, { cause: error });
|
|
106
|
+
}
|
|
107
|
+
const { manifest, payloadEntries } = await this.readPublication(commit, signal);
|
|
108
|
+
let blobs: Buffer[];
|
|
109
|
+
try {
|
|
110
|
+
blobs = await readGitBlobs(
|
|
111
|
+
payloadEntries.map((entry) => entry.object),
|
|
112
|
+
{
|
|
113
|
+
gitDir: this.cacheDir,
|
|
114
|
+
signal,
|
|
115
|
+
timeoutMs: this.commandTimeoutMs,
|
|
116
|
+
allowFileProtocol: this.allowLocalRemotes,
|
|
117
|
+
maxOutputBytes: manifest.files.reduce((total, file) => total + file.size, 0),
|
|
118
|
+
},
|
|
119
|
+
);
|
|
120
|
+
} catch (error) {
|
|
121
|
+
if (error instanceof Error && /exceeds/u.test(error.message)) {
|
|
122
|
+
throw new Error("Git snapshot file content exceeds its manifest size.", { cause: error });
|
|
123
|
+
}
|
|
124
|
+
throw this.redactedError(error);
|
|
125
|
+
}
|
|
126
|
+
throwIfAborted(signal);
|
|
127
|
+
const files: SnapshotFile[] = manifest.files.map((file, index) => {
|
|
128
|
+
const content = blobs[index];
|
|
129
|
+
if (!content || content.byteLength !== file.size || sha256(content) !== file.sha256) {
|
|
130
|
+
throw new Error(`Git snapshot file checksum or size mismatch: ${file.path}`);
|
|
131
|
+
}
|
|
132
|
+
return { path: file.path, contentBase64: content.toString("base64"), sha256: file.sha256 };
|
|
133
|
+
});
|
|
134
|
+
const snapshot: Snapshot = {
|
|
135
|
+
version: manifest.snapshotVersion,
|
|
136
|
+
id: manifest.snapshotId,
|
|
137
|
+
createdAt: manifest.createdAt,
|
|
138
|
+
machine: manifest.machine,
|
|
139
|
+
profile: manifest.profile,
|
|
140
|
+
...(manifest.snapshotSyncSessions === undefined
|
|
141
|
+
? {}
|
|
142
|
+
: { syncSessions: manifest.snapshotSyncSessions }),
|
|
143
|
+
files,
|
|
144
|
+
};
|
|
145
|
+
validateGitSnapshot(snapshot, manifest, this.config.destination.namespace);
|
|
146
|
+
return snapshot;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async publishSnapshot(
|
|
150
|
+
snapshot: Snapshot,
|
|
151
|
+
expected: ExpectedRemoteHead,
|
|
152
|
+
options: PublishSnapshotOptions = {},
|
|
153
|
+
): Promise<PublishSnapshotResult> {
|
|
154
|
+
throwIfAborted(options.signal);
|
|
155
|
+
const files = prepareGitSnapshot(snapshot, this.config.destination.namespace);
|
|
156
|
+
const observed = await this.fetchRemoteHead(options.signal);
|
|
157
|
+
if (!matchesExpected(observed, expected, this.identity)) {
|
|
158
|
+
throw new SyncBackendConflictError(
|
|
159
|
+
"Git remote changed while preparing publication. Run /sync status and retry.",
|
|
160
|
+
{ currentHead: observed ? await this.headForSha(observed, options.signal) : undefined },
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
throwIfAborted(options.signal);
|
|
164
|
+
const manifest: GitManifest = {
|
|
165
|
+
version: GIT_MANIFEST_VERSION,
|
|
166
|
+
snapshotVersion: snapshot.version,
|
|
167
|
+
snapshotId: snapshot.id,
|
|
168
|
+
createdAt: snapshot.createdAt,
|
|
169
|
+
machine: snapshot.machine,
|
|
170
|
+
profile: snapshot.profile,
|
|
171
|
+
syncSessions:
|
|
172
|
+
snapshot.syncSessions === true ||
|
|
173
|
+
snapshot.files.some((file) => file.path.startsWith("sessions/")),
|
|
174
|
+
...(snapshot.syncSessions === undefined
|
|
175
|
+
? {}
|
|
176
|
+
: { snapshotSyncSessions: snapshot.syncSessions }),
|
|
177
|
+
files: files.map(({ path: filePath, sha256: fileSha, size }) => ({
|
|
178
|
+
path: filePath,
|
|
179
|
+
sha256: fileSha,
|
|
180
|
+
size,
|
|
181
|
+
})),
|
|
182
|
+
};
|
|
183
|
+
let candidate: string;
|
|
184
|
+
try {
|
|
185
|
+
candidate = await this.createCommit(snapshot, files, manifest, observed, options.signal);
|
|
186
|
+
} catch (error) {
|
|
187
|
+
throw this.redactedError(error);
|
|
188
|
+
}
|
|
189
|
+
throwIfAborted(options.signal);
|
|
190
|
+
options.onCommit?.();
|
|
191
|
+
|
|
192
|
+
const ref = this.remoteRef();
|
|
193
|
+
const lease = `--force-with-lease=${ref}:${observed ?? ""}`;
|
|
194
|
+
let pushError: unknown;
|
|
195
|
+
try {
|
|
196
|
+
await this.git(
|
|
197
|
+
[
|
|
198
|
+
"push",
|
|
199
|
+
"--porcelain",
|
|
200
|
+
"--no-verify",
|
|
201
|
+
lease,
|
|
202
|
+
this.config.profile.remote,
|
|
203
|
+
`${candidate}:${ref}`,
|
|
204
|
+
],
|
|
205
|
+
{ timeoutMs: this.postCommitTimeoutMs },
|
|
206
|
+
);
|
|
207
|
+
await this.afterPushForTest?.();
|
|
208
|
+
} catch (error) {
|
|
209
|
+
pushError = error;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
let current: string | undefined;
|
|
213
|
+
try {
|
|
214
|
+
current = await this.fetchRemoteHead(AbortSignal.timeout(this.postCommitTimeoutMs));
|
|
215
|
+
} catch (error) {
|
|
216
|
+
throw new SyncBackendPublicationOutcomeUnknownError(
|
|
217
|
+
`Git publication outcome is unknown: ${this.safeError(pushError ?? error)}`,
|
|
218
|
+
{ cause: pushError ?? error },
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
if (current !== candidate) {
|
|
222
|
+
if (pushError && current === observed) {
|
|
223
|
+
throw new Error(
|
|
224
|
+
`Git publication failed without updating the owned branch: ${this.safeError(pushError)}`,
|
|
225
|
+
{
|
|
226
|
+
cause: pushError,
|
|
227
|
+
},
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
throw new SyncBackendConflictError(
|
|
231
|
+
pushError
|
|
232
|
+
? `Git publication lease was rejected: ${this.safeError(pushError)}`
|
|
233
|
+
: "Git remote changed immediately after publication.",
|
|
234
|
+
{
|
|
235
|
+
phase: "after-commit",
|
|
236
|
+
currentHead: current ? await this.headForSha(current) : undefined,
|
|
237
|
+
candidateMayHaveBeenActive: true,
|
|
238
|
+
cause: pushError instanceof Error ? pushError : undefined,
|
|
239
|
+
},
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
const head = await this.headForSha(candidate);
|
|
243
|
+
return { head, warnings: [] };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async listHistory(signal?: AbortSignal): Promise<RemoteHistoryEntry[]> {
|
|
247
|
+
const sha = await this.fetchRemoteHead(signal);
|
|
248
|
+
if (!sha) return [];
|
|
249
|
+
const result = await this.git(
|
|
250
|
+
["rev-list", "--first-parent", "--reverse", "--max-count=100", sha],
|
|
251
|
+
{ signal },
|
|
252
|
+
);
|
|
253
|
+
const commits = result.stdout.toString("utf8").trim().split("\n").filter(Boolean);
|
|
254
|
+
const entries: RemoteHistoryEntry[] = [];
|
|
255
|
+
for (const commit of commits) {
|
|
256
|
+
const { manifest } = await this.readPublication(commit, signal);
|
|
257
|
+
entries.push({
|
|
258
|
+
snapshotRef: commit,
|
|
259
|
+
snapshotId: manifest.snapshotId,
|
|
260
|
+
createdAt: manifest.createdAt,
|
|
261
|
+
machine: manifest.machine,
|
|
262
|
+
syncSessions: manifest.syncSessions,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
return entries;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
async diagnose(signal?: AbortSignal): Promise<BackendDiagnostic[]> {
|
|
269
|
+
const diagnostics: BackendDiagnostic[] = [];
|
|
270
|
+
try {
|
|
271
|
+
const version = await runGit(["--version"], {
|
|
272
|
+
signal,
|
|
273
|
+
timeoutMs: this.commandTimeoutMs,
|
|
274
|
+
});
|
|
275
|
+
const versionText = version.stdout.toString("utf8").trim();
|
|
276
|
+
const supported = isSupportedGitVersion(versionText);
|
|
277
|
+
diagnostics.push({
|
|
278
|
+
key: "git-version",
|
|
279
|
+
level: supported ? "info" : "error",
|
|
280
|
+
message: supported
|
|
281
|
+
? versionText
|
|
282
|
+
: `${versionText || "unknown Git version"}; pi-sync requires Git 2.30 or newer`,
|
|
283
|
+
});
|
|
284
|
+
} catch (error) {
|
|
285
|
+
return [{ key: "git-version", level: "error", message: this.safeError(error) }];
|
|
286
|
+
}
|
|
287
|
+
try {
|
|
288
|
+
const head = await this.readHead(signal);
|
|
289
|
+
if (head) await this.readSnapshot(head.snapshotRef, signal);
|
|
290
|
+
diagnostics.push({
|
|
291
|
+
key: "git-remote",
|
|
292
|
+
level: "info",
|
|
293
|
+
message: head
|
|
294
|
+
? `git remote: reachable; owned branch ${this.config.destination.branch} is valid`
|
|
295
|
+
: `git remote: reachable; owned branch ${this.config.destination.branch} is not created yet`,
|
|
296
|
+
});
|
|
297
|
+
diagnostics.push({
|
|
298
|
+
key: "git-cache",
|
|
299
|
+
level: "info",
|
|
300
|
+
message: "git cache: private bare repository is healthy",
|
|
301
|
+
});
|
|
302
|
+
} catch (error) {
|
|
303
|
+
diagnostics.push({
|
|
304
|
+
key: "git-remote",
|
|
305
|
+
level: "error",
|
|
306
|
+
message: `git remote: ${this.safeError(error)}`,
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
return diagnostics;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
private async resolveSnapshotReference(reference: string, head: string, signal?: AbortSignal) {
|
|
313
|
+
if (isCommitSha(reference) || /^[0-9a-f]{64}$/u.test(reference)) {
|
|
314
|
+
requireCommitSha(reference);
|
|
315
|
+
return reference;
|
|
316
|
+
}
|
|
317
|
+
if (!reference || reference.length > 512 || !/^[A-Za-z0-9._-]+$/u.test(reference)) {
|
|
318
|
+
throw new Error("Invalid Git publication reference.");
|
|
319
|
+
}
|
|
320
|
+
const result = await this.git(["rev-list", "--first-parent", "--max-count=100", head], {
|
|
321
|
+
signal,
|
|
322
|
+
});
|
|
323
|
+
const commits = result.stdout.toString("utf8").trim().split("\n").filter(Boolean);
|
|
324
|
+
const matches: string[] = [];
|
|
325
|
+
for (const commit of commits) {
|
|
326
|
+
const { manifest } = await this.readPublication(commit, signal);
|
|
327
|
+
if (manifest.snapshotId === reference) matches.push(commit);
|
|
328
|
+
}
|
|
329
|
+
if (matches.length === 0) {
|
|
330
|
+
throw new Error(`Git snapshot publication was not found: ${reference}`);
|
|
331
|
+
}
|
|
332
|
+
if (matches.length > 1) {
|
|
333
|
+
throw new Error(
|
|
334
|
+
`Git snapshot id is ambiguous; use a commit reference from /sync history: ${reference}`,
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
return matches[0] as string;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
private async headForSha(sha: string, signal?: AbortSignal) {
|
|
341
|
+
const { manifest } = await this.readPublication(sha, signal);
|
|
342
|
+
return remoteHead(sha, manifest, this.identity);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
private async fetchRemoteHead(signal?: AbortSignal) {
|
|
346
|
+
await this.ensureCache(signal);
|
|
347
|
+
const result = await this.git(
|
|
348
|
+
["ls-remote", "--refs", this.config.profile.remote, this.remoteRef()],
|
|
349
|
+
{ signal },
|
|
350
|
+
);
|
|
351
|
+
const line = result.stdout.toString("utf8").trim();
|
|
352
|
+
if (!line) return undefined;
|
|
353
|
+
const [sha, ref, ...extra] = line.split(/\s+/u);
|
|
354
|
+
if (extra.length > 0 || ref !== this.remoteRef() || !sha) {
|
|
355
|
+
throw new Error("Git remote returned a malformed owned-ref response.");
|
|
356
|
+
}
|
|
357
|
+
requireCommitSha(sha);
|
|
358
|
+
await this.afterLsRemoteForTest?.();
|
|
359
|
+
return withGitCacheMutation(
|
|
360
|
+
this.cacheDir,
|
|
361
|
+
async () => {
|
|
362
|
+
const localRef = `refs/pisync/fetch/${process.pid}-${randomUUID()}`;
|
|
363
|
+
try {
|
|
364
|
+
await this.git(
|
|
365
|
+
[
|
|
366
|
+
"fetch",
|
|
367
|
+
"--no-tags",
|
|
368
|
+
"--force",
|
|
369
|
+
this.config.profile.remote,
|
|
370
|
+
`${this.remoteRef()}:${localRef}`,
|
|
371
|
+
],
|
|
372
|
+
{ signal },
|
|
373
|
+
);
|
|
374
|
+
const fetched = (await this.git(["rev-parse", "--verify", localRef], { signal })).stdout
|
|
375
|
+
.toString("utf8")
|
|
376
|
+
.trim();
|
|
377
|
+
requireCommitSha(fetched);
|
|
378
|
+
return fetched;
|
|
379
|
+
} finally {
|
|
380
|
+
await this.git(["update-ref", "-d", localRef], { timeoutMs: 5_000 }).catch(
|
|
381
|
+
() => undefined,
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
},
|
|
385
|
+
signal,
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
private async readManifest(commit: string, signal?: AbortSignal): Promise<GitManifest> {
|
|
390
|
+
requireCommitSha(commit);
|
|
391
|
+
const bytes = await this.showFile(commit, this.manifestPath(), signal, MAX_GIT_MANIFEST_BYTES);
|
|
392
|
+
let parsed: unknown;
|
|
393
|
+
try {
|
|
394
|
+
parsed = JSON.parse(bytes.toString("utf8"));
|
|
395
|
+
} catch (error) {
|
|
396
|
+
throw new Error("Git publication manifest is malformed.", { cause: error });
|
|
397
|
+
}
|
|
398
|
+
return requireGitManifest(parsed);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
private showFile(
|
|
402
|
+
commit: string,
|
|
403
|
+
filePath: string,
|
|
404
|
+
signal?: AbortSignal,
|
|
405
|
+
maxOutputBytes?: number,
|
|
406
|
+
) {
|
|
407
|
+
return this.git(["show", `${commit}:${filePath}`], { signal, maxOutputBytes }).then(
|
|
408
|
+
(result) => result.stdout,
|
|
409
|
+
);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
private async createCommit(
|
|
413
|
+
snapshot: Snapshot,
|
|
414
|
+
files: PreparedGitFile[],
|
|
415
|
+
manifest: GitManifest,
|
|
416
|
+
parent: string | undefined,
|
|
417
|
+
signal?: AbortSignal,
|
|
418
|
+
) {
|
|
419
|
+
const manifestBytes = Buffer.from(`${JSON.stringify(manifest)}\n`, "utf8");
|
|
420
|
+
if (manifestBytes.byteLength > MAX_GIT_MANIFEST_BYTES) {
|
|
421
|
+
throw new Error(`Git publication manifest exceeds the ${MAX_GIT_MANIFEST_BYTES}-byte limit.`);
|
|
422
|
+
}
|
|
423
|
+
await this.ensureCache(signal);
|
|
424
|
+
const temporaryDirectory = await fs.mkdtemp(path.join(path.dirname(this.cacheDir), ".index-"));
|
|
425
|
+
const indexPath = path.join(temporaryDirectory, "index");
|
|
426
|
+
const payloadDirectory = path.join(temporaryDirectory, "payloads");
|
|
427
|
+
const env = { GIT_INDEX_FILE: indexPath };
|
|
428
|
+
try {
|
|
429
|
+
await fs.mkdir(payloadDirectory, { mode: 0o700 });
|
|
430
|
+
const uniqueFiles = [...new Map(files.map((file) => [file.sha256, file])).values()].sort(
|
|
431
|
+
(left, right) => left.sha256.localeCompare(right.sha256),
|
|
432
|
+
);
|
|
433
|
+
for (const file of uniqueFiles) {
|
|
434
|
+
throwIfAborted(signal);
|
|
435
|
+
await fs.writeFile(path.join(payloadDirectory, file.sha256), file.content, {
|
|
436
|
+
flag: "wx",
|
|
437
|
+
mode: 0o600,
|
|
438
|
+
});
|
|
439
|
+
}
|
|
440
|
+
await this.afterPayloadWriteForTest?.();
|
|
441
|
+
throwIfAborted(signal);
|
|
442
|
+
const hashed = await this.git(["hash-object", "-w", "--no-filters", "--stdin-paths"], {
|
|
443
|
+
cwd: payloadDirectory,
|
|
444
|
+
input: uniqueFiles.map((file) => file.sha256).join("\n") + (uniqueFiles.length ? "\n" : ""),
|
|
445
|
+
signal,
|
|
446
|
+
maxOutputBytes: Math.max(1024, uniqueFiles.length * 64),
|
|
447
|
+
});
|
|
448
|
+
const objectIds = hashed.stdout.toString("utf8").trim().split("\n").filter(Boolean);
|
|
449
|
+
if (objectIds.length !== uniqueFiles.length || objectIds.some((id) => !isCommitSha(id))) {
|
|
450
|
+
throw new Error("Git hash-object returned a malformed payload response.");
|
|
451
|
+
}
|
|
452
|
+
const objectsBySha256 = new Map(
|
|
453
|
+
uniqueFiles.map((file, index) => [file.sha256, objectIds[index] as string]),
|
|
454
|
+
);
|
|
455
|
+
const manifestBlob = (
|
|
456
|
+
await this.git(["hash-object", "-w", "--stdin"], { input: manifestBytes, signal })
|
|
457
|
+
).stdout
|
|
458
|
+
.toString("utf8")
|
|
459
|
+
.trim();
|
|
460
|
+
if (!isCommitSha(manifestBlob)) throw new Error("Git returned an invalid manifest blob id.");
|
|
461
|
+
await this.git(["read-tree", "--empty"], { env, signal });
|
|
462
|
+
const indexLines = [
|
|
463
|
+
`100644 ${manifestBlob}\t${this.manifestPath()}`,
|
|
464
|
+
...files.map((file) => {
|
|
465
|
+
const object = objectsBySha256.get(file.sha256);
|
|
466
|
+
if (!object) throw new Error("Git payload object is missing after hashing.");
|
|
467
|
+
return `100644 ${object}\t${this.filePath(file.path)}`;
|
|
468
|
+
}),
|
|
469
|
+
];
|
|
470
|
+
await this.git(["update-index", "-z", "--index-info"], {
|
|
471
|
+
env,
|
|
472
|
+
signal,
|
|
473
|
+
input: Buffer.from(`${indexLines.join("\0")}\0`, "utf8"),
|
|
474
|
+
});
|
|
475
|
+
const tree = (await this.git(["write-tree"], { env, signal })).stdout.toString("utf8").trim();
|
|
476
|
+
const date = Number.isNaN(Date.parse(snapshot.createdAt))
|
|
477
|
+
? new Date().toISOString()
|
|
478
|
+
: snapshot.createdAt;
|
|
479
|
+
const commit = await this.git(
|
|
480
|
+
["commit-tree", tree, ...(parent ? ["-p", parent] : []), "-F", "-"],
|
|
481
|
+
{
|
|
482
|
+
signal,
|
|
483
|
+
input: `pi-sync snapshot ${snapshot.id}\n`,
|
|
484
|
+
env: {
|
|
485
|
+
GIT_AUTHOR_NAME: "pi-sync",
|
|
486
|
+
GIT_AUTHOR_EMAIL: "pi-sync@localhost",
|
|
487
|
+
GIT_COMMITTER_NAME: "pi-sync",
|
|
488
|
+
GIT_COMMITTER_EMAIL: "pi-sync@localhost",
|
|
489
|
+
GIT_AUTHOR_DATE: date,
|
|
490
|
+
GIT_COMMITTER_DATE: date,
|
|
491
|
+
},
|
|
492
|
+
},
|
|
493
|
+
);
|
|
494
|
+
const sha = commit.stdout.toString("utf8").trim();
|
|
495
|
+
requireCommitSha(sha);
|
|
496
|
+
return sha;
|
|
497
|
+
} finally {
|
|
498
|
+
await fs.rm(temporaryDirectory, { recursive: true, force: true });
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
private ensureCache(signal?: AbortSignal) {
|
|
503
|
+
if (!this.cacheReady) {
|
|
504
|
+
const operation = withGitCacheMutation(
|
|
505
|
+
this.cacheDir,
|
|
506
|
+
() => this.initializeCache(signal),
|
|
507
|
+
signal,
|
|
508
|
+
);
|
|
509
|
+
const wrapped = operation.catch((error) => {
|
|
510
|
+
if (this.cacheReady === wrapped) this.cacheReady = undefined;
|
|
511
|
+
throw this.redactedError(error);
|
|
512
|
+
});
|
|
513
|
+
this.cacheReady = wrapped;
|
|
514
|
+
}
|
|
515
|
+
return this.cacheReady;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
private async initializeCache(signal?: AbortSignal) {
|
|
519
|
+
const version = await runGit(["--version"], {
|
|
520
|
+
signal,
|
|
521
|
+
timeoutMs: this.commandTimeoutMs,
|
|
522
|
+
});
|
|
523
|
+
const versionText = version.stdout.toString("utf8").trim();
|
|
524
|
+
if (!isSupportedGitVersion(versionText)) {
|
|
525
|
+
throw new Error(
|
|
526
|
+
`${versionText || "Unknown Git version"}; pi-sync requires Git 2.30 or newer.`,
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
const parent = path.dirname(this.cacheDir);
|
|
530
|
+
const cacheParent = path.dirname(this.cacheRoot);
|
|
531
|
+
await assertNotSymlink(cacheParent, "Git cache parent");
|
|
532
|
+
await assertNotSymlink(this.cacheRoot, "Git cache root");
|
|
533
|
+
await fs.mkdir(this.cacheRoot, { recursive: true, mode: 0o700 });
|
|
534
|
+
await assertNotSymlink(cacheParent, "Git cache parent");
|
|
535
|
+
await assertNotSymlink(this.cacheRoot, "Git cache root");
|
|
536
|
+
await assertNotSymlink(parent, "Git cache identity directory");
|
|
537
|
+
await fs.mkdir(parent, { recursive: true, mode: 0o700 });
|
|
538
|
+
await assertNotSymlink(parent, "Git cache identity directory");
|
|
539
|
+
let recreate = false;
|
|
540
|
+
try {
|
|
541
|
+
const stat = await fs.lstat(this.cacheDir);
|
|
542
|
+
if (stat.isSymbolicLink()) throw new Error("Refusing symlinked Git cache.");
|
|
543
|
+
if (!stat.isDirectory()) recreate = true;
|
|
544
|
+
else {
|
|
545
|
+
try {
|
|
546
|
+
recreate = !(await this.cacheUsesSha1(signal));
|
|
547
|
+
} catch {
|
|
548
|
+
recreate = true;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
} catch (error) {
|
|
552
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
553
|
+
}
|
|
554
|
+
if (recreate) await fs.rm(this.cacheDir, { recursive: true, force: true });
|
|
555
|
+
try {
|
|
556
|
+
await fs.access(this.cacheDir);
|
|
557
|
+
} catch {
|
|
558
|
+
try {
|
|
559
|
+
await runGit(["init", "--bare", "--object-format=sha1", this.cacheDir], {
|
|
560
|
+
signal,
|
|
561
|
+
timeoutMs: this.commandTimeoutMs,
|
|
562
|
+
allowFileProtocol: this.allowLocalRemotes,
|
|
563
|
+
});
|
|
564
|
+
} catch (initError) {
|
|
565
|
+
const concurrent = await this.cacheUsesSha1(signal).catch(() => false);
|
|
566
|
+
if (!concurrent) throw initError;
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
if (process.platform !== "win32") await fs.chmod(parent, 0o700);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
private async cacheUsesSha1(signal?: AbortSignal) {
|
|
573
|
+
const result = await this.git(["rev-parse", "--is-bare-repository", "--show-object-format"], {
|
|
574
|
+
signal,
|
|
575
|
+
});
|
|
576
|
+
return result.stdout.toString("utf8").trim() === "true\nsha1";
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
private git(
|
|
580
|
+
args: string[],
|
|
581
|
+
options: {
|
|
582
|
+
cwd?: string;
|
|
583
|
+
input?: Buffer | string;
|
|
584
|
+
env?: NodeJS.ProcessEnv;
|
|
585
|
+
signal?: AbortSignal;
|
|
586
|
+
timeoutMs?: number;
|
|
587
|
+
maxOutputBytes?: number;
|
|
588
|
+
} = {},
|
|
589
|
+
) {
|
|
590
|
+
return runGit(args, {
|
|
591
|
+
gitDir: this.cacheDir,
|
|
592
|
+
allowFileProtocol: this.allowLocalRemotes,
|
|
593
|
+
timeoutMs: options.timeoutMs ?? this.commandTimeoutMs,
|
|
594
|
+
...options,
|
|
595
|
+
}).catch((error) => {
|
|
596
|
+
throw this.redactedError(error);
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
private remoteRef() {
|
|
601
|
+
return `refs/heads/${this.config.destination.branch}`;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
private publicationPath() {
|
|
605
|
+
return this.config.destination.directory;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
private manifestPath() {
|
|
609
|
+
return posixJoin(this.publicationPath(), "manifest.json");
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
private filePath(filePath: string) {
|
|
613
|
+
return posixJoin(this.publicationPath(), "files", filePath);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
private async readPublication(commit: string, signal?: AbortSignal) {
|
|
617
|
+
const manifest = await this.readManifest(commit, signal);
|
|
618
|
+
const entries = await this.readPublicationTree(commit, signal);
|
|
619
|
+
const payloadEntries = validateGitPublicationTree(
|
|
620
|
+
entries,
|
|
621
|
+
manifest,
|
|
622
|
+
this.manifestPath(),
|
|
623
|
+
(filePath) => this.filePath(filePath),
|
|
624
|
+
);
|
|
625
|
+
return { manifest, payloadEntries };
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
private async readPublicationTree(commit: string, signal?: AbortSignal) {
|
|
629
|
+
const result = await this.git(["ls-tree", "-r", "-z", commit], {
|
|
630
|
+
signal,
|
|
631
|
+
maxOutputBytes: MAX_GIT_TREE_OUTPUT_BYTES,
|
|
632
|
+
});
|
|
633
|
+
return parseGitTree(result.stdout);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
private redactedError(error: unknown) {
|
|
637
|
+
if (error instanceof Error && error.name === "AbortError") return error;
|
|
638
|
+
return new Error(this.safeError(error));
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
private safeError(error: unknown) {
|
|
642
|
+
const raw =
|
|
643
|
+
error instanceof GitCommandError
|
|
644
|
+
? error.stderr || error.message
|
|
645
|
+
: error instanceof Error
|
|
646
|
+
? error.message
|
|
647
|
+
: String(error);
|
|
648
|
+
return redactGitError(raw, this.config.profile.remote, this.cacheDir);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
export function gitBackendIdentity(config: ResolvedGitBackend) {
|
|
653
|
+
let remoteIdentity: string;
|
|
654
|
+
try {
|
|
655
|
+
remoteIdentity = normalizeGitRemoteIdentity(config.profile.remote);
|
|
656
|
+
} catch {
|
|
657
|
+
remoteIdentity = config.profile.remote;
|
|
658
|
+
}
|
|
659
|
+
const canonical = JSON.stringify([
|
|
660
|
+
remoteIdentity,
|
|
661
|
+
config.destination.branch,
|
|
662
|
+
config.destination.directory,
|
|
663
|
+
]);
|
|
664
|
+
return `git:${sha256(Buffer.from(canonical))}`;
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
function gitDestination(config: ResolvedGitBackend) {
|
|
668
|
+
let host = "Git remote";
|
|
669
|
+
const remote = config.profile.remote;
|
|
670
|
+
if (remote.includes("://")) {
|
|
671
|
+
try {
|
|
672
|
+
host = new URL(remote).host;
|
|
673
|
+
} catch {
|
|
674
|
+
host = "Git remote";
|
|
675
|
+
}
|
|
676
|
+
} else {
|
|
677
|
+
const match = /^(?:[^@]+@)?(?<host>\[[^\]]+\]|[^:]+):/u.exec(remote);
|
|
678
|
+
if (match?.groups?.host) host = match.groups.host;
|
|
679
|
+
}
|
|
680
|
+
return `${host} · ${config.destination.branch}:${config.destination.directory}`;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
function remoteHead(sha: string, manifest: GitManifest, identity: string): RemoteHead {
|
|
684
|
+
return {
|
|
685
|
+
snapshotRef: sha,
|
|
686
|
+
snapshotId: manifest.snapshotId,
|
|
687
|
+
revision: `${identity}:${sha}`,
|
|
688
|
+
createdAt: manifest.createdAt,
|
|
689
|
+
machine: manifest.machine,
|
|
690
|
+
syncSessions: manifest.syncSessions,
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function matchesExpected(
|
|
695
|
+
current: string | undefined,
|
|
696
|
+
expected: ExpectedRemoteHead,
|
|
697
|
+
identity: string,
|
|
698
|
+
) {
|
|
699
|
+
if (expected.kind === "missing") return current === undefined;
|
|
700
|
+
try {
|
|
701
|
+
return current === decodeRevision(expected.revision, identity);
|
|
702
|
+
} catch {
|
|
703
|
+
return false;
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
function decodeRevision(revision: string, identity: string) {
|
|
708
|
+
const prefix = `${identity}:`;
|
|
709
|
+
const sha = revision.startsWith(prefix) ? revision.slice(prefix.length) : "";
|
|
710
|
+
if (!/^[0-9a-f]{40}$/u.test(sha)) throw new Error("Invalid Git remote revision.");
|
|
711
|
+
return sha;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
function isCommitSha(value: string) {
|
|
715
|
+
return /^[0-9a-f]{40}$/u.test(value);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
function requireCommitSha(value: string) {
|
|
719
|
+
if (/^[0-9a-f]{64}$/u.test(value)) {
|
|
720
|
+
throw new Error("Unsupported Git SHA-256 repository; pi-sync currently requires SHA-1 refs.");
|
|
721
|
+
}
|
|
722
|
+
if (!/^[0-9a-f]{40}$/u.test(value)) throw new Error("Invalid Git publication reference.");
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
export function isSupportedGitVersion(value: string) {
|
|
726
|
+
const match = /git version (\d+)\.(\d+)/u.exec(value);
|
|
727
|
+
if (!match) return false;
|
|
728
|
+
const major = Number(match[1]);
|
|
729
|
+
const minor = Number(match[2]);
|
|
730
|
+
return major > 2 || (major === 2 && minor >= 30);
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
function assertGitDestination(config: ResolvedGitBackend) {
|
|
734
|
+
try {
|
|
735
|
+
if (
|
|
736
|
+
normalizeGitBranch(config.destination.branch) !== config.destination.branch ||
|
|
737
|
+
normalizeGitDirectory(config.destination.directory) !== config.destination.directory
|
|
738
|
+
) {
|
|
739
|
+
throw new Error("Git storage location is not normalized.");
|
|
740
|
+
}
|
|
741
|
+
validateGitNamespace(config.destination.namespace);
|
|
742
|
+
} catch (error) {
|
|
743
|
+
throw new Error("Invalid Git storage location.", { cause: error });
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
function assertProductionRemote(remote: string) {
|
|
748
|
+
let normalized: string | undefined;
|
|
749
|
+
try {
|
|
750
|
+
normalized = normalizeGitRemote(remote);
|
|
751
|
+
} catch (error) {
|
|
752
|
+
throw new Error(error instanceof Error ? error.message : "Invalid Git remote.", {
|
|
753
|
+
cause: error,
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
if (!normalized || normalized !== remote)
|
|
757
|
+
throw new Error("Invalid or non-normalized Git remote.");
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
async function withGitCacheMutation<T>(
|
|
761
|
+
cacheDir: string,
|
|
762
|
+
run: () => Promise<T>,
|
|
763
|
+
signal?: AbortSignal,
|
|
764
|
+
): Promise<T> {
|
|
765
|
+
const previous = gitCacheMutationQueues.get(cacheDir) ?? Promise.resolve();
|
|
766
|
+
const operation = previous
|
|
767
|
+
.catch(() => undefined)
|
|
768
|
+
.then(() => {
|
|
769
|
+
throwIfAborted(signal);
|
|
770
|
+
return run();
|
|
771
|
+
});
|
|
772
|
+
const tail = operation.then(
|
|
773
|
+
() => undefined,
|
|
774
|
+
() => undefined,
|
|
775
|
+
);
|
|
776
|
+
gitCacheMutationQueues.set(cacheDir, tail);
|
|
777
|
+
void tail.then(() => {
|
|
778
|
+
if (gitCacheMutationQueues.get(cacheDir) === tail) gitCacheMutationQueues.delete(cacheDir);
|
|
779
|
+
});
|
|
780
|
+
if (!signal) return operation;
|
|
781
|
+
throwIfAborted(signal);
|
|
782
|
+
let rejectAbort: ((reason: unknown) => void) | undefined;
|
|
783
|
+
const aborted = new Promise<never>((_resolve, reject) => {
|
|
784
|
+
rejectAbort = reject;
|
|
785
|
+
});
|
|
786
|
+
const onAbort = () => rejectAbort?.(abortReason(signal));
|
|
787
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
788
|
+
if (signal.aborted) onAbort();
|
|
789
|
+
try {
|
|
790
|
+
return await Promise.race([operation, aborted]);
|
|
791
|
+
} finally {
|
|
792
|
+
signal.removeEventListener("abort", onAbort);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
async function assertNotSymlink(target: string, label: string) {
|
|
797
|
+
try {
|
|
798
|
+
const stat = await fs.lstat(target);
|
|
799
|
+
if (stat.isSymbolicLink()) throw new Error(`Refusing symlinked ${label}.`);
|
|
800
|
+
} catch (error) {
|
|
801
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function redactGitError(value: string, remote: string, cacheDir: string) {
|
|
806
|
+
return (
|
|
807
|
+
value
|
|
808
|
+
.replaceAll(remote, "<git-remote>")
|
|
809
|
+
.replaceAll(cacheDir, "<git-cache>")
|
|
810
|
+
.replace(/https:\/\/[^/@\s]+@/gu, "https://<credentials>@")
|
|
811
|
+
.replace(/\b(password|token|authorization)=\S+/giu, "$1=<redacted>")
|
|
812
|
+
.replace(/\bBearer\s+\S+/giu, "Bearer <redacted>")
|
|
813
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: sanitize untrusted process output.
|
|
814
|
+
.replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ")
|
|
815
|
+
.trim()
|
|
816
|
+
.slice(0, 4096)
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function sha256(value: Buffer) {
|
|
821
|
+
return createHash("sha256").update(value).digest("hex");
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
function abortReason(signal: AbortSignal) {
|
|
825
|
+
return signal.reason instanceof Error
|
|
826
|
+
? signal.reason
|
|
827
|
+
: new DOMException("The operation was aborted", "AbortError");
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
function throwIfAborted(signal?: AbortSignal) {
|
|
831
|
+
if (signal?.aborted) throw abortReason(signal);
|
|
832
|
+
}
|