@narumitw/pi-sync 0.25.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +238 -121
- package/package.json +7 -2
- package/src/command.ts +162 -24
- package/src/config.ts +395 -89
- package/src/file-selection.ts +280 -0
- package/src/index.ts +1 -0
- package/src/lock.ts +3 -3
- package/src/manager-ui.ts +961 -0
- package/src/s3-client.ts +47 -13
- package/src/settings-management.ts +307 -0
- package/src/snapshot-apply.ts +41 -17
- package/src/snapshot-transaction.ts +201 -0
- package/src/snapshot.ts +76 -95
- package/src/sync-format.ts +168 -0
- package/src/sync-policy.ts +129 -0
- package/src/sync-state.ts +167 -0
- package/src/sync.ts +357 -351
- package/src/types.ts +47 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { agentDir, stateDir } from "./config.js";
|
|
5
|
+
import { assertWithinRoot, isPathInside } from "./paths.js";
|
|
6
|
+
import { sessionStorageRoot } from "./snapshot.js";
|
|
7
|
+
import type { SnapshotApplyPlan } from "./types.js";
|
|
8
|
+
|
|
9
|
+
const JOURNAL_VERSION = 1;
|
|
10
|
+
|
|
11
|
+
interface TransactionEntry {
|
|
12
|
+
target: string;
|
|
13
|
+
backupName: string;
|
|
14
|
+
kind: "missing" | "file" | "directory" | "symlink";
|
|
15
|
+
linkTarget?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
interface TransactionJournal {
|
|
19
|
+
version: number;
|
|
20
|
+
root: string;
|
|
21
|
+
sessionRoot?: string;
|
|
22
|
+
entries: TransactionEntry[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function applySnapshotTransaction(
|
|
26
|
+
plan: SnapshotApplyPlan,
|
|
27
|
+
options: { sessionDir?: string } = {},
|
|
28
|
+
) {
|
|
29
|
+
await recoverPendingSnapshotTransactions();
|
|
30
|
+
const transaction = await prepareTransaction(plan, options.sessionDir);
|
|
31
|
+
try {
|
|
32
|
+
for (const target of plan.deletes) {
|
|
33
|
+
await fs.rm(target, { force: true, recursive: true });
|
|
34
|
+
}
|
|
35
|
+
for (const item of plan.writes) {
|
|
36
|
+
await fs.writeFile(item.target, item.content);
|
|
37
|
+
}
|
|
38
|
+
await fs.rm(transaction.directory, { recursive: true, force: true });
|
|
39
|
+
} catch (error) {
|
|
40
|
+
try {
|
|
41
|
+
await restoreTransaction(transaction.directory, transaction.journal);
|
|
42
|
+
} catch (recoveryError) {
|
|
43
|
+
throw new AggregateError(
|
|
44
|
+
[error, recoveryError],
|
|
45
|
+
`Snapshot apply failed and automatic recovery also failed. Transaction retained at ${transaction.directory}.`,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export async function recoverPendingSnapshotTransactions() {
|
|
53
|
+
const directory = transactionRoot();
|
|
54
|
+
let entries: import("node:fs").Dirent[];
|
|
55
|
+
try {
|
|
56
|
+
entries = await fs.readdir(directory, { withFileTypes: true });
|
|
57
|
+
} catch (error) {
|
|
58
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
62
|
+
if (!entry.isDirectory()) continue;
|
|
63
|
+
const transactionDirectory = path.join(directory, entry.name);
|
|
64
|
+
const journalPath = path.join(transactionDirectory, "journal.json");
|
|
65
|
+
let journal: TransactionJournal;
|
|
66
|
+
try {
|
|
67
|
+
journal = JSON.parse(await fs.readFile(journalPath, "utf8")) as TransactionJournal;
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
|
70
|
+
await fs.rm(transactionDirectory, { recursive: true, force: true });
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
throw new Error(`Cannot recover malformed pi-sync transaction: ${journalPath}`, {
|
|
74
|
+
cause: error,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
await restoreTransaction(transactionDirectory, journal);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function prepareTransaction(plan: SnapshotApplyPlan, sessionDir?: string) {
|
|
82
|
+
const root = path.resolve(agentDir());
|
|
83
|
+
const sessionRoot = sessionDir ? path.resolve(sessionStorageRoot(root, sessionDir)) : undefined;
|
|
84
|
+
const directory = path.join(transactionRoot(), randomUUID());
|
|
85
|
+
const backupDirectory = path.join(directory, "before");
|
|
86
|
+
await fs.mkdir(backupDirectory, { recursive: true, mode: 0o700 });
|
|
87
|
+
const targets = [...new Set([...plan.deletes, ...plan.writes.map((item) => item.target)])].sort();
|
|
88
|
+
const entries: TransactionEntry[] = [];
|
|
89
|
+
for (let index = 0; index < targets.length; index += 1) {
|
|
90
|
+
const target = targets[index];
|
|
91
|
+
if (!target) continue;
|
|
92
|
+
assertAllowedTarget(root, sessionRoot, target);
|
|
93
|
+
const backupName = `${index}`;
|
|
94
|
+
const backupPath = path.join(backupDirectory, backupName);
|
|
95
|
+
try {
|
|
96
|
+
const stat = await fs.lstat(target);
|
|
97
|
+
if (stat.isSymbolicLink()) {
|
|
98
|
+
entries.push({
|
|
99
|
+
target,
|
|
100
|
+
backupName,
|
|
101
|
+
kind: "symlink",
|
|
102
|
+
linkTarget: await fs.readlink(target),
|
|
103
|
+
});
|
|
104
|
+
} else if (stat.isDirectory()) {
|
|
105
|
+
await fs.cp(target, backupPath, {
|
|
106
|
+
recursive: true,
|
|
107
|
+
dereference: false,
|
|
108
|
+
preserveTimestamps: true,
|
|
109
|
+
});
|
|
110
|
+
entries.push({ target, backupName, kind: "directory" });
|
|
111
|
+
} else if (stat.isFile()) {
|
|
112
|
+
await fs.copyFile(target, backupPath);
|
|
113
|
+
await fs.chmod(backupPath, stat.mode);
|
|
114
|
+
entries.push({ target, backupName, kind: "file" });
|
|
115
|
+
} else {
|
|
116
|
+
throw new Error(`Unsupported existing snapshot target: ${target}`);
|
|
117
|
+
}
|
|
118
|
+
} catch (error) {
|
|
119
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
120
|
+
entries.push({ target, backupName, kind: "missing" });
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
const journal: TransactionJournal = {
|
|
124
|
+
version: JOURNAL_VERSION,
|
|
125
|
+
root,
|
|
126
|
+
sessionRoot,
|
|
127
|
+
entries,
|
|
128
|
+
};
|
|
129
|
+
await fs.writeFile(
|
|
130
|
+
path.join(directory, "journal.json"),
|
|
131
|
+
`${JSON.stringify(journal, null, "\t")}\n`,
|
|
132
|
+
{
|
|
133
|
+
mode: 0o600,
|
|
134
|
+
},
|
|
135
|
+
);
|
|
136
|
+
return { directory, journal };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function restoreTransaction(directory: string, journal: TransactionJournal) {
|
|
140
|
+
validateJournal(directory, journal);
|
|
141
|
+
for (const entry of [...journal.entries].sort(
|
|
142
|
+
(left, right) => right.target.length - left.target.length,
|
|
143
|
+
)) {
|
|
144
|
+
await fs.rm(entry.target, { recursive: true, force: true });
|
|
145
|
+
}
|
|
146
|
+
for (const entry of journal.entries) {
|
|
147
|
+
if (entry.kind === "missing") continue;
|
|
148
|
+
await fs.mkdir(path.dirname(entry.target), { recursive: true });
|
|
149
|
+
const backupPath = path.join(directory, "before", entry.backupName);
|
|
150
|
+
assertWithinRoot(directory, backupPath);
|
|
151
|
+
if (entry.kind === "file") await fs.copyFile(backupPath, entry.target);
|
|
152
|
+
else if (entry.kind === "directory") {
|
|
153
|
+
await fs.cp(backupPath, entry.target, {
|
|
154
|
+
recursive: true,
|
|
155
|
+
dereference: false,
|
|
156
|
+
preserveTimestamps: true,
|
|
157
|
+
});
|
|
158
|
+
} else if (entry.kind === "symlink" && entry.linkTarget !== undefined) {
|
|
159
|
+
await fs.symlink(entry.linkTarget, entry.target);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
await fs.rm(directory, { recursive: true, force: true });
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function validateJournal(directory: string, journal: TransactionJournal) {
|
|
166
|
+
if (journal.version !== JOURNAL_VERSION || !Array.isArray(journal.entries)) {
|
|
167
|
+
throw new Error(`Unsupported pi-sync transaction journal: ${directory}`);
|
|
168
|
+
}
|
|
169
|
+
const expectedRoot = path.resolve(agentDir());
|
|
170
|
+
if (path.resolve(journal.root) !== expectedRoot) {
|
|
171
|
+
throw new Error(`Transaction root no longer matches the Pi agent directory: ${directory}`);
|
|
172
|
+
}
|
|
173
|
+
for (const entry of journal.entries) {
|
|
174
|
+
if (
|
|
175
|
+
!entry ||
|
|
176
|
+
typeof entry.target !== "string" ||
|
|
177
|
+
typeof entry.backupName !== "string" ||
|
|
178
|
+
!/^\d+$/u.test(entry.backupName)
|
|
179
|
+
) {
|
|
180
|
+
throw new Error(`Invalid pi-sync transaction entry: ${directory}`);
|
|
181
|
+
}
|
|
182
|
+
assertAllowedTarget(journal.root, journal.sessionRoot, entry.target);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function assertAllowedTarget(root: string, sessionRoot: string | undefined, target: string) {
|
|
187
|
+
const resolved = path.resolve(target);
|
|
188
|
+
if (isPathInside(root, resolved)) {
|
|
189
|
+
assertWithinRoot(root, resolved);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
if (sessionRoot && isPathInside(sessionRoot, resolved)) {
|
|
193
|
+
assertWithinRoot(sessionRoot, resolved);
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
throw new Error(`Transaction target is outside configured roots: ${target}`);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function transactionRoot() {
|
|
200
|
+
return path.join(stateDir(), "transactions");
|
|
201
|
+
}
|
package/src/snapshot.ts
CHANGED
|
@@ -5,20 +5,25 @@ import os from "node:os";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { agentDir, configuredSessionDir } from "./config.js";
|
|
7
7
|
import { isDeniedPath, posixJoin, safeJoin, toPosix } from "./paths.js";
|
|
8
|
+
|
|
8
9
|
export { isDeniedPath } from "./paths.js";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
canonicalSnapshotPathForConfig,
|
|
13
|
+
DEFAULT_SYNC_FILES,
|
|
14
|
+
extraFilePathsByLower,
|
|
15
|
+
isConfiguredSnapshotPath,
|
|
16
|
+
isPreservableUnmanagedSnapshotPath,
|
|
17
|
+
normalizeExtraFiles,
|
|
18
|
+
normalizeSyncFiles,
|
|
19
|
+
} from "./sync-policy.js";
|
|
20
|
+
import type { Snapshot, SnapshotFile, SnapshotOptions, SyncConfig } from "./types.js";
|
|
21
|
+
|
|
22
|
+
export { canonicalSnapshotPathForConfig, isConfiguredSnapshotPath } from "./sync-policy.js";
|
|
15
23
|
|
|
16
24
|
const VERSION = 1;
|
|
17
|
-
const TOP_LEVEL_FILES
|
|
18
|
-
const TOP_LEVEL_DIRS = new Set(
|
|
19
|
-
const TOP_LEVEL_FILE_PATHS = new Map([...TOP_LEVEL_FILES].map((name) => [name.toLowerCase(), name]));
|
|
20
|
-
const TOP_LEVEL_FILE_NAMES = new Set(TOP_LEVEL_FILE_PATHS.keys());
|
|
21
|
-
const RESERVED_TOP_LEVEL_NAMES = new Set([...TOP_LEVEL_DIRS, "sessions"]);
|
|
25
|
+
const TOP_LEVEL_FILES: readonly string[] = DEFAULT_SYNC_FILES.filter((name) => name.includes("."));
|
|
26
|
+
const TOP_LEVEL_DIRS = new Set<string>(DEFAULT_SYNC_FILES.filter((name) => !name.includes(".")));
|
|
22
27
|
const SECRET_PATTERNS = [
|
|
23
28
|
/AWS_SECRET_ACCESS_KEY\s*[=:]\s*['"]?[A-Za-z0-9/+]{35,}/i,
|
|
24
29
|
/(ANTHROPIC|OPENAI|GEMINI|GOOGLE|FIRECRAWL|GITHUB|CLOUDFLARE|R2|S3)_[A-Z0-9_]*(KEY|TOKEN|SECRET)\s*[=:]\s*['"]?[^\s'"]{12,}/i,
|
|
@@ -33,27 +38,6 @@ function expandHome(value: string) {
|
|
|
33
38
|
return value;
|
|
34
39
|
}
|
|
35
40
|
|
|
36
|
-
export function normalizeExtraFiles(value: unknown) {
|
|
37
|
-
if (!Array.isArray(value)) return [];
|
|
38
|
-
const seen = new Set<string>();
|
|
39
|
-
return value
|
|
40
|
-
.filter((item): item is string => typeof item === "string")
|
|
41
|
-
.map((item) => item.trim())
|
|
42
|
-
.filter((item) => {
|
|
43
|
-
const lower = item.toLowerCase();
|
|
44
|
-
if (
|
|
45
|
-
item === "" || item === "." || item === ".." || item.includes("/") || item.includes("\\") ||
|
|
46
|
-
TOP_LEVEL_FILE_NAMES.has(lower) || isDeniedPath(item) || RESERVED_TOP_LEVEL_NAMES.has(lower) || seen.has(lower)
|
|
47
|
-
) return false;
|
|
48
|
-
seen.add(lower);
|
|
49
|
-
return true;
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export function extraFilePathsByLower(value: unknown) {
|
|
54
|
-
return new Map(normalizeExtraFiles(value).map((fileName) => [fileName.toLowerCase(), fileName]));
|
|
55
|
-
}
|
|
56
|
-
|
|
57
41
|
function selectTopLevelFileEntry(entries: Dirent[], fileName: string) {
|
|
58
42
|
const exact = entries.find((entry) => entry.isFile() && entry.name === fileName);
|
|
59
43
|
if (exact) return exact;
|
|
@@ -70,15 +54,24 @@ function sha256(value: Buffer) {
|
|
|
70
54
|
function isSafeSnapshotPath(relativePath: string) {
|
|
71
55
|
if (relativePath.includes("\\")) return false;
|
|
72
56
|
const normalized = toPosix(relativePath);
|
|
73
|
-
return
|
|
74
|
-
|
|
57
|
+
return (
|
|
58
|
+
Boolean(normalized) &&
|
|
59
|
+
normalized !== "." &&
|
|
60
|
+
normalized !== ".." &&
|
|
61
|
+
!normalized.startsWith("../") &&
|
|
62
|
+
!path.posix.isAbsolute(normalized) &&
|
|
63
|
+
path.posix.normalize(normalized) === normalized &&
|
|
64
|
+
!isDeniedPath(normalized)
|
|
65
|
+
);
|
|
75
66
|
}
|
|
76
67
|
|
|
77
68
|
function snapshotsMatch(left: Snapshot, right: Snapshot) {
|
|
78
69
|
const leftHashes = new Map(left.files.map((file) => [file.path, file.sha256]));
|
|
79
70
|
const rightHashes = new Map(right.files.map((file) => [file.path, file.sha256]));
|
|
80
|
-
return
|
|
81
|
-
|
|
71
|
+
return (
|
|
72
|
+
leftHashes.size === rightHashes.size &&
|
|
73
|
+
[...leftHashes].every(([filePath, hash]) => rightHashes.get(filePath) === hash)
|
|
74
|
+
);
|
|
82
75
|
}
|
|
83
76
|
|
|
84
77
|
function snapshotId() {
|
|
@@ -91,6 +84,7 @@ export async function createSnapshot(
|
|
|
91
84
|
): Promise<Snapshot> {
|
|
92
85
|
const syncSessions = Boolean(options.syncSessions);
|
|
93
86
|
const files = await collectFiles(agentDir(), {
|
|
87
|
+
syncFiles: options.syncFiles,
|
|
94
88
|
syncSessions,
|
|
95
89
|
sessionDir: options.sessionDir ?? (await configuredSessionDir()),
|
|
96
90
|
extraFiles: options.extraFiles,
|
|
@@ -112,12 +106,14 @@ export async function collectFiles(
|
|
|
112
106
|
): Promise<SnapshotFile[]> {
|
|
113
107
|
const results: SnapshotFile[] = [];
|
|
114
108
|
const entries = await fs.readdir(root, { withFileTypes: true });
|
|
109
|
+
const selectedFiles = new Set<string>(normalizeSyncFiles(options.syncFiles));
|
|
115
110
|
for (const entry of entries) {
|
|
116
|
-
if (entry.isDirectory() && TOP_LEVEL_DIRS.has(entry.name)) {
|
|
111
|
+
if (entry.isDirectory() && TOP_LEVEL_DIRS.has(entry.name) && selectedFiles.has(entry.name)) {
|
|
117
112
|
await collectDirectory(results, root, entry.name);
|
|
118
113
|
}
|
|
119
114
|
}
|
|
120
115
|
for (const fileName of TOP_LEVEL_FILES) {
|
|
116
|
+
if (!selectedFiles.has(fileName)) continue;
|
|
121
117
|
const entry = selectTopLevelFileEntry(entries, fileName);
|
|
122
118
|
if (entry) await addFile(results, root, entry.name, fileName);
|
|
123
119
|
}
|
|
@@ -168,7 +164,11 @@ async function addFile(
|
|
|
168
164
|
if (!isSafeSnapshotPath(snapshotPath)) return;
|
|
169
165
|
const absolutePath = safeJoin(root, relativePath);
|
|
170
166
|
const content = await fs.readFile(absolutePath);
|
|
171
|
-
results.push({
|
|
167
|
+
results.push({
|
|
168
|
+
path: snapshotPath,
|
|
169
|
+
contentBase64: content.toString("base64"),
|
|
170
|
+
sha256: sha256(content),
|
|
171
|
+
});
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
export function isSessionPath(relativePath: string) {
|
|
@@ -181,11 +181,18 @@ export function isSessionFilePath(relativePath: string) {
|
|
|
181
181
|
}
|
|
182
182
|
|
|
183
183
|
export function sessionStorageRoot(root: string, configuredSessionDir?: string) {
|
|
184
|
-
return configuredSessionDir
|
|
184
|
+
return configuredSessionDir
|
|
185
|
+
? path.resolve(expandHome(configuredSessionDir))
|
|
186
|
+
: path.resolve(root, "sessions");
|
|
185
187
|
}
|
|
186
188
|
|
|
187
|
-
export function sessionSnapshotPathFromAbsolute(
|
|
188
|
-
|
|
189
|
+
export function sessionSnapshotPathFromAbsolute(
|
|
190
|
+
sessionFile: string,
|
|
191
|
+
configuredSessionDir?: string,
|
|
192
|
+
) {
|
|
193
|
+
const relativePath = toPosix(
|
|
194
|
+
path.relative(sessionStorageRoot(agentDir(), configuredSessionDir), sessionFile),
|
|
195
|
+
);
|
|
189
196
|
if (!relativePath || relativePath.startsWith("../") || path.posix.isAbsolute(relativePath)) {
|
|
190
197
|
return undefined;
|
|
191
198
|
}
|
|
@@ -195,23 +202,21 @@ export function sessionSnapshotPathFromAbsolute(sessionFile: string, configuredS
|
|
|
195
202
|
|
|
196
203
|
export function snapshotTarget(root: string, relativePath: string, configuredSessionDir?: string) {
|
|
197
204
|
if (isSessionPath(relativePath)) {
|
|
198
|
-
return safeJoin(
|
|
205
|
+
return safeJoin(
|
|
206
|
+
sessionStorageRoot(root, configuredSessionDir),
|
|
207
|
+
relativePath.slice("sessions/".length),
|
|
208
|
+
);
|
|
199
209
|
}
|
|
200
210
|
return safeJoin(root, relativePath);
|
|
201
211
|
}
|
|
202
212
|
|
|
203
|
-
function isPathInside(parent: string, child: string) {
|
|
204
|
-
const relativePath = path.relative(parent, child);
|
|
205
|
-
return relativePath === "" || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
|
|
206
|
-
}
|
|
207
|
-
|
|
208
213
|
export function snapshotIncludesSessions(snapshot: Snapshot) {
|
|
209
214
|
return snapshot.syncSessions === true || snapshot.files.some((file) => isSessionPath(file.path));
|
|
210
215
|
}
|
|
211
216
|
|
|
212
217
|
export function filterSnapshotForConfigPolicy(
|
|
213
218
|
snapshot: Snapshot,
|
|
214
|
-
config: Pick<SyncConfig, "syncSessions" | "extraFiles">,
|
|
219
|
+
config: Pick<SyncConfig, "syncFiles" | "syncSessions" | "extraFiles">,
|
|
215
220
|
options: { regenerateId?: boolean } = {},
|
|
216
221
|
) {
|
|
217
222
|
const extraFilePaths = extraFilePathsByLower(config.extraFiles);
|
|
@@ -230,23 +235,9 @@ export function filterSnapshotForConfigPolicy(
|
|
|
230
235
|
};
|
|
231
236
|
}
|
|
232
237
|
|
|
233
|
-
export function isConfiguredSnapshotPath(
|
|
234
|
-
relativePath: string,
|
|
235
|
-
config: Pick<SyncConfig, "syncSessions">,
|
|
236
|
-
extraFiles: Set<string>,
|
|
237
|
-
) {
|
|
238
|
-
const normalized = toPosix(relativePath);
|
|
239
|
-
if (isSessionPath(normalized)) return config.syncSessions;
|
|
240
|
-
if (!normalized.includes("/")) {
|
|
241
|
-
const lower = normalized.toLowerCase();
|
|
242
|
-
return TOP_LEVEL_FILE_NAMES.has(lower) || extraFiles.has(lower);
|
|
243
|
-
}
|
|
244
|
-
return TOP_LEVEL_DIRS.has(normalized.slice(0, normalized.indexOf("/")));
|
|
245
|
-
}
|
|
246
|
-
|
|
247
238
|
function canonicalizeSnapshotFilesForConfig(
|
|
248
239
|
files: SnapshotFile[],
|
|
249
|
-
config: Pick<SyncConfig, "syncSessions">,
|
|
240
|
+
config: Pick<SyncConfig, "syncFiles" | "syncSessions">,
|
|
250
241
|
extraFiles: Set<string>,
|
|
251
242
|
extraFilePaths: Map<string, string>,
|
|
252
243
|
) {
|
|
@@ -257,14 +248,17 @@ function canonicalizeSnapshotFilesForConfig(
|
|
|
257
248
|
>();
|
|
258
249
|
for (const file of files) {
|
|
259
250
|
const normalized = toPosix(file.path);
|
|
260
|
-
if (
|
|
251
|
+
if (
|
|
252
|
+
!isSafeSnapshotPath(file.path) ||
|
|
253
|
+
!isConfiguredSnapshotPath(normalized, config, extraFiles)
|
|
254
|
+
) {
|
|
261
255
|
continue;
|
|
262
256
|
}
|
|
263
|
-
|
|
264
|
-
if (!topLevelPath) {
|
|
257
|
+
if (normalized.includes("/")) {
|
|
265
258
|
configuredFiles.push(normalized === file.path ? file : { ...file, path: normalized });
|
|
266
259
|
continue;
|
|
267
260
|
}
|
|
261
|
+
const topLevelPath = canonicalSnapshotPathForConfig(normalized, extraFilePaths);
|
|
268
262
|
const candidate = {
|
|
269
263
|
exact: normalized === topLevelPath,
|
|
270
264
|
file: { ...file, path: topLevelPath },
|
|
@@ -281,17 +275,6 @@ function canonicalizeSnapshotFilesForConfig(
|
|
|
281
275
|
].sort((left, right) => left.path.localeCompare(right.path));
|
|
282
276
|
}
|
|
283
277
|
|
|
284
|
-
function canonicalTopLevelFilePath(relativePath: string, extraFilePaths: Map<string, string>) {
|
|
285
|
-
const normalized = toPosix(relativePath);
|
|
286
|
-
if (normalized.includes("/")) return undefined;
|
|
287
|
-
const lower = normalized.toLowerCase();
|
|
288
|
-
return TOP_LEVEL_FILE_PATHS.get(lower) ?? extraFilePaths.get(lower);
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
export function canonicalSnapshotPathForConfig(relativePath: string, extraFilePaths: Map<string, string>) {
|
|
292
|
-
return canonicalTopLevelFilePath(relativePath, extraFilePaths) ?? toPosix(relativePath);
|
|
293
|
-
}
|
|
294
|
-
|
|
295
278
|
function isPreferredExtraCandidate(
|
|
296
279
|
left: { exact: boolean; originalPath: string },
|
|
297
280
|
right: { exact: boolean; originalPath: string },
|
|
@@ -332,38 +315,36 @@ export function scanSnapshot(snapshot: Snapshot) {
|
|
|
332
315
|
export function mergeRemotePreservedFiles(
|
|
333
316
|
local: Snapshot,
|
|
334
317
|
remote: Snapshot,
|
|
335
|
-
config: Pick<SyncConfig, "syncSessions" | "extraFiles">,
|
|
318
|
+
config: Pick<SyncConfig, "syncFiles" | "syncSessions" | "extraFiles">,
|
|
336
319
|
) {
|
|
337
|
-
const
|
|
338
|
-
const
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
const
|
|
342
|
-
const
|
|
320
|
+
const localPathNames = new Set(local.files.map((file) => file.path.toLowerCase()));
|
|
321
|
+
const configuredExtraFiles = new Set(
|
|
322
|
+
normalizeExtraFiles(config.extraFiles).map((file) => file.toLowerCase()),
|
|
323
|
+
);
|
|
324
|
+
const preservedPathNames = new Set<string>();
|
|
325
|
+
const preserved = remote.files.filter((file) => {
|
|
343
326
|
const normalized = toPosix(file.path);
|
|
344
327
|
const lower = normalized.toLowerCase();
|
|
345
328
|
if (
|
|
346
|
-
|
|
347
|
-
|
|
329
|
+
localPathNames.has(lower) ||
|
|
330
|
+
preservedPathNames.has(lower) ||
|
|
348
331
|
!isSafeSnapshotPath(file.path) ||
|
|
349
|
-
normalized
|
|
350
|
-
|
|
351
|
-
RESERVED_TOP_LEVEL_NAMES.has(lower) ||
|
|
352
|
-
extraFileNames.has(lower) ||
|
|
353
|
-
seenRemoteExtraNames.has(lower)
|
|
332
|
+
isConfiguredSnapshotPath(normalized, config, configuredExtraFiles) ||
|
|
333
|
+
!isPreservableUnmanagedSnapshotPath(normalized)
|
|
354
334
|
) {
|
|
355
335
|
return false;
|
|
356
336
|
}
|
|
357
|
-
|
|
337
|
+
preservedPathNames.add(lower);
|
|
358
338
|
return true;
|
|
359
339
|
});
|
|
360
|
-
if (
|
|
340
|
+
if (preserved.length === 0) return local;
|
|
361
341
|
return {
|
|
362
|
-
...
|
|
342
|
+
...local,
|
|
363
343
|
id: snapshotId(),
|
|
364
344
|
createdAt: new Date().toISOString(),
|
|
365
345
|
machine: os.hostname(),
|
|
366
|
-
|
|
346
|
+
syncSessions: snapshotIncludesSessions(local) || snapshotIncludesSessions(remote),
|
|
347
|
+
files: [...local.files, ...preserved].sort((left, right) =>
|
|
367
348
|
left.path.localeCompare(right.path),
|
|
368
349
|
),
|
|
369
350
|
};
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { agentDir } from "./config.js";
|
|
3
|
+
import { fileHashMap } from "./sync-state.js";
|
|
4
|
+
import type { LatestPointer, RemoteObject, Snapshot, SyncConfig } from "./types.js";
|
|
5
|
+
|
|
6
|
+
export function formatDiff(local: Snapshot, remote: Snapshot) {
|
|
7
|
+
const localMap = fileHashMap(local);
|
|
8
|
+
const remoteMap = fileHashMap(remote);
|
|
9
|
+
const allPaths = [...new Set([...Object.keys(localMap), ...Object.keys(remoteMap)])].sort();
|
|
10
|
+
const lines = [
|
|
11
|
+
`local: ${local.files.length} files`,
|
|
12
|
+
`remote: ${remote.id} (${remote.files.length} files)`,
|
|
13
|
+
"",
|
|
14
|
+
];
|
|
15
|
+
let changed = 0;
|
|
16
|
+
for (const filePath of allPaths) {
|
|
17
|
+
if (!localMap[filePath]) {
|
|
18
|
+
lines.push(`Remote only: ${filePath}`);
|
|
19
|
+
changed += 1;
|
|
20
|
+
} else if (!remoteMap[filePath]) {
|
|
21
|
+
lines.push(`Local only: ${filePath}`);
|
|
22
|
+
changed += 1;
|
|
23
|
+
} else if (localMap[filePath] !== remoteMap[filePath]) {
|
|
24
|
+
lines.push(`Different: ${filePath}`);
|
|
25
|
+
changed += 1;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (changed === 0) lines.push("No file differences.");
|
|
29
|
+
return lines.join("\n");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function formatSnapshotOnlyDiff(title: string, snapshot: Snapshot) {
|
|
33
|
+
return [`${title}: ${snapshot.id}`, ...snapshot.files.map((file) => `Add: ${file.path}`)].join(
|
|
34
|
+
"\n",
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function formatPushSummary(
|
|
39
|
+
config: SyncConfig,
|
|
40
|
+
upload: Snapshot,
|
|
41
|
+
latest: RemoteObject<LatestPointer>,
|
|
42
|
+
preservedRemoteFileCount = 0,
|
|
43
|
+
remote?: Snapshot,
|
|
44
|
+
) {
|
|
45
|
+
return [
|
|
46
|
+
`Target: ${safeTerminalText(config.target ?? "default")}`,
|
|
47
|
+
`Destination: ${formatDestination(config)}`,
|
|
48
|
+
`Upload ${upload.files.length} files from ${safeTerminalText(agentDir())}.`,
|
|
49
|
+
`Sessions: ${upload.syncSessions ? "included — may contain private conversations" : "not included"}`,
|
|
50
|
+
latest.value ? `Remote latest: ${latest.value.snapshot}` : "Remote latest: empty",
|
|
51
|
+
"Publication effect: latest.json will point to the new immutable snapshot.",
|
|
52
|
+
formatPublicationPreview(remote, upload),
|
|
53
|
+
preservedRemoteFileCount > 0
|
|
54
|
+
? `Possible secrets in locally managed files were scanned before this prompt; ${preservedRemoteFileCount} preserved remote file(s) were not rescanned.`
|
|
55
|
+
: "Possible secrets were scanned before this prompt.",
|
|
56
|
+
].join("\n");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function formatApplyPreview(local: Snapshot, remote: Snapshot) {
|
|
60
|
+
return formatDirectionalChanges(local, remote, {
|
|
61
|
+
add: "Add locally",
|
|
62
|
+
update: "Update locally",
|
|
63
|
+
remove: "Remove locally",
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function formatPullSummary(
|
|
68
|
+
config: SyncConfig,
|
|
69
|
+
local: Snapshot,
|
|
70
|
+
remote: Snapshot,
|
|
71
|
+
protectedSessionCount: number,
|
|
72
|
+
) {
|
|
73
|
+
return [
|
|
74
|
+
`Target: ${safeTerminalText(config.target ?? "default")}`,
|
|
75
|
+
`Destination: ${formatDestination(config)}`,
|
|
76
|
+
`Snapshot: ${safeTerminalText(remote.id)}`,
|
|
77
|
+
`Sessions: ${remote.syncSessions ? "included — may contain private conversations" : "not included"}`,
|
|
78
|
+
`Protected live sessions: ${protectedSessionCount || "none"}`,
|
|
79
|
+
formatApplyPreview(local, remote),
|
|
80
|
+
"A local backup is created before these writes/deletes. The remote active snapshot is unchanged.",
|
|
81
|
+
].join("\n");
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function formatRollbackSummary(
|
|
85
|
+
config: SyncConfig,
|
|
86
|
+
local: Snapshot,
|
|
87
|
+
remote: Snapshot,
|
|
88
|
+
requestedSnapshot: string,
|
|
89
|
+
protectedSessionCount: number,
|
|
90
|
+
) {
|
|
91
|
+
return [
|
|
92
|
+
`Target: ${safeTerminalText(config.target ?? "default")}`,
|
|
93
|
+
`Destination: ${formatDestination(config)}`,
|
|
94
|
+
`Snapshot: ${safeTerminalText(requestedSnapshot)}`,
|
|
95
|
+
`Sessions: ${remote.syncSessions ? "included — may contain private conversations" : "not included"}`,
|
|
96
|
+
`Protected live sessions: ${protectedSessionCount || "none"}`,
|
|
97
|
+
formatApplyPreview(local, remote),
|
|
98
|
+
"A local backup is created before applying; the remote latest pointer will change.",
|
|
99
|
+
].join("\n");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function formatPublicationPreview(remote: Snapshot | undefined, upload: Snapshot) {
|
|
103
|
+
if (!remote) {
|
|
104
|
+
return ["Remote is empty.", ...upload.files.map((file) => `Add remotely: ${file.path}`)].join(
|
|
105
|
+
"\n",
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
return formatDirectionalChanges(remote, upload, {
|
|
109
|
+
add: "Add remotely",
|
|
110
|
+
update: "Update remotely",
|
|
111
|
+
remove: "Remove remotely",
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function formatDirectionalChanges(
|
|
116
|
+
before: Snapshot,
|
|
117
|
+
after: Snapshot,
|
|
118
|
+
labels: { add: string; update: string; remove: string },
|
|
119
|
+
) {
|
|
120
|
+
const beforeMap = fileHashMap(before);
|
|
121
|
+
const afterMap = fileHashMap(after);
|
|
122
|
+
const paths = [...new Set([...Object.keys(beforeMap), ...Object.keys(afterMap)])].sort();
|
|
123
|
+
const lines: string[] = [];
|
|
124
|
+
for (const filePath of paths) {
|
|
125
|
+
if (!beforeMap[filePath]) lines.push(`${labels.add}: ${filePath}`);
|
|
126
|
+
else if (!afterMap[filePath]) lines.push(`${labels.remove}: ${filePath}`);
|
|
127
|
+
else if (beforeMap[filePath] !== afterMap[filePath])
|
|
128
|
+
lines.push(`${labels.update}: ${filePath}`);
|
|
129
|
+
}
|
|
130
|
+
if (lines.length === 0) lines.push("No file changes.");
|
|
131
|
+
return lines.join("\n");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function formatDestination(config: SyncConfig) {
|
|
135
|
+
let host = config.endpoint;
|
|
136
|
+
try {
|
|
137
|
+
host = new URL(config.endpoint).hostname;
|
|
138
|
+
} catch {
|
|
139
|
+
// Preserve a sanitized invalid endpoint for actionable repair output.
|
|
140
|
+
}
|
|
141
|
+
return safeTerminalText(`${host} · ${config.bucket}/${config.prefix}/profiles/${config.profile}`);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export function countPreservedRemoteFiles(local: Snapshot, upload: Snapshot) {
|
|
145
|
+
const localPaths = new Set(local.files.map((file) => file.path));
|
|
146
|
+
return upload.files.filter((file) => !localPaths.has(file.path)).length;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function safeTerminalText(value: string) {
|
|
150
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: Escape untrusted terminal controls.
|
|
151
|
+
return value.replace(/[\u0000-\u001f\u007f-\u009f]/gu, "?");
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export function redact(value: string) {
|
|
155
|
+
return value.length <= 8 ? "configured" : `${value.slice(0, 4)}…${value.slice(-4)}`;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export function remoteIdentity(remote: RemoteObject<LatestPointer>) {
|
|
159
|
+
return remote.missing ? "missing" : (remote.value?.snapshot ?? "unknown");
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function sha256(value: Buffer) {
|
|
163
|
+
return createHash("sha256").update(value).digest("hex");
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function errorMessage(error: unknown) {
|
|
167
|
+
return safeTerminalText(error instanceof Error ? error.message : String(error));
|
|
168
|
+
}
|