@ericsanchezok/synergy-presets 0.0.0-stage → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +30 -0
- package/README.md +74 -2
- package/dist/modules/access/index.js +21 -0
- package/dist/modules/cli/data/index.js +31 -0
- package/dist/modules/cli/data/merge.js +251 -0
- package/dist/modules/cli/data/move.js +300 -0
- package/dist/modules/cli/data/pack.js +130 -0
- package/dist/modules/cli/data/transfer.js +251 -0
- package/dist/modules/cli/data/workspace-transfer.js +215 -0
- package/dist/modules/cli/generate.js +1 -0
- package/dist/modules/cli/server.js +3 -0
- package/dist/modules/cli/web.js +1 -0
- package/dist/modules/cli-adapter.js +10 -0
- package/dist/modules/cli-commands.js +8 -0
- package/dist/modules/component.js +19 -0
- package/dist/modules/components.js +62 -0
- package/dist/modules/configuration.js +38 -0
- package/dist/modules/daemon-entry.js +19 -0
- package/dist/modules/features.js +4 -0
- package/dist/modules/index.js +39 -0
- package/dist/modules/registration.js +15 -0
- package/dist/modules/server/routes.js +13 -0
- package/dist/modules/server/runtime-handle.js +42 -0
- package/dist/modules/server/runtime.js +22 -0
- package/dist/modules/server/web-app.js +7 -0
- package/package.json +188 -4
- package/src/access/index.ts +23 -0
- package/src/cli/data/index.ts +34 -0
- package/src/cli/data/merge.ts +334 -0
- package/src/cli/data/move.ts +369 -0
- package/src/cli/data/pack.ts +153 -0
- package/src/cli/data/transfer.ts +273 -0
- package/src/cli/data/workspace-transfer.ts +234 -0
- package/src/cli/generate.ts +1 -0
- package/src/cli/server.ts +4 -0
- package/src/cli/web.ts +1 -0
- package/src/cli-adapter.ts +14 -0
- package/src/cli-commands.ts +9 -0
- package/src/component.ts +21 -0
- package/src/components.ts +65 -0
- package/src/configuration.ts +38 -0
- package/src/daemon-entry.ts +21 -0
- package/src/features.ts +4 -0
- package/src/index.ts +45 -0
- package/src/registration.ts +17 -0
- package/src/server/routes.ts +12 -0
- package/src/server/runtime-handle.ts +46 -0
- package/src/server/runtime.ts +30 -0
- package/src/server/web-app.ts +8 -0
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import { logo } from "@ericsanchezok/synergy-cli/branding";
|
|
2
|
+
import { getLibraryInfo, resolveLibraryDB, mergeLibraryDB, } from "@ericsanchezok/synergy-library/cli/data";
|
|
3
|
+
import { DataTransfer } from "./transfer.js";
|
|
4
|
+
import fs from "fs/promises";
|
|
5
|
+
import path from "path";
|
|
6
|
+
import os from "os";
|
|
7
|
+
import * as prompts from "@clack/prompts";
|
|
8
|
+
import { cmd } from "@ericsanchezok/synergy-util/cli-command";
|
|
9
|
+
import { UI } from "@ericsanchezok/synergy-util/terminal";
|
|
10
|
+
import { archiveExclusions, CATEGORIES, scanCategories, formatSize, shortenPath, dirExists, checkDiskSpace, isDirEmpty, copyDirSkipExisting, dataRoot, } from "@ericsanchezok/synergy-cli/cli/cmd/data/shared";
|
|
11
|
+
export async function executeMove(opts) {
|
|
12
|
+
const sourceRoot = dataRoot();
|
|
13
|
+
if (!(await dirExists(sourceRoot))) {
|
|
14
|
+
prompts.log.error(`No synergy data found at ${shortenPath(sourceRoot)}`);
|
|
15
|
+
prompts.outro("Nothing to move");
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
const catStats = await scanCategories(sourceRoot);
|
|
19
|
+
let totalSize = 0;
|
|
20
|
+
let totalFiles = 0;
|
|
21
|
+
for (const stats of catStats.values()) {
|
|
22
|
+
totalSize += stats.size;
|
|
23
|
+
totalFiles += stats.fileCount;
|
|
24
|
+
}
|
|
25
|
+
prompts.log.info(`Source: ${shortenPath(sourceRoot)} (${formatSize(totalSize)}, ${totalFiles.toLocaleString()} files)`);
|
|
26
|
+
// Step 1: Target directory
|
|
27
|
+
// User provides a base path (SYNERGY_HOME). The actual data root is basePath/.synergy
|
|
28
|
+
// because root() = path.join(SYNERGY_HOME, ".synergy").
|
|
29
|
+
let homePath;
|
|
30
|
+
if (opts.target) {
|
|
31
|
+
homePath = path.resolve(opts.target);
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
const input = await prompts.text({
|
|
35
|
+
message: "Where should the data be moved to?",
|
|
36
|
+
placeholder: path.join(os.homedir(), ".local", "share", "synergy-data"),
|
|
37
|
+
validate: (v) => {
|
|
38
|
+
if (!v?.trim())
|
|
39
|
+
return "Please enter a path";
|
|
40
|
+
return undefined;
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
if (prompts.isCancel(input)) {
|
|
44
|
+
prompts.cancel("Cancelled");
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
homePath = path.resolve(input);
|
|
48
|
+
}
|
|
49
|
+
const targetPath = homePath.endsWith(".synergy") ? homePath : path.join(homePath, ".synergy");
|
|
50
|
+
if (targetPath === sourceRoot) {
|
|
51
|
+
prompts.log.error("Target is the same as current location");
|
|
52
|
+
prompts.outro("Nothing to move");
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
await DataTransfer.validateHomes([sourceRoot, targetPath]);
|
|
56
|
+
// Check disk space
|
|
57
|
+
const diskOk = await checkDiskSpace(homePath, totalSize);
|
|
58
|
+
const availStr = diskOk.available != null ? formatSize(diskOk.available) : "unknown";
|
|
59
|
+
if (!diskOk.ok) {
|
|
60
|
+
prompts.log.error(`Insufficient disk space at target (${availStr} available, ${formatSize(totalSize)} needed)`);
|
|
61
|
+
prompts.outro("Move aborted");
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
// Check if target has existing data
|
|
65
|
+
const targetExists = await dirExists(targetPath);
|
|
66
|
+
const targetEmpty = targetExists ? await isDirEmpty(targetPath).catch(() => true) : true;
|
|
67
|
+
if (targetExists && !targetEmpty) {
|
|
68
|
+
const targetStats = await scanCategories(targetPath);
|
|
69
|
+
let targetSize = 0;
|
|
70
|
+
for (const stats of targetStats.values())
|
|
71
|
+
targetSize += stats.size;
|
|
72
|
+
prompts.log.info(`Target: ${shortenPath(targetPath)} (${formatSize(targetSize)} existing)`);
|
|
73
|
+
prompts.log.warn("Target already contains data. Existing files will be kept — nothing will be overwritten.");
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
prompts.log.info(`Target: ${shortenPath(targetPath)} (${availStr} available)`);
|
|
77
|
+
}
|
|
78
|
+
// Step 2: Select categories
|
|
79
|
+
const selectable = CATEGORIES.filter((c) => !c.required);
|
|
80
|
+
const selected = await prompts.multiselect({
|
|
81
|
+
message: "What should be moved? (Space to toggle, Enter to confirm)",
|
|
82
|
+
options: selectable.map((cat) => ({
|
|
83
|
+
value: cat.key,
|
|
84
|
+
label: cat.label,
|
|
85
|
+
hint: formatSize(catStats.get(cat.key)?.size ?? 0),
|
|
86
|
+
})),
|
|
87
|
+
initialValues: selectable.filter((c) => c.defaultValue).map((c) => c.key),
|
|
88
|
+
required: false,
|
|
89
|
+
});
|
|
90
|
+
if (prompts.isCancel(selected)) {
|
|
91
|
+
prompts.cancel("Cancelled");
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const selectedKeys = new Set([...selected, ...CATEGORIES.filter((c) => c.required).map((c) => c.key)]);
|
|
95
|
+
const selectedCategories = CATEGORIES.filter((c) => selectedKeys.has(c.key));
|
|
96
|
+
const selectedSize = selectedCategories.reduce((sum, c) => sum + (catStats.get(c.key)?.size ?? 0), 0);
|
|
97
|
+
// Show plan
|
|
98
|
+
UI.empty();
|
|
99
|
+
prompts.log.message("Move plan:");
|
|
100
|
+
for (const cat of CATEGORIES) {
|
|
101
|
+
const included = selectedKeys.has(cat.key);
|
|
102
|
+
const icon = included ? "●" : "○";
|
|
103
|
+
const dim = included ? "" : UI.Style.TEXT_DIM;
|
|
104
|
+
for (const subdir of cat.subdirs) {
|
|
105
|
+
const subStats = await scanDir(path.join(sourceRoot, subdir)).catch(() => ({ size: 0 }));
|
|
106
|
+
prompts.log.info(` ${icon} ${dim}${subdir.padEnd(12)}${UI.Style.TEXT_NORMAL} → ${shortenPath(path.join(targetPath, subdir))} ${UI.Style.TEXT_DIM}(${formatSize(subStats.size)})`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
if (opts.dryRun) {
|
|
110
|
+
prompts.outro("Dry run — no changes made");
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
// Step 3: Confirm
|
|
114
|
+
const confirm = await prompts.confirm({
|
|
115
|
+
message: `Move ${formatSize(selectedSize)} to ${shortenPath(targetPath)}?`,
|
|
116
|
+
initialValue: true,
|
|
117
|
+
});
|
|
118
|
+
if (confirm !== true || prompts.isCancel(confirm)) {
|
|
119
|
+
prompts.cancel("Cancelled");
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
await using homes = await DataTransfer.lockHomes([sourceRoot, targetPath]);
|
|
123
|
+
// Step 5: Handle library.db if core is selected
|
|
124
|
+
let libraryStrategy = "skip";
|
|
125
|
+
const sourceLibrary = await resolveLibraryDB(sourceRoot);
|
|
126
|
+
const targetLibrary = path.join(targetPath, "data", "library.db");
|
|
127
|
+
if (selectedKeys.has("core") && (await dirExists(sourceLibrary))) {
|
|
128
|
+
const targetLibraryExists = await dirExists(targetLibrary);
|
|
129
|
+
if (targetLibraryExists) {
|
|
130
|
+
const srcInfo = await getLibraryInfo(sourceLibrary);
|
|
131
|
+
const tgtInfo = await getLibraryInfo(targetLibrary);
|
|
132
|
+
if (srcInfo.dimensions && tgtInfo.dimensions && srcInfo.dimensions !== tgtInfo.dimensions) {
|
|
133
|
+
prompts.log.warn("Vector dimension mismatch between source and target library:");
|
|
134
|
+
prompts.log.info(` Source: ${srcInfo.dimensions}d${srcInfo.embeddingModel ? ` (${srcInfo.embeddingModel})` : ""}`);
|
|
135
|
+
prompts.log.info(` Target: ${tgtInfo.dimensions}d${tgtInfo.embeddingModel ? ` (${tgtInfo.embeddingModel})` : ""}`);
|
|
136
|
+
const choice = await prompts.select({
|
|
137
|
+
message: "How should library data be handled?",
|
|
138
|
+
options: [
|
|
139
|
+
{
|
|
140
|
+
value: "text_only",
|
|
141
|
+
label: "Merge text only, discard source vectors",
|
|
142
|
+
hint: "Source memories added without vector search until re-embedded",
|
|
143
|
+
},
|
|
144
|
+
{ value: "skip", label: "Skip library entirely", hint: "Source memories are not imported" },
|
|
145
|
+
{
|
|
146
|
+
value: "replace_vectors",
|
|
147
|
+
label: "Replace: use source vectors, drop target vectors",
|
|
148
|
+
hint: "Your existing memories lose vector search",
|
|
149
|
+
},
|
|
150
|
+
],
|
|
151
|
+
});
|
|
152
|
+
if (prompts.isCancel(choice)) {
|
|
153
|
+
prompts.cancel("Cancelled");
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
libraryStrategy = choice;
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
libraryStrategy = "text_only"; // dimensions match or one side has no vec tables
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// If target has no library.db, the file copy will handle it
|
|
163
|
+
}
|
|
164
|
+
return DataTransfer.withFiles([sourceRoot, targetPath], async () => {
|
|
165
|
+
// Step 6: Execute move
|
|
166
|
+
UI.empty();
|
|
167
|
+
const errors = [];
|
|
168
|
+
let libraryMerged = false;
|
|
169
|
+
const copyOrder = selectedCategories.toSorted((a, b) => Number(a.subdirs.includes("data")) - Number(b.subdirs.includes("data")));
|
|
170
|
+
for (const cat of copyOrder) {
|
|
171
|
+
for (const subdir of cat.subdirs) {
|
|
172
|
+
const src = path.join(sourceRoot, subdir);
|
|
173
|
+
const dst = path.join(targetPath, subdir);
|
|
174
|
+
if (!(await dirExists(src)))
|
|
175
|
+
continue;
|
|
176
|
+
// Special handling for library.db inside data/
|
|
177
|
+
if (subdir === "data" && libraryStrategy !== "skip") {
|
|
178
|
+
const targetLibraryExists = await dirExists(targetLibrary);
|
|
179
|
+
if (targetLibraryExists && selectedKeys.has("core")) {
|
|
180
|
+
// Merge library via SQL, then copy the rest of data/ skipping library
|
|
181
|
+
const librarySpinner = prompts.spinner();
|
|
182
|
+
librarySpinner.start("Merging library.db...");
|
|
183
|
+
try {
|
|
184
|
+
const result = await mergeLibraryDB(sourceLibrary, targetLibrary, libraryStrategy);
|
|
185
|
+
librarySpinner.stop(`Merged library: ${result.memoriesMerged} memories, ${result.experiencesMerged} experiences${result.vecDropped ? " (vectors handled per strategy)" : ""}`);
|
|
186
|
+
libraryMerged = true;
|
|
187
|
+
}
|
|
188
|
+
catch (e) {
|
|
189
|
+
librarySpinner.error("Failed to merge library.db");
|
|
190
|
+
errors.push(`library.db: ${e instanceof Error ? e.message : String(e)}`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const catSize = catStats.get(cat.key)?.size ?? 0;
|
|
195
|
+
const spinner = prompts.spinner();
|
|
196
|
+
spinner.start(`Moving ${subdir}/ (${formatSize(catSize)})...`);
|
|
197
|
+
try {
|
|
198
|
+
const result = subdir === "data"
|
|
199
|
+
? await DataTransfer.merge(sourceRoot, targetPath, { trusted: true })
|
|
200
|
+
: await copyDirSkipExisting(src, dst, (p) => {
|
|
201
|
+
const pct = Math.round(((p.copied + p.skipped) / p.total) * 100);
|
|
202
|
+
spinner.message(`Moving ${subdir}/ ${pct}% — ${shortenPath(p.currentFile)}`);
|
|
203
|
+
}, undefined, undefined, archiveExclusions(subdir));
|
|
204
|
+
const skippedNote = result.skipped > 0 ? ` (${result.skipped} existing files kept)` : "";
|
|
205
|
+
spinner.stop(`Moved ${subdir}/${skippedNote}`);
|
|
206
|
+
}
|
|
207
|
+
catch (e) {
|
|
208
|
+
spinner.error(`Failed to move ${subdir}/`);
|
|
209
|
+
errors.push(`${subdir}: ${e instanceof Error ? e.message : String(e)}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
// Step 7: Write marker
|
|
214
|
+
if (errors.length === 0) {
|
|
215
|
+
const markerPath = path.join(targetPath, ".synergy-home");
|
|
216
|
+
await Bun.write(markerPath, `# Created by 'synergy data move' on ${new Date().toISOString()}\n`);
|
|
217
|
+
}
|
|
218
|
+
// Step 8: Remove original if requested
|
|
219
|
+
if (errors.length === 0 && opts.removeOriginal) {
|
|
220
|
+
const rmSpinner = prompts.spinner();
|
|
221
|
+
rmSpinner.start("Removing original data...");
|
|
222
|
+
try {
|
|
223
|
+
await fs.rm(sourceRoot, { recursive: true, force: true });
|
|
224
|
+
rmSpinner.stop("Original data removed");
|
|
225
|
+
}
|
|
226
|
+
catch (e) {
|
|
227
|
+
rmSpinner.error("Failed to remove original data");
|
|
228
|
+
prompts.log.warn("Remove manually after verification: " + shortenPath(sourceRoot));
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
// Report
|
|
232
|
+
UI.empty();
|
|
233
|
+
if (errors.length > 0) {
|
|
234
|
+
process.exitCode = 1;
|
|
235
|
+
prompts.log.warn("Move completed with errors:");
|
|
236
|
+
for (const err of errors)
|
|
237
|
+
prompts.log.error(` ${err}`);
|
|
238
|
+
prompts.log.info("Original data preserved at " + shortenPath(sourceRoot));
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
prompts.log.success("Data moved to " + shortenPath(targetPath));
|
|
242
|
+
}
|
|
243
|
+
if (!opts.removeOriginal || errors.length > 0) {
|
|
244
|
+
prompts.log.info(`Original data preserved at ${shortenPath(sourceRoot)}`);
|
|
245
|
+
}
|
|
246
|
+
prompts.log.info(`Run \`synergy data set-home ${shortenPath(homePath)}\` to switch to the new location`);
|
|
247
|
+
prompts.log.info("Restart any running synergy servers to use the new location");
|
|
248
|
+
prompts.outro("Done");
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
async function scanDir(dir) {
|
|
252
|
+
let size = 0;
|
|
253
|
+
async function walk(current) {
|
|
254
|
+
const entries = await fs.readdir(current, { withFileTypes: true }).catch(() => []);
|
|
255
|
+
for (const entry of entries) {
|
|
256
|
+
const full = path.join(current, entry.name);
|
|
257
|
+
if (entry.isDirectory()) {
|
|
258
|
+
await walk(full);
|
|
259
|
+
}
|
|
260
|
+
else if (entry.isFile()) {
|
|
261
|
+
const stat = await fs.stat(full).catch(() => null);
|
|
262
|
+
if (stat)
|
|
263
|
+
size += stat.size;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
await walk(dir);
|
|
268
|
+
return { size };
|
|
269
|
+
}
|
|
270
|
+
export const DataMoveCommand = cmd({
|
|
271
|
+
command: "move <target>",
|
|
272
|
+
describe: "move synergy data to a new location",
|
|
273
|
+
builder: (yargs) => yargs
|
|
274
|
+
.positional("target", {
|
|
275
|
+
type: "string",
|
|
276
|
+
describe: "target directory path",
|
|
277
|
+
demandOption: true,
|
|
278
|
+
})
|
|
279
|
+
.option("remove-original", {
|
|
280
|
+
type: "boolean",
|
|
281
|
+
default: false,
|
|
282
|
+
describe: "remove original data after successful move",
|
|
283
|
+
})
|
|
284
|
+
.option("dry-run", {
|
|
285
|
+
type: "boolean",
|
|
286
|
+
default: false,
|
|
287
|
+
describe: "show plan without executing",
|
|
288
|
+
}),
|
|
289
|
+
handler: async (args) => {
|
|
290
|
+
UI.empty();
|
|
291
|
+
UI.println(logo(" "));
|
|
292
|
+
UI.empty();
|
|
293
|
+
prompts.intro("Move Synergy Data");
|
|
294
|
+
await executeMove({
|
|
295
|
+
target: args.target,
|
|
296
|
+
removeOriginal: args.removeOriginal,
|
|
297
|
+
dryRun: args.dryRun,
|
|
298
|
+
});
|
|
299
|
+
},
|
|
300
|
+
});
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { logo } from "@ericsanchezok/synergy-cli/branding";
|
|
2
|
+
import { getLibraryInfo, resolveLibraryDB } from "@ericsanchezok/synergy-library/cli/data";
|
|
3
|
+
import { DataTransfer } from "./transfer.js";
|
|
4
|
+
import fs from "fs/promises";
|
|
5
|
+
import path from "path";
|
|
6
|
+
import os from "os";
|
|
7
|
+
import * as prompts from "@clack/prompts";
|
|
8
|
+
import { cmd } from "@ericsanchezok/synergy-util/cli-command";
|
|
9
|
+
import { UI } from "@ericsanchezok/synergy-util/terminal";
|
|
10
|
+
import { archiveExclusions, copyDirSkipExisting, CATEGORIES, scanCategories, formatSize, shortenPath, dataRoot, } from "@ericsanchezok/synergy-cli/cli/cmd/data/shared";
|
|
11
|
+
export const DataPackCommand = cmd({
|
|
12
|
+
command: "pack [output]",
|
|
13
|
+
describe: "pack synergy data into a zip archive",
|
|
14
|
+
builder: (yargs) => yargs.positional("output", {
|
|
15
|
+
type: "string",
|
|
16
|
+
describe: "output zip file path",
|
|
17
|
+
default: "",
|
|
18
|
+
}),
|
|
19
|
+
handler: async (args) => {
|
|
20
|
+
const root = dataRoot();
|
|
21
|
+
const outputArg = args.output;
|
|
22
|
+
UI.empty();
|
|
23
|
+
UI.println(logo(" "));
|
|
24
|
+
UI.empty();
|
|
25
|
+
prompts.intro("Pack Synergy Data");
|
|
26
|
+
const catStats = await scanCategories(root);
|
|
27
|
+
let totalSize = 0;
|
|
28
|
+
for (const stats of catStats.values())
|
|
29
|
+
totalSize += stats.size;
|
|
30
|
+
prompts.log.info(`Location: ${shortenPath(root)} (${formatSize(totalSize)})`);
|
|
31
|
+
// Step 1: Select categories
|
|
32
|
+
const selectable = CATEGORIES.filter((c) => !c.required);
|
|
33
|
+
const selected = await prompts.multiselect({
|
|
34
|
+
message: "What should be packed? (Space to toggle, Enter to confirm)",
|
|
35
|
+
options: selectable.map((cat) => ({
|
|
36
|
+
value: cat.key,
|
|
37
|
+
label: cat.label,
|
|
38
|
+
hint: formatSize(catStats.get(cat.key)?.size ?? 0),
|
|
39
|
+
})),
|
|
40
|
+
initialValues: selectable.filter((c) => c.defaultValue).map((c) => c.key),
|
|
41
|
+
required: false,
|
|
42
|
+
});
|
|
43
|
+
if (prompts.isCancel(selected)) {
|
|
44
|
+
prompts.cancel("Cancelled");
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const selectedKeys = new Set([...selected, ...CATEGORIES.filter((c) => c.required).map((c) => c.key)]);
|
|
48
|
+
const selectedCategories = CATEGORIES.filter((c) => selectedKeys.has(c.key));
|
|
49
|
+
// Step 2: Determine output path
|
|
50
|
+
const dateStr = new Date().toISOString().slice(0, 10);
|
|
51
|
+
const defaultName = `synergy-data-${dateStr}.zip`;
|
|
52
|
+
const outputPath = outputArg
|
|
53
|
+
? path.resolve(outputArg.endsWith(".zip") ? outputArg : `${outputArg}.zip`)
|
|
54
|
+
: path.join(os.homedir(), defaultName);
|
|
55
|
+
// Step 3: Build manifest
|
|
56
|
+
const libraryInfo = await getLibraryInfo(await resolveLibraryDB(root));
|
|
57
|
+
const manifest = {
|
|
58
|
+
version: 1,
|
|
59
|
+
createdAt: new Date().toISOString(),
|
|
60
|
+
library: libraryInfo.exists
|
|
61
|
+
? {
|
|
62
|
+
dimensions: libraryInfo.dimensions,
|
|
63
|
+
embeddingModel: libraryInfo.embeddingModel,
|
|
64
|
+
memoryCount: libraryInfo.memoryCount,
|
|
65
|
+
experienceCount: libraryInfo.experienceCount,
|
|
66
|
+
}
|
|
67
|
+
: null,
|
|
68
|
+
};
|
|
69
|
+
// Step 4: Pack
|
|
70
|
+
const spinner = prompts.spinner();
|
|
71
|
+
spinner.start("Packing data...");
|
|
72
|
+
try {
|
|
73
|
+
const packed = await createDataArchive(root, outputPath, selectedCategories.flatMap((category) => category.subdirs), manifest);
|
|
74
|
+
const packedSize = (await fs.stat(packed)).size;
|
|
75
|
+
spinner.stop(`Packed to ${shortenPath(packed)} (${formatSize(packedSize)})`);
|
|
76
|
+
}
|
|
77
|
+
catch (e) {
|
|
78
|
+
spinner.error("Packing failed");
|
|
79
|
+
prompts.log.error(`Failed to pack: ${e instanceof Error ? e.message : String(e)}`);
|
|
80
|
+
prompts.outro("Failed");
|
|
81
|
+
process.exitCode = 1;
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
prompts.outro("Done");
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
export async function createDataArchive(root, output, directories, manifest) {
|
|
88
|
+
await using homes = await DataTransfer.lockHomes([root]);
|
|
89
|
+
const stage = await fs.mkdtemp(path.join(os.tmpdir(), "synergy-pack-"));
|
|
90
|
+
try {
|
|
91
|
+
await Bun.write(path.join(stage, "manifest.json"), JSON.stringify(manifest, null, 2));
|
|
92
|
+
const included = ["manifest.json"];
|
|
93
|
+
for (const directory of directories) {
|
|
94
|
+
const source = path.join(root, directory);
|
|
95
|
+
const exists = await fs.stat(source).then(() => true, (error) => {
|
|
96
|
+
if (error.code === "ENOENT")
|
|
97
|
+
return false;
|
|
98
|
+
throw error;
|
|
99
|
+
});
|
|
100
|
+
if (!exists)
|
|
101
|
+
continue;
|
|
102
|
+
const destination = path.join(stage, directory);
|
|
103
|
+
if (directory === "data")
|
|
104
|
+
await DataTransfer.pack(root, destination);
|
|
105
|
+
else
|
|
106
|
+
await copyDirSkipExisting(source, destination, undefined, undefined, undefined, archiveExclusions(directory));
|
|
107
|
+
included.push(directory);
|
|
108
|
+
}
|
|
109
|
+
const zip = Bun.which("zip");
|
|
110
|
+
const filename = zip ? output : output.replace(/\.zip$/, ".tar.gz");
|
|
111
|
+
await fs.mkdir(path.dirname(filename), { recursive: true });
|
|
112
|
+
const temporary = path.join(path.dirname(filename), `.synergy-pack-${crypto.randomUUID()}${zip ? ".zip" : ".tar.gz"}`);
|
|
113
|
+
try {
|
|
114
|
+
const command = zip ? [zip, "-q", "-r", "-y", temporary, ...included] : ["tar", "-czf", temporary, ...included];
|
|
115
|
+
const child = Bun.spawn(command, { cwd: stage, stdout: "ignore", stderr: "pipe" });
|
|
116
|
+
const errors = new Response(child.stderr).text();
|
|
117
|
+
if ((await child.exited) !== 0)
|
|
118
|
+
throw new Error(`Archive creation failed: ${await errors}`);
|
|
119
|
+
await errors;
|
|
120
|
+
await fs.rename(temporary, filename);
|
|
121
|
+
}
|
|
122
|
+
finally {
|
|
123
|
+
await fs.rm(temporary, { force: true });
|
|
124
|
+
}
|
|
125
|
+
return filename;
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
await fs.rm(stage, { recursive: true, force: true });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { upgradeImportedRecord } from "@ericsanchezok/synergy-harness/migration/import";
|
|
2
|
+
import { installedWorkerCommand } from "@ericsanchezok/synergy-util/installed-launcher";
|
|
3
|
+
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { existsSync } from "node:fs";
|
|
7
|
+
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { StorageBootstrap } from "@ericsanchezok/synergy-harness/storage/bootstrap";
|
|
9
|
+
import { authorityRecordRoots, StoragePortable } from "@ericsanchezok/synergy-harness/storage/portable";
|
|
10
|
+
import { legacyRecordKey } from "@ericsanchezok/synergy-harness/storage/legacy-import";
|
|
11
|
+
import { Storage } from "@ericsanchezok/synergy-harness/storage/storage";
|
|
12
|
+
import { Usage } from "@ericsanchezok/synergy-harness/usage";
|
|
13
|
+
import { NoteStore } from "@ericsanchezok/synergy-note";
|
|
14
|
+
import { StorageCompat } from "@ericsanchezok/synergy-harness/storage/compat";
|
|
15
|
+
import { Session } from "@ericsanchezok/synergy-harness/session";
|
|
16
|
+
import { SnapshotArchive } from "@ericsanchezok/synergy-harness/session/snapshot-archive";
|
|
17
|
+
import { WorkspaceHomeTransfer } from "./workspace-transfer.js";
|
|
18
|
+
import { RuntimeContext } from "@ericsanchezok/synergy-harness/lifecycle/context";
|
|
19
|
+
import { WorkspaceAccess } from "@ericsanchezok/synergy-harness/workspace/access";
|
|
20
|
+
import { archiveExclusions, copyDirSkipExisting, } from "@ericsanchezok/synergy-cli/cli/cmd/data/shared";
|
|
21
|
+
const derived = new Set([
|
|
22
|
+
"session_index",
|
|
23
|
+
"endpoint_session",
|
|
24
|
+
"sessions_page_index",
|
|
25
|
+
"session_child_index",
|
|
26
|
+
"session_nav_v2",
|
|
27
|
+
]);
|
|
28
|
+
const local = new Set([
|
|
29
|
+
"storage_meta",
|
|
30
|
+
"storage_import",
|
|
31
|
+
"storage_import_files",
|
|
32
|
+
"storage_staging",
|
|
33
|
+
"storage_transfer",
|
|
34
|
+
"compat_import",
|
|
35
|
+
"compat_catalog",
|
|
36
|
+
]);
|
|
37
|
+
export var DataTransfer;
|
|
38
|
+
(function (DataTransfer) {
|
|
39
|
+
const fileClaims = RuntimeContext.createAsyncContext();
|
|
40
|
+
async function canonical(directory) {
|
|
41
|
+
try {
|
|
42
|
+
return await fs.realpath(directory);
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
const parent = path.dirname(directory);
|
|
46
|
+
if (error.code !== "ENOENT" || parent === directory)
|
|
47
|
+
throw error;
|
|
48
|
+
return path.join(await canonical(parent), path.basename(directory));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function contains(root, filename) {
|
|
52
|
+
const relative = path.relative(root, filename);
|
|
53
|
+
return relative === "" || (relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
|
|
54
|
+
}
|
|
55
|
+
async function withFiles(roots, fn) {
|
|
56
|
+
const resolved = await Promise.all(roots.map((root) => canonical(path.resolve(root))));
|
|
57
|
+
const current = fileClaims.getStore();
|
|
58
|
+
if (current) {
|
|
59
|
+
if (!resolved.every((root) => current.some((claimed) => contains(claimed, root))))
|
|
60
|
+
throw new Error("Nested Home transfer escaped its file ownership");
|
|
61
|
+
return fn();
|
|
62
|
+
}
|
|
63
|
+
return WorkspaceAccess.maintenance(() => WorkspaceAccess.retire(resolved, () => fileClaims.run(resolved, fn)));
|
|
64
|
+
}
|
|
65
|
+
DataTransfer.withFiles = withFiles;
|
|
66
|
+
async function validateHomes(roots) {
|
|
67
|
+
const resolved = await Promise.all(roots.map((root) => canonical(path.resolve(root))));
|
|
68
|
+
for (let i = 0; i < resolved.length; i++)
|
|
69
|
+
for (let j = 0; j < resolved.length; j++) {
|
|
70
|
+
if (i === j)
|
|
71
|
+
continue;
|
|
72
|
+
if (contains(resolved[i], resolved[j]))
|
|
73
|
+
throw new Error("Source and target Homes must not overlap");
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
DataTransfer.validateHomes = validateHomes;
|
|
77
|
+
async function lockHomes(roots) {
|
|
78
|
+
await validateHomes(roots);
|
|
79
|
+
for (const root of [...new Set(roots.map((root) => path.resolve(root)))].sort()) {
|
|
80
|
+
const entry = fileURLToPath(new URL("../../index.js", import.meta.url));
|
|
81
|
+
const child = Bun.spawn({
|
|
82
|
+
cmd: installedWorkerCommand(process.env, "__storage-maintenance-runner") ??
|
|
83
|
+
(existsSync(entry)
|
|
84
|
+
? [process.execPath, "run", entry, "__storage-maintenance-runner"]
|
|
85
|
+
: [process.execPath, "__storage-maintenance-runner"]),
|
|
86
|
+
env: { ...process.env, SYNERGY_HOME: path.dirname(root), SYNERGY_RUNTIME_ROOT: root },
|
|
87
|
+
stdout: "ignore",
|
|
88
|
+
stderr: "pipe",
|
|
89
|
+
});
|
|
90
|
+
const error = new Response(child.stderr).text();
|
|
91
|
+
const code = await child.exited;
|
|
92
|
+
if (code !== 0)
|
|
93
|
+
throw new Error(`Data upgrade failed: ${await error}`);
|
|
94
|
+
await error;
|
|
95
|
+
}
|
|
96
|
+
return SnapshotArchive.lockHomes(roots);
|
|
97
|
+
}
|
|
98
|
+
DataTransfer.lockHomes = lockHomes;
|
|
99
|
+
function exclude(relative, skipped = new Set(), packs) {
|
|
100
|
+
const segments = relative.split(path.sep);
|
|
101
|
+
if (archiveExclusions("data").includes(segments[0]))
|
|
102
|
+
return true;
|
|
103
|
+
if (segments[0] === "agent-artifacts" && segments.length > 1 && packs && !packs.has(segments[1]))
|
|
104
|
+
return true;
|
|
105
|
+
if (segments[0] === "sessions" && skipped.has(segments[2] ?? ""))
|
|
106
|
+
return true;
|
|
107
|
+
return Boolean(legacyRecordKey(segments.join("/")));
|
|
108
|
+
}
|
|
109
|
+
async function artifactPacks(handle, accept = () => true) {
|
|
110
|
+
return handle.store.snapshot(async (tx) => {
|
|
111
|
+
const packs = new Set();
|
|
112
|
+
for await (const entry of tx.artifacts())
|
|
113
|
+
if (accept(entry.key))
|
|
114
|
+
packs.add(entry.location.pack);
|
|
115
|
+
return packs;
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
async function pack(sourceRoot, destination) {
|
|
119
|
+
return withFiles([sourceRoot, destination], () => packRecords(sourceRoot, destination));
|
|
120
|
+
}
|
|
121
|
+
DataTransfer.pack = pack;
|
|
122
|
+
async function packRecords(sourceRoot, destination) {
|
|
123
|
+
const handle = await StorageBootstrap.inspect(sourceRoot);
|
|
124
|
+
if (!handle)
|
|
125
|
+
throw new Error("Source storage has not been initialized");
|
|
126
|
+
try {
|
|
127
|
+
await StoragePortable.exportFile(handle.store, path.join(destination, "agent-records.ndjson"));
|
|
128
|
+
await SnapshotArchive.merge(handle.artifactDirectory, destination, { metadata: false });
|
|
129
|
+
const packs = await artifactPacks(handle);
|
|
130
|
+
const result = await copyDirSkipExisting(handle.artifactDirectory, destination, undefined, undefined, undefined, (relative) => exclude(relative, undefined, packs));
|
|
131
|
+
await Storage.validateArtifacts({ store: handle.store, artifactDirectory: destination });
|
|
132
|
+
return result;
|
|
133
|
+
}
|
|
134
|
+
finally {
|
|
135
|
+
await handle.store.close();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
async function merge(sourceRoot, targetRoot, options = {}) {
|
|
139
|
+
return withFiles([sourceRoot, targetRoot], () => mergeRecords(sourceRoot, targetRoot, options));
|
|
140
|
+
}
|
|
141
|
+
DataTransfer.merge = merge;
|
|
142
|
+
async function mergeRecords(sourceRoot, targetRoot, options) {
|
|
143
|
+
await validateHomes([sourceRoot, targetRoot]);
|
|
144
|
+
const source = await StorageBootstrap.inspect(sourceRoot);
|
|
145
|
+
if (!source)
|
|
146
|
+
throw new Error("Source storage has not been initialized");
|
|
147
|
+
let target;
|
|
148
|
+
try {
|
|
149
|
+
await StorageCompat.assertConverged(source.store);
|
|
150
|
+
target = await StorageBootstrap.prepare({ root: targetRoot });
|
|
151
|
+
await StorageCompat.assertConverged(target.store);
|
|
152
|
+
const skipped = new Set();
|
|
153
|
+
for (const scopeID of await target.store.scan(["sessions"]))
|
|
154
|
+
for (const id of await target.store.scan(["sessions", scopeID]))
|
|
155
|
+
skipped.add(id);
|
|
156
|
+
for (const scopeID of await target.store.scan(["snapshot-v2"]))
|
|
157
|
+
for (const id of await target.store.scan(["snapshot-v2", scopeID, "owners"]))
|
|
158
|
+
skipped.add(id);
|
|
159
|
+
const id = randomUUID();
|
|
160
|
+
const backup = path.join(targetRoot, "data", "storage", "transfers", id);
|
|
161
|
+
// Retain the entire source, including skipped aggregates, before move can remove its home.
|
|
162
|
+
await pack(sourceRoot, path.join(backup, "data"));
|
|
163
|
+
const acceptArtifact = (key) => {
|
|
164
|
+
const owner = sessionOwner({ key });
|
|
165
|
+
return (!local.has(key[0]) &&
|
|
166
|
+
!derived.has(key[0]) &&
|
|
167
|
+
(options.trusted || !authorityRecordRoots.has(key[0])) &&
|
|
168
|
+
(!owner || !skipped.has(owner)));
|
|
169
|
+
};
|
|
170
|
+
const packs = await artifactPacks(source, acceptArtifact);
|
|
171
|
+
const copied = await copyDirSkipExisting(source.artifactDirectory, path.join(targetRoot, "data"), options.progress, undefined, undefined, (relative) => exclude(relative, skipped, packs));
|
|
172
|
+
await SnapshotArchive.merge(source.artifactDirectory, path.join(targetRoot, "data"), { metadata: false, skipped });
|
|
173
|
+
await Storage.validateArtifacts({ store: source.store, artifactDirectory: path.join(targetRoot, "data") }, { accept: acceptArtifact });
|
|
174
|
+
const conflicts = new Set();
|
|
175
|
+
const workspaces = await WorkspaceHomeTransfer.prepare(source.store, target.store, acceptArtifact, options.trusted ? { sourceRoot, targetRoot } : undefined);
|
|
176
|
+
const result = await StoragePortable.importFile(target.store, path.join(backup, "data", "agent-records.ndjson"), {
|
|
177
|
+
operationID: id,
|
|
178
|
+
accept: async (entry, tx) => {
|
|
179
|
+
if (entry.type === "event")
|
|
180
|
+
return false;
|
|
181
|
+
if (entry.type === "receipt")
|
|
182
|
+
return true;
|
|
183
|
+
if (local.has(entry.key[0]) || derived.has(entry.key[0]))
|
|
184
|
+
return false;
|
|
185
|
+
if (workspaces && WorkspaceHomeTransfer.roots.has(entry.key[0]))
|
|
186
|
+
return false;
|
|
187
|
+
// Grants, consent and trust decisions never cross homes through an
|
|
188
|
+
// untrusted merge: an imported approval would silently satisfy the
|
|
189
|
+
// consent prompt for a later plugin install. Same-home relocation
|
|
190
|
+
// (data move) opts in explicitly.
|
|
191
|
+
if (!options.trusted && authorityRecordRoots.has(entry.key[0]))
|
|
192
|
+
return false;
|
|
193
|
+
const owner = sessionOwner(entry);
|
|
194
|
+
if (owner && skipped.has(owner)) {
|
|
195
|
+
conflicts.add(owner);
|
|
196
|
+
return false;
|
|
197
|
+
}
|
|
198
|
+
if (entry.type === "artifact") {
|
|
199
|
+
try {
|
|
200
|
+
await tx.artifact(entry.key);
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
if (error instanceof Storage.NotFoundError)
|
|
205
|
+
return true;
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return (await tx.readMany([entry.key]))[0] === undefined;
|
|
210
|
+
},
|
|
211
|
+
transform: (entry) => {
|
|
212
|
+
if (entry.type !== "record")
|
|
213
|
+
return entry;
|
|
214
|
+
const value = upgradeImportedRecord(entry.key, entry.value);
|
|
215
|
+
return {
|
|
216
|
+
...entry,
|
|
217
|
+
value: workspaces && acceptArtifact(entry.key) ? workspaces.record(entry.key, value) : value,
|
|
218
|
+
};
|
|
219
|
+
},
|
|
220
|
+
afterImport: async (tx) => {
|
|
221
|
+
await workspaces?.publish(tx);
|
|
222
|
+
await Session.rebuildStorageIndexes(tx);
|
|
223
|
+
await NoteStore.invalidateStorageIndexes(tx);
|
|
224
|
+
await Usage.reconcileTransfer(tx);
|
|
225
|
+
await tx.write(["storage_transfer", id], {
|
|
226
|
+
version: 1,
|
|
227
|
+
complete: true,
|
|
228
|
+
skippedSessionIDs: [...conflicts],
|
|
229
|
+
backup: path.relative(targetRoot, backup),
|
|
230
|
+
created: Date.now(),
|
|
231
|
+
});
|
|
232
|
+
},
|
|
233
|
+
});
|
|
234
|
+
await Storage.writeJsonAtomic(path.join(backup, "report.json"), JSON.stringify({ version: 1, ...result, skippedSessionIDs: [...conflicts] }), { private: true, durable: true });
|
|
235
|
+
return { ...copied, skipped: copied.skipped + conflicts.size, skippedSessions: conflicts.size };
|
|
236
|
+
}
|
|
237
|
+
finally {
|
|
238
|
+
await target?.store.close();
|
|
239
|
+
await source.store.close();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
})(DataTransfer || (DataTransfer = {}));
|
|
243
|
+
function sessionOwner(entry) {
|
|
244
|
+
const key = entry.key;
|
|
245
|
+
if (key[0] === "sessions" || key[0].startsWith("session_search_") || key[0] === "session_message_order_v1")
|
|
246
|
+
return key[2];
|
|
247
|
+
if (key[0] === "snapshot-v2" && ["owners", "migrations", "deletions"].includes(key[2]))
|
|
248
|
+
return key[3];
|
|
249
|
+
if (key[0] === "storage_recovery" && key[1] === "sessions")
|
|
250
|
+
return key[2];
|
|
251
|
+
}
|