@xiangyixie/dsh-superagent 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/sync.js ADDED
@@ -0,0 +1,456 @@
1
+ /**
2
+ * Approximate multi-device sync for the state directory, carried by a git
3
+ * repository. There is no server: the remote is whatever git remote the user
4
+ * already has, and `git` itself is the only dependency.
5
+ *
6
+ * What the state directory actually looks like decides the conflict surface,
7
+ * so the layout matters here:
8
+ *
9
+ * - `runs/<runId>.json` is one whole file per run (see store.js), rewritten in
10
+ * place on every save. Two machines only touch the same file when they work
11
+ * on the same run id, so the per-run split — not one shared JSONL — is what
12
+ * keeps ordinary concurrent runs from colliding at all.
13
+ * - `memory/<scope>.jsonl` is append-only per scope. Two machines appending to
14
+ * one scope are the realistic conflict: git sees two different lines added at
15
+ * the same tail position.
16
+ * - `<runId>.lock`, `<runId>.cancel`, `.owner`, `control.json` and
17
+ * `notifications.json` describe one machine or one process. They are ignored,
18
+ * and `control.json` is a bearer token, so it must never be committed.
19
+ *
20
+ * A conflict is never resolved automatically: the merge is left in place with
21
+ * both sides, the paths are returned, and the caller reports them.
22
+ *
23
+ * @module dsh-superagent/sync
24
+ */
25
+ import { execFile } from 'node:child_process'
26
+ import fsp from 'node:fs/promises'
27
+ import os from 'node:os'
28
+ import path from 'node:path'
29
+
30
+ /**
31
+ * Patterns kept out of the synced repository. Each one either names a secret,
32
+ * a per-machine identity, or a side file that is meaningless on another host.
33
+ */
34
+ export const IGNORE_PATTERNS = [
35
+ '.owner',
36
+ 'control.json',
37
+ '.env',
38
+ '.env.*',
39
+ '*.key',
40
+ '*.pem',
41
+ '*.lock',
42
+ '*.cancel',
43
+ '*.corrupt-*',
44
+ '*.tmp',
45
+ 'notifications.json',
46
+ '*.log',
47
+ ]
48
+
49
+ /** Markers that let a later `sync init` refresh only its own block. */
50
+ export const IGNORE_START = '# >>> dsh-superagent sync — machine-local files below (managed block)'
51
+ export const IGNORE_END = '# <<< dsh-superagent sync'
52
+
53
+ /** Why the ignored files are ignored, printed by `sync init`. */
54
+ export const IGNORE_REASONS = {
55
+ '.owner': 'hostname of the machine that mounted the directory',
56
+ 'control.json': 'bearer token for the loopback control plane',
57
+ '.env': 'environment files can hold credentials',
58
+ '.env.*': 'environment files can hold credentials',
59
+ '*.key': 'private keys',
60
+ '*.pem': 'certificates and keys',
61
+ '*.lock': 'a run lock names one live process on one machine',
62
+ '*.cancel': 'a cancel request is consumed by the owner and is not history',
63
+ '*.corrupt-*': 'quarantined bytes from one machine, kept locally',
64
+ '*.tmp': 'half-written temp file from an interrupted save',
65
+ 'notifications.json': 'rolling-hour notification counter for this machine',
66
+ '*.log': 'logs describe this machine only',
67
+ }
68
+
69
+ /**
70
+ * @returns {string} the managed block, including its markers and a trailing newline.
71
+ */
72
+ export function buildIgnoreBlock() {
73
+ return `${IGNORE_START}\n${IGNORE_PATTERNS.join('\n')}\n${IGNORE_END}\n`
74
+ }
75
+
76
+ /**
77
+ * Add or refresh the managed block without touching anything outside it, so a
78
+ * user's own entries in `.gitignore` survive a second `sync init`.
79
+ * @param {string} [existing] - current `.gitignore` contents.
80
+ * @returns {{ text: string, changed: boolean }} new contents and whether they differ.
81
+ */
82
+ export function mergeIgnore(existing = '') {
83
+ const text = String(existing)
84
+ const block = buildIgnoreBlock()
85
+ const start = text.indexOf(IGNORE_START)
86
+ const end = text.indexOf(IGNORE_END)
87
+ let next
88
+ if (start !== -1 && end > start) {
89
+ next = `${text.slice(0, start)}${block}${text.slice(end + IGNORE_END.length).replace(/^\r?\n/, '')}`
90
+ } else {
91
+ const base = text.replace(/\s+$/, '')
92
+ next = base ? `${base}\n\n${block}` : block
93
+ }
94
+ const normalize = value => value.replace(/\s+$/, '')
95
+ return { text: next, changed: normalize(next) !== normalize(text) }
96
+ }
97
+
98
+ /** Message prefix for a command that could not run at all. */
99
+ function gitFailure(args, result) {
100
+ const output = `${result.stderr}${result.stdout}`.trim()
101
+ return new Error(`git ${args[0]} failed${output ? `: ${output.split('\n').slice(0, 3).join(' ')}` : ''}`)
102
+ }
103
+
104
+ /**
105
+ * Build a sync handle bound to one state directory.
106
+ * @param {string} root - absolute state root (the directory that becomes a repository).
107
+ * @param {object} [options] - overrides.
108
+ * @param {string} [options.gitBin] - git executable; defaults to `git` on PATH.
109
+ * @param {string} [options.hostname] - machine name recorded in commit messages.
110
+ * @param {() => number} [options.now] - clock, injectable for tests.
111
+ * @returns {object} `init`, `pull`, `push`, `status`.
112
+ */
113
+ export function createSync(root, options = {}) {
114
+ const stateRoot = path.resolve(root)
115
+ const gitBin = options.gitBin ?? 'git'
116
+ const hostname = options.hostname ?? os.hostname()
117
+ const now = options.now ?? (() => Date.now())
118
+ const stamp = () => new Date(now()).toISOString()
119
+
120
+ /**
121
+ * Run git once. A non-zero exit is returned rather than thrown, because
122
+ * merge and push rejections are answers, not crashes.
123
+ */
124
+ function exec(args) {
125
+ return new Promise(resolve => {
126
+ execFile(gitBin, args, {
127
+ cwd: stateRoot,
128
+ windowsHide: true,
129
+ encoding: 'utf8',
130
+ maxBuffer: 16 * 1024 * 1024,
131
+ env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
132
+ }, (error, stdout, stderr) => {
133
+ if (!error) return resolve({ code: 0, stdout: stdout ?? '', stderr: stderr ?? '' })
134
+ if (error.code === 'ENOENT') {
135
+ return resolve({ code: 127, stdout: '', stderr: 'git is not installed or not on PATH' })
136
+ }
137
+ const code = typeof error.code === 'number' ? error.code : 1
138
+ resolve({ code, stdout: stdout ?? '', stderr: stderr ?? error.message })
139
+ })
140
+ })
141
+ }
142
+
143
+ /** Same as `exec`, but a non-zero exit throws with git's own first lines. */
144
+ async function must(args) {
145
+ const result = await exec(args)
146
+ if (result.code !== 0) throw gitFailure(args, result)
147
+ return result
148
+ }
149
+
150
+ const repoDir = path.join(stateRoot, '.git')
151
+
152
+ async function exists(file) {
153
+ try {
154
+ await fsp.access(file)
155
+ return true
156
+ } catch {
157
+ return false
158
+ }
159
+ }
160
+
161
+ async function ensureRepo() {
162
+ if (!(await exists(repoDir))) {
163
+ throw new Error(`${stateRoot} is not a git repository; run "sync init <git-remote>" there first`)
164
+ }
165
+ }
166
+
167
+ async function remoteUrl() {
168
+ const result = await exec(['remote', 'get-url', 'origin'])
169
+ return result.code === 0 ? result.stdout.trim() || null : null
170
+ }
171
+
172
+ /** @returns {Promise<string | null>} short branch name, or null on detached HEAD. */
173
+ async function branchName() {
174
+ const result = await exec(['symbolic-ref', '--short', 'HEAD'])
175
+ return result.code === 0 ? result.stdout.trim() || null : null
176
+ }
177
+
178
+ /** @returns {Promise<string | null>} e.g. `origin/main`, when an upstream is configured. */
179
+ async function upstreamRef() {
180
+ const result = await exec(['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'])
181
+ return result.code === 0 ? result.stdout.trim() || null : null
182
+ }
183
+
184
+ async function refExists(ref) {
185
+ return (await exec(['rev-parse', '--verify', '--quiet', ref])).code === 0
186
+ }
187
+
188
+ /** @returns {Promise<string[]>} paths git left with conflict markers. */
189
+ async function conflictFiles() {
190
+ const result = await exec(['diff', '--name-only', '--diff-filter=U'])
191
+ if (result.code !== 0) return []
192
+ return result.stdout.split('\n').map(line => line.trim()).filter(Boolean)
193
+ }
194
+
195
+ async function stashEntries() {
196
+ const result = await exec(['stash', 'list', '--format=%gd%x09%s'])
197
+ if (result.code !== 0) return []
198
+ return result.stdout.split('\n').filter(Boolean).map(line => {
199
+ const [ref, subject = ''] = line.split('\t')
200
+ return { ref, subject }
201
+ })
202
+ }
203
+
204
+ /**
205
+ * Stage everything and commit if anything changed.
206
+ * @param {string} message - commit subject.
207
+ * @returns {Promise<string | null>} commit sha, or null when nothing was staged.
208
+ */
209
+ async function commitAll(message) {
210
+ await must(['add', '-A'])
211
+ const staged = await must(['diff', '--cached', '--name-only'])
212
+ if (!staged.stdout.trim()) return null
213
+ await must(['commit', '-m', message])
214
+ return (await must(['rev-parse', 'HEAD'])).stdout.trim()
215
+ }
216
+
217
+ /** Write the managed block into `<root>/.gitignore`. */
218
+ async function writeIgnoreFile() {
219
+ const file = path.join(stateRoot, '.gitignore')
220
+ let current = ''
221
+ try {
222
+ current = await fsp.readFile(file, 'utf8')
223
+ } catch {
224
+ // No file yet: the block becomes the whole file.
225
+ }
226
+ const merged = mergeIgnore(current)
227
+ if (merged.changed) await fsp.writeFile(file, merged.text, 'utf8')
228
+ return { file, changed: merged.changed, patterns: IGNORE_PATTERNS }
229
+ }
230
+
231
+ /**
232
+ * The notebook directory ships a nested `.gitignore` containing `*`, which
233
+ * would keep every note out of this repository too. Inside a state directory
234
+ * that is its own repository the notes are the point, so the default is
235
+ * replaced; a file the user customized is left alone.
236
+ * @returns {Promise<boolean>} whether notes will be synced.
237
+ */
238
+ async function enableMemorySync() {
239
+ const dir = path.join(stateRoot, 'memory')
240
+ const file = path.join(dir, '.gitignore')
241
+ let current = null
242
+ try {
243
+ current = await fsp.readFile(file, 'utf8')
244
+ } catch {
245
+ // Created below.
246
+ }
247
+ if (current !== null && current.replace(/\s+/g, '') !== '*') return false
248
+ await fsp.mkdir(dir, { recursive: true })
249
+ await fsp.writeFile(
250
+ file,
251
+ '# managed by dsh-superagent sync: the notebook is what crosses machines\n*\n!.gitignore\n!*.jsonl\n',
252
+ 'utf8',
253
+ )
254
+ return true
255
+ }
256
+
257
+ return {
258
+ root: stateRoot,
259
+
260
+ /**
261
+ * Turn the state directory into a repository, write the ignore rules, and
262
+ * make the first commit. Safe to run again: it refreshes the remote URL and
263
+ * the managed ignore block instead of failing.
264
+ * @param {{ remote: string }} params - remote URL or path for `origin`.
265
+ */
266
+ async init({ remote }) {
267
+ const value = typeof remote === 'string' ? remote.trim() : ''
268
+ if (!value) throw new Error('sync init needs a git remote (an HTTPS URL, an SSH URL, or a path to a bare repository)')
269
+ await fsp.mkdir(stateRoot, { recursive: true })
270
+ const created = !(await exists(repoDir))
271
+ if (created) await must(['init'])
272
+ // A machine with no ~/.gitconfig cannot commit; a repository-local
273
+ // fallback identity keeps the first commit from failing on that alone.
274
+ const hasEmail = (await exec(['config', '--get', 'user.email'])).code === 0
275
+ const hasName = (await exec(['config', '--get', 'user.name'])).code === 0
276
+ if (!hasEmail) await must(['config', 'user.email', 'dsh-superagent-sync@localhost'])
277
+ if (!hasName) await must(['config', 'user.name', 'dsh-superagent sync'])
278
+
279
+ const existing = await remoteUrl()
280
+ let remoteAction = 'unchanged'
281
+ if (!existing) {
282
+ await must(['remote', 'add', 'origin', value])
283
+ remoteAction = 'added'
284
+ } else if (existing !== value) {
285
+ await must(['remote', 'set-url', 'origin', value])
286
+ remoteAction = 'updated'
287
+ }
288
+
289
+ const ignore = await writeIgnoreFile()
290
+ const memorySynced = await enableMemorySync()
291
+ const commit = await commitAll(`sync init: ${hostname} ${stamp()}`)
292
+ const branch = await branchName()
293
+ return {
294
+ root: stateRoot,
295
+ created,
296
+ branch,
297
+ remote: value,
298
+ remoteAction,
299
+ ignoreChanged: ignore.changed,
300
+ ignoredPatterns: IGNORE_PATTERNS,
301
+ memorySynced,
302
+ commit,
303
+ }
304
+ },
305
+
306
+ /**
307
+ * Fetch the remote and merge it. Uncommitted local work is stashed first and
308
+ * reapplied afterwards, so a pull never silently discards an edit. A real
309
+ * conflict is left in the working tree with both sides; nothing is chosen.
310
+ */
311
+ async pull() {
312
+ await ensureRepo()
313
+ // A second pull before the first merge was resolved would fail inside
314
+ // `stash`; report the same conflicts again instead.
315
+ const pending = await conflictFiles()
316
+ if (pending.length > 0) {
317
+ return { stash: null, merged: false, conflicts: pending, ref: await upstreamRef(), output: 'a merge is already in progress' }
318
+ }
319
+ const remote = await remoteUrl()
320
+ if (!remote) throw new Error('this repository has no origin remote; run "sync init <git-remote>" to set one')
321
+
322
+ const fetched = await exec(['fetch', '--quiet', 'origin'])
323
+ if (fetched.code !== 0) throw gitFailure(['fetch', 'origin'], fetched)
324
+
325
+ let ref = await upstreamRef()
326
+ if (!ref) {
327
+ const branch = await branchName()
328
+ if (!branch) throw new Error('HEAD is detached, so there is no branch to pull into')
329
+ const candidate = `origin/${branch}`
330
+ if (await refExists(candidate)) ref = candidate
331
+ }
332
+ if (!ref) return { stash: null, merged: false, conflicts: [], note: 'the remote has no branch for this repository yet' }
333
+
334
+ // Only stash after the fetch succeeded: a failed fetch must leave the
335
+ // working tree exactly as it was.
336
+ const status = await must(['status', '--porcelain'])
337
+ let stash = null
338
+ if (status.stdout.trim()) {
339
+ const message = `dsh-sync auto-stash ${stamp()} on ${hostname}`
340
+ const pushed = await exec(['stash', 'push', '-u', '-m', message])
341
+ if (pushed.code !== 0) throw gitFailure(['stash', 'push'], pushed)
342
+ const entries = await stashEntries()
343
+ const entry = entries.find(item => item.subject === message) ?? entries[0]
344
+ stash = {
345
+ ref: entry?.ref ?? 'stash@{0}',
346
+ message,
347
+ files: status.stdout.split('\n').filter(Boolean).length,
348
+ reapplied: false,
349
+ }
350
+ }
351
+
352
+ const merge = await exec(['merge', '--no-edit', '--allow-unrelated-histories', ref])
353
+ const conflicts = await conflictFiles()
354
+ if (conflicts.length > 0) {
355
+ // Both sides stay in the tree. The stash is deliberately not reapplied
356
+ // on top of a half-merged tree: that would only add conflicts to review.
357
+ return { stash, merged: false, conflicts, ref, output: `${merge.stdout}${merge.stderr}`.trim() }
358
+ }
359
+ if (merge.code !== 0) throw gitFailure(['merge', ref], merge)
360
+
361
+ if (!(await upstreamRef())) await exec(['branch', `--set-upstream-to=${ref}`])
362
+
363
+ if (stash) {
364
+ const pop = await exec(['stash', 'pop'])
365
+ stash.reapplied = pop.code === 0
366
+ if (!stash.reapplied) {
367
+ stash.kept = true
368
+ stash.popError = `${pop.stdout}${pop.stderr}`.trim()
369
+ }
370
+ }
371
+ return {
372
+ stash,
373
+ merged: !/Already up to date/i.test(merge.stdout),
374
+ conflicts: [],
375
+ ref,
376
+ output: merge.stdout.trim(),
377
+ }
378
+ },
379
+
380
+ /**
381
+ * Commit local state and push it. A rejection is reported, not merged: the
382
+ * two halves stay separate commands so the user sees the merge.
383
+ * @param {{ message?: string }} [params] - optional commit subject.
384
+ */
385
+ async push({ message } = {}) {
386
+ await ensureRepo()
387
+ const remote = await remoteUrl()
388
+ if (!remote) throw new Error('this repository has no origin remote; run "sync init <git-remote>" to set one')
389
+ const branch = await branchName()
390
+ if (!branch) throw new Error('HEAD is detached, so there is no branch to push')
391
+ const pending = await conflictFiles()
392
+ if (pending.length > 0) {
393
+ const error = new Error(`the state directory still has ${pending.length} unresolved conflict(s): ${pending.join(', ')}`)
394
+ error.conflicts = pending
395
+ throw error
396
+ }
397
+
398
+ const commit = await commitAll(message ?? `sync: ${hostname} ${stamp()}`)
399
+ const pushed = await exec(['push', '-u', 'origin', `HEAD:refs/heads/${branch}`])
400
+ if (pushed.code !== 0) {
401
+ const output = `${pushed.stdout}${pushed.stderr}`.trim()
402
+ const rejected = /rejected|non-fast-forward|fetch first/i.test(output)
403
+ return { commit, pushed: false, rejected, branch, output }
404
+ }
405
+ const upstream = await upstreamRef()
406
+ return { commit, pushed: true, branch, upstream, output: `${pushed.stdout}${pushed.stderr}`.trim() }
407
+ },
408
+
409
+ /**
410
+ * Local vs remote position and unfinished business.
411
+ * @param {{ fetch?: boolean }} [params] - set `fetch: false` to read last-known refs offline.
412
+ */
413
+ async status({ fetch = true } = {}) {
414
+ await ensureRepo()
415
+ const remote = await remoteUrl()
416
+ const branch = await branchName()
417
+ let fetched = false
418
+ let fetchError = null
419
+ if (fetch && remote) {
420
+ const result = await exec(['fetch', '--quiet', 'origin'])
421
+ fetched = result.code === 0
422
+ if (!fetched) fetchError = `${result.stderr}${result.stdout}`.trim()
423
+ }
424
+ const upstream = await upstreamRef()
425
+ let ahead = null
426
+ let behind = null
427
+ if (upstream) {
428
+ // --left-right --count prints "<upstream only>\t<HEAD only>".
429
+ const counts = await exec(['rev-list', '--left-right', '--count', `${upstream}...HEAD`])
430
+ if (counts.code === 0) {
431
+ const [left, right] = counts.stdout.trim().split(/\s+/).map(Number)
432
+ behind = Number.isFinite(left) ? left : null
433
+ ahead = Number.isFinite(right) ? right : null
434
+ }
435
+ }
436
+ const dirty = await must(['status', '--porcelain'])
437
+ const conflicts = await conflictFiles()
438
+ const stashes = await stashEntries()
439
+ const head = await exec(['log', '-1', '--format=%h %s'])
440
+ return {
441
+ root: stateRoot,
442
+ remote,
443
+ branch,
444
+ upstream,
445
+ ahead,
446
+ behind,
447
+ dirty: dirty.stdout.split('\n').filter(Boolean).length,
448
+ conflicts,
449
+ stashes,
450
+ head: head.code === 0 ? head.stdout.trim() : null,
451
+ fetched,
452
+ fetchError,
453
+ }
454
+ },
455
+ }
456
+ }