@lifestreamdynamics/vault-cli 1.3.11 → 1.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/audit.js +1 -1
- package/dist/commands/hooks.js +43 -9
- package/dist/commands/sync.js +29 -8
- package/dist/commands/webhooks.js +43 -8
- package/dist/sync/daemon-worker.js +8 -3
- package/dist/sync/engine.d.ts +47 -7
- package/dist/sync/engine.js +124 -33
- package/dist/sync/ignore.js +1 -1
- package/dist/sync/remote-poller.js +96 -55
- package/dist/sync/state.js +1 -1
- package/dist/sync/types.d.ts +2 -0
- package/dist/sync/watcher.js +18 -11
- package/package.json +2 -2
package/dist/commands/audit.js
CHANGED
|
@@ -2,7 +2,7 @@ import chalk from 'chalk';
|
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import os from 'node:os';
|
|
5
|
-
import { AuditLogger } from '@lifestreamdynamics/vault-sdk';
|
|
5
|
+
import { AuditLogger } from '@lifestreamdynamics/vault-sdk/audit';
|
|
6
6
|
import { addGlobalFlags, resolveFlags } from '../utils/flags.js';
|
|
7
7
|
import { createOutput, handleError } from '../utils/output.js';
|
|
8
8
|
const DEFAULT_LOG_PATH = path.join(os.homedir(), '.lsvault', 'audit.log');
|
package/dist/commands/hooks.js
CHANGED
|
@@ -3,6 +3,21 @@ import { getClientAsync } from '../client.js';
|
|
|
3
3
|
import { addGlobalFlags, resolveFlags } from '../utils/flags.js';
|
|
4
4
|
import { createOutput, handleError } from '../utils/output.js';
|
|
5
5
|
import { resolveVaultId } from '../utils/resolve-vault.js';
|
|
6
|
+
/**
|
|
7
|
+
* Valid hook trigger events. Unlike webhooks, hooks do NOT accept the `*`
|
|
8
|
+
* wildcard. Inlined from the internal vault-shared package (not published to
|
|
9
|
+
* npm) so the standalone CLI build has no unpublishable workspace dependency.
|
|
10
|
+
* Keep in sync with VAULT_EVENT_TYPES in packages/shared/src/constants.ts.
|
|
11
|
+
*/
|
|
12
|
+
const VAULT_EVENT_TYPES = [
|
|
13
|
+
'document.created', 'document.updated', 'document.deleted', 'document.moved', 'document.copied',
|
|
14
|
+
'directory.created', 'document.overdue', 'document.due-soon',
|
|
15
|
+
'calendar.event.created', 'calendar.event.updated', 'calendar.event.deleted', 'calendar.event.due',
|
|
16
|
+
'calendar.event.overdue', 'calendar.event.status_changed',
|
|
17
|
+
'booking.created', 'booking.confirmed', 'booking.cancelled', 'booking.no_show', 'booking.completed',
|
|
18
|
+
'booking.reminder', 'booking.rescheduled',
|
|
19
|
+
'calendar.event.participant.added', 'calendar.event.participant.responded',
|
|
20
|
+
];
|
|
6
21
|
export function registerHookCommands(program) {
|
|
7
22
|
const hooks = program.command('hooks').description('Manage vault event hooks');
|
|
8
23
|
addGlobalFlags(hooks.command('list')
|
|
@@ -49,17 +64,35 @@ export function registerHookCommands(program) {
|
|
|
49
64
|
.description('Create a new hook')
|
|
50
65
|
.argument('<vaultId>', 'Vault ID or slug')
|
|
51
66
|
.argument('<name>', 'Hook name')
|
|
52
|
-
.requiredOption('--trigger <event>', 'Trigger event (
|
|
67
|
+
.requiredOption('--trigger <event>', 'Trigger event (e.g. document.created, calendar.event.created, booking.confirmed)')
|
|
53
68
|
.requiredOption('--action <type>', 'Action type (webhook, ai_prompt, document_operation, auto_calendar_event, auto_booking_process)')
|
|
54
69
|
.requiredOption('--config <json>', 'Action configuration as JSON')
|
|
55
70
|
.option('--filter <json>', 'Trigger filter as JSON')
|
|
56
71
|
.addHelpText('after', `
|
|
57
|
-
VALID TRIGGER EVENTS
|
|
58
|
-
document.created
|
|
59
|
-
document.updated
|
|
60
|
-
document.deleted
|
|
61
|
-
document.moved
|
|
62
|
-
document.copied
|
|
72
|
+
VALID TRIGGER EVENTS (no wildcard — hooks fire on a single event type)
|
|
73
|
+
document.created Document was created
|
|
74
|
+
document.updated Document content was updated
|
|
75
|
+
document.deleted Document was deleted
|
|
76
|
+
document.moved Document was moved or renamed
|
|
77
|
+
document.copied Document was copied
|
|
78
|
+
directory.created Directory was created
|
|
79
|
+
document.overdue Document is past its due date
|
|
80
|
+
document.due-soon Document is due soon
|
|
81
|
+
calendar.event.created Calendar event was created
|
|
82
|
+
calendar.event.updated Calendar event was updated
|
|
83
|
+
calendar.event.deleted Calendar event was deleted
|
|
84
|
+
calendar.event.due Calendar event is due
|
|
85
|
+
calendar.event.overdue Calendar event is overdue
|
|
86
|
+
calendar.event.status_changed Calendar event status changed
|
|
87
|
+
booking.created Booking was created
|
|
88
|
+
booking.confirmed Booking was confirmed
|
|
89
|
+
booking.cancelled Booking was cancelled
|
|
90
|
+
booking.no_show Booking was marked no-show
|
|
91
|
+
booking.completed Booking was completed
|
|
92
|
+
booking.reminder Booking reminder sent
|
|
93
|
+
booking.rescheduled Booking was rescheduled
|
|
94
|
+
calendar.event.participant.added Participant added to event
|
|
95
|
+
calendar.event.participant.responded Participant responded to event
|
|
63
96
|
|
|
64
97
|
VALID ACTION TYPES
|
|
65
98
|
webhook Send an HTTP notification to a URL
|
|
@@ -71,6 +104,7 @@ VALID ACTION TYPES
|
|
|
71
104
|
EXAMPLES
|
|
72
105
|
lsvault hooks create <vaultId> my-hook --trigger document.created --action webhook --config '{"url":"https://example.com/hook"}'
|
|
73
106
|
lsvault hooks create <vaultId> ai-tag --trigger document.created --action ai_prompt --config '{"prompt":"Suggest tags"}'
|
|
107
|
+
lsvault hooks create <vaultId> booking-hook --trigger booking.confirmed --action webhook --config '{"url":"https://example.com/hook"}'
|
|
74
108
|
lsvault hooks create <vaultId> move-docs --trigger document.created --action document_operation --config '{"operation":"move","targetPath":"inbox/"}'`))
|
|
75
109
|
.action(async (vaultId, name, _opts) => {
|
|
76
110
|
const flags = resolveFlags(_opts);
|
|
@@ -95,12 +129,12 @@ EXAMPLES
|
|
|
95
129
|
return;
|
|
96
130
|
}
|
|
97
131
|
}
|
|
98
|
-
const VALID_TRIGGERS =
|
|
132
|
+
const VALID_TRIGGERS = VAULT_EVENT_TYPES;
|
|
99
133
|
const VALID_ACTIONS = ['webhook', 'ai_prompt', 'document_operation', 'auto_calendar_event', 'auto_booking_process'];
|
|
100
134
|
const trigger = String(_opts.trigger);
|
|
101
135
|
const action = String(_opts.action);
|
|
102
136
|
if (!VALID_TRIGGERS.includes(trigger)) {
|
|
103
|
-
out.error(`Invalid trigger "${trigger}". Valid values:
|
|
137
|
+
out.error(`Invalid trigger "${trigger}". Valid values: ${VALID_TRIGGERS.join(', ')}`);
|
|
104
138
|
process.exitCode = 1;
|
|
105
139
|
return;
|
|
106
140
|
}
|
package/dist/commands/sync.js
CHANGED
|
@@ -169,16 +169,24 @@ Sync modes:
|
|
|
169
169
|
const ignorePatterns = resolveIgnorePatterns(config.ignore, config.localPath);
|
|
170
170
|
const lastState = loadSyncState(config.id);
|
|
171
171
|
out.startSpinner('Scanning local files...');
|
|
172
|
-
const localFiles = scanLocalFiles(config.localPath, ignorePatterns);
|
|
172
|
+
const localFiles = scanLocalFiles(config.localPath, ignorePatterns, lastState);
|
|
173
173
|
out.debug(`Found ${Object.keys(localFiles).length} local files`);
|
|
174
174
|
out.startSpinner('Scanning remote files...');
|
|
175
|
-
const
|
|
175
|
+
const remoteResult = await scanRemoteFiles(client, config.vaultId, ignorePatterns, { remote: lastState.remote ?? {}, remoteListEtag: lastState.remoteListEtag });
|
|
176
|
+
const remoteFiles = remoteResult.files;
|
|
176
177
|
out.debug(`Found ${Object.keys(remoteFiles).length} remote files`);
|
|
177
178
|
out.startSpinner('Computing diff...');
|
|
178
179
|
const diff = computePullDiff(localFiles, remoteFiles, lastState);
|
|
179
180
|
const unchanged = Object.keys(remoteFiles).length - diff.downloads.length;
|
|
180
181
|
const totalOps = diff.downloads.length + diff.deletes.length;
|
|
182
|
+
// Persist the new list ETag regardless of whether there are changes
|
|
183
|
+
if (remoteResult.listEtag) {
|
|
184
|
+
lastState.remoteListEtag = remoteResult.listEtag;
|
|
185
|
+
}
|
|
181
186
|
if (totalOps === 0) {
|
|
187
|
+
if (remoteResult.listEtag) {
|
|
188
|
+
saveSyncState(lastState);
|
|
189
|
+
}
|
|
182
190
|
out.succeedSpinner('Everything is up to date');
|
|
183
191
|
if (flags.output === 'json') {
|
|
184
192
|
out.record({
|
|
@@ -218,7 +226,9 @@ Sync modes:
|
|
|
218
226
|
if (progress.phase === 'transferring' && progress.currentFile) {
|
|
219
227
|
out.startSpinner(`[${progress.current}/${progress.total}] ${progress.currentFile}`);
|
|
220
228
|
}
|
|
221
|
-
}, concurrency)
|
|
229
|
+
}, concurrency, (file) => {
|
|
230
|
+
out.startSpinner(`Rate limited — waiting and retrying… (${file})`);
|
|
231
|
+
});
|
|
222
232
|
if (result.errors.length > 0) {
|
|
223
233
|
out.failSpinner(`Pull completed with ${result.errors.length} error(s)`);
|
|
224
234
|
for (const err of result.errors) {
|
|
@@ -259,16 +269,24 @@ Sync modes:
|
|
|
259
269
|
const ignorePatterns = resolveIgnorePatterns(config.ignore, config.localPath);
|
|
260
270
|
const lastState = loadSyncState(config.id);
|
|
261
271
|
out.startSpinner('Scanning local files...');
|
|
262
|
-
const localFiles = scanLocalFiles(config.localPath, ignorePatterns);
|
|
272
|
+
const localFiles = scanLocalFiles(config.localPath, ignorePatterns, lastState);
|
|
263
273
|
out.debug(`Found ${Object.keys(localFiles).length} local files`);
|
|
264
274
|
out.startSpinner('Scanning remote files...');
|
|
265
|
-
const
|
|
275
|
+
const remoteResult = await scanRemoteFiles(client, config.vaultId, ignorePatterns, { remote: lastState.remote ?? {}, remoteListEtag: lastState.remoteListEtag });
|
|
276
|
+
const remoteFiles = remoteResult.files;
|
|
266
277
|
out.debug(`Found ${Object.keys(remoteFiles).length} remote files`);
|
|
267
278
|
out.startSpinner('Computing diff...');
|
|
268
279
|
const diff = computePushDiff(localFiles, remoteFiles, lastState);
|
|
269
280
|
const unchanged = Object.keys(localFiles).length - diff.uploads.length;
|
|
270
281
|
const totalOps = diff.uploads.length + diff.deletes.length;
|
|
282
|
+
// Persist the new list ETag regardless of whether there are changes
|
|
283
|
+
if (remoteResult.listEtag) {
|
|
284
|
+
lastState.remoteListEtag = remoteResult.listEtag;
|
|
285
|
+
}
|
|
271
286
|
if (totalOps === 0) {
|
|
287
|
+
if (remoteResult.listEtag) {
|
|
288
|
+
saveSyncState(lastState);
|
|
289
|
+
}
|
|
272
290
|
out.succeedSpinner('Everything is up to date');
|
|
273
291
|
if (flags.output === 'json') {
|
|
274
292
|
out.record({
|
|
@@ -308,7 +326,9 @@ Sync modes:
|
|
|
308
326
|
if (progress.phase === 'transferring' && progress.currentFile) {
|
|
309
327
|
out.startSpinner(`[${progress.current}/${progress.total}] ${progress.currentFile}`);
|
|
310
328
|
}
|
|
311
|
-
}, concurrency)
|
|
329
|
+
}, concurrency, (file) => {
|
|
330
|
+
out.startSpinner(`Rate limited — waiting and retrying… (${file})`);
|
|
331
|
+
});
|
|
312
332
|
if (result.errors.length > 0) {
|
|
313
333
|
out.failSpinner(`Push completed with ${result.errors.length} error(s)`);
|
|
314
334
|
for (const err of result.errors) {
|
|
@@ -348,8 +368,9 @@ Sync modes:
|
|
|
348
368
|
const ignorePatterns = resolveIgnorePatterns(config.ignore, config.localPath);
|
|
349
369
|
const lastState = loadSyncState(config.id);
|
|
350
370
|
out.startSpinner('Scanning...');
|
|
351
|
-
const localFiles = scanLocalFiles(config.localPath, ignorePatterns);
|
|
352
|
-
const
|
|
371
|
+
const localFiles = scanLocalFiles(config.localPath, ignorePatterns, lastState);
|
|
372
|
+
const remoteResult = await scanRemoteFiles(client, config.vaultId, ignorePatterns, { remote: lastState.remote ?? {}, remoteListEtag: lastState.remoteListEtag });
|
|
373
|
+
const remoteFiles = remoteResult.files;
|
|
353
374
|
const pullDiff = computePullDiff(localFiles, remoteFiles, lastState);
|
|
354
375
|
const pushDiff = computePushDiff(localFiles, remoteFiles, lastState);
|
|
355
376
|
out.stopSpinner();
|
|
@@ -3,6 +3,22 @@ import { getClientAsync } from '../client.js';
|
|
|
3
3
|
import { addGlobalFlags, resolveFlags } from '../utils/flags.js';
|
|
4
4
|
import { createOutput, handleError } from '../utils/output.js';
|
|
5
5
|
import { resolveVaultId } from '../utils/resolve-vault.js';
|
|
6
|
+
/**
|
|
7
|
+
* Valid webhook event names, including the `*` wildcard. Inlined from the
|
|
8
|
+
* internal vault-shared package (not published to npm) so the standalone CLI
|
|
9
|
+
* build has no unpublishable workspace dependency. Keep in sync with
|
|
10
|
+
* WEBHOOK_EVENT_TYPES in packages/shared/src/constants.ts.
|
|
11
|
+
*/
|
|
12
|
+
const WEBHOOK_EVENT_TYPES = [
|
|
13
|
+
'*',
|
|
14
|
+
'document.created', 'document.updated', 'document.deleted', 'document.moved', 'document.copied',
|
|
15
|
+
'directory.created', 'document.overdue', 'document.due-soon',
|
|
16
|
+
'calendar.event.created', 'calendar.event.updated', 'calendar.event.deleted', 'calendar.event.due',
|
|
17
|
+
'calendar.event.overdue', 'calendar.event.status_changed',
|
|
18
|
+
'booking.created', 'booking.confirmed', 'booking.cancelled', 'booking.no_show', 'booking.completed',
|
|
19
|
+
'booking.reminder', 'booking.rescheduled',
|
|
20
|
+
'calendar.event.participant.added', 'calendar.event.participant.responded',
|
|
21
|
+
];
|
|
6
22
|
export function registerWebhookCommands(program) {
|
|
7
23
|
const webhooks = program.command('webhooks').description('Manage vault webhooks');
|
|
8
24
|
addGlobalFlags(webhooks.command('list')
|
|
@@ -46,19 +62,38 @@ export function registerWebhookCommands(program) {
|
|
|
46
62
|
.description('Create a new webhook')
|
|
47
63
|
.argument('<vaultId>', 'Vault ID or slug')
|
|
48
64
|
.argument('<url>', 'Webhook endpoint URL')
|
|
49
|
-
.option('--events <events>', 'Comma-separated events (document.created,
|
|
65
|
+
.option('--events <events>', 'Comma-separated events (document.created, calendar.event.created, booking.created, *, etc.)', 'document.created,document.updated,document.deleted')
|
|
50
66
|
.addHelpText('after', `
|
|
51
67
|
VALID EVENT NAMES
|
|
52
|
-
document.created
|
|
53
|
-
document.updated
|
|
54
|
-
document.deleted
|
|
55
|
-
document.moved
|
|
56
|
-
document.copied
|
|
57
|
-
|
|
68
|
+
document.created Document was created
|
|
69
|
+
document.updated Document content was updated
|
|
70
|
+
document.deleted Document was deleted
|
|
71
|
+
document.moved Document was moved or renamed
|
|
72
|
+
document.copied Document was copied
|
|
73
|
+
directory.created Directory was created
|
|
74
|
+
document.overdue Document is past its due date
|
|
75
|
+
document.due-soon Document is due soon
|
|
76
|
+
calendar.event.created Calendar event was created
|
|
77
|
+
calendar.event.updated Calendar event was updated
|
|
78
|
+
calendar.event.deleted Calendar event was deleted
|
|
79
|
+
calendar.event.due Calendar event is due
|
|
80
|
+
calendar.event.overdue Calendar event is overdue
|
|
81
|
+
calendar.event.status_changed Calendar event status changed
|
|
82
|
+
booking.created Booking was created
|
|
83
|
+
booking.confirmed Booking was confirmed
|
|
84
|
+
booking.cancelled Booking was cancelled
|
|
85
|
+
booking.no_show Booking was marked no-show
|
|
86
|
+
booking.completed Booking was completed
|
|
87
|
+
booking.reminder Booking reminder sent
|
|
88
|
+
booking.rescheduled Booking was rescheduled
|
|
89
|
+
calendar.event.participant.added Participant added to event
|
|
90
|
+
calendar.event.participant.responded Participant responded to event
|
|
91
|
+
* All events
|
|
58
92
|
|
|
59
93
|
EXAMPLES
|
|
60
94
|
lsvault webhooks create <vaultId> https://example.com/hook
|
|
61
95
|
lsvault webhooks create <vaultId> https://example.com/hook --events "document.created,document.deleted"
|
|
96
|
+
lsvault webhooks create <vaultId> https://example.com/hook --events "calendar.event.created,booking.created"
|
|
62
97
|
lsvault webhooks create <vaultId> https://example.com/hook --events "*"`))
|
|
63
98
|
.action(async (vaultId, url, _opts) => {
|
|
64
99
|
const flags = resolveFlags(_opts);
|
|
@@ -68,7 +103,7 @@ EXAMPLES
|
|
|
68
103
|
process.exitCode = 1;
|
|
69
104
|
return;
|
|
70
105
|
}
|
|
71
|
-
const VALID_EVENTS =
|
|
106
|
+
const VALID_EVENTS = WEBHOOK_EVENT_TYPES;
|
|
72
107
|
const events = String(_opts.events || 'document.created,document.updated,document.deleted').split(',').map((e) => e.trim());
|
|
73
108
|
const invalid = events.filter(e => !VALID_EVENTS.includes(e));
|
|
74
109
|
if (invalid.length > 0) {
|
|
@@ -11,7 +11,7 @@ import { removePid } from './daemon.js';
|
|
|
11
11
|
import { loadConfigAsync } from '../config.js';
|
|
12
12
|
import { LifestreamVaultClient } from '@lifestreamdynamics/vault-sdk';
|
|
13
13
|
import { scanLocalFiles, scanRemoteFiles, computePushDiff, computePullDiff, executePush, executePull } from './engine.js';
|
|
14
|
-
import { loadSyncState } from './state.js';
|
|
14
|
+
import { loadSyncState, saveSyncState } from './state.js';
|
|
15
15
|
const managed = [];
|
|
16
16
|
function log(msg) {
|
|
17
17
|
const ts = new Date().toISOString();
|
|
@@ -50,8 +50,13 @@ async function start() {
|
|
|
50
50
|
log(`Reconciling ${config.id.slice(0, 8)} (${config.mode} mode)...`);
|
|
51
51
|
const ignorePatterns = resolveIgnorePatterns(config.ignore, config.localPath);
|
|
52
52
|
const lastState = loadSyncState(config.id);
|
|
53
|
-
const localFiles = scanLocalFiles(config.localPath, ignorePatterns);
|
|
54
|
-
const
|
|
53
|
+
const localFiles = scanLocalFiles(config.localPath, ignorePatterns, lastState);
|
|
54
|
+
const remoteResult = await scanRemoteFiles(client, config.vaultId, ignorePatterns, { remote: lastState.remote ?? {}, remoteListEtag: lastState.remoteListEtag });
|
|
55
|
+
const remoteFiles = remoteResult.files;
|
|
56
|
+
if (remoteResult.listEtag) {
|
|
57
|
+
lastState.remoteListEtag = remoteResult.listEtag;
|
|
58
|
+
saveSyncState(lastState);
|
|
59
|
+
}
|
|
55
60
|
let pushed = 0;
|
|
56
61
|
let pulled = 0;
|
|
57
62
|
let deleted = 0;
|
package/dist/sync/engine.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LifestreamVaultClient } from '@lifestreamdynamics/vault-sdk';
|
|
2
|
-
import type { SyncConfig, FileState } from './types.js';
|
|
2
|
+
import type { SyncConfig, SyncState, FileState } from './types.js';
|
|
3
3
|
import { computePullDiff, computePushDiff, type SyncDiff, type SyncDiffEntry } from './diff.js';
|
|
4
4
|
export interface SyncProgress {
|
|
5
5
|
phase: 'scanning' | 'computing' | 'transferring' | 'complete';
|
|
@@ -23,13 +23,40 @@ export interface SyncResult {
|
|
|
23
23
|
/**
|
|
24
24
|
* Scan local directory recursively for .md files.
|
|
25
25
|
* Returns a map of relative doc paths -> FileState.
|
|
26
|
+
*
|
|
27
|
+
* When `lastState` is provided, files whose stat mtime and size match the
|
|
28
|
+
* persisted entry skip the readFileSync + hash step entirely (D-2 fast-path).
|
|
26
29
|
*/
|
|
27
|
-
export declare function scanLocalFiles(localPath: string, ignorePatterns: string[]): Record<string, FileState>;
|
|
30
|
+
export declare function scanLocalFiles(localPath: string, ignorePatterns: string[], lastState?: SyncState): Record<string, FileState>;
|
|
31
|
+
/** Result type for {@link scanRemoteFiles}. */
|
|
32
|
+
export interface ScanRemoteResult {
|
|
33
|
+
/** Map of doc paths -> FileState for all non-ignored remote files. */
|
|
34
|
+
files: Record<string, FileState>;
|
|
35
|
+
/** List ETag from this response, for use in the next call. */
|
|
36
|
+
listEtag: string;
|
|
37
|
+
/** True when the server confirmed the vault is unchanged (304 fast-path). */
|
|
38
|
+
vaultUnchanged: boolean;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Scan remote vault for document list using the syncList fast-path.
|
|
42
|
+
*
|
|
43
|
+
* When `knownState` is provided (with hashes and optionally a prior listEtag),
|
|
44
|
+
* the server may respond 304 and `vaultUnchanged` will be true — in that case
|
|
45
|
+
* `files` is rebuilt from `knownState.remote` without any per-doc network call.
|
|
46
|
+
*
|
|
47
|
+
* When `knownState` is omitted a full list is always fetched (backward-compat).
|
|
48
|
+
*/
|
|
49
|
+
export declare function scanRemoteFiles(client: LifestreamVaultClient, vaultId: string, ignorePatterns: string[], knownState?: {
|
|
50
|
+
remote: Record<string, FileState>;
|
|
51
|
+
remoteListEtag?: string;
|
|
52
|
+
}): Promise<ScanRemoteResult>;
|
|
28
53
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
54
|
+
* Called when the SDK signals a 429 response while transferring a file.
|
|
55
|
+
* The SDK will back off and retry automatically; the CLI uses this to
|
|
56
|
+
* update the spinner text so the user sees "Rate limited — waiting and retrying…"
|
|
57
|
+
* rather than an apparently frozen transfer.
|
|
31
58
|
*/
|
|
32
|
-
export
|
|
59
|
+
export type ThrottleCallback = (file: string) => void;
|
|
33
60
|
/**
|
|
34
61
|
* Validates and clamps a user-supplied concurrency value. Throws on invalid
|
|
35
62
|
* values so the CLI can surface a clear error before kicking off any I/O.
|
|
@@ -38,9 +65,22 @@ export declare function resolveConcurrency(value: number | undefined): number;
|
|
|
38
65
|
/**
|
|
39
66
|
* Execute a pull operation: download remote changes to local.
|
|
40
67
|
*/
|
|
41
|
-
export declare function executePull(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback, concurrency?: number): Promise<SyncResult>;
|
|
68
|
+
export declare function executePull(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback, concurrency?: number, onThrottle?: ThrottleCallback): Promise<SyncResult>;
|
|
42
69
|
/**
|
|
43
70
|
* Execute a push operation: upload local changes to remote.
|
|
44
71
|
*/
|
|
45
|
-
export declare function executePush(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback, concurrency?: number): Promise<SyncResult>;
|
|
72
|
+
export declare function executePush(client: LifestreamVaultClient, config: SyncConfig, diff: SyncDiff, onProgress?: ProgressCallback, concurrency?: number, onThrottle?: ThrottleCallback): Promise<SyncResult>;
|
|
73
|
+
/**
|
|
74
|
+
* Returns true when an error represents an HTTP 429 (Too Many Requests / rate
|
|
75
|
+
* limited) response. The SDK retries 429s transparently; a 429 reaching here
|
|
76
|
+
* means all retry attempts were exhausted.
|
|
77
|
+
*
|
|
78
|
+
* Prefers the structured status code the SDK attaches (RateLimitError sets
|
|
79
|
+
* `statusCode = 429`) and only falls back to a narrow message match. The
|
|
80
|
+
* message fallback is intentionally strict — it does NOT match "rate limit" or
|
|
81
|
+
* "throttle" loosely, since those words appear in unrelated errors (e.g. an
|
|
82
|
+
* authorization message mentioning a rate-limited account) and a false positive
|
|
83
|
+
* would suppress a real error.
|
|
84
|
+
*/
|
|
85
|
+
export declare function isThrottleError(err: unknown): boolean;
|
|
46
86
|
export { computePullDiff, computePushDiff, type SyncDiff, type SyncDiffEntry };
|
package/dist/sync/engine.js
CHANGED
|
@@ -11,8 +11,11 @@ import { computePullDiff, computePushDiff } from './diff.js';
|
|
|
11
11
|
/**
|
|
12
12
|
* Scan local directory recursively for .md files.
|
|
13
13
|
* Returns a map of relative doc paths -> FileState.
|
|
14
|
+
*
|
|
15
|
+
* When `lastState` is provided, files whose stat mtime and size match the
|
|
16
|
+
* persisted entry skip the readFileSync + hash step entirely (D-2 fast-path).
|
|
14
17
|
*/
|
|
15
|
-
export function scanLocalFiles(localPath, ignorePatterns) {
|
|
18
|
+
export function scanLocalFiles(localPath, ignorePatterns, lastState) {
|
|
16
19
|
const files = {};
|
|
17
20
|
function walk(dir, prefix) {
|
|
18
21
|
if (!fs.existsSync(dir))
|
|
@@ -28,14 +31,21 @@ export function scanLocalFiles(localPath, ignorePatterns) {
|
|
|
28
31
|
else if (entry.isFile() && entry.name.endsWith('.md')) {
|
|
29
32
|
if (!shouldIgnore(relPath, ignorePatterns)) {
|
|
30
33
|
const absPath = path.join(dir, entry.name);
|
|
31
|
-
const content = fs.readFileSync(absPath);
|
|
32
34
|
const stat = fs.statSync(absPath);
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
hash
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
const stored = lastState?.local?.[relPath];
|
|
36
|
+
if (stored && stat.size === stored.size && stat.mtime.toISOString() === stored.mtime) {
|
|
37
|
+
// mtime+size unchanged — reuse persisted hash, skip file read
|
|
38
|
+
files[relPath] = stored;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
const content = fs.readFileSync(absPath);
|
|
42
|
+
files[relPath] = {
|
|
43
|
+
path: relPath,
|
|
44
|
+
hash: hashFileContent(content),
|
|
45
|
+
mtime: stat.mtime.toISOString(),
|
|
46
|
+
size: stat.size,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
39
49
|
}
|
|
40
50
|
}
|
|
41
51
|
}
|
|
@@ -44,23 +54,61 @@ export function scanLocalFiles(localPath, ignorePatterns) {
|
|
|
44
54
|
return files;
|
|
45
55
|
}
|
|
46
56
|
/**
|
|
47
|
-
* Scan remote vault for document list.
|
|
48
|
-
*
|
|
57
|
+
* Scan remote vault for document list using the syncList fast-path.
|
|
58
|
+
*
|
|
59
|
+
* When `knownState` is provided (with hashes and optionally a prior listEtag),
|
|
60
|
+
* the server may respond 304 and `vaultUnchanged` will be true — in that case
|
|
61
|
+
* `files` is rebuilt from `knownState.remote` without any per-doc network call.
|
|
62
|
+
*
|
|
63
|
+
* When `knownState` is omitted a full list is always fetched (backward-compat).
|
|
49
64
|
*/
|
|
50
|
-
export async function scanRemoteFiles(client, vaultId, ignorePatterns) {
|
|
51
|
-
const
|
|
65
|
+
export async function scanRemoteFiles(client, vaultId, ignorePatterns, knownState) {
|
|
66
|
+
const sdkKnownState = {
|
|
67
|
+
hashes: knownState
|
|
68
|
+
? Object.fromEntries(Object.entries(knownState.remote).map(([k, v]) => [k, v.hash]))
|
|
69
|
+
: {},
|
|
70
|
+
listEtag: knownState?.remoteListEtag,
|
|
71
|
+
};
|
|
72
|
+
const sync = await client.documents.syncList(vaultId, sdkKnownState);
|
|
73
|
+
if (sync.vaultUnchanged) {
|
|
74
|
+
// Server confirmed nothing changed — rebuild files from persisted state.
|
|
75
|
+
const files = {};
|
|
76
|
+
if (knownState) {
|
|
77
|
+
for (const [docPath, fs_] of Object.entries(knownState.remote)) {
|
|
78
|
+
if (!shouldIgnore(docPath, ignorePatterns)) {
|
|
79
|
+
files[docPath] = fs_;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return { files, listEtag: sync.listEtag, vaultUnchanged: true };
|
|
84
|
+
}
|
|
85
|
+
// Build files from changes + unchanged paths.
|
|
52
86
|
const files = {};
|
|
53
|
-
for (const
|
|
54
|
-
if (!shouldIgnore(
|
|
55
|
-
files[
|
|
56
|
-
path:
|
|
57
|
-
hash:
|
|
58
|
-
mtime:
|
|
59
|
-
|
|
87
|
+
for (const change of sync.changes) {
|
|
88
|
+
if (!shouldIgnore(change.path, ignorePatterns)) {
|
|
89
|
+
files[change.path] = {
|
|
90
|
+
path: change.path,
|
|
91
|
+
hash: change.contentHash,
|
|
92
|
+
mtime: change.fileModifiedAt,
|
|
93
|
+
// sizeBytes is not provided by syncList change objects; use 0 as a
|
|
94
|
+
// fallback — size is only used for progress-bar estimation in the diff.
|
|
95
|
+
size: 0,
|
|
60
96
|
};
|
|
61
97
|
}
|
|
62
98
|
}
|
|
63
|
-
|
|
99
|
+
for (const unchangedPath of sync.unchanged) {
|
|
100
|
+
if (!shouldIgnore(unchangedPath, ignorePatterns)) {
|
|
101
|
+
// Reuse the persisted FileState so size is preserved for progress reporting.
|
|
102
|
+
const stored = knownState?.remote?.[unchangedPath];
|
|
103
|
+
files[unchangedPath] = stored ?? {
|
|
104
|
+
path: unchangedPath,
|
|
105
|
+
hash: sdkKnownState.hashes[unchangedPath] ?? '',
|
|
106
|
+
mtime: '',
|
|
107
|
+
size: 0,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return { files, listEtag: sync.listEtag, vaultUnchanged: false };
|
|
64
112
|
}
|
|
65
113
|
/**
|
|
66
114
|
* Write a file atomically using a temp file + rename.
|
|
@@ -95,7 +143,7 @@ export function resolveConcurrency(value) {
|
|
|
95
143
|
* Handles result initialization, state loading, progress callbacks,
|
|
96
144
|
* quota error handling, state saving, and lastSync update.
|
|
97
145
|
*/
|
|
98
|
-
async function executeSyncOperation(config, diff, handlers, onProgress, concurrency = DEFAULT_TRANSFER_CONCURRENCY) {
|
|
146
|
+
async function executeSyncOperation(config, diff, handlers, onProgress, concurrency = DEFAULT_TRANSFER_CONCURRENCY, onThrottle) {
|
|
99
147
|
const result = {
|
|
100
148
|
filesUploaded: 0,
|
|
101
149
|
filesDownloaded: 0,
|
|
@@ -122,7 +170,7 @@ async function executeSyncOperation(config, diff, handlers, onProgress, concurre
|
|
|
122
170
|
totalBytes: diff.totalBytes,
|
|
123
171
|
});
|
|
124
172
|
try {
|
|
125
|
-
const content = await handlers.transferFile(entry, config);
|
|
173
|
+
const content = await handlers.transferFile(entry, config, onThrottle);
|
|
126
174
|
result[handlers.transferCounterKey]++;
|
|
127
175
|
result.bytesTransferred += entry.sizeBytes;
|
|
128
176
|
// Update state
|
|
@@ -140,6 +188,11 @@ async function executeSyncOperation(config, diff, handlers, onProgress, concurre
|
|
|
140
188
|
if (isQuotaError(message)) {
|
|
141
189
|
stopSubmitting = true;
|
|
142
190
|
}
|
|
191
|
+
// 429 errors that reach here have already exhausted SDK-level retries.
|
|
192
|
+
// Stop submitting new work to avoid hammering a still-throttled API.
|
|
193
|
+
if (isThrottleError(err)) {
|
|
194
|
+
stopSubmitting = true;
|
|
195
|
+
}
|
|
143
196
|
}
|
|
144
197
|
}
|
|
145
198
|
// Bounded async pool. Workers race for entries off the queue tail; once
|
|
@@ -190,13 +243,13 @@ async function executeSyncOperation(config, diff, handlers, onProgress, concurre
|
|
|
190
243
|
/**
|
|
191
244
|
* Execute a pull operation: download remote changes to local.
|
|
192
245
|
*/
|
|
193
|
-
export async function executePull(client, config, diff, onProgress, concurrency) {
|
|
246
|
+
export async function executePull(client, config, diff, onProgress, concurrency, onThrottle) {
|
|
194
247
|
return executeSyncOperation(config, diff, {
|
|
195
248
|
transfers: diff.downloads,
|
|
196
249
|
deletes: diff.deletes,
|
|
197
250
|
transferCounterKey: 'filesDownloaded',
|
|
198
|
-
async transferFile(entry, cfg) {
|
|
199
|
-
const { content } = await retryWithBackoff(() => client.documents.get(cfg.vaultId, entry.path));
|
|
251
|
+
async transferFile(entry, cfg, throttleCallback) {
|
|
252
|
+
const { content } = await retryWithBackoff(() => client.documents.get(cfg.vaultId, entry.path), throttleCallback ? () => throttleCallback(entry.path) : undefined);
|
|
200
253
|
const localFile = path.join(cfg.localPath, entry.path);
|
|
201
254
|
const localDir = path.dirname(localFile);
|
|
202
255
|
if (!fs.existsSync(localDir)) {
|
|
@@ -211,31 +264,40 @@ export async function executePull(client, config, diff, onProgress, concurrency)
|
|
|
211
264
|
fs.unlinkSync(localFile);
|
|
212
265
|
}
|
|
213
266
|
},
|
|
214
|
-
}, onProgress, concurrency);
|
|
267
|
+
}, onProgress, concurrency, onThrottle);
|
|
215
268
|
}
|
|
216
269
|
/**
|
|
217
270
|
* Execute a push operation: upload local changes to remote.
|
|
218
271
|
*/
|
|
219
|
-
export async function executePush(client, config, diff, onProgress, concurrency) {
|
|
272
|
+
export async function executePush(client, config, diff, onProgress, concurrency, onThrottle) {
|
|
220
273
|
return executeSyncOperation(config, diff, {
|
|
221
274
|
transfers: diff.uploads,
|
|
222
275
|
deletes: diff.deletes,
|
|
223
276
|
transferCounterKey: 'filesUploaded',
|
|
224
|
-
async transferFile(entry, cfg) {
|
|
277
|
+
async transferFile(entry, cfg, throttleCallback) {
|
|
225
278
|
const localFile = path.join(cfg.localPath, entry.path);
|
|
226
279
|
const content = fs.readFileSync(localFile, 'utf-8');
|
|
227
|
-
await retryWithBackoff(() => client.documents.put(cfg.vaultId, entry.path, content));
|
|
280
|
+
await retryWithBackoff(() => client.documents.put(cfg.vaultId, entry.path, content), throttleCallback ? () => throttleCallback(entry.path) : undefined);
|
|
228
281
|
return content;
|
|
229
282
|
},
|
|
230
283
|
async deleteFile(entry, cfg) {
|
|
231
284
|
await retryWithBackoff(() => client.documents.delete(cfg.vaultId, entry.path));
|
|
232
285
|
},
|
|
233
|
-
}, onProgress, concurrency);
|
|
286
|
+
}, onProgress, concurrency, onThrottle);
|
|
234
287
|
}
|
|
235
288
|
/**
|
|
236
|
-
* Retry a function with exponential backoff (max 3 retries)
|
|
289
|
+
* Retry a function with exponential backoff (max 3 retries) for transient
|
|
290
|
+
* network errors. Throttle (429) and quota/permission errors are NOT retried
|
|
291
|
+
* here — the SDK already retries 429s transparently (ky retry config), and
|
|
292
|
+
* quota/permission errors are not recoverable by retrying.
|
|
293
|
+
*
|
|
294
|
+
* @param onThrottle - Optional callback to invoke when a 429 is observed.
|
|
295
|
+
* The SDK will retry automatically; this is called so the CLI can update
|
|
296
|
+
* its spinner text to "Rate limited — waiting and retrying…".
|
|
297
|
+
* The parameter value is the file path being transferred.
|
|
237
298
|
*/
|
|
238
|
-
async function retryWithBackoff(fn,
|
|
299
|
+
async function retryWithBackoff(fn, onThrottle) {
|
|
300
|
+
const maxRetries = 3;
|
|
239
301
|
let lastError;
|
|
240
302
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
241
303
|
try {
|
|
@@ -244,7 +306,14 @@ async function retryWithBackoff(fn, maxRetries = 3) {
|
|
|
244
306
|
catch (err) {
|
|
245
307
|
lastError = err;
|
|
246
308
|
const message = err instanceof Error ? err.message : String(err);
|
|
247
|
-
//
|
|
309
|
+
// Throttle errors: the SDK already exhausted its own retry budget with
|
|
310
|
+
// proper Retry-After backoff. Don't layer another retry loop on top —
|
|
311
|
+
// that would ignore the server's backoff signal and hammer the API.
|
|
312
|
+
if (isThrottleError(err)) {
|
|
313
|
+
onThrottle?.('');
|
|
314
|
+
throw err;
|
|
315
|
+
}
|
|
316
|
+
// Don't retry on other non-transient errors
|
|
248
317
|
if (isQuotaError(message) || isPermissionError(message)) {
|
|
249
318
|
throw err;
|
|
250
319
|
}
|
|
@@ -256,6 +325,28 @@ async function retryWithBackoff(fn, maxRetries = 3) {
|
|
|
256
325
|
}
|
|
257
326
|
throw lastError;
|
|
258
327
|
}
|
|
328
|
+
/**
|
|
329
|
+
* Returns true when an error represents an HTTP 429 (Too Many Requests / rate
|
|
330
|
+
* limited) response. The SDK retries 429s transparently; a 429 reaching here
|
|
331
|
+
* means all retry attempts were exhausted.
|
|
332
|
+
*
|
|
333
|
+
* Prefers the structured status code the SDK attaches (RateLimitError sets
|
|
334
|
+
* `statusCode = 429`) and only falls back to a narrow message match. The
|
|
335
|
+
* message fallback is intentionally strict — it does NOT match "rate limit" or
|
|
336
|
+
* "throttle" loosely, since those words appear in unrelated errors (e.g. an
|
|
337
|
+
* authorization message mentioning a rate-limited account) and a false positive
|
|
338
|
+
* would suppress a real error.
|
|
339
|
+
*/
|
|
340
|
+
export function isThrottleError(err) {
|
|
341
|
+
if (err && typeof err === 'object') {
|
|
342
|
+
const code = err.statusCode
|
|
343
|
+
?? err.status;
|
|
344
|
+
if (code === 429)
|
|
345
|
+
return true;
|
|
346
|
+
}
|
|
347
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
348
|
+
return /\b429\b|too many requests/i.test(message);
|
|
349
|
+
}
|
|
259
350
|
function isQuotaError(message) {
|
|
260
351
|
return /quota|storage limit|limit exceeded/i.test(message);
|
|
261
352
|
}
|
package/dist/sync/ignore.js
CHANGED
|
@@ -50,6 +50,7 @@ export function resolveIgnorePatterns(configIgnore, localPath) {
|
|
|
50
50
|
* The docPath should be a relative path using forward slashes.
|
|
51
51
|
*/
|
|
52
52
|
export function shouldIgnore(docPath, patterns) {
|
|
53
|
+
const basename = path.posix.basename(docPath);
|
|
53
54
|
for (const pattern of patterns) {
|
|
54
55
|
// Directory patterns (ending with /)
|
|
55
56
|
if (pattern.endsWith('/')) {
|
|
@@ -63,7 +64,6 @@ export function shouldIgnore(docPath, patterns) {
|
|
|
63
64
|
return true;
|
|
64
65
|
}
|
|
65
66
|
// Also check basename for file-level patterns (e.g., ".DS_Store" matches "sub/.DS_Store")
|
|
66
|
-
const basename = path.posix.basename(docPath);
|
|
67
67
|
if (minimatch(basename, pattern, { dot: true })) {
|
|
68
68
|
return true;
|
|
69
69
|
}
|
|
@@ -8,6 +8,7 @@ import { shouldIgnore } from './ignore.js';
|
|
|
8
8
|
import { loadSyncState, saveSyncState, hashFileContent, buildRemoteFileState } from './state.js';
|
|
9
9
|
import { updateLastSync } from './config.js';
|
|
10
10
|
import { resolveConflict, detectConflict, createConflictFile, formatConflictLog } from './conflict.js';
|
|
11
|
+
import { isThrottleError } from './engine.js';
|
|
11
12
|
/**
|
|
12
13
|
* Creates and starts a remote poller for a sync configuration.
|
|
13
14
|
* Returns a stop function.
|
|
@@ -22,62 +23,91 @@ export function createRemotePoller(client, config, options) {
|
|
|
22
23
|
return; // Skip if previous poll still in progress
|
|
23
24
|
polling = true;
|
|
24
25
|
try {
|
|
25
|
-
const remoteDocs = await client.documents.list(config.vaultId);
|
|
26
26
|
const state = loadSyncState(config.id);
|
|
27
27
|
let changes = 0;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
28
|
+
let stateMutated = false;
|
|
29
|
+
// Build the known state from persisted remote hashes + stored list ETag.
|
|
30
|
+
const known = {
|
|
31
|
+
hashes: Object.fromEntries(Object.entries(state.remote).map(([k, v]) => [k, v.hash])),
|
|
32
|
+
listEtag: state.remoteListEtag,
|
|
33
|
+
};
|
|
34
|
+
const sync = await client.documents.syncList(config.vaultId, known);
|
|
35
|
+
// Steady-state: server confirmed nothing changed — skip all per-doc work.
|
|
36
|
+
if (sync.vaultUnchanged)
|
|
37
|
+
return;
|
|
38
|
+
// Persist the new list ETag so the next poll can 304.
|
|
39
|
+
state.remoteListEtag = sync.listEtag;
|
|
40
|
+
stateMutated = true;
|
|
41
|
+
// Process added/changed documents.
|
|
42
|
+
for (const change of sync.changes) {
|
|
43
|
+
if (shouldIgnore(change.path, ignorePatterns))
|
|
43
44
|
continue;
|
|
45
|
+
const lastRemote = state.remote[change.path];
|
|
46
|
+
let content;
|
|
47
|
+
let remoteHash;
|
|
48
|
+
if (lastRemote) {
|
|
49
|
+
// Conditional GET — server can still 304 us if our hash is current
|
|
50
|
+
// (rare: syncList classified this as 'changed' but the GET hash matches
|
|
51
|
+
// our last-known hash — possible race between list and get).
|
|
52
|
+
const result = await client.documents.get(config.vaultId, change.path, {
|
|
53
|
+
ifNoneMatch: `"${lastRemote.hash}"`,
|
|
54
|
+
});
|
|
55
|
+
if (result.notModified) {
|
|
56
|
+
// Server confirmed our copy is current despite differing list hash.
|
|
57
|
+
// Update mtime only; no file write needed.
|
|
58
|
+
if (lastRemote.mtime !== change.fileModifiedAt) {
|
|
59
|
+
state.remote[change.path] = { ...lastRemote, mtime: change.fileModifiedAt };
|
|
60
|
+
stateMutated = true;
|
|
61
|
+
}
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
content = result.content;
|
|
65
|
+
remoteHash = hashFileContent(content);
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// First-time entry: unconditional GET.
|
|
69
|
+
const fetched = await client.documents.get(config.vaultId, change.path);
|
|
70
|
+
content = fetched.content;
|
|
71
|
+
remoteHash = hashFileContent(content);
|
|
44
72
|
}
|
|
45
|
-
const localFile = path.join(config.localPath,
|
|
73
|
+
const localFile = path.join(config.localPath, change.path);
|
|
46
74
|
const localExists = fs.existsSync(localFile);
|
|
47
75
|
if (localExists) {
|
|
48
76
|
const localContent = fs.readFileSync(localFile, 'utf-8');
|
|
49
77
|
const localHash = hashFileContent(localContent);
|
|
50
78
|
if (localHash === remoteHash) {
|
|
51
79
|
// Content is already the same — just update state
|
|
52
|
-
state.local[
|
|
53
|
-
state.remote[
|
|
80
|
+
state.local[change.path] = { path: change.path, hash: localHash, mtime: new Date().toISOString(), size: Buffer.byteLength(localContent) };
|
|
81
|
+
state.remote[change.path] = buildRemoteFileState(change.path, content, change.fileModifiedAt);
|
|
82
|
+
stateMutated = true;
|
|
54
83
|
continue;
|
|
55
84
|
}
|
|
56
85
|
// Check for conflict
|
|
57
|
-
const lastLocal = state.local[
|
|
58
|
-
const localState = { path:
|
|
59
|
-
const remoteState = { path:
|
|
86
|
+
const lastLocal = state.local[change.path];
|
|
87
|
+
const localState = { path: change.path, hash: localHash, mtime: fs.statSync(localFile).mtime.toISOString(), size: Buffer.byteLength(localContent) };
|
|
88
|
+
const remoteState = { path: change.path, hash: remoteHash, mtime: change.fileModifiedAt, size: Buffer.byteLength(content) };
|
|
60
89
|
if (detectConflict(localState, remoteState, lastLocal, lastRemote)) {
|
|
61
90
|
const resolution = resolveConflict(config.onConflict, localState, remoteState);
|
|
62
91
|
let conflictFile = null;
|
|
63
92
|
if (resolution === 'remote') {
|
|
64
|
-
conflictFile = createConflictFile(config.localPath,
|
|
65
|
-
onLocalWrite?.(
|
|
93
|
+
conflictFile = createConflictFile(config.localPath, change.path, localContent, 'local');
|
|
94
|
+
onLocalWrite?.(change.path);
|
|
66
95
|
const tmpConflict = localFile + '.tmp';
|
|
67
96
|
fs.writeFileSync(tmpConflict, content, 'utf-8');
|
|
68
97
|
fs.renameSync(tmpConflict, localFile);
|
|
69
|
-
log(`Conflict: ${
|
|
98
|
+
log(`Conflict: ${change.path} — used remote, saved local as ${conflictFile}`);
|
|
70
99
|
}
|
|
71
100
|
else {
|
|
72
|
-
conflictFile = createConflictFile(config.localPath,
|
|
73
|
-
await client.documents.put(config.vaultId,
|
|
74
|
-
log(`Conflict: ${
|
|
101
|
+
conflictFile = createConflictFile(config.localPath, change.path, content, 'remote');
|
|
102
|
+
await client.documents.put(config.vaultId, change.path, localContent);
|
|
103
|
+
log(`Conflict: ${change.path} — used local, saved remote as ${conflictFile}`);
|
|
75
104
|
}
|
|
76
|
-
onConflictLog?.(formatConflictLog(
|
|
77
|
-
state.local[
|
|
78
|
-
state.remote[
|
|
79
|
-
? buildRemoteFileState(
|
|
80
|
-
: buildRemoteFileState(
|
|
105
|
+
onConflictLog?.(formatConflictLog(change.path, resolution, conflictFile));
|
|
106
|
+
state.local[change.path] = resolution === 'remote' ? remoteState : localState;
|
|
107
|
+
state.remote[change.path] = resolution === 'remote'
|
|
108
|
+
? buildRemoteFileState(change.path, content, change.fileModifiedAt)
|
|
109
|
+
: buildRemoteFileState(change.path, localContent, new Date().toISOString());
|
|
110
|
+
stateMutated = true;
|
|
81
111
|
changes++;
|
|
82
112
|
continue;
|
|
83
113
|
}
|
|
@@ -87,44 +117,55 @@ export function createRemotePoller(client, config, options) {
|
|
|
87
117
|
if (!fs.existsSync(dir)) {
|
|
88
118
|
fs.mkdirSync(dir, { recursive: true });
|
|
89
119
|
}
|
|
90
|
-
onLocalWrite?.(
|
|
120
|
+
onLocalWrite?.(change.path);
|
|
91
121
|
const tmpFile = localFile + '.tmp';
|
|
92
122
|
fs.writeFileSync(tmpFile, content, 'utf-8');
|
|
93
123
|
fs.renameSync(tmpFile, localFile);
|
|
94
|
-
log(`Pulled: ${
|
|
124
|
+
log(`Pulled: ${change.path}`);
|
|
95
125
|
changes++;
|
|
96
|
-
state.local[
|
|
97
|
-
path:
|
|
126
|
+
state.local[change.path] = {
|
|
127
|
+
path: change.path,
|
|
98
128
|
hash: remoteHash,
|
|
99
129
|
mtime: new Date().toISOString(),
|
|
100
130
|
size: Buffer.byteLength(content),
|
|
101
131
|
};
|
|
102
|
-
state.remote[
|
|
132
|
+
state.remote[change.path] = buildRemoteFileState(change.path, content, change.fileModifiedAt);
|
|
133
|
+
stateMutated = true;
|
|
103
134
|
}
|
|
104
|
-
//
|
|
105
|
-
for (const
|
|
106
|
-
if (shouldIgnore(
|
|
135
|
+
// Process remote deletions.
|
|
136
|
+
for (const removedPath of sync.removed) {
|
|
137
|
+
if (shouldIgnore(removedPath, ignorePatterns))
|
|
107
138
|
continue;
|
|
108
|
-
const
|
|
109
|
-
if (
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
log(`Deleted local: ${docPath} (removed from remote)`);
|
|
114
|
-
changes++;
|
|
115
|
-
}
|
|
116
|
-
delete state.local[docPath];
|
|
117
|
-
delete state.remote[docPath];
|
|
139
|
+
const localFile = path.join(config.localPath, removedPath);
|
|
140
|
+
if (fs.existsSync(localFile)) {
|
|
141
|
+
fs.unlinkSync(localFile);
|
|
142
|
+
log(`Deleted local: ${removedPath} (removed from remote)`);
|
|
143
|
+
changes++;
|
|
118
144
|
}
|
|
145
|
+
delete state.local[removedPath];
|
|
146
|
+
delete state.remote[removedPath];
|
|
147
|
+
stateMutated = true;
|
|
119
148
|
}
|
|
120
|
-
if (changes > 0) {
|
|
149
|
+
if (changes > 0 || stateMutated) {
|
|
121
150
|
saveSyncState(state);
|
|
122
|
-
|
|
123
|
-
|
|
151
|
+
if (changes > 0) {
|
|
152
|
+
updateLastSync(config.id);
|
|
153
|
+
log(`Poll complete: ${changes} change(s)`);
|
|
154
|
+
}
|
|
124
155
|
}
|
|
125
156
|
}
|
|
126
157
|
catch (err) {
|
|
127
|
-
|
|
158
|
+
if (isThrottleError(err)) {
|
|
159
|
+
// The SDK already retried the request with Retry-After backoff and
|
|
160
|
+
// exhausted its retry budget. Log a warning rather than invoking
|
|
161
|
+
// onError so the daemon loop does NOT immediately re-poll on top of
|
|
162
|
+
// the backoff that the SDK already applied. The next scheduled poll
|
|
163
|
+
// (after intervalMs) will pick up the changes.
|
|
164
|
+
log('Rate limited by server — will retry on next scheduled poll');
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
onError?.(err instanceof Error ? err : new Error(String(err)));
|
|
168
|
+
}
|
|
128
169
|
}
|
|
129
170
|
finally {
|
|
130
171
|
polling = false;
|
package/dist/sync/state.js
CHANGED
|
@@ -46,7 +46,7 @@ export function saveSyncState(state) {
|
|
|
46
46
|
fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 });
|
|
47
47
|
}
|
|
48
48
|
state.updatedAt = new Date().toISOString();
|
|
49
|
-
fs.writeFileSync(stateFilePath(state.syncId), JSON.stringify(state
|
|
49
|
+
fs.writeFileSync(stateFilePath(state.syncId), JSON.stringify(state) + '\n', { mode: 0o600 });
|
|
50
50
|
}
|
|
51
51
|
/**
|
|
52
52
|
* Delete sync state for a given sync configuration.
|
package/dist/sync/types.d.ts
CHANGED
|
@@ -51,6 +51,8 @@ export interface SyncState {
|
|
|
51
51
|
local: Record<string, FileState>;
|
|
52
52
|
/** Map of document path -> file state for remote files */
|
|
53
53
|
remote: Record<string, FileState>;
|
|
54
|
+
/** ETag from the most recent successful list response (for conditional polling). */
|
|
55
|
+
remoteListEtag?: string;
|
|
54
56
|
/** ISO 8601 timestamp when state was last updated */
|
|
55
57
|
updatedAt: string;
|
|
56
58
|
}
|
package/dist/sync/watcher.js
CHANGED
|
@@ -101,18 +101,25 @@ export function createWatcher(client, config, options) {
|
|
|
101
101
|
// Check remote for conflicts in bidirectional mode
|
|
102
102
|
if (config.mode === 'sync' && lastRemote) {
|
|
103
103
|
try {
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
104
|
+
const result = await client.documents.get(config.vaultId, docPath, {
|
|
105
|
+
ifNoneMatch: `"${lastRemote.hash}"`,
|
|
106
|
+
});
|
|
107
|
+
if (!result.notModified) {
|
|
108
|
+
// Server returned the body — check if hash differs from our last-known state.
|
|
109
|
+
const remoteHash = hashFileContent(result.content);
|
|
110
|
+
if (remoteHash !== lastRemote.hash) {
|
|
111
|
+
const conflictResult = await handleConflict({
|
|
112
|
+
absPath, docPath, localContent: content, localHash,
|
|
113
|
+
lastLocal, lastRemote,
|
|
114
|
+
remoteContent: result.content, remoteHash,
|
|
115
|
+
remoteUpdatedAt: result.document.updatedAt, state,
|
|
116
|
+
});
|
|
117
|
+
if (conflictResult !== 'skip')
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
// 200 + matching hash: rare list/cache mismatch — fall through to push.
|
|
115
121
|
}
|
|
122
|
+
// 304: remote unchanged since last sync — no conflict possible. Fall through to push.
|
|
116
123
|
}
|
|
117
124
|
catch {
|
|
118
125
|
// Remote check failed — proceed with push
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lifestreamdynamics/vault-cli",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.4",
|
|
4
4
|
"description": "Command-line interface for Lifestream Vault",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22"
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"prepublishOnly": "npm run build && npm test"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@lifestreamdynamics/vault-sdk": "^2.
|
|
47
|
+
"@lifestreamdynamics/vault-sdk": "^2.2.0",
|
|
48
48
|
"chalk": "^5.4.0",
|
|
49
49
|
"chokidar": "^4.0.3",
|
|
50
50
|
"commander": "^13.0.0",
|