@indigoai-us/hq-cloud 5.23.0 → 5.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +20 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +18 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +46 -2
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +77 -20
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +278 -61
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +484 -3
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +27 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +56 -2
- package/src/bin/sync-runner.ts +39 -0
- package/src/cli/share.test.ts +577 -3
- package/src/cli/share.ts +395 -85
- package/src/cli/sync.ts +28 -0
package/dist/cli/share.d.ts
CHANGED
|
@@ -8,6 +8,25 @@ import type { EntityContext, VaultServiceConfig } from "../types.js";
|
|
|
8
8
|
import type { UploadAuthor } from "../s3.js";
|
|
9
9
|
import type { ConflictStrategy } from "./conflict.js";
|
|
10
10
|
import type { SyncProgressEvent } from "./sync.js";
|
|
11
|
+
/**
|
|
12
|
+
* Cheap pure check — pass the relative key OR a basename; either works. Used
|
|
13
|
+
* in both the file walker (basename matching) and the delete-plan walker
|
|
14
|
+
* (relative-key matching). The regex matches anywhere in the string, which is
|
|
15
|
+
* fine: the `.conflict-<ISO>-<hash>.` token is unambiguous.
|
|
16
|
+
*/
|
|
17
|
+
declare function isEphemeralPath(p: string): boolean;
|
|
18
|
+
/**
|
|
19
|
+
* Test-only export. Kept under a `_testing` namespace so the module's public
|
|
20
|
+
* surface stays focused on `share()` / `ShareOptions` / `ShareResult` while
|
|
21
|
+
* regression-critical regex contracts (the conflict-mirror pattern) can be
|
|
22
|
+
* pinned by direct unit tests without round-tripping through share().
|
|
23
|
+
*
|
|
24
|
+
* Do NOT import from `_testing` outside of tests in this package.
|
|
25
|
+
*/
|
|
26
|
+
export declare const _testing: {
|
|
27
|
+
isEphemeralPath: typeof isEphemeralPath;
|
|
28
|
+
EPHEMERAL_PATH_PATTERN: RegExp;
|
|
29
|
+
};
|
|
11
30
|
export interface ShareOptions {
|
|
12
31
|
/** Path(s) to share (files or directories) */
|
|
13
32
|
paths: string[];
|
|
@@ -87,29 +106,44 @@ export interface ShareOptions {
|
|
|
87
106
|
* convert into remote `DeleteObject` calls. Only consulted when
|
|
88
107
|
* `propagateDeletes === true`.
|
|
89
108
|
*
|
|
90
|
-
* - `"
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
109
|
+
* - `"currency-gated"` (safest; default scheduled for 5.25 after soak):
|
|
110
|
+
* for each candidate, issue a remote HEAD and compare the current
|
|
111
|
+
* remote ETag against the journal's
|
|
112
|
+
* last-recorded `remoteEtag`. Match → safe-to-delete (this machine is
|
|
113
|
+
* current for the file, so the local deletion reflects an intentional
|
|
114
|
+
* removal AFTER seeing the latest remote version). Mismatch → refuse
|
|
115
|
+
* and emit `delete-refused-stale-etag`; the journal entry is left
|
|
116
|
+
* intact so the next pull leg re-pulls via the same hasRemoteChanged
|
|
117
|
+
* path. 404 → tombstone: drop the journal entry, no DeleteObject (the
|
|
118
|
+
* remote was already gone). Strictly safer than `owned-only` because
|
|
119
|
+
* it gates on per-file proof of currency rather than direction-of-
|
|
120
|
+
* origin — files that arrived via `/update-hq` (direction:"down") can
|
|
121
|
+
* legitimately be deleted by the device that pulled them, as long as
|
|
122
|
+
* no other device has touched them since.
|
|
123
|
+
* - `"owned-only"` (current default in 5.24): only entries whose journal
|
|
124
|
+
* `direction === "up"` are eligible. That is, only files this machine
|
|
125
|
+
* previously uploaded can be remotely deleted on its behalf. Entries
|
|
126
|
+
* recorded as pulled from elsewhere are never delete-propagated.
|
|
127
|
+
* Default in 5.24 while currency-gated soaks; scheduled to lose the
|
|
128
|
+
* default in 5.25. Downside: any file that arrived via `/update-hq`
|
|
129
|
+
* or another device's push is stuck on remote forever once locally
|
|
130
|
+
* removed, because no device "owns" it under this rule.
|
|
98
131
|
* - `"all"`: legacy behaviour — every in-scope journal entry whose
|
|
99
|
-
* local file is missing is eligible (regardless of direction
|
|
100
|
-
* bidirectional runner's first-push and any tool that
|
|
101
|
-
* mirror a destructive local checkout opts in here
|
|
132
|
+
* local file is missing is eligible (regardless of direction or
|
|
133
|
+
* currency). The bidirectional runner's first-push and any tool that
|
|
134
|
+
* wants to mirror a destructive local checkout opts in here
|
|
135
|
+
* explicitly. Use with care — a stale device can erase peer uploads.
|
|
102
136
|
*
|
|
103
|
-
* Independently of this policy, an entry is also dropped from the
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
137
|
+
* Independently of this policy, an entry is also dropped from the plan
|
|
138
|
+
* when (a) it matches `EPHEMERAL_PATH_PATTERN` (conflict mirrors never
|
|
139
|
+
* propagate), or (b) neither the file-shape nor the directory-shape probe
|
|
140
|
+
* of `shouldSync` accepts the path — i.e. the current ignore filter would
|
|
141
|
+
* have skipped the path on pull. That symmetry blocks the failure mode
|
|
142
|
+
* where a path was filtered locally but lived in the vault (and the
|
|
143
|
+
* journal) from an older HQ layout or a different machine, causing the
|
|
144
|
+
* next push to erase it.
|
|
111
145
|
*/
|
|
112
|
-
propagateDeletePolicy?: "owned-only" | "all";
|
|
146
|
+
propagateDeletePolicy?: "currency-gated" | "owned-only" | "all";
|
|
113
147
|
/**
|
|
114
148
|
* Identity stamped onto each uploaded object's S3 user metadata
|
|
115
149
|
* (`created-by`, `created-by-sub`, `created-at`). The hq-console vault UI
|
|
@@ -144,8 +178,30 @@ export interface ShareResult {
|
|
|
144
178
|
* Number of remote `DeleteObject` calls that succeeded this run. Always 0
|
|
145
179
|
* when `propagateDeletes` is false. The corresponding journal entries are
|
|
146
180
|
* removed in the same pass so the next sync sees the key as truly gone.
|
|
181
|
+
* Does NOT include tombstones (remote was already 404; no DELETE was
|
|
182
|
+
* issued — see `filesTombstoned`) or refused-stale entries (currency-
|
|
183
|
+
* gated refused because remote etag drifted — see `filesRefusedStale`).
|
|
147
184
|
*/
|
|
148
185
|
filesDeleted: number;
|
|
186
|
+
/**
|
|
187
|
+
* Number of journal entries dropped because the remote was already 404 at
|
|
188
|
+
* HEAD time (cleaned out-of-band — e.g. someone hand-deleted via the S3
|
|
189
|
+
* console, or another tool ran a destructive operation). No `DeleteObject`
|
|
190
|
+
* was issued for these; the journal converges with reality. Always 0 when
|
|
191
|
+
* `propagateDeletes` is false or `propagateDeletePolicy !== "currency-gated"`.
|
|
192
|
+
*/
|
|
193
|
+
filesTombstoned: number;
|
|
194
|
+
/**
|
|
195
|
+
* Number of delete candidates refused by the `currency-gated` policy
|
|
196
|
+
* because the remote object's current ETag no longer matches the journal's
|
|
197
|
+
* recorded one (some other device modified the file since this device last
|
|
198
|
+
* synced it) — OR because the journal entry is a legacy record with no
|
|
199
|
+
* `remoteEtag` to compare against. Neither S3 nor the journal is mutated
|
|
200
|
+
* for these; the next pull leg re-pulls naturally via `hasRemoteChanged`.
|
|
201
|
+
* Always 0 when `propagateDeletes` is false or policy is not
|
|
202
|
+
* `currency-gated`.
|
|
203
|
+
*/
|
|
204
|
+
filesRefusedStale: number;
|
|
149
205
|
/**
|
|
150
206
|
* Paths (company-relative) that were detected as push conflicts. Mirrors
|
|
151
207
|
* `SyncResult.conflictPaths` so push and pull surface conflicts the same
|
|
@@ -158,4 +214,5 @@ export interface ShareResult {
|
|
|
158
214
|
* Share local file(s) to the entity vault.
|
|
159
215
|
*/
|
|
160
216
|
export declare function share(options: ShareOptions): Promise<ShareResult>;
|
|
217
|
+
export {};
|
|
161
218
|
//# sourceMappingURL=share.d.ts.map
|
package/dist/cli/share.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/cli/share.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AAGlF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAY7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AA+
|
|
1
|
+
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/cli/share.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AAGlF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAY7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAuBnD;;;;;GAKG;AACH,iBAAS,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAE3C;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,QAAQ;;;CAGpB,CAAC;AA+IF,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC;;;;;;;;;;;;;;;;;;OAkBG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;;;;;;OASG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;OAgBG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;;OAOG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;;OASG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAmVvE"}
|
package/dist/cli/share.js
CHANGED
|
@@ -11,6 +11,46 @@ import { uploadFile, uploadSymlink, headRemoteFile, deleteRemoteFile } from "../
|
|
|
11
11
|
import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, removeEntry, normalizeEtag, } from "../journal.js";
|
|
12
12
|
import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
|
|
13
13
|
import { resolveConflict } from "./conflict.js";
|
|
14
|
+
/**
|
|
15
|
+
* Local-only ephemeral artifacts: conflict-mirror files written by the pull
|
|
16
|
+
* leg whenever a 3-way merge keeps local AND wants to preserve the remote
|
|
17
|
+
* version for inspection. Format: `<orig>.conflict-<ISO-utc>-<machineHash>.<ext>`
|
|
18
|
+
* (e.g. `.claude/CLAUDE.md.conflict-2026-05-13T19-40-40Z-e5797a.md`).
|
|
19
|
+
*
|
|
20
|
+
* These files MUST never round-trip to S3 — they're local-only safety backups
|
|
21
|
+
* the user reviews and deletes once the merge is resolved. Pre-fix, the push
|
|
22
|
+
* walker happily uploaded them, the journal recorded them, and the
|
|
23
|
+
* `owned-only` delete policy then refused to clean them up when the user
|
|
24
|
+
* deleted them locally (because pull-confirmation had stamped them as
|
|
25
|
+
* `direction: "down"`). Net effect: a permanent litter ratchet on remote.
|
|
26
|
+
*
|
|
27
|
+
* Wire-points: (1) push walker — `collectFiles` / `walkDir` skip these so
|
|
28
|
+
* they never upload; (2) `computeDeletePlan` — skip these so an already-
|
|
29
|
+
* journaled mirror that's been deleted locally doesn't get included in the
|
|
30
|
+
* regular delete plan (the dedicated reconcile path handles existing litter).
|
|
31
|
+
*/
|
|
32
|
+
const EPHEMERAL_PATH_PATTERN = /\.conflict-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z-[a-f0-9]+\./;
|
|
33
|
+
/**
|
|
34
|
+
* Cheap pure check — pass the relative key OR a basename; either works. Used
|
|
35
|
+
* in both the file walker (basename matching) and the delete-plan walker
|
|
36
|
+
* (relative-key matching). The regex matches anywhere in the string, which is
|
|
37
|
+
* fine: the `.conflict-<ISO>-<hash>.` token is unambiguous.
|
|
38
|
+
*/
|
|
39
|
+
function isEphemeralPath(p) {
|
|
40
|
+
return EPHEMERAL_PATH_PATTERN.test(p);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Test-only export. Kept under a `_testing` namespace so the module's public
|
|
44
|
+
* surface stays focused on `share()` / `ShareOptions` / `ShareResult` while
|
|
45
|
+
* regression-critical regex contracts (the conflict-mirror pattern) can be
|
|
46
|
+
* pinned by direct unit tests without round-tripping through share().
|
|
47
|
+
*
|
|
48
|
+
* Do NOT import from `_testing` outside of tests in this package.
|
|
49
|
+
*/
|
|
50
|
+
export const _testing = {
|
|
51
|
+
isEphemeralPath,
|
|
52
|
+
EPHEMERAL_PATH_PATTERN,
|
|
53
|
+
};
|
|
14
54
|
/**
|
|
15
55
|
* Pure Stage-1 pass for push: walk the candidate file list, hash each one,
|
|
16
56
|
* apply the size-limit and skip-unchanged gates, and return a classified
|
|
@@ -94,12 +134,15 @@ function computePushPlan(filesToShare, journal, skipUnchanged) {
|
|
|
94
134
|
*/
|
|
95
135
|
export async function share(options) {
|
|
96
136
|
const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, skipUnchanged, propagateDeletes } = options;
|
|
97
|
-
// Default to
|
|
98
|
-
// but the caller hasn't pinned a policy.
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
// to
|
|
137
|
+
// Default to "owned-only" — the pre-5.24 behavior — when delete-propagation
|
|
138
|
+
// is on but the caller hasn't pinned a policy. Staged-default rollout
|
|
139
|
+
// (see CHANGELOG / PR for hq-cloud 5.24.0): 5.24 ships the currency-gated
|
|
140
|
+
// CODE PATH plus the conflict-mirror exclusion (which is policy-
|
|
141
|
+
// independent and immediately stops new litter), but holds the default
|
|
142
|
+
// flip to a later release after soak. Opt into the safer policy now via
|
|
143
|
+
// `propagateDeletePolicy: "currency-gated"` (explicit) or
|
|
144
|
+
// `HQ_SYNC_DELETE_POLICY=currency-gated` (env, honored by sync-runner).
|
|
145
|
+
// The default flip to `"currency-gated"` is scheduled for 5.25.0.
|
|
103
146
|
const propagateDeletePolicy = options.propagateDeletePolicy ?? "owned-only";
|
|
104
147
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
105
148
|
// Exactly-one-of contract: either we vend (vaultConfig) or the caller did
|
|
@@ -150,6 +193,13 @@ export async function share(options) {
|
|
|
150
193
|
let bytesUploaded = 0;
|
|
151
194
|
let filesSkipped = 0;
|
|
152
195
|
let filesDeleted = 0;
|
|
196
|
+
// Tombstone and refused-stale counts mirror the deletePlan buckets so the
|
|
197
|
+
// ShareResult can report them without the caller having to count events.
|
|
198
|
+
// Populated only after Stage 3 runs (deletePlan is computed first, then
|
|
199
|
+
// mutated through the execution loop) — initial zero handles the
|
|
200
|
+
// propagateDeletes=false path.
|
|
201
|
+
let filesTombstoned = 0;
|
|
202
|
+
let filesRefusedStale = 0;
|
|
153
203
|
const conflictPaths = [];
|
|
154
204
|
// Collect all files to share
|
|
155
205
|
const filesToShare = collectFiles(paths, hqRoot, syncRoot, shouldSync);
|
|
@@ -167,8 +217,8 @@ export async function share(options) {
|
|
|
167
217
|
? resolveDeleteScopeRoots(paths, hqRoot, syncRoot)
|
|
168
218
|
: [];
|
|
169
219
|
const deletePlan = propagateDeletes === true
|
|
170
|
-
? computeDeletePlan(journal, syncRoot, deleteScopeRoots, shouldSync, propagateDeletePolicy)
|
|
171
|
-
: [];
|
|
220
|
+
? await computeDeletePlan(journal, syncRoot, deleteScopeRoots, shouldSync, propagateDeletePolicy, ctx)
|
|
221
|
+
: { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
172
222
|
emit({
|
|
173
223
|
type: "plan",
|
|
174
224
|
// share() is push-only; pull counts are sourced from sync()'s plan event.
|
|
@@ -180,7 +230,11 @@ export async function share(options) {
|
|
|
180
230
|
// Push conflicts require a remote HEAD; we don't yet do that in Stage 1,
|
|
181
231
|
// so this stays 0. V1.5 (single LIST) will let us classify them up-front.
|
|
182
232
|
filesToConflict: 0,
|
|
183
|
-
|
|
233
|
+
// Reported count is the deletes we're actually going to issue — does NOT
|
|
234
|
+
// include tombstones (no S3 call) or refused-stale (no journal change).
|
|
235
|
+
// Refusals surface as their own event stream so consumers that care can
|
|
236
|
+
// render a "kept on remote: N" line separately.
|
|
237
|
+
filesToDelete: deletePlan.toDelete.length,
|
|
184
238
|
});
|
|
185
239
|
// Stage 2: execute. Skip items pre-classified as no-ops, then for each
|
|
186
240
|
// upload candidate run the HEAD + 3-way conflict check + actual PUT.
|
|
@@ -240,6 +294,11 @@ export async function share(options) {
|
|
|
240
294
|
bytesUploaded,
|
|
241
295
|
filesSkipped,
|
|
242
296
|
filesDeleted,
|
|
297
|
+
// Abort path: delete stage never runs, so tombstone + refused-
|
|
298
|
+
// stale counts are necessarily zero. Explicit fields keep the
|
|
299
|
+
// ShareResult shape stable for consumers that destructure.
|
|
300
|
+
filesTombstoned,
|
|
301
|
+
filesRefusedStale,
|
|
243
302
|
conflictPaths,
|
|
244
303
|
aborted: true,
|
|
245
304
|
};
|
|
@@ -292,12 +351,28 @@ export async function share(options) {
|
|
|
292
351
|
});
|
|
293
352
|
}
|
|
294
353
|
}
|
|
295
|
-
// Stage 3: propagate deletes.
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
//
|
|
300
|
-
|
|
354
|
+
// Stage 3: propagate deletes. Three buckets, three actions:
|
|
355
|
+
//
|
|
356
|
+
// 1. `toDelete` — write a delete-marker (versioning is enabled on the
|
|
357
|
+
// bucket so the delete is soft and prior versions remain recoverable)
|
|
358
|
+
// and remove the journal entry so the next sync sees the key as
|
|
359
|
+
// truly gone on this machine. A failed DeleteObject leaves both
|
|
360
|
+
// the journal entry and remote object intact — the next run retries.
|
|
361
|
+
//
|
|
362
|
+
// 2. `toTombstone` — the remote was 404 at HEAD time (cleaned up out
|
|
363
|
+
// of band, e.g. someone hand-deleted via console). No DeleteObject
|
|
364
|
+
// needed; just drop the journal entry so the journal converges with
|
|
365
|
+
// reality. Emit a synthetic `progress` event with `deleted: true`
|
|
366
|
+
// and bytes=0 so consumers see the convergence.
|
|
367
|
+
//
|
|
368
|
+
// 3. `refusedStale` — under `currency-gated`, the remote's current
|
|
369
|
+
// ETag no longer matches the journal's recorded one. Some other
|
|
370
|
+
// device modified the file since this device last synced it. Keep
|
|
371
|
+
// the remote intact; keep the journal entry intact. The next pull
|
|
372
|
+
// leg of `sync now` re-pulls naturally via the existing
|
|
373
|
+
// `hasRemoteChanged` path. Emit a dedicated event so UIs can
|
|
374
|
+
// surface the refusal without inferring it from absence.
|
|
375
|
+
for (const relativePath of deletePlan.toDelete) {
|
|
301
376
|
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
302
377
|
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
303
378
|
}
|
|
@@ -322,6 +397,27 @@ export async function share(options) {
|
|
|
322
397
|
});
|
|
323
398
|
}
|
|
324
399
|
}
|
|
400
|
+
for (const relativePath of deletePlan.toTombstone) {
|
|
401
|
+
removeEntry(journal, relativePath);
|
|
402
|
+
filesTombstoned++;
|
|
403
|
+
emit({
|
|
404
|
+
type: "progress",
|
|
405
|
+
path: relativePath,
|
|
406
|
+
bytes: 0,
|
|
407
|
+
deleted: true,
|
|
408
|
+
message: "tombstone (remote already 404)",
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
for (const refused of deletePlan.refusedStale) {
|
|
412
|
+
filesRefusedStale++;
|
|
413
|
+
emit({
|
|
414
|
+
type: "delete-refused-stale-etag",
|
|
415
|
+
path: refused.key,
|
|
416
|
+
journalEtag: refused.journalEtag,
|
|
417
|
+
remoteEtag: refused.remoteEtag,
|
|
418
|
+
reason: refused.reason,
|
|
419
|
+
});
|
|
420
|
+
}
|
|
325
421
|
// See cli/sync.ts: stamp lastSync on completion so a no-op share still
|
|
326
422
|
// ticks the "Last sync" indicator.
|
|
327
423
|
journal.lastSync = new Date().toISOString();
|
|
@@ -331,6 +427,8 @@ export async function share(options) {
|
|
|
331
427
|
bytesUploaded,
|
|
332
428
|
filesSkipped,
|
|
333
429
|
filesDeleted,
|
|
430
|
+
filesTombstoned,
|
|
431
|
+
filesRefusedStale,
|
|
334
432
|
conflictPaths,
|
|
335
433
|
aborted: false,
|
|
336
434
|
};
|
|
@@ -347,7 +445,12 @@ function defaultConsoleLogger(event) {
|
|
|
347
445
|
}
|
|
348
446
|
else if (event.type === "progress") {
|
|
349
447
|
if (event.deleted) {
|
|
350
|
-
|
|
448
|
+
// Append `message` when present (e.g. tombstone events carry
|
|
449
|
+
// "tombstone (remote already 404)"). Without this, tombstones and
|
|
450
|
+
// real deletes render byte-identically in the tty stream, and
|
|
451
|
+
// operators have no way to distinguish from logs alone.
|
|
452
|
+
const suffix = event.message ? ` — ${event.message}` : "";
|
|
453
|
+
console.log(` ✗ ${event.path} (deleted)${suffix}`);
|
|
351
454
|
}
|
|
352
455
|
else if (event.message) {
|
|
353
456
|
console.log(` ✓ ${event.path} — "${event.message}"`);
|
|
@@ -362,6 +465,17 @@ function defaultConsoleLogger(event) {
|
|
|
362
465
|
else if (event.type === "error") {
|
|
363
466
|
console.error(` ✗ ${event.path} — ${event.message}`);
|
|
364
467
|
}
|
|
468
|
+
else if (event.type === "delete-refused-stale-etag") {
|
|
469
|
+
// Branch on `reason`, not on the sentinel etag strings, so legacy
|
|
470
|
+
// entries render with a clear explanation instead of "<legacy-no-etag>"
|
|
471
|
+
// leaking into operator-visible output.
|
|
472
|
+
if (event.reason === "legacy-no-etag") {
|
|
473
|
+
console.error(` ⚠ no-etag-on-record, kept on remote: ${event.path} (journal entry predates etag tracking)`);
|
|
474
|
+
}
|
|
475
|
+
else {
|
|
476
|
+
console.error(` ⚠ stale-etag, kept on remote: ${event.path} (journal=${event.journalEtag}, remote=${event.remoteEtag})`);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
365
479
|
}
|
|
366
480
|
/**
|
|
367
481
|
* Resolve active company from .hq/config.json or parent directory chain.
|
|
@@ -395,6 +509,12 @@ function collectFiles(paths, hqRoot, syncRoot, filter) {
|
|
|
395
509
|
const results = [];
|
|
396
510
|
for (const p of paths) {
|
|
397
511
|
const absolutePath = path.isAbsolute(p) ? p : path.resolve(hqRoot, p);
|
|
512
|
+
// Ephemeral artifacts (conflict mirrors) — see EPHEMERAL_PATH_PATTERN doc.
|
|
513
|
+
// Caller may pass one explicitly; we still refuse to upload it. Basename
|
|
514
|
+
// check matches the walkDir gate so behavior is identical whether the
|
|
515
|
+
// mirror is the user-supplied path or found during directory recursion.
|
|
516
|
+
if (isEphemeralPath(path.basename(absolutePath)))
|
|
517
|
+
continue;
|
|
398
518
|
// existsSync follows symlinks: a dangling top-level link will report
|
|
399
519
|
// not-existing and be skipped here. lstatSync below handles the
|
|
400
520
|
// valid-link case directly without needing the existsSync gate.
|
|
@@ -468,6 +588,12 @@ function walkDir(dir, syncRoot, filter) {
|
|
|
468
588
|
return results;
|
|
469
589
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
470
590
|
for (const entry of entries) {
|
|
591
|
+
// Ephemeral artifacts (conflict mirrors) are local-only safety backups
|
|
592
|
+
// that MUST NEVER round-trip to S3. Check basename here so the filter
|
|
593
|
+
// applies regardless of which company root contains them. See
|
|
594
|
+
// EPHEMERAL_PATH_PATTERN doc for the full rationale.
|
|
595
|
+
if (isEphemeralPath(entry.name))
|
|
596
|
+
continue;
|
|
471
597
|
const absolutePath = path.join(dir, entry.name);
|
|
472
598
|
const isDir = entry.isDirectory();
|
|
473
599
|
// Symlinks need their own filter probe BEFORE the regular gate.
|
|
@@ -598,33 +724,92 @@ function resolveDeleteScopeRoots(paths, hqRoot, syncRoot) {
|
|
|
598
724
|
}
|
|
599
725
|
return Array.from(prefixes);
|
|
600
726
|
}
|
|
727
|
+
/**
|
|
728
|
+
* Concurrency cap for the per-file HEAD-O-meter (currency-gated). Sequential
|
|
729
|
+
* HEADs would add ~N×(50-200ms) to a sync — for the 261-mirror real-world
|
|
730
|
+
* case that's 15-50s of latency. 16-way concurrency keeps S3 well within
|
|
731
|
+
* per-prefix burst limits (~3,500 GET/HEAD/sec/prefix is the documented
|
|
732
|
+
* floor) and bounded under the AWS-SDK default agent's max-sockets so we
|
|
733
|
+
* don't compete with the in-flight upload pool.
|
|
734
|
+
*/
|
|
735
|
+
const DELETE_PLAN_HEAD_CONCURRENCY = 16;
|
|
601
736
|
/**
|
|
602
737
|
* Walk every journal key in `scopeRoots` whose local file is missing from
|
|
603
|
-
* disk and
|
|
604
|
-
*
|
|
738
|
+
* disk and bucket each candidate into the right action per `policy`. Hard
|
|
739
|
+
* filters that drop a candidate entirely (no bucket) — regardless of policy:
|
|
605
740
|
*
|
|
606
|
-
* 1. Its key
|
|
607
|
-
* 2. Its local file
|
|
608
|
-
*
|
|
741
|
+
* 1. Its key must match (or sit beneath) one of the `scopeRoots` prefixes.
|
|
742
|
+
* 2. Its local file must be missing from disk (lstat ENOENT). We use
|
|
743
|
+
* `lstat` (not `existsSync`) so a dangling symlink — a link whose
|
|
744
|
+
* target has been removed but whose link file is still on disk —
|
|
745
|
+
* counts as "still present locally" and is NOT delete-propagated.
|
|
746
|
+
* Pre-fix, existsSync followed the link, returned false, and the
|
|
747
|
+
* entry was queued for remote DeleteObject in the same sync that
|
|
748
|
+
* had just uploaded it via `uploadSymlink` — the link round-tripped
|
|
749
|
+
* as "upload, then delete" in one cycle. ENOENT means truly absent
|
|
750
|
+
* → eligible; other lstat errors propagate.
|
|
751
|
+
* 3. The current ignore filter (`shouldSync`) accepts the key — paths
|
|
609
752
|
* filtered out by `.hqignore` / `.gitignore` / `DEFAULT_IGNORES` are
|
|
610
|
-
* never delete-propagated.
|
|
611
|
-
*
|
|
612
|
-
*
|
|
613
|
-
*
|
|
614
|
-
*
|
|
615
|
-
* is
|
|
616
|
-
*
|
|
617
|
-
*
|
|
618
|
-
*
|
|
619
|
-
*
|
|
620
|
-
*
|
|
753
|
+
* never delete-propagated. Closes the failure mode where a path lives
|
|
754
|
+
* in the vault (and journal) but the local walk skips it because of
|
|
755
|
+
* asymmetric ignore rules.
|
|
756
|
+
*
|
|
757
|
+
* Dual-hint probe: by the time we're considering this entry for
|
|
758
|
+
* remote deletion, the local file is already gone — we have no way to
|
|
759
|
+
* know whether it was a regular file or a symlink record. A single
|
|
760
|
+
* `isDir=false` probe would silently keep the remote record alive
|
|
761
|
+
* whenever the only matching `.hqinclude` allowlist pattern is dir-
|
|
762
|
+
* only (e.g. `companies/*\/knowledge/`), since gitignore's slash
|
|
763
|
+
* semantics reject the slashless probe. The same dual-hint pattern in
|
|
764
|
+
* `walkDir`/`collectFiles` (push) and `computePullPlan` (pull) applies
|
|
765
|
+
* symmetrically here. Pure path lookup, no I/O.
|
|
766
|
+
* 4. The key does NOT match `EPHEMERAL_PATH_PATTERN`. Conflict mirrors
|
|
767
|
+
* are local-only artifacts that should never have been journaled in
|
|
768
|
+
* the first place; the dedicated reconcile command sweeps already-
|
|
769
|
+
* journaled mirrors. Excluding them here keeps a regular `sync now`
|
|
770
|
+
* from accidentally deleting a mirror another device is still
|
|
771
|
+
* reviewing.
|
|
772
|
+
*
|
|
773
|
+
* Then per-policy bucketing:
|
|
774
|
+
*
|
|
775
|
+
* - `"currency-gated"` (default, safest): issue a HEAD against the remote.
|
|
776
|
+
* 200 + `normalizeEtag(remote) === entry.remoteEtag` → `toDelete`.
|
|
777
|
+
* 200 + mismatch → `refusedStale` (peer drift; let pull re-pull).
|
|
778
|
+
* 404 → `toTombstone` (remote was cleaned out-of-band).
|
|
779
|
+
* If the journal entry has no recorded `remoteEtag` (legacy entries
|
|
780
|
+
* written before etag tracking), the candidate falls back to
|
|
781
|
+
* `refusedStale` with `reason: "legacy-no-etag"` — we can't prove
|
|
782
|
+
* currency without an etag, so refusal is the safe direction. The
|
|
783
|
+
* journal entry survives so a future sync with a recorded etag can
|
|
784
|
+
* re-evaluate.
|
|
785
|
+
*
|
|
786
|
+
* HEAD calls are batched at `DELETE_PLAN_HEAD_CONCURRENCY` so a large
|
|
787
|
+
* candidate set (e.g. a one-shot reconcile sweep) doesn't serialize
|
|
788
|
+
* into N×RTT latency. The candidate set is materialized into a list
|
|
789
|
+
* first (synchronous filters above), then the HEAD pass runs in
|
|
790
|
+
* bounded-parallel chunks.
|
|
791
|
+
*
|
|
792
|
+
* Note: there is a TOCTOU window between this HEAD and the eventual
|
|
793
|
+
* `deleteRemoteFile` call in the share() execution loop. If a peer
|
|
794
|
+
* overwrites the object in that window (~50-200ms), the resulting
|
|
795
|
+
* delete-marker lands on a newer version than we verified. S3
|
|
796
|
+
* versioning makes the worst case recoverable (prior versions are
|
|
797
|
+
* retained), and the conditional-delete primitive does not exist on
|
|
798
|
+
* S3 DeleteObject — only PutObject/CopyObject accept `IfMatch`. The
|
|
799
|
+
* window is bounded, not zero. Realtime sync (separate work) reduces
|
|
800
|
+
* it further by keeping the journal continuously fresh.
|
|
801
|
+
* - `"owned-only"`: include only entries with `direction === "up"`. No
|
|
802
|
+
* HEAD round-trip. Goes to `toDelete`. Legacy fallback.
|
|
803
|
+
* - `"all"`: include every candidate. No HEAD, no direction check. Goes
|
|
804
|
+
* to `toDelete`. Caller has explicitly opted out of safety gates.
|
|
621
805
|
*
|
|
622
806
|
* Empty `scopeRoots` ⇒ empty plan (caller didn't opt in).
|
|
623
807
|
*/
|
|
624
|
-
function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSync, policy) {
|
|
808
|
+
async function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSync, policy, ctx) {
|
|
809
|
+
const plan = { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
625
810
|
if (scopeRoots.length === 0)
|
|
626
|
-
return
|
|
627
|
-
const
|
|
811
|
+
return plan;
|
|
812
|
+
const headCandidates = [];
|
|
628
813
|
for (const [relativeKey, entry] of Object.entries(journal.files)) {
|
|
629
814
|
const inScope = scopeRoots.some((root) => root === "" ||
|
|
630
815
|
relativeKey === root ||
|
|
@@ -632,14 +817,6 @@ function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSync, policy) {
|
|
|
632
817
|
if (!inScope)
|
|
633
818
|
continue;
|
|
634
819
|
const localPath = path.join(syncRoot, relativeKey);
|
|
635
|
-
// lstat (not existsSync) so a dangling symlink — a link whose
|
|
636
|
-
// target has been removed but whose link file is still on disk —
|
|
637
|
-
// counts as "still present locally" and is NOT delete-propagated.
|
|
638
|
-
// Pre-fix, existsSync followed the link, returned false, and the
|
|
639
|
-
// entry was queued for remote DeleteObject in the same sync that
|
|
640
|
-
// had just uploaded it via uploadSymlink. The link round-tripped
|
|
641
|
-
// as "upload, then delete" in one cycle. ENOENT means truly
|
|
642
|
-
// absent → eligible; other lstat errors propagate.
|
|
643
820
|
let presentLocally = true;
|
|
644
821
|
try {
|
|
645
822
|
fs.lstatSync(localPath);
|
|
@@ -657,27 +834,67 @@ function computeDeletePlan(journal, syncRoot, scopeRoots, shouldSync, policy) {
|
|
|
657
834
|
}
|
|
658
835
|
if (presentLocally)
|
|
659
836
|
continue;
|
|
660
|
-
// (3) Symmetric filter guard. `shouldSync` is constructed from the same
|
|
661
|
-
// hqRoot the pull leg uses, so a key the pull would have skipped
|
|
662
|
-
// ("ignored") is also one we must not delete-propagate.
|
|
663
|
-
//
|
|
664
|
-
// Dual-hint probe: by the time we're considering this entry for
|
|
665
|
-
// remote deletion, the local file is already gone — we have no
|
|
666
|
-
// way to know whether it was a regular file or a symlink record.
|
|
667
|
-
// A single isDir=false probe would silently keep the remote
|
|
668
|
-
// record alive whenever the only matching .hqinclude allowlist
|
|
669
|
-
// pattern is dir-only (e.g. `companies/*/knowledge/`), since
|
|
670
|
-
// gitignore's slash semantics reject the slashless probe. The
|
|
671
|
-
// same dual-hint pattern in walkDir/collectFiles (push) and
|
|
672
|
-
// computePullPlan (pull) applies symmetrically here. Pure path
|
|
673
|
-
// lookup, no I/O.
|
|
674
837
|
if (!shouldSync(localPath, false) && !shouldSync(localPath, true))
|
|
675
838
|
continue;
|
|
676
|
-
// (
|
|
677
|
-
|
|
839
|
+
// Ephemeral artifacts (conflict mirrors) never propagate-delete via the
|
|
840
|
+
// normal path — see EPHEMERAL_PATH_PATTERN doc.
|
|
841
|
+
if (isEphemeralPath(relativeKey))
|
|
842
|
+
continue;
|
|
843
|
+
if (policy === "all") {
|
|
844
|
+
plan.toDelete.push(relativeKey);
|
|
845
|
+
continue;
|
|
846
|
+
}
|
|
847
|
+
if (policy === "owned-only") {
|
|
848
|
+
if (entry.direction !== "up")
|
|
849
|
+
continue;
|
|
850
|
+
plan.toDelete.push(relativeKey);
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
// currency-gated: queue for HEAD unless the entry is legacy (no etag).
|
|
854
|
+
const journalEtag = entry.remoteEtag;
|
|
855
|
+
if (!journalEtag) {
|
|
856
|
+
plan.refusedStale.push({
|
|
857
|
+
key: relativeKey,
|
|
858
|
+
journalEtag: "<legacy-no-etag>",
|
|
859
|
+
remoteEtag: "<unknown>",
|
|
860
|
+
reason: "legacy-no-etag",
|
|
861
|
+
});
|
|
678
862
|
continue;
|
|
679
|
-
|
|
863
|
+
}
|
|
864
|
+
headCandidates.push({ key: relativeKey, journalEtag });
|
|
865
|
+
}
|
|
866
|
+
// Stage 2: bounded-parallel HEAD pass. Promise.all over chunks of size
|
|
867
|
+
// `DELETE_PLAN_HEAD_CONCURRENCY` so a large candidate set doesn't
|
|
868
|
+
// serialize into N round-trips, and so we don't burst past the AWS-SDK
|
|
869
|
+
// default agent's per-host socket cap. Each result is bucketed
|
|
870
|
+
// independently — one failed HEAD doesn't poison the others (errors
|
|
871
|
+
// propagate from the chunk's Promise.all and are surfaced by share()'s
|
|
872
|
+
// outer try/catch, mirroring the existing pre-share error handling).
|
|
873
|
+
for (let i = 0; i < headCandidates.length; i += DELETE_PLAN_HEAD_CONCURRENCY) {
|
|
874
|
+
const chunk = headCandidates.slice(i, i + DELETE_PLAN_HEAD_CONCURRENCY);
|
|
875
|
+
const results = await Promise.all(chunk.map(async (c) => ({
|
|
876
|
+
candidate: c,
|
|
877
|
+
remote: await headRemoteFile(ctx, c.key),
|
|
878
|
+
})));
|
|
879
|
+
for (const { candidate, remote } of results) {
|
|
880
|
+
if (remote === null) {
|
|
881
|
+
plan.toTombstone.push(candidate.key);
|
|
882
|
+
continue;
|
|
883
|
+
}
|
|
884
|
+
const currentEtag = normalizeEtag(remote.etag);
|
|
885
|
+
if (currentEtag === candidate.journalEtag) {
|
|
886
|
+
plan.toDelete.push(candidate.key);
|
|
887
|
+
}
|
|
888
|
+
else {
|
|
889
|
+
plan.refusedStale.push({
|
|
890
|
+
key: candidate.key,
|
|
891
|
+
journalEtag: candidate.journalEtag,
|
|
892
|
+
remoteEtag: currentEtag,
|
|
893
|
+
reason: "stale-etag",
|
|
894
|
+
});
|
|
895
|
+
}
|
|
896
|
+
}
|
|
680
897
|
}
|
|
681
|
-
return
|
|
898
|
+
return plan;
|
|
682
899
|
}
|
|
683
900
|
//# sourceMappingURL=share.js.map
|