@indigoai-us/hq-cloud 6.14.17 → 6.14.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +35 -0
- package/dist/active-company.d.ts +43 -0
- package/dist/active-company.d.ts.map +1 -0
- package/dist/active-company.js +132 -0
- package/dist/active-company.js.map +1 -0
- package/dist/active-company.test.d.ts +2 -0
- package/dist/active-company.test.d.ts.map +1 -0
- package/dist/active-company.test.js +149 -0
- package/dist/active-company.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +7 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +21 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +27 -1
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +33 -4
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.d.ts +2 -0
- package/dist/bin/sync-runner-planning.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.test.js +115 -0
- package/dist/bin/sync-runner-planning.test.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +10 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +201 -25
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts +1 -1
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-routes.js +14 -2
- package/dist/bin/sync-runner-watch-routes.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +39 -9
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +114 -20
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +852 -8
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-core.js +27 -1
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-drop-dir-symlink.test.d.ts +2 -0
- package/dist/cli/rescue-drop-dir-symlink.test.d.ts.map +1 -0
- package/dist/cli/rescue-drop-dir-symlink.test.js +206 -0
- package/dist/cli/rescue-drop-dir-symlink.test.js.map +1 -0
- package/dist/cli/share.d.ts +15 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -55
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +119 -6
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +19 -25
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +121 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +3 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +13 -2
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +26 -1
- package/dist/journal.test.js.map +1 -1
- package/dist/manifest-reconcile.d.ts +118 -5
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +319 -62
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/manifest-reconcile.test.js +824 -2
- package/dist/manifest-reconcile.test.js.map +1 -1
- package/dist/personal-vault.d.ts +12 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +20 -0
- package/dist/personal-vault.js.map +1 -1
- package/dist/qmd-reindex.d.ts +88 -11
- package/dist/qmd-reindex.d.ts.map +1 -1
- package/dist/qmd-reindex.js +354 -65
- package/dist/qmd-reindex.js.map +1 -1
- package/dist/qmd-reindex.test.d.ts +4 -0
- package/dist/qmd-reindex.test.d.ts.map +1 -1
- package/dist/qmd-reindex.test.js +285 -3
- package/dist/qmd-reindex.test.js.map +1 -1
- package/dist/s3.d.ts +13 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +34 -1
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +60 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/telemetry.d.ts +22 -0
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +79 -0
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +117 -1
- package/dist/telemetry.test.js.map +1 -1
- package/dist/watcher.d.ts +26 -1
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +75 -12
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/pnpm-workspace.yaml +1 -1
- package/src/active-company.test.ts +188 -0
- package/src/active-company.ts +168 -0
- package/src/bin/sync-runner-company.ts +29 -1
- package/src/bin/sync-runner-planning.test.ts +131 -0
- package/src/bin/sync-runner-planning.ts +60 -7
- package/src/bin/sync-runner-watch-loop.ts +322 -32
- package/src/bin/sync-runner-watch-routes.ts +14 -1
- package/src/bin/sync-runner.test.ts +1021 -16
- package/src/bin/sync-runner.ts +179 -28
- package/src/cli/rescue-core.ts +26 -1
- package/src/cli/rescue-drop-dir-symlink.test.ts +224 -0
- package/src/cli/share.test.ts +150 -6
- package/src/cli/share.ts +143 -48
- package/src/cli/sync.test.ts +150 -1
- package/src/cli/sync.ts +25 -29
- package/src/journal.test.ts +45 -0
- package/src/journal.ts +18 -1
- package/src/manifest-reconcile.test.ts +1019 -3
- package/src/manifest-reconcile.ts +424 -66
- package/src/personal-vault.ts +23 -0
- package/src/qmd-reindex.test.ts +324 -4
- package/src/qmd-reindex.ts +414 -76
- package/src/s3.test.ts +79 -0
- package/src/s3.ts +48 -1
- package/src/telemetry.test.ts +128 -0
- package/src/telemetry.ts +80 -0
- package/src/watcher.ts +111 -14
- package/test/e2e/watcher-real-chokidar.test.ts +62 -2
- package/test/e2e/watcher-recursive-backend.test.ts +67 -1
- package/test/joiner-manifest-reconcile.integration.test.ts +283 -0
|
@@ -29,12 +29,35 @@ export interface RunnerTarget {
|
|
|
29
29
|
slug: string;
|
|
30
30
|
name?: string;
|
|
31
31
|
bucketName?: string;
|
|
32
|
+
/**
|
|
33
|
+
* Entity liveness as observed AT PLAN TIME, copied off the `entity.get`
|
|
34
|
+
* result this plan was built from. Carried so a downstream consumer can make
|
|
35
|
+
* a liveness decision without paying a second round trip.
|
|
36
|
+
*
|
|
37
|
+
* These deliberately do NOT gate SYNC. A suspended company still syncs — that
|
|
38
|
+
* is user-visible behavior and not ours to change here. They exist so the
|
|
39
|
+
* MANIFEST decision (see `manifest-reconcile.ts`) can be as strict as it was
|
|
40
|
+
* when it re-fetched the entity itself. Absent fields mean "unknown", and
|
|
41
|
+
* every consumer of them is required to fail closed.
|
|
42
|
+
*/
|
|
43
|
+
entityType?: string;
|
|
44
|
+
entityStatus?: string;
|
|
32
45
|
personalMode?: boolean;
|
|
33
46
|
journalSlug?: string;
|
|
34
47
|
}
|
|
35
48
|
|
|
49
|
+
/**
|
|
50
|
+
* Why a run cannot proceed to a fanout.
|
|
51
|
+
*
|
|
52
|
+
* - `no-memberships` — the caller resolved fine but belongs to no company.
|
|
53
|
+
* Usually a joiner whose invite is still pending acceptance.
|
|
54
|
+
* - `no-person-entity` — the caller is signed in but has no personal entity to
|
|
55
|
+
* sync into, so even `--personal` has no target.
|
|
56
|
+
*/
|
|
57
|
+
export type SetupNeededReason = "no-memberships" | "no-person-entity";
|
|
58
|
+
|
|
36
59
|
export type MembershipResolution =
|
|
37
|
-
| { status: "setup-needed" }
|
|
60
|
+
| { status: "setup-needed"; reason: SetupNeededReason; pendingInviteCount?: number }
|
|
38
61
|
| { status: "memberships"; memberships: Pick<Membership, "companyUid">[] };
|
|
39
62
|
|
|
40
63
|
export async function resolveMembershipsForRun(options: {
|
|
@@ -44,11 +67,12 @@ export async function resolveMembershipsForRun(options: {
|
|
|
44
67
|
client: VaultClientSurface;
|
|
45
68
|
claims: IdentityClaims | null;
|
|
46
69
|
stderr: { write: (chunk: string) => boolean | void };
|
|
70
|
+
/** Returns how many invites were still pending after the claim attempt. */
|
|
47
71
|
runClaimDance: (
|
|
48
72
|
client: VaultClientSurface,
|
|
49
73
|
claims: IdentityClaims,
|
|
50
74
|
stderr: { write: (chunk: string) => boolean | void },
|
|
51
|
-
) => Promise<
|
|
75
|
+
) => Promise<number>;
|
|
52
76
|
listMemberships: (
|
|
53
77
|
client: VaultClientSurface,
|
|
54
78
|
) => Promise<Pick<Membership, "companyUid">[]>;
|
|
@@ -58,8 +82,9 @@ export async function resolveMembershipsForRun(options: {
|
|
|
58
82
|
}
|
|
59
83
|
|
|
60
84
|
if (options.companies) {
|
|
85
|
+
let pendingInviteCount = 0;
|
|
61
86
|
if (options.claims) {
|
|
62
|
-
await options.runClaimDance(
|
|
87
|
+
pendingInviteCount = await options.runClaimDance(
|
|
63
88
|
options.client,
|
|
64
89
|
options.claims,
|
|
65
90
|
options.stderr,
|
|
@@ -68,7 +93,15 @@ export async function resolveMembershipsForRun(options: {
|
|
|
68
93
|
|
|
69
94
|
const memberships = await options.listMemberships(options.client);
|
|
70
95
|
if (memberships.length === 0) {
|
|
71
|
-
|
|
96
|
+
// Pending invites that did NOT become memberships are the single most
|
|
97
|
+
// useful thing we can tell this user: they are not "solo", they have an
|
|
98
|
+
// invite waiting to be accepted. Reported only on the empty path, where
|
|
99
|
+
// it explains the emptiness.
|
|
100
|
+
return {
|
|
101
|
+
status: "setup-needed",
|
|
102
|
+
reason: "no-memberships",
|
|
103
|
+
...(pendingInviteCount > 0 ? { pendingInviteCount } : {}),
|
|
104
|
+
};
|
|
72
105
|
}
|
|
73
106
|
return { status: "memberships", memberships };
|
|
74
107
|
}
|
|
@@ -88,18 +121,31 @@ export async function buildFanoutPlan(options: {
|
|
|
88
121
|
claims: IdentityClaims | null;
|
|
89
122
|
resolveSkipPersonal: (flag: boolean) => boolean;
|
|
90
123
|
}): Promise<
|
|
91
|
-
| { status: "setup-needed" }
|
|
124
|
+
| { status: "setup-needed"; reason: SetupNeededReason }
|
|
92
125
|
| { status: "plan"; plan: RunnerTarget[] }
|
|
93
126
|
> {
|
|
94
127
|
const plan: RunnerTarget[] = [];
|
|
95
128
|
for (const m of options.memberships) {
|
|
96
129
|
let slug = m.companyUid;
|
|
97
130
|
let name: string | undefined;
|
|
131
|
+
// Carried onto the target so downstream consumers (manifest reconciliation)
|
|
132
|
+
// can use the bucket, and judge liveness, without re-fetching the entity we
|
|
133
|
+
// already resolved here. On the degraded catch path below these stay
|
|
134
|
+
// undefined, which every consumer must read as "unknown" and reject.
|
|
135
|
+
let bucketName: string | undefined;
|
|
136
|
+
let entityType: string | undefined;
|
|
137
|
+
let entityStatus: string | undefined;
|
|
98
138
|
try {
|
|
99
139
|
const info = await options.client.entity.get(m.companyUid);
|
|
100
140
|
if (!info) continue;
|
|
101
141
|
slug = info.slug || m.companyUid;
|
|
102
142
|
name = info.name;
|
|
143
|
+
bucketName = info.bucketName;
|
|
144
|
+
// Recorded, never acted on here: a suspended or unexpectedly-typed
|
|
145
|
+
// company still gets a sync leg exactly as before. Only the manifest
|
|
146
|
+
// decision consumes these.
|
|
147
|
+
entityType = info.type;
|
|
148
|
+
entityStatus = info.status;
|
|
103
149
|
} catch (err) {
|
|
104
150
|
if (err instanceof VaultAuthError) throw err;
|
|
105
151
|
// A stale membership can outlive its company entity. Do not schedule a
|
|
@@ -117,7 +163,14 @@ export async function buildFanoutPlan(options: {
|
|
|
117
163
|
}
|
|
118
164
|
// Best-effort — keep UID as the display identifier.
|
|
119
165
|
}
|
|
120
|
-
plan.push({
|
|
166
|
+
plan.push({
|
|
167
|
+
uid: m.companyUid,
|
|
168
|
+
slug,
|
|
169
|
+
...(name ? { name } : {}),
|
|
170
|
+
...(bucketName ? { bucketName } : {}),
|
|
171
|
+
...(entityType ? { entityType } : {}),
|
|
172
|
+
...(entityStatus ? { entityStatus } : {}),
|
|
173
|
+
});
|
|
121
174
|
}
|
|
122
175
|
|
|
123
176
|
if (
|
|
@@ -153,7 +206,7 @@ export async function buildFanoutPlan(options: {
|
|
|
153
206
|
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
154
207
|
});
|
|
155
208
|
} else if (options.personal) {
|
|
156
|
-
return { status: "setup-needed" };
|
|
209
|
+
return { status: "setup-needed", reason: "no-person-entity" };
|
|
157
210
|
}
|
|
158
211
|
}
|
|
159
212
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as os from "os";
|
|
2
2
|
import * as path from "path";
|
|
3
|
+
import * as fs from "fs";
|
|
3
4
|
import { inventoryManagedSnapshots } from "../backup-prune.js";
|
|
4
5
|
import { VaultClient } from "../index.js";
|
|
5
6
|
import { pickCanonicalPersonEntity } from "../vault-client.js";
|
|
@@ -16,8 +17,17 @@ import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
|
16
17
|
import {
|
|
17
18
|
TreeWatcher,
|
|
18
19
|
systemClock,
|
|
20
|
+
type LocalDeleteSnapshot,
|
|
21
|
+
type TreeChange,
|
|
19
22
|
type TreeChangeBatch,
|
|
20
23
|
} from "../watcher.js";
|
|
24
|
+
import {
|
|
25
|
+
clearLocalDeleteIntent,
|
|
26
|
+
markLocalDeleteIntent,
|
|
27
|
+
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
28
|
+
readJournal,
|
|
29
|
+
writeJournal,
|
|
30
|
+
} from "../journal.js";
|
|
21
31
|
import {
|
|
22
32
|
NoopPushReceiver,
|
|
23
33
|
type PushReceiver,
|
|
@@ -37,7 +47,12 @@ import {
|
|
|
37
47
|
routeKey,
|
|
38
48
|
type WatchRoute,
|
|
39
49
|
} from "./sync-runner-watch-routes.js";
|
|
40
|
-
import type {
|
|
50
|
+
import type {
|
|
51
|
+
Direction,
|
|
52
|
+
RunnerLoopDeps,
|
|
53
|
+
RunnerPassOutcome,
|
|
54
|
+
WatcherSurface,
|
|
55
|
+
} from "./sync-runner.js";
|
|
41
56
|
|
|
42
57
|
const FULL_RECONCILE_EVERY_TICKS = 10;
|
|
43
58
|
|
|
@@ -88,7 +103,9 @@ export interface ParsedLoopArgs {
|
|
|
88
103
|
}
|
|
89
104
|
|
|
90
105
|
export interface WatchLoopRuntime {
|
|
91
|
-
runPassWithOperationLockAlreadyHeld: (
|
|
106
|
+
runPassWithOperationLockAlreadyHeld: (
|
|
107
|
+
passArgv: string[],
|
|
108
|
+
) => Promise<number | RunnerPassOutcome>;
|
|
92
109
|
authRequiredPassExit: number;
|
|
93
110
|
defaultGetIdTokenClaims: () => ({ email?: string } & TelemetryClaims) | null;
|
|
94
111
|
defaultGetAccessToken: () => Promise<string>;
|
|
@@ -96,6 +113,31 @@ export interface WatchLoopRuntime {
|
|
|
96
113
|
region: string;
|
|
97
114
|
}
|
|
98
115
|
|
|
116
|
+
type JournalFiles = ReturnType<typeof readJournal>["files"];
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Select journal entries covered by one watcher delete. Exact file unlinks
|
|
120
|
+
* deliberately avoid enumeration; only directory unlinks expand descendants.
|
|
121
|
+
*/
|
|
122
|
+
export function journalEntriesForWatcherDelete(
|
|
123
|
+
files: JournalFiles,
|
|
124
|
+
key: string,
|
|
125
|
+
kind: "unlink" | "unlinkDir",
|
|
126
|
+
): Array<[string, JournalFiles[string]]> {
|
|
127
|
+
if (kind === "unlink") {
|
|
128
|
+
const entry = files[key];
|
|
129
|
+
return entry ? [[key, entry]] : [];
|
|
130
|
+
}
|
|
131
|
+
return Object.entries(files).filter(
|
|
132
|
+
([candidate]) =>
|
|
133
|
+
key === "" || candidate === key || candidate.startsWith(`${key}/`),
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function passExitCode(outcome: number | RunnerPassOutcome): number {
|
|
138
|
+
return typeof outcome === "number" ? outcome : outcome.exitCode;
|
|
139
|
+
}
|
|
140
|
+
|
|
99
141
|
export async function runOneShotWithOperationLock(
|
|
100
142
|
argv: string[],
|
|
101
143
|
parsed: ParsedLoopArgs,
|
|
@@ -105,9 +147,11 @@ export async function runOneShotWithOperationLock(
|
|
|
105
147
|
const runOnce =
|
|
106
148
|
deps.runPass ?? runtime.runPassWithOperationLockAlreadyHeld;
|
|
107
149
|
try {
|
|
108
|
-
return
|
|
109
|
-
|
|
110
|
-
|
|
150
|
+
return passExitCode(
|
|
151
|
+
await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
|
|
152
|
+
timeoutSec: parsed.lockTimeoutSec,
|
|
153
|
+
}),
|
|
154
|
+
);
|
|
111
155
|
} catch (err) {
|
|
112
156
|
if (err instanceof OperationLockedError) {
|
|
113
157
|
process.stderr.write(err.message + "\n");
|
|
@@ -154,8 +198,10 @@ export async function runWatchLoop(
|
|
|
154
198
|
if (a === "--poll-remote-ms") return false;
|
|
155
199
|
if (a === "--event-push") return false;
|
|
156
200
|
if (a === "--scope-path") return false;
|
|
201
|
+
if (a === "--delete-scope-path") return false;
|
|
157
202
|
if (i > 0 && argv[i - 1] === "--poll-remote-ms") return false;
|
|
158
203
|
if (i > 0 && argv[i - 1] === "--scope-path") return false;
|
|
204
|
+
if (i > 0 && argv[i - 1] === "--delete-scope-path") return false;
|
|
159
205
|
return true;
|
|
160
206
|
});
|
|
161
207
|
// Authoritative direction from parseArgs — never re-derive from argv (that
|
|
@@ -183,13 +229,18 @@ export async function runWatchLoop(
|
|
|
183
229
|
}
|
|
184
230
|
}
|
|
185
231
|
|
|
186
|
-
|
|
232
|
+
type PassPreparation = () => void;
|
|
233
|
+
const runPassWithLock = async (
|
|
234
|
+
passArgvForRun: string[],
|
|
235
|
+
prepare?: PassPreparation,
|
|
236
|
+
): Promise<number | RunnerPassOutcome> => {
|
|
187
237
|
try {
|
|
188
238
|
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
189
239
|
timeoutSec: 0,
|
|
190
240
|
onWaitStart: () => undefined,
|
|
191
241
|
});
|
|
192
242
|
try {
|
|
243
|
+
prepare?.();
|
|
193
244
|
return await runPass(passArgvForRun);
|
|
194
245
|
} finally {
|
|
195
246
|
handle.release();
|
|
@@ -206,7 +257,10 @@ export async function runWatchLoop(
|
|
|
206
257
|
return await withOperationLock(
|
|
207
258
|
hqRoot,
|
|
208
259
|
"sync",
|
|
209
|
-
() =>
|
|
260
|
+
() => {
|
|
261
|
+
prepare?.();
|
|
262
|
+
return runPass(passArgvForRun);
|
|
263
|
+
},
|
|
210
264
|
{ timeoutSec: parsed.lockTimeoutSec },
|
|
211
265
|
);
|
|
212
266
|
} catch (err) {
|
|
@@ -219,10 +273,11 @@ export async function runWatchLoop(
|
|
|
219
273
|
};
|
|
220
274
|
|
|
221
275
|
let stopped = false;
|
|
222
|
-
let activePass: Promise<number> | null = null;
|
|
276
|
+
let activePass: Promise<number | RunnerPassOutcome> | null = null;
|
|
223
277
|
type QueuedPass = {
|
|
224
278
|
argv: string[];
|
|
225
|
-
|
|
279
|
+
prepare?: PassPreparation;
|
|
280
|
+
resolve: (outcome: number | RunnerPassOutcome) => void;
|
|
226
281
|
reject: (err: unknown) => void;
|
|
227
282
|
};
|
|
228
283
|
const pendingPasses: QueuedPass[] = [];
|
|
@@ -239,13 +294,16 @@ export async function runWatchLoop(
|
|
|
239
294
|
}
|
|
240
295
|
const next = pendingPasses.shift();
|
|
241
296
|
if (!next) return;
|
|
242
|
-
const current = startGuardedPass(next.argv);
|
|
297
|
+
const current = startGuardedPass(next.argv, next.prepare);
|
|
243
298
|
void current.then(next.resolve, next.reject);
|
|
244
299
|
};
|
|
245
|
-
const startGuardedPass = (
|
|
300
|
+
const startGuardedPass = (
|
|
301
|
+
passArgvForRun: string[],
|
|
302
|
+
prepare?: PassPreparation,
|
|
303
|
+
): Promise<number | RunnerPassOutcome> => {
|
|
246
304
|
const current = stopped
|
|
247
305
|
? Promise.resolve(0)
|
|
248
|
-
: runPassWithLock(passArgvForRun);
|
|
306
|
+
: runPassWithLock(passArgvForRun, prepare);
|
|
249
307
|
activePass = current;
|
|
250
308
|
void current
|
|
251
309
|
.finally(() => {
|
|
@@ -257,13 +315,16 @@ export async function runWatchLoop(
|
|
|
257
315
|
.catch(() => undefined);
|
|
258
316
|
return current;
|
|
259
317
|
};
|
|
260
|
-
const runGuarded = (
|
|
318
|
+
const runGuarded = (
|
|
319
|
+
passArgvForRun: string[],
|
|
320
|
+
prepare?: PassPreparation,
|
|
321
|
+
): Promise<number | RunnerPassOutcome> => {
|
|
261
322
|
if (activePass === null && pendingPasses.length === 0) {
|
|
262
|
-
return startGuardedPass(passArgvForRun);
|
|
323
|
+
return startGuardedPass(passArgvForRun, prepare);
|
|
263
324
|
}
|
|
264
325
|
|
|
265
|
-
return new Promise<number>((resolve, reject) => {
|
|
266
|
-
pendingPasses.push({ argv: passArgvForRun, resolve, reject });
|
|
326
|
+
return new Promise<number | RunnerPassOutcome>((resolve, reject) => {
|
|
327
|
+
pendingPasses.push({ argv: passArgvForRun, prepare, resolve, reject });
|
|
267
328
|
drainQueuedPasses();
|
|
268
329
|
});
|
|
269
330
|
};
|
|
@@ -271,6 +332,7 @@ export async function runWatchLoop(
|
|
|
271
332
|
let watcher: WatcherSurface | null = null;
|
|
272
333
|
let detachSignal: (() => void) | null = null;
|
|
273
334
|
const pendingWatcherPaths = new Map<string, string>();
|
|
335
|
+
const pendingWatcherChanges = new Map<string, TreeChange>();
|
|
274
336
|
let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
|
|
275
337
|
let pendingWatcherBareChange = false;
|
|
276
338
|
let pendingWatcherOverflowed = false;
|
|
@@ -290,6 +352,8 @@ export async function runWatchLoop(
|
|
|
290
352
|
: null;
|
|
291
353
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
292
354
|
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
355
|
+
const change = batch.changes?.get(absolutePath);
|
|
356
|
+
if (change) pendingWatcherChanges.set(absolutePath, change);
|
|
293
357
|
}
|
|
294
358
|
if (batch.overflowed) {
|
|
295
359
|
pendingWatcherOverflowed = true;
|
|
@@ -300,7 +364,9 @@ export async function runWatchLoop(
|
|
|
300
364
|
}
|
|
301
365
|
if (changedRelPath) {
|
|
302
366
|
pendingWatcherOriginalBatch = null;
|
|
303
|
-
|
|
367
|
+
const absolutePath = path.join(hqRoot, changedRelPath);
|
|
368
|
+
pendingWatcherPaths.set(absolutePath, changedRelPath);
|
|
369
|
+
pendingWatcherChanges.delete(absolutePath);
|
|
304
370
|
return;
|
|
305
371
|
}
|
|
306
372
|
pendingWatcherOriginalBatch = null;
|
|
@@ -316,6 +382,9 @@ export async function runWatchLoop(
|
|
|
316
382
|
: pendingWatcherPaths.size > 0 || pendingWatcherOverflowed
|
|
317
383
|
? {
|
|
318
384
|
paths: new Map(pendingWatcherPaths),
|
|
385
|
+
...(pendingWatcherChanges.size > 0
|
|
386
|
+
? { changes: new Map(pendingWatcherChanges) }
|
|
387
|
+
: {}),
|
|
319
388
|
...(pendingWatcherOverflowed
|
|
320
389
|
? {
|
|
321
390
|
overflowed: true,
|
|
@@ -327,6 +396,7 @@ export async function runWatchLoop(
|
|
|
327
396
|
: null;
|
|
328
397
|
const bare = pendingWatcherBareChange;
|
|
329
398
|
pendingWatcherPaths.clear();
|
|
399
|
+
pendingWatcherChanges.clear();
|
|
330
400
|
pendingWatcherOriginalBatch = null;
|
|
331
401
|
pendingWatcherBareChange = false;
|
|
332
402
|
pendingWatcherOverflowed = false;
|
|
@@ -341,15 +411,178 @@ export async function runWatchLoop(
|
|
|
341
411
|
}
|
|
342
412
|
};
|
|
343
413
|
|
|
414
|
+
const journalCoordinates = (
|
|
415
|
+
relativePath: string,
|
|
416
|
+
): { slug: string; key: string } | null => {
|
|
417
|
+
const route = routeChangeToTarget(relativePath);
|
|
418
|
+
if (!route) return null;
|
|
419
|
+
if (route.kind === "personal") {
|
|
420
|
+
return { slug: PERSONAL_VAULT_JOURNAL_SLUG, key: relativePath };
|
|
421
|
+
}
|
|
422
|
+
const prefix = `companies/${route.slug}`;
|
|
423
|
+
return {
|
|
424
|
+
slug: route.slug,
|
|
425
|
+
key:
|
|
426
|
+
relativePath === prefix
|
|
427
|
+
? ""
|
|
428
|
+
: relativePath.slice(prefix.length + 1),
|
|
429
|
+
};
|
|
430
|
+
};
|
|
431
|
+
|
|
432
|
+
const captureLocalDeleteSnapshots = (
|
|
433
|
+
relativePath: string,
|
|
434
|
+
kind: "unlink" | "unlinkDir",
|
|
435
|
+
): LocalDeleteSnapshot[] => {
|
|
436
|
+
const coordinates = journalCoordinates(relativePath);
|
|
437
|
+
const route = routeChangeToTarget(relativePath);
|
|
438
|
+
if (!coordinates || !route) return [];
|
|
439
|
+
const journal = readJournal(coordinates.slug);
|
|
440
|
+
const snapshots: LocalDeleteSnapshot[] = [];
|
|
441
|
+
const captureEntry = (
|
|
442
|
+
key: string,
|
|
443
|
+
entry: (typeof journal.files)[string] | undefined,
|
|
444
|
+
): void => {
|
|
445
|
+
if (!entry?.remoteEtag || !entry.kind || entry.removedAt !== undefined) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
const snapshot: LocalDeleteSnapshot = {
|
|
449
|
+
journalSlug: coordinates.slug,
|
|
450
|
+
journalPath: key,
|
|
451
|
+
deleteScopeRoot: coordinates.key,
|
|
452
|
+
absolutePath:
|
|
453
|
+
route.kind === "company"
|
|
454
|
+
? path.join(hqRoot, "companies", route.slug, key)
|
|
455
|
+
: path.join(hqRoot, key),
|
|
456
|
+
remoteEtag: entry.remoteEtag,
|
|
457
|
+
localHash: entry.hash,
|
|
458
|
+
localKind: entry.kind,
|
|
459
|
+
};
|
|
460
|
+
if (
|
|
461
|
+
markLocalDeleteIntent(
|
|
462
|
+
journal,
|
|
463
|
+
key,
|
|
464
|
+
snapshot.localHash,
|
|
465
|
+
snapshot.localKind,
|
|
466
|
+
snapshot.remoteEtag,
|
|
467
|
+
)
|
|
468
|
+
) {
|
|
469
|
+
snapshots.push(snapshot);
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
|
|
473
|
+
// Chokidar emits one unlink per file during rm -rf. Avoid walking the
|
|
474
|
+
// whole shard for every exact-key event; directory unlinks alone need
|
|
475
|
+
// descendant expansion.
|
|
476
|
+
for (const [key, entry] of journalEntriesForWatcherDelete(
|
|
477
|
+
journal.files,
|
|
478
|
+
coordinates.key,
|
|
479
|
+
kind,
|
|
480
|
+
)) {
|
|
481
|
+
captureEntry(key, entry);
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// Persist at observation time, before the debounced batch can queue
|
|
485
|
+
// behind an in-flight pull. The pull planner treats this current,
|
|
486
|
+
// version-bound intent as authoritative and will not resurrect the path.
|
|
487
|
+
if (snapshots.length > 0) {
|
|
488
|
+
writeJournal(coordinates.slug, journal);
|
|
489
|
+
}
|
|
490
|
+
return snapshots;
|
|
491
|
+
};
|
|
492
|
+
|
|
493
|
+
const prepareWatcherChanges = (changes: Map<string, TreeChange>): void => {
|
|
494
|
+
const journals = new Map<string, ReturnType<typeof readJournal>>();
|
|
495
|
+
const dirty = new Set<string>();
|
|
496
|
+
const journalFor = (slug: string) => {
|
|
497
|
+
const loaded = journals.get(slug);
|
|
498
|
+
if (loaded) return loaded;
|
|
499
|
+
const journal = readJournal(slug);
|
|
500
|
+
journals.set(slug, journal);
|
|
501
|
+
return journal;
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
for (const [absolutePath, change] of changes.entries()) {
|
|
505
|
+
if (change.kind === "unlink" || change.kind === "unlinkDir") {
|
|
506
|
+
for (const snapshot of change.deleteSnapshots ?? []) {
|
|
507
|
+
let absent = false;
|
|
508
|
+
try {
|
|
509
|
+
fs.lstatSync(snapshot.absolutePath);
|
|
510
|
+
} catch (err: unknown) {
|
|
511
|
+
const code =
|
|
512
|
+
err && typeof err === "object" && "code" in err
|
|
513
|
+
? (err as { code?: string }).code
|
|
514
|
+
: undefined;
|
|
515
|
+
if (code !== "ENOENT") continue;
|
|
516
|
+
absent = true;
|
|
517
|
+
}
|
|
518
|
+
if (!absent) {
|
|
519
|
+
if (
|
|
520
|
+
clearLocalDeleteIntent(
|
|
521
|
+
journalFor(snapshot.journalSlug),
|
|
522
|
+
snapshot.journalPath,
|
|
523
|
+
)
|
|
524
|
+
) {
|
|
525
|
+
dirty.add(snapshot.journalSlug);
|
|
526
|
+
}
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
if (
|
|
530
|
+
markLocalDeleteIntent(
|
|
531
|
+
journalFor(snapshot.journalSlug),
|
|
532
|
+
snapshot.journalPath,
|
|
533
|
+
snapshot.localHash,
|
|
534
|
+
snapshot.localKind,
|
|
535
|
+
snapshot.remoteEtag,
|
|
536
|
+
)
|
|
537
|
+
) {
|
|
538
|
+
dirty.add(snapshot.journalSlug);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
const relativePath = path
|
|
545
|
+
.relative(hqRoot, absolutePath)
|
|
546
|
+
.split(path.sep)
|
|
547
|
+
.join("/");
|
|
548
|
+
const coordinates = journalCoordinates(relativePath);
|
|
549
|
+
if (
|
|
550
|
+
coordinates &&
|
|
551
|
+
clearLocalDeleteIntent(
|
|
552
|
+
journalFor(coordinates.slug),
|
|
553
|
+
coordinates.key,
|
|
554
|
+
)
|
|
555
|
+
) {
|
|
556
|
+
dirty.add(coordinates.slug);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
for (const slug of dirty) {
|
|
561
|
+
writeJournal(slug, journals.get(slug)!);
|
|
562
|
+
}
|
|
563
|
+
};
|
|
564
|
+
|
|
344
565
|
const groupBatchByRoute = (
|
|
345
566
|
batch: TreeChangeBatch,
|
|
346
567
|
): Map<
|
|
347
568
|
string,
|
|
348
|
-
{
|
|
569
|
+
{
|
|
570
|
+
route: WatchRoute;
|
|
571
|
+
paths: Map<string, string>;
|
|
572
|
+
changes: Map<string, TreeChange>;
|
|
573
|
+
relPaths: string[];
|
|
574
|
+
deleteScopeRoots: string[];
|
|
575
|
+
}
|
|
349
576
|
> => {
|
|
350
577
|
const grouped = new Map<
|
|
351
578
|
string,
|
|
352
|
-
{
|
|
579
|
+
{
|
|
580
|
+
route: WatchRoute;
|
|
581
|
+
paths: Map<string, string>;
|
|
582
|
+
changes: Map<string, TreeChange>;
|
|
583
|
+
relPaths: string[];
|
|
584
|
+
deleteScopeRoots: string[];
|
|
585
|
+
}
|
|
353
586
|
>();
|
|
354
587
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
355
588
|
const route = routeChangeToTarget(relativePath);
|
|
@@ -357,9 +590,28 @@ export async function runWatchLoop(
|
|
|
357
590
|
const key = routeKey(route);
|
|
358
591
|
const existing =
|
|
359
592
|
grouped.get(key) ??
|
|
360
|
-
{
|
|
593
|
+
{
|
|
594
|
+
route,
|
|
595
|
+
paths: new Map<string, string>(),
|
|
596
|
+
changes: new Map<string, TreeChange>(),
|
|
597
|
+
relPaths: [],
|
|
598
|
+
deleteScopeRoots: [],
|
|
599
|
+
};
|
|
361
600
|
existing.paths.set(absolutePath, relativePath);
|
|
601
|
+
const change = batch.changes?.get(absolutePath);
|
|
602
|
+
if (change) existing.changes.set(absolutePath, change);
|
|
362
603
|
existing.relPaths = [...new Set([...existing.paths.values()])];
|
|
604
|
+
existing.deleteScopeRoots = [
|
|
605
|
+
...new Set(
|
|
606
|
+
[...existing.changes.values()].flatMap((item) =>
|
|
607
|
+
(item.deleteSnapshots ?? []).flatMap((snapshot) =>
|
|
608
|
+
snapshot.deleteScopeRoot === undefined
|
|
609
|
+
? []
|
|
610
|
+
: [snapshot.deleteScopeRoot],
|
|
611
|
+
),
|
|
612
|
+
),
|
|
613
|
+
),
|
|
614
|
+
];
|
|
363
615
|
grouped.set(key, existing);
|
|
364
616
|
}
|
|
365
617
|
return grouped;
|
|
@@ -381,11 +633,38 @@ export async function runWatchLoop(
|
|
|
381
633
|
group.route,
|
|
382
634
|
group.relPaths,
|
|
383
635
|
passArgv,
|
|
636
|
+
group.deleteScopeRoots,
|
|
384
637
|
);
|
|
385
638
|
try {
|
|
386
|
-
const result = await runGuarded(
|
|
387
|
-
|
|
388
|
-
|
|
639
|
+
const result = await runGuarded(
|
|
640
|
+
scopedArgv,
|
|
641
|
+
group.changes.size > 0
|
|
642
|
+
? () => prepareWatcherChanges(group.changes)
|
|
643
|
+
: undefined,
|
|
644
|
+
);
|
|
645
|
+
if (passExitCode(result) === 0) {
|
|
646
|
+
const refusedPaths = new Set(
|
|
647
|
+
typeof result === "number"
|
|
648
|
+
? []
|
|
649
|
+
: (result.result?.pushPathResults ?? [])
|
|
650
|
+
.filter((pathResult) => pathResult.status === "refused")
|
|
651
|
+
.map((pathResult) => pathResult.relativePath),
|
|
652
|
+
);
|
|
653
|
+
publishBatchIfAvailable(
|
|
654
|
+
new Map(
|
|
655
|
+
[...group.paths.entries()].filter(
|
|
656
|
+
([absolutePath, relativePath]) => {
|
|
657
|
+
if (refusedPaths.has(relativePath)) return false;
|
|
658
|
+
if (group.changes.get(absolutePath)?.kind !== "unlinkDir") {
|
|
659
|
+
return true;
|
|
660
|
+
}
|
|
661
|
+
return ![...refusedPaths].some((refusedPath) =>
|
|
662
|
+
refusedPath.startsWith(`${relativePath}/`),
|
|
663
|
+
);
|
|
664
|
+
},
|
|
665
|
+
),
|
|
666
|
+
),
|
|
667
|
+
);
|
|
389
668
|
} else {
|
|
390
669
|
restorePendingWatcherPaths(group.paths);
|
|
391
670
|
}
|
|
@@ -399,7 +678,9 @@ export async function runWatchLoop(
|
|
|
399
678
|
}
|
|
400
679
|
|
|
401
680
|
if (!drainRunsPull) return 0;
|
|
402
|
-
return
|
|
681
|
+
return passExitCode(
|
|
682
|
+
await runGuarded(buildScopedDrainPullArgv(passArgv)),
|
|
683
|
+
);
|
|
403
684
|
};
|
|
404
685
|
|
|
405
686
|
let receiver: PushReceiver | null = null;
|
|
@@ -416,8 +697,14 @@ export async function runWatchLoop(
|
|
|
416
697
|
debounceMs: opts.debounceMs,
|
|
417
698
|
clock: opts.clock,
|
|
418
699
|
personalMode: !companiesMode,
|
|
700
|
+
captureLocalDeleteSnapshots: opts.captureLocalDeleteSnapshots,
|
|
419
701
|
}));
|
|
420
|
-
watcher = createWatcher({
|
|
702
|
+
watcher = createWatcher({
|
|
703
|
+
hqRoot,
|
|
704
|
+
debounceMs,
|
|
705
|
+
clock,
|
|
706
|
+
captureLocalDeleteSnapshots,
|
|
707
|
+
});
|
|
421
708
|
|
|
422
709
|
watcher.onChange((changedRelPath, batch) => {
|
|
423
710
|
if (stopped) return;
|
|
@@ -431,8 +718,10 @@ export async function runWatchLoop(
|
|
|
431
718
|
if (!route) return;
|
|
432
719
|
const targetedArgv = buildTargetedPullArgv(route, passArgv);
|
|
433
720
|
const result = await runGuarded(targetedArgv);
|
|
434
|
-
if (result !== 0) {
|
|
435
|
-
throw new Error(
|
|
721
|
+
if (passExitCode(result) !== 0) {
|
|
722
|
+
throw new Error(
|
|
723
|
+
`targeted pull failed with exit code ${passExitCode(result)}`,
|
|
724
|
+
);
|
|
436
725
|
}
|
|
437
726
|
};
|
|
438
727
|
const createReceiver = deps.createReceiver ?? (() => new NoopPushReceiver());
|
|
@@ -537,12 +826,13 @@ export async function runWatchLoop(
|
|
|
537
826
|
const result = shouldRunFull
|
|
538
827
|
? await runGuarded(passArgv)
|
|
539
828
|
: await runScopedDrain(pending.batch);
|
|
540
|
-
|
|
829
|
+
const exitCode = passExitCode(result);
|
|
830
|
+
if (exitCode === runtime.authRequiredPassExit) {
|
|
541
831
|
// The pass already emitted auth-error. Stop the unattended loop but
|
|
542
832
|
// keep the process exit successful so desktop/manual consumers treat
|
|
543
833
|
// this as a reauth state, not a crash.
|
|
544
834
|
return 0;
|
|
545
|
-
} else if (
|
|
835
|
+
} else if (exitCode === TRANSIENT_NETWORK_EXIT) {
|
|
546
836
|
// The machine was briefly offline (the discovery fetch failed after
|
|
547
837
|
// retries). That is NOT a crash — stay alive and retry on the next
|
|
548
838
|
// poll. Returning it here would exit the watcher and the menubar would
|
|
@@ -551,7 +841,7 @@ export async function runWatchLoop(
|
|
|
551
841
|
process.stderr.write(
|
|
552
842
|
"hq-sync-runner: watch pass skipped — transient network failure; retrying next poll\n",
|
|
553
843
|
);
|
|
554
|
-
} else if (
|
|
844
|
+
} else if (exitCode === PARTIAL_SYNC_EXIT) {
|
|
555
845
|
// The pass completed but some per-file transfers threw. Also NOT a
|
|
556
846
|
// crash: the per-file errors were already emitted as ndjson `error`
|
|
557
847
|
// events (menubar breadcrumbs + journal), and exiting here just gets
|
|
@@ -562,8 +852,8 @@ export async function runWatchLoop(
|
|
|
562
852
|
process.stderr.write(
|
|
563
853
|
"hq-sync-runner: watch pass completed with per-file errors; staying alive, retrying next poll\n",
|
|
564
854
|
);
|
|
565
|
-
} else if (
|
|
566
|
-
return
|
|
855
|
+
} else if (exitCode !== 0) {
|
|
856
|
+
return exitCode;
|
|
567
857
|
}
|
|
568
858
|
if (Date.now() >= nextBackupPruneAt) {
|
|
569
859
|
// Advance before invoking: a failed/unavailable backstop cannot become
|