@indigoai-us/hq-cloud 6.11.10 → 6.11.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +2 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +231 -52
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +330 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +16 -1
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -1
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +58 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.js +229 -15
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-exec-bit-preserve.test.d.ts +2 -0
- package/dist/cli/rescue-exec-bit-preserve.test.d.ts.map +1 -0
- package/dist/cli/rescue-exec-bit-preserve.test.js +169 -0
- package/dist/cli/rescue-exec-bit-preserve.test.js.map +1 -0
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +100 -32
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +188 -59
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +487 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +148 -29
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts +8 -0
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +17 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +6 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +62 -13
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +189 -0
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +6 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +13 -15
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +36 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/watcher.d.ts +36 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +152 -30
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner.test.ts +396 -11
- package/src/bin/sync-runner.ts +254 -52
- package/src/cli/reindex.test.ts +47 -1
- package/src/cli/reindex.ts +17 -1
- package/src/cli/rescue-classify-ordering.test.ts +61 -0
- package/src/cli/rescue-core.ts +261 -15
- package/src/cli/rescue-exec-bit-preserve.test.ts +187 -0
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +103 -34
- package/src/cli/sync.test.ts +594 -1
- package/src/cli/sync.ts +229 -65
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +182 -30
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +18 -3
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +205 -0
- package/src/remote-pull.ts +77 -14
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +38 -1
- package/src/sync/push-receiver.ts +15 -18
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +209 -33
package/src/bin/sync-runner.ts
CHANGED
|
@@ -107,6 +107,7 @@ import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
|
|
|
107
107
|
import { reindexAfterSync } from "../qmd-reindex.js";
|
|
108
108
|
import { pruneConflictIndex } from "../lib/conflict-index.js";
|
|
109
109
|
import {
|
|
110
|
+
acquireOperationLock,
|
|
110
111
|
withOperationLock,
|
|
111
112
|
OperationLockedError,
|
|
112
113
|
OPERATION_LOCKED_EXIT,
|
|
@@ -486,6 +487,8 @@ export interface RunnerDeps {
|
|
|
486
487
|
createVaultClient?: (config: VaultServiceConfig) => VaultClientSurface;
|
|
487
488
|
/** Sync function. Defaults to `cli/sync.sync`. */
|
|
488
489
|
sync?: (options: SyncOptions) => Promise<SyncResult>;
|
|
490
|
+
/** Internal: set when runRunner is invoked under the per-root operation lock. */
|
|
491
|
+
operationLockAlreadyHeld?: boolean;
|
|
489
492
|
/** Share function (push phase). Defaults to `cli/share.share`. */
|
|
490
493
|
share?: (options: ShareOptions) => Promise<ShareResult>;
|
|
491
494
|
/**
|
|
@@ -1169,6 +1172,7 @@ export async function runRunner(
|
|
|
1169
1172
|
// rows land on the right company regardless of which phase emitted them.
|
|
1170
1173
|
// Also updates `state` for `progress` events so the rollup has accurate
|
|
1171
1174
|
// partial counts even if the sync function throws before returning.
|
|
1175
|
+
let companyHadTransferError = false;
|
|
1172
1176
|
const tagAndEmit = (event: SyncProgressEvent): void => {
|
|
1173
1177
|
if (event.type === "plan") {
|
|
1174
1178
|
emit({
|
|
@@ -1212,6 +1216,11 @@ export async function runRunner(
|
|
|
1212
1216
|
resolution: event.resolution,
|
|
1213
1217
|
});
|
|
1214
1218
|
} else if (event.type === "error") {
|
|
1219
|
+
companyHadTransferError = true;
|
|
1220
|
+
errors.push({
|
|
1221
|
+
company: companyLabel,
|
|
1222
|
+
message: event.path ? `${event.path}: ${event.message}` : event.message,
|
|
1223
|
+
});
|
|
1215
1224
|
emit({
|
|
1216
1225
|
type: "error",
|
|
1217
1226
|
company: companyLabel,
|
|
@@ -1440,6 +1449,9 @@ export async function runRunner(
|
|
|
1440
1449
|
teamSyncedSlugs,
|
|
1441
1450
|
}
|
|
1442
1451
|
: {}),
|
|
1452
|
+
...(deps.operationLockAlreadyHeld
|
|
1453
|
+
? { operationLockAlreadyHeld: true }
|
|
1454
|
+
: {}),
|
|
1443
1455
|
onEvent: tagAndEmit,
|
|
1444
1456
|
});
|
|
1445
1457
|
}
|
|
@@ -1474,7 +1486,11 @@ export async function runRunner(
|
|
|
1474
1486
|
state.bytesDownloaded = pullResult.bytesDownloaded;
|
|
1475
1487
|
state.filesUploaded = pushResult.filesUploaded;
|
|
1476
1488
|
state.bytesUploaded = pushResult.bytesUploaded;
|
|
1477
|
-
state.status =
|
|
1489
|
+
state.status = companyHadTransferError
|
|
1490
|
+
? "errored"
|
|
1491
|
+
: aborted
|
|
1492
|
+
? "aborted"
|
|
1493
|
+
: "complete";
|
|
1478
1494
|
|
|
1479
1495
|
emit({
|
|
1480
1496
|
type: "complete",
|
|
@@ -1882,6 +1898,26 @@ export function buildTargetedPushArgv(
|
|
|
1882
1898
|
return ["--companies", "--direction", "push", ...carried];
|
|
1883
1899
|
}
|
|
1884
1900
|
|
|
1901
|
+
type WatchRoute = NonNullable<ReturnType<typeof routeChangeToTarget>>;
|
|
1902
|
+
|
|
1903
|
+
function routeKey(route: WatchRoute): string {
|
|
1904
|
+
return route.kind === "company" ? `company:${route.slug}` : "personal";
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
function routesForBatch(batch: TreeChangeBatch): Map<string, WatchRoute> {
|
|
1908
|
+
const routes = new Map<string, WatchRoute>();
|
|
1909
|
+
for (const relPath of batch.paths.values()) {
|
|
1910
|
+
const route = routeChangeToTarget(relPath);
|
|
1911
|
+
if (!route) continue;
|
|
1912
|
+
routes.set(routeKey(route), route);
|
|
1913
|
+
}
|
|
1914
|
+
return routes;
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
function buildFullFanoutPushArgv(baseArgv: string[]): string[] {
|
|
1918
|
+
return ["--companies", "--direction", "push", ...carriedFlags(baseArgv)];
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1885
1921
|
/**
|
|
1886
1922
|
* Build the argv for a targeted PULL pass from a routed change (US-009 — the
|
|
1887
1923
|
* receiver's pull-on-event path). Mirrors {@link buildTargetedPushArgv} but
|
|
@@ -1908,14 +1944,12 @@ export async function runRunnerWithLoop(
|
|
|
1908
1944
|
argv: string[],
|
|
1909
1945
|
deps: RunnerLoopDeps = {},
|
|
1910
1946
|
): Promise<number> {
|
|
1947
|
+
const parsed = parseArgs(argv);
|
|
1911
1948
|
if (!argv.includes("--watch")) {
|
|
1912
1949
|
// One-shot cloud sync — take the per-root operation lock so it is mutually
|
|
1913
|
-
// exclusive with rescue/reindex.
|
|
1914
|
-
//
|
|
1915
|
-
//
|
|
1916
|
-
// through here). If args don't parse, fall through to `runRunner` so it
|
|
1917
|
-
// surfaces the parse error rather than us masking it with a lock failure.
|
|
1918
|
-
const parsed = parseArgs(argv);
|
|
1950
|
+
// exclusive with rescue/reindex. If args don't parse, fall through to
|
|
1951
|
+
// `runRunner` so it surfaces the parse error rather than us masking it
|
|
1952
|
+
// with a lock failure.
|
|
1919
1953
|
if ("error" in parsed) return runRunner(argv);
|
|
1920
1954
|
// The actual sync pass — same seam the watch loop uses (deps.runPass),
|
|
1921
1955
|
// so a test can assert "waits for a short-lived holder, THEN proceeds to
|
|
@@ -1924,7 +1958,10 @@ export async function runRunnerWithLoop(
|
|
|
1924
1958
|
// (feedback_28a1833f): instant-sync one-shots used to exit 17 and die on
|
|
1925
1959
|
// a lock conflict with the ~1-min reindex hook; they now WAIT (default)
|
|
1926
1960
|
// and proceed once the short holder releases.
|
|
1927
|
-
const runOnce =
|
|
1961
|
+
const runOnce =
|
|
1962
|
+
deps.runPass ??
|
|
1963
|
+
((passArgv: string[]) =>
|
|
1964
|
+
runRunner(passArgv, { operationLockAlreadyHeld: true }));
|
|
1928
1965
|
try {
|
|
1929
1966
|
return await withOperationLock(parsed.hqRoot, "sync", () => runOnce(argv), {
|
|
1930
1967
|
timeoutSec: parsed.lockTimeoutSec,
|
|
@@ -1944,10 +1981,14 @@ export async function runRunnerWithLoop(
|
|
|
1944
1981
|
throw err;
|
|
1945
1982
|
}
|
|
1946
1983
|
}
|
|
1984
|
+
if ("error" in parsed) return runRunner(argv);
|
|
1947
1985
|
const sleep =
|
|
1948
1986
|
deps.sleep ??
|
|
1949
1987
|
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
1950
|
-
const runPass =
|
|
1988
|
+
const runPass =
|
|
1989
|
+
deps.runPass ??
|
|
1990
|
+
((passArgv: string[]) =>
|
|
1991
|
+
runRunner(passArgv, { operationLockAlreadyHeld: true }));
|
|
1951
1992
|
const pollIdx = argv.indexOf("--poll-remote-ms");
|
|
1952
1993
|
const pollMs =
|
|
1953
1994
|
pollIdx >= 0 && argv[pollIdx + 1] ? Number(argv[pollIdx + 1]) : 600_000;
|
|
@@ -1961,9 +2002,7 @@ export async function runRunnerWithLoop(
|
|
|
1961
2002
|
// watch filter correctly in companies mode. personalMode is only for a
|
|
1962
2003
|
// personal-vault-as-root run, where companies/ et al. genuinely aren't synced.
|
|
1963
2004
|
const companiesMode = argv.includes("--companies");
|
|
1964
|
-
const
|
|
1965
|
-
const hqRoot =
|
|
1966
|
-
hqIdx >= 0 && argv[hqIdx + 1] ? argv[hqIdx + 1] : DEFAULT_HQ_ROOT;
|
|
2005
|
+
const hqRoot = parsed.hqRoot;
|
|
1967
2006
|
|
|
1968
2007
|
// Strip the loop-only flags before delegating: the parser inside runRunner
|
|
1969
2008
|
// accepts --watch/--poll-remote-ms/--event-push, but we don't want a per-
|
|
@@ -1976,31 +2015,181 @@ export async function runRunnerWithLoop(
|
|
|
1976
2015
|
return true;
|
|
1977
2016
|
});
|
|
1978
2017
|
|
|
1979
|
-
|
|
1980
|
-
// The poll loop AND watcher-triggered targeted pushes funnel through this
|
|
1981
|
-
// mutex so a watcher push never overlaps an in-flight pass (PRD AC). A
|
|
1982
|
-
// trigger that arrives while a pass runs is collapsed by WatchPushDriver's
|
|
1983
|
-
// own pending-while-pushing logic, then re-armed after the pass settles.
|
|
1984
|
-
let inFlight = false;
|
|
1985
|
-
let stopped = false;
|
|
1986
|
-
const runGuarded = async (
|
|
1987
|
-
pass: () => Promise<number>,
|
|
1988
|
-
): Promise<number | "skipped"> => {
|
|
1989
|
-
if (inFlight) return "skipped";
|
|
1990
|
-
inFlight = true;
|
|
2018
|
+
if (parsed.lockTimeoutSec === 0) {
|
|
1991
2019
|
try {
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
2020
|
+
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
2021
|
+
timeoutSec: 0,
|
|
2022
|
+
onWaitStart: () => undefined,
|
|
2023
|
+
});
|
|
2024
|
+
handle.release();
|
|
2025
|
+
} catch (err) {
|
|
2026
|
+
if (err instanceof OperationLockedError) {
|
|
2027
|
+
process.stderr.write(err.message + "\n");
|
|
2028
|
+
return OPERATION_LOCKED_EXIT;
|
|
2029
|
+
}
|
|
2030
|
+
throw err;
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
|
|
2034
|
+
const runPassWithLock = async (passArgvForRun: string[]): Promise<number> => {
|
|
2035
|
+
try {
|
|
2036
|
+
const handle = acquireOperationLock(hqRoot, "sync", {
|
|
2037
|
+
timeoutSec: 0,
|
|
2038
|
+
onWaitStart: () => undefined,
|
|
2039
|
+
});
|
|
2040
|
+
try {
|
|
2041
|
+
return await runPass(passArgvForRun);
|
|
2042
|
+
} finally {
|
|
2043
|
+
handle.release();
|
|
2044
|
+
}
|
|
2045
|
+
} catch (err) {
|
|
2046
|
+
if (!(err instanceof OperationLockedError)) throw err;
|
|
2047
|
+
if (parsed.lockTimeoutSec === 0) {
|
|
2048
|
+
process.stderr.write(err.message + "\n");
|
|
2049
|
+
return OPERATION_LOCKED_EXIT;
|
|
2050
|
+
}
|
|
1995
2051
|
}
|
|
2052
|
+
|
|
2053
|
+
try {
|
|
2054
|
+
return await withOperationLock(
|
|
2055
|
+
hqRoot,
|
|
2056
|
+
"sync",
|
|
2057
|
+
() => runPass(passArgvForRun),
|
|
2058
|
+
{ timeoutSec: parsed.lockTimeoutSec },
|
|
2059
|
+
);
|
|
2060
|
+
} catch (err) {
|
|
2061
|
+
if (err instanceof OperationLockedError) {
|
|
2062
|
+
process.stderr.write(err.message + "\n");
|
|
2063
|
+
return OPERATION_LOCKED_EXIT;
|
|
2064
|
+
}
|
|
2065
|
+
throw err;
|
|
2066
|
+
}
|
|
2067
|
+
};
|
|
2068
|
+
|
|
2069
|
+
// ---- shared pass queue -----------------------------------------------
|
|
2070
|
+
// The poll loop, pull-on-event receiver, and watcher-triggered pushes all
|
|
2071
|
+
// funnel through this queue so local/remote triggers never overlap, and a
|
|
2072
|
+
// trigger arriving during an active pass runs immediately after it instead
|
|
2073
|
+
// of being dropped.
|
|
2074
|
+
let stopped = false;
|
|
2075
|
+
let activePass: Promise<number> | null = null;
|
|
2076
|
+
type QueuedPass = {
|
|
2077
|
+
argv: string[];
|
|
2078
|
+
resolve: (code: number) => void;
|
|
2079
|
+
reject: (err: unknown) => void;
|
|
2080
|
+
};
|
|
2081
|
+
const pendingPasses: QueuedPass[] = [];
|
|
2082
|
+
const resolveStoppedQueue = (): void => {
|
|
2083
|
+
while (pendingPasses.length > 0) {
|
|
2084
|
+
pendingPasses.shift()?.resolve(0);
|
|
2085
|
+
}
|
|
2086
|
+
};
|
|
2087
|
+
const drainQueuedPasses = (): void => {
|
|
2088
|
+
if (activePass !== null) return;
|
|
2089
|
+
if (stopped) {
|
|
2090
|
+
resolveStoppedQueue();
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
const next = pendingPasses.shift();
|
|
2094
|
+
if (!next) return;
|
|
2095
|
+
const current = startGuardedPass(next.argv);
|
|
2096
|
+
void current.then(next.resolve, next.reject);
|
|
2097
|
+
};
|
|
2098
|
+
const startGuardedPass = (passArgvForRun: string[]): Promise<number> => {
|
|
2099
|
+
const current = stopped
|
|
2100
|
+
? Promise.resolve(0)
|
|
2101
|
+
: runPassWithLock(passArgvForRun);
|
|
2102
|
+
activePass = current;
|
|
2103
|
+
void current
|
|
2104
|
+
.finally(() => {
|
|
2105
|
+
if (activePass === current) {
|
|
2106
|
+
activePass = null;
|
|
2107
|
+
drainQueuedPasses();
|
|
2108
|
+
}
|
|
2109
|
+
})
|
|
2110
|
+
.catch(() => undefined);
|
|
2111
|
+
return current;
|
|
2112
|
+
};
|
|
2113
|
+
const runGuarded = (passArgvForRun: string[]): Promise<number> => {
|
|
2114
|
+
if (activePass === null && pendingPasses.length === 0) {
|
|
2115
|
+
return startGuardedPass(passArgvForRun);
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
return new Promise<number>((resolve, reject) => {
|
|
2119
|
+
pendingPasses.push({ argv: passArgvForRun, resolve, reject });
|
|
2120
|
+
drainQueuedPasses();
|
|
2121
|
+
});
|
|
1996
2122
|
};
|
|
1997
2123
|
|
|
1998
2124
|
// ---- event-push wiring (Phase 1) -------------------------------------
|
|
1999
2125
|
let watcher: WatcherSurface | null = null;
|
|
2000
2126
|
let driver: WatchPushDriver | null = null;
|
|
2001
2127
|
let detachSignal: (() => void) | null = null;
|
|
2002
|
-
|
|
2003
|
-
let
|
|
2128
|
+
const pendingWatcherPaths = new Map<string, string>();
|
|
2129
|
+
let pendingWatcherOriginalBatch: TreeChangeBatch | null = null;
|
|
2130
|
+
let pendingWatcherBareChange = false;
|
|
2131
|
+
let pendingWatcherOverflowed = false;
|
|
2132
|
+
let pendingWatcherDroppedPaths = 0;
|
|
2133
|
+
let pendingWatcherDroppedBytes = 0;
|
|
2134
|
+
const addPendingWatcherChange = (
|
|
2135
|
+
changedRelPath?: string,
|
|
2136
|
+
batch?: TreeChangeBatch,
|
|
2137
|
+
): void => {
|
|
2138
|
+
if (batch) {
|
|
2139
|
+
pendingWatcherOriginalBatch =
|
|
2140
|
+
pendingWatcherPaths.size === 0 &&
|
|
2141
|
+
!pendingWatcherBareChange &&
|
|
2142
|
+
!pendingWatcherOverflowed &&
|
|
2143
|
+
!batch.overflowed
|
|
2144
|
+
? batch
|
|
2145
|
+
: null;
|
|
2146
|
+
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
2147
|
+
pendingWatcherPaths.set(absolutePath, relativePath);
|
|
2148
|
+
}
|
|
2149
|
+
if (batch.overflowed) {
|
|
2150
|
+
pendingWatcherOverflowed = true;
|
|
2151
|
+
pendingWatcherDroppedPaths += batch.droppedPaths ?? 0;
|
|
2152
|
+
pendingWatcherDroppedBytes += batch.droppedBytes ?? 0;
|
|
2153
|
+
}
|
|
2154
|
+
return;
|
|
2155
|
+
}
|
|
2156
|
+
if (changedRelPath) {
|
|
2157
|
+
pendingWatcherOriginalBatch = null;
|
|
2158
|
+
pendingWatcherPaths.set(path.join(hqRoot, changedRelPath), changedRelPath);
|
|
2159
|
+
return;
|
|
2160
|
+
}
|
|
2161
|
+
pendingWatcherOriginalBatch = null;
|
|
2162
|
+
pendingWatcherBareChange = true;
|
|
2163
|
+
};
|
|
2164
|
+
const takePendingWatcherChange = (): {
|
|
2165
|
+
rel: string | null;
|
|
2166
|
+
batch: TreeChangeBatch | null;
|
|
2167
|
+
} => {
|
|
2168
|
+
const batch =
|
|
2169
|
+
pendingWatcherOriginalBatch !== null && !pendingWatcherOverflowed
|
|
2170
|
+
? pendingWatcherOriginalBatch
|
|
2171
|
+
: pendingWatcherPaths.size > 0 || pendingWatcherOverflowed
|
|
2172
|
+
? {
|
|
2173
|
+
paths: new Map(pendingWatcherPaths),
|
|
2174
|
+
...(pendingWatcherOverflowed
|
|
2175
|
+
? {
|
|
2176
|
+
overflowed: true,
|
|
2177
|
+
droppedPaths: pendingWatcherDroppedPaths,
|
|
2178
|
+
droppedBytes: pendingWatcherDroppedBytes,
|
|
2179
|
+
}
|
|
2180
|
+
: {}),
|
|
2181
|
+
}
|
|
2182
|
+
: null;
|
|
2183
|
+
const rel = [...pendingWatcherPaths.values()][0] ?? null;
|
|
2184
|
+
const fallbackRel = rel ?? (pendingWatcherBareChange ? null : null);
|
|
2185
|
+
pendingWatcherPaths.clear();
|
|
2186
|
+
pendingWatcherOriginalBatch = null;
|
|
2187
|
+
pendingWatcherBareChange = false;
|
|
2188
|
+
pendingWatcherOverflowed = false;
|
|
2189
|
+
pendingWatcherDroppedPaths = 0;
|
|
2190
|
+
pendingWatcherDroppedBytes = 0;
|
|
2191
|
+
return { rel: fallbackRel, batch };
|
|
2192
|
+
};
|
|
2004
2193
|
// ---- pull-on-event receiver (Phase 2, US-009) ------------------------
|
|
2005
2194
|
// Started after the watcher, disposed before the watcher (mirror of the
|
|
2006
2195
|
// PushTransport ordering). Dormant by default: the default factory returns
|
|
@@ -2037,24 +2226,36 @@ export async function runRunnerWithLoop(
|
|
|
2037
2226
|
clock,
|
|
2038
2227
|
push: async () => {
|
|
2039
2228
|
if (stopped) return;
|
|
2040
|
-
|
|
2041
|
-
//
|
|
2042
|
-
//
|
|
2043
|
-
|
|
2044
|
-
const
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2229
|
+
// Snapshot accumulated watcher work BEFORE the await. Changes landing
|
|
2230
|
+
// mid-pass are accumulated for the NEXT pass instead of overwriting
|
|
2231
|
+
// this pass's publish target.
|
|
2232
|
+
const { rel, batch: batchForPublish } = takePendingWatcherChange();
|
|
2233
|
+
const batchRoutes = batchForPublish
|
|
2234
|
+
? routesForBatch(batchForPublish)
|
|
2235
|
+
: new Map<string, WatchRoute>();
|
|
2236
|
+
let targetedArgv: string[];
|
|
2237
|
+
if (batchForPublish?.overflowed || batchRoutes.size > 1) {
|
|
2238
|
+
targetedArgv = buildFullFanoutPushArgv(passArgv);
|
|
2239
|
+
} else {
|
|
2240
|
+
const route =
|
|
2241
|
+
batchRoutes.size === 1
|
|
2242
|
+
? [...batchRoutes.values()][0]
|
|
2243
|
+
: rel
|
|
2244
|
+
? routeChangeToTarget(rel)
|
|
2245
|
+
: { kind: "personal" as const };
|
|
2246
|
+
if (!route) return;
|
|
2247
|
+
targetedArgv = buildTargetedPushArgv(route, passArgv);
|
|
2248
|
+
}
|
|
2249
|
+
const result = await runGuarded(targetedArgv);
|
|
2052
2250
|
// Phase 3 (US-017): publish PushEvents only AFTER the targeted push
|
|
2053
2251
|
// pass succeeded — an event must never announce bytes that are not
|
|
2054
|
-
// in S3 yet. A
|
|
2055
|
-
//
|
|
2056
|
-
// single-path batch when the watcher emitted a bare path
|
|
2057
|
-
|
|
2252
|
+
// in S3 yet. A failed pass publishes nothing; queued passes run after
|
|
2253
|
+
// the active pass instead of dropping the watcher-triggered push. Fall
|
|
2254
|
+
// back to a single-path batch when the watcher emitted a bare path
|
|
2255
|
+
// signal. Overflowed batches deliberately publish nothing because the
|
|
2256
|
+
// exact path set was dropped; the full fanout push plus cadence poll is
|
|
2257
|
+
// the resync signal.
|
|
2258
|
+
if (result === 0 && !stopped && eventSync && !batchForPublish?.overflowed) {
|
|
2058
2259
|
const batch: TreeChangeBatch | null =
|
|
2059
2260
|
batchForPublish ??
|
|
2060
2261
|
(rel ? { paths: new Map([[path.join(hqRoot, rel), rel]]) } : null);
|
|
@@ -2070,8 +2271,7 @@ export async function runRunnerWithLoop(
|
|
|
2070
2271
|
// the bare-signal TreeWatcher leaves it null → personal-vault route.
|
|
2071
2272
|
watcher.onChange((changedRelPath, batch) => {
|
|
2072
2273
|
if (stopped) return;
|
|
2073
|
-
|
|
2074
|
-
lastBatch = batch ?? null;
|
|
2274
|
+
addPendingWatcherChange(changedRelPath, batch);
|
|
2075
2275
|
driver?.notifyChange();
|
|
2076
2276
|
});
|
|
2077
2277
|
watcher.start();
|
|
@@ -2088,7 +2288,10 @@ export async function runRunnerWithLoop(
|
|
|
2088
2288
|
const route = routeChangeToTarget(ctx.event.relativePath);
|
|
2089
2289
|
if (!route) return;
|
|
2090
2290
|
const targetedArgv = buildTargetedPullArgv(route, passArgv);
|
|
2091
|
-
await runGuarded(
|
|
2291
|
+
const result = await runGuarded(targetedArgv);
|
|
2292
|
+
if (result !== 0) {
|
|
2293
|
+
throw new Error(`targeted pull failed with exit code ${result}`);
|
|
2294
|
+
}
|
|
2092
2295
|
};
|
|
2093
2296
|
const createReceiver =
|
|
2094
2297
|
deps.createReceiver ?? (() => new NoopPushReceiver());
|
|
@@ -2167,6 +2370,7 @@ export async function runRunnerWithLoop(
|
|
|
2167
2370
|
const shutdown = (): void => {
|
|
2168
2371
|
if (stopped) return;
|
|
2169
2372
|
stopped = true;
|
|
2373
|
+
resolveStoppedQueue();
|
|
2170
2374
|
// Dispose the receiver FIRST (mirror of the PushTransport ordering:
|
|
2171
2375
|
// inbound subscription torn down before the watcher) so no new
|
|
2172
2376
|
// pull-on-event fires mid-teardown. dispose() is async (it drains the
|
|
@@ -2213,10 +2417,8 @@ export async function runRunnerWithLoop(
|
|
|
2213
2417
|
|
|
2214
2418
|
try {
|
|
2215
2419
|
while (!stopped) {
|
|
2216
|
-
const result = await runGuarded(
|
|
2217
|
-
|
|
2218
|
-
// benign — the next iteration retries after the poll interval.
|
|
2219
|
-
if (typeof result === "number" && result !== 0) {
|
|
2420
|
+
const result = await runGuarded(passArgv);
|
|
2421
|
+
if (result !== 0) {
|
|
2220
2422
|
return result;
|
|
2221
2423
|
}
|
|
2222
2424
|
// Sleep the poll interval, but wake early on shutdown so SIGTERM stops
|
package/src/cli/reindex.test.ts
CHANGED
|
@@ -8,13 +8,36 @@
|
|
|
8
8
|
* re-deriving the implementation internals.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
11
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
|
12
12
|
import * as fs from "fs";
|
|
13
13
|
import * as path from "path";
|
|
14
14
|
import * as os from "os";
|
|
15
15
|
import { reindex } from "./reindex.js";
|
|
16
16
|
import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
|
|
17
17
|
|
|
18
|
+
// HQ-B0: simulate the Windows filesystem rejecting a ':' in a path segment.
|
|
19
|
+
// The flag is off by default, so every other test sees the real `mkdirSync`;
|
|
20
|
+
// the regression test below flips it on only for its own run.
|
|
21
|
+
const hoisted = vi.hoisted(() => ({ failNamespacedMkdir: false }));
|
|
22
|
+
vi.mock("fs", async (importOriginal) => {
|
|
23
|
+
const actual = await importOriginal<typeof import("fs")>();
|
|
24
|
+
const mkdirSync = ((p: unknown, opts: unknown) => {
|
|
25
|
+
if (
|
|
26
|
+
hoisted.failNamespacedMkdir &&
|
|
27
|
+
typeof p === "string" &&
|
|
28
|
+
/[:][^/\\]*$/.test(p) // a colon in the final path segment
|
|
29
|
+
) {
|
|
30
|
+
throw Object.assign(
|
|
31
|
+
new Error(`ENOENT: no such file or directory, mkdir '${p}'`),
|
|
32
|
+
{ code: "ENOENT" },
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
return (actual.mkdirSync as (p: unknown, opts: unknown) => unknown)(p, opts);
|
|
36
|
+
}) as typeof actual.mkdirSync;
|
|
37
|
+
const patched = { ...actual, mkdirSync };
|
|
38
|
+
return { ...patched, default: patched };
|
|
39
|
+
});
|
|
40
|
+
|
|
18
41
|
describe("reindex", () => {
|
|
19
42
|
let root: string;
|
|
20
43
|
let stateDir: string;
|
|
@@ -223,4 +246,27 @@ describe("reindex", () => {
|
|
|
223
246
|
reindex({ repoRoot: root });
|
|
224
247
|
expect(fs.existsSync(lockPathFor(root))).toBe(false);
|
|
225
248
|
});
|
|
249
|
+
|
|
250
|
+
// ── HQ-B0: wrapper dir name uses ':' which is illegal on Windows ─────────
|
|
251
|
+
// `<ns>:<skill>` wrapper dirs contain a colon — fine on macOS/Linux, but a
|
|
252
|
+
// reserved drive/ADS separator on Windows, where mkdirSync throws ENOENT.
|
|
253
|
+
// A single un-creatable wrapper must not abort the entire reindex run.
|
|
254
|
+
|
|
255
|
+
it("does not abort reindex when a namespaced wrapper dir cannot be created (':' illegal on Windows)", () => {
|
|
256
|
+
writeSkill("core/skills/demo");
|
|
257
|
+
writeSkill("companies/acme/skills/widget");
|
|
258
|
+
|
|
259
|
+
hoisted.failNamespacedMkdir = true;
|
|
260
|
+
try {
|
|
261
|
+
// Without the guard the wrapper mkdir throws ENOENT and aborts the whole
|
|
262
|
+
// command; with it, the run completes and the bad wrappers are skipped.
|
|
263
|
+
expect(reindex({ repoRoot: root }).status).toBe(0);
|
|
264
|
+
} finally {
|
|
265
|
+
hoisted.failNamespacedMkdir = false;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// The un-creatable wrappers are skipped (not half-written).
|
|
269
|
+
expect(fs.existsSync(path.join(root, ".claude/skills/core:demo"))).toBe(false);
|
|
270
|
+
expect(fs.existsSync(path.join(root, ".claude/skills/acme:widget"))).toBe(false);
|
|
271
|
+
});
|
|
226
272
|
});
|
package/src/cli/reindex.ts
CHANGED
|
@@ -270,7 +270,23 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
|
|
|
270
270
|
/* best-effort */
|
|
271
271
|
}
|
|
272
272
|
}
|
|
273
|
-
|
|
273
|
+
try {
|
|
274
|
+
fs.mkdirSync(wrapper, { recursive: true });
|
|
275
|
+
} catch (err) {
|
|
276
|
+
// Namespaced wrappers embed a ':' in the directory name
|
|
277
|
+
// (`<ns>:<skill>`). That is a legal filename character on macOS/Linux
|
|
278
|
+
// but a reserved drive/ADS separator on Windows, so mkdir there fails
|
|
279
|
+
// with ENOENT. Skip this one wrapper with a clear message instead of
|
|
280
|
+
// aborting the whole reindex (the skill's source folder is untouched).
|
|
281
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
282
|
+
warn(
|
|
283
|
+
`reindex: could not create skill wrapper '${wrapperName}' ` +
|
|
284
|
+
`(${code ?? "error"}: ${(err as Error).message}). Namespaced wrappers ` +
|
|
285
|
+
`use a ':' in the directory name, which is not a legal filename ` +
|
|
286
|
+
`character on this platform (e.g. Windows); skipping this skill.`,
|
|
287
|
+
);
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
274
290
|
|
|
275
291
|
// Symlink every (non-hidden) entry in the source skill folder. The
|
|
276
292
|
// wrapper lives three levels below REPO_ROOT.
|
|
@@ -74,6 +74,41 @@ const lexists = (p: string) => {
|
|
|
74
74
|
}
|
|
75
75
|
};
|
|
76
76
|
|
|
77
|
+
function hasRecoveryManifestFor(root: string, rel: string): boolean {
|
|
78
|
+
let found = false;
|
|
79
|
+
const interesting = /(recover|recovery|rollback|transaction|manifest|resume|rescue)/i;
|
|
80
|
+
const walk = (dir: string) => {
|
|
81
|
+
if (found) return;
|
|
82
|
+
let entries: fs.Dirent[];
|
|
83
|
+
try {
|
|
84
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
85
|
+
} catch {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
89
|
+
for (const ent of entries) {
|
|
90
|
+
if (found) return;
|
|
91
|
+
const abs = path.join(dir, ent.name);
|
|
92
|
+
if (ent.isDirectory()) {
|
|
93
|
+
walk(abs);
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
if (!ent.isFile()) continue;
|
|
97
|
+
const relPath = abs.slice(root.length + 1);
|
|
98
|
+
if (!interesting.test(relPath)) continue;
|
|
99
|
+
let body = "";
|
|
100
|
+
try {
|
|
101
|
+
body = fs.readFileSync(abs, "utf-8");
|
|
102
|
+
} catch {
|
|
103
|
+
body = "";
|
|
104
|
+
}
|
|
105
|
+
found = body.includes(rel) && interesting.test(`${relPath}\n${body}`);
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
walk(root);
|
|
109
|
+
return found;
|
|
110
|
+
}
|
|
111
|
+
|
|
77
112
|
describe.skipIf(!gitAvailable)("rescue classify-before-delete + dir-symlink handling", () => {
|
|
78
113
|
let workDir: string;
|
|
79
114
|
let upstream: string;
|
|
@@ -108,6 +143,8 @@ describe.skipIf(!gitAvailable)("rescue classify-before-delete + dir-symlink hand
|
|
|
108
143
|
git(workDir, "init", "-b", "main", "upstream");
|
|
109
144
|
fs.writeFileSync(path.join(upstream, "core/a.md"), "v1\n");
|
|
110
145
|
fs.writeFileSync(path.join(upstream, "core/b.md"), "beta\n");
|
|
146
|
+
fs.mkdirSync(path.join(upstream, "core/blocker"), { recursive: true });
|
|
147
|
+
fs.writeFileSync(path.join(upstream, "core/blocker/second.md"), "upstream\n");
|
|
111
148
|
git(upstream, "add", "-A");
|
|
112
149
|
git(upstream, "commit", "-m", "floor");
|
|
113
150
|
floorSha = git(upstream, "rev-parse", "HEAD");
|
|
@@ -267,6 +304,30 @@ exec ${JSON.stringify(realGit)} "$@"
|
|
|
267
304
|
expect(fs.existsSync(path.join(hqRoot, "core/zzz.md"))).toBe(true);
|
|
268
305
|
});
|
|
269
306
|
|
|
307
|
+
it("F12: rolls back or records recovery when an apply action fails after earlier destructive actions", () => {
|
|
308
|
+
const hqRoot = makeHqRoot();
|
|
309
|
+
fs.mkdirSync(path.join(hqRoot, "core/blocker"), { recursive: true });
|
|
310
|
+
fs.writeFileSync(path.join(hqRoot, "core/blocker/second.md"), "local edit\n");
|
|
311
|
+
fs.writeFileSync(path.join(hqRoot, "personal/blocker"), "not a directory\n");
|
|
312
|
+
|
|
313
|
+
const firstDeletedByApply = path.join(hqRoot, "core/a.md");
|
|
314
|
+
const firstOriginal = fs.readFileSync(firstDeletedByApply, "utf-8");
|
|
315
|
+
const r = runRescueCapture(liveArgs(hqRoot), baseEnv);
|
|
316
|
+
const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
|
|
317
|
+
|
|
318
|
+
const applyFailed = r.threw !== undefined || (r.status !== undefined && r.status !== 0);
|
|
319
|
+
expect(applyFailed, out).toBe(true);
|
|
320
|
+
expect(fs.existsSync(path.join(hqRoot, "core/blocker/second.md")), out).toBe(true);
|
|
321
|
+
|
|
322
|
+
const firstActionRolledBack =
|
|
323
|
+
fs.existsSync(firstDeletedByApply) &&
|
|
324
|
+
fs.readFileSync(firstDeletedByApply, "utf-8") === firstOriginal;
|
|
325
|
+
expect(
|
|
326
|
+
firstActionRolledBack || hasRecoveryManifestFor(hqRoot, "core/a.md"),
|
|
327
|
+
`${out}\ncore/a.md was deleted without rollback or a recovery/transaction manifest`,
|
|
328
|
+
).toBe(true);
|
|
329
|
+
});
|
|
330
|
+
|
|
270
331
|
it("dry-run classifies the dir-symlink identically to the live run and changes nothing", () => {
|
|
271
332
|
const hqRoot = makeHqRoot();
|
|
272
333
|
const target = path.join(hqRoot, "personal/knowledge/ad-creative-engine");
|