@indigoai-us/hq-cloud 6.15.7 → 6.15.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +2 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +2 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +3 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +65 -31
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +72 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +7 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +132 -11
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +11 -2
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +7 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +5 -2
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +11 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/s3.d.ts +9 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +19 -0
- package/dist/s3.js.map +1 -1
- package/dist/s3.symlink-materialize.test.js +26 -1
- package/dist/s3.symlink-materialize.test.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/share.d.ts
CHANGED
|
@@ -209,6 +209,11 @@ export interface ShareOptions {
|
|
|
209
209
|
* This is the seam `hq-sync-runner` uses to stream ndjson for push events.
|
|
210
210
|
*/
|
|
211
211
|
onEvent?: (event: SyncProgressEvent) => void;
|
|
212
|
+
/**
|
|
213
|
+
* Watch-runner liveness callback for long synchronous file walks and hash
|
|
214
|
+
* planning. The caller owns throttling; one-shot CLI callers omit it.
|
|
215
|
+
*/
|
|
216
|
+
onHeartbeat?: () => void;
|
|
212
217
|
/**
|
|
213
218
|
* When true, files whose local hash matches the journal entry from the
|
|
214
219
|
* last sync are skipped (no remote HEAD, no upload). This is the gate
|
|
@@ -519,7 +524,7 @@ export interface ShareResult {
|
|
|
519
524
|
pathResults?: SharePathResult[];
|
|
520
525
|
aborted: boolean;
|
|
521
526
|
}
|
|
522
|
-
export type ShareDeleteRefusalReason = "stale-etag" | "legacy-no-etag" | "bulk-asymmetry" | "divergent-local" | "missing-delete-intent" | "intent-changed" | "recreated-locally" | "transfer-error";
|
|
527
|
+
export type ShareDeleteRefusalReason = "stale-etag" | "legacy-no-etag" | "bulk-asymmetry" | "divergent-local" | "missing-delete-intent" | "intent-changed" | "recreated-locally" | "marker-mismatch" | "transfer-error";
|
|
523
528
|
export interface SharePathResult {
|
|
524
529
|
path: string;
|
|
525
530
|
status: "accepted" | "refused";
|
|
@@ -592,6 +597,7 @@ type CollectedEntry = {
|
|
|
592
597
|
interface CollectHooks {
|
|
593
598
|
onUnreachablePath?: (namedPath: string, reason: UnreachablePathReason) => void;
|
|
594
599
|
onLinkedSubtree?: (rel: string) => void;
|
|
600
|
+
onHeartbeat?: () => void;
|
|
595
601
|
}
|
|
596
602
|
/**
|
|
597
603
|
* Resolve a caller-supplied push path to an absolute path.
|
package/dist/cli/share.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/cli/share.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/cli/share.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAe,MAAM,aAAa,CAAC;AAkBlF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AA8B7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAwE/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,eAAO,MAAM,sBAAsB,QACmD,CAAC;AAEvF;;;;;;;;;GASG;AACH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,kBAAkB,QAAqB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAElD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,GAAG,OAAO,CAEtF;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,QAAQ;;;;;;;;CAQpB,CAAC;AAgNF,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC;;;;;;;;;;;;;;;;;;OAkBG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB;;;;;;;;;OASG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;OAgBG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;;;;;;OAWG;IACH,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC/C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC1C;AAED,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG,SAAS,GAAG,iBAAiB,GAAG,iBAAiB,CAAC;AAEtF;;;;;GAKG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;IAClD,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;gBAE5C,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC,EAAE,QAAQ,EAAE,MAAM;CAuBtF;AAED,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;;OAOG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;;OASG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;;;OAQG;IACH,sBAAsB,EAAE,MAAM,EAAE,CAAC;IACjC;;;;;;;OAOG;IACH,0BAA0B,EAAE,MAAM,CAAC;IACnC;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;OAQG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;;OAKG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;;;;;;;;OAiBG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B;;;;;;;OAOG;IACH,wBAAwB,EAAE,MAAM,EAAE,CAAC;IACnC;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,eAAe,EAAE,CAAC;IAChC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,wBAAwB,GAChC,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,uBAAuB,GACvB,gBAAgB,GAChB,mBAAmB,GACnB,iBAAiB,GACjB,gBAAgB,CAAC;AAErB,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IAC/B,SAAS,EAAE,QAAQ,GAAG,WAAW,CAAC;IAClC,MAAM,CAAC,EAAE,wBAAwB,CAAC;CACnC;AAmDD;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAC5C,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,EACzD,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAC3C,aAAa,CAAC,EAAE,MAAM,IAAI,GACzB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,CAW/C;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA6CvE;AAssCD;;;GAGG;AACH,iBAAS,oBAAoB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI,CAgI5D;AAED;;;;;;;;;GASG;AACH,KAAK,cAAc,GACf;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,UAAU,YAAY;IACpB,iBAAiB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,eAAe,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,iBAAS,gBAAgB,CACvB,CAAC,EAAE,MAAM,EACT,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAsB,GAC1B,MAAM,CAuBR;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,iBAAS,qBAAqB,CAC5B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,gBAAgB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GACvC,OAAO,CAwBT;AA8HD;;;;;;;;;;;GAWG;AACH,iBAAS,YAAY,CACnB,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,EAC/C,KAAK,GAAE,YAAiB,GACvB,cAAc,EAAE,CAkGlB"}
|
package/dist/cli/share.js
CHANGED
|
@@ -10,7 +10,7 @@ import * as path from "path";
|
|
|
10
10
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
11
11
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
12
12
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
|
|
13
|
-
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, } from "../s3.js";
|
|
13
|
+
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, } from "../s3.js";
|
|
14
14
|
import * as crypto from "crypto";
|
|
15
15
|
import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
|
|
16
16
|
import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
@@ -247,9 +247,10 @@ export const _testing = {
|
|
|
247
247
|
* detecting a push conflict requires a remote HEAD that we defer to Stage 2.
|
|
248
248
|
* Consumers that want a conflict count get it from the `complete` event.
|
|
249
249
|
*/
|
|
250
|
-
function computePushPlan(filesToShare, journal, skipUnchanged) {
|
|
250
|
+
function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
|
|
251
251
|
const items = [];
|
|
252
252
|
for (const entry of filesToShare) {
|
|
253
|
+
onHeartbeat?.();
|
|
253
254
|
const { absolutePath, relativePath } = entry;
|
|
254
255
|
// Symlinks bypass the size-limit gate (the wire body is small,
|
|
255
256
|
// bounded by target length) and hash the target through the
|
|
@@ -482,7 +483,7 @@ export async function share(options) {
|
|
|
482
483
|
if (uploadResult.aborted) {
|
|
483
484
|
return buildShareResult(run, counters, filesRefusedStalePaths, uploadResult.abortFlightConflictPaths, pathResults, true);
|
|
484
485
|
}
|
|
485
|
-
await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths, pathResults);
|
|
486
|
+
await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths, pathResults, !uploadResult.hadUploadFailure && uploadResult.workerErrors.length === 0);
|
|
486
487
|
finalizeShareJournal(run);
|
|
487
488
|
throwUploadWorkerErrors(uploadResult.workerErrors);
|
|
488
489
|
return buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths, pathResults, false);
|
|
@@ -648,6 +649,7 @@ async function buildSharePlans(run) {
|
|
|
648
649
|
}
|
|
649
650
|
},
|
|
650
651
|
onLinkedSubtree: (rel) => run.linkedSubtreeSet.add(rel),
|
|
652
|
+
onHeartbeat: run.options.onHeartbeat,
|
|
651
653
|
});
|
|
652
654
|
// Ask #2 of feedback_a51cb63d: "error, not warn-skip, when the named file
|
|
653
655
|
// exists locally but is unreachable by the resolver". This is the throw that
|
|
@@ -732,7 +734,7 @@ async function buildSharePlans(run) {
|
|
|
732
734
|
});
|
|
733
735
|
return false;
|
|
734
736
|
});
|
|
735
|
-
const pushPlan = computePushPlan(filesToShare, run.journal, run.skipUnchanged === true);
|
|
737
|
+
const pushPlan = computePushPlan(filesToShare, run.journal, run.skipUnchanged === true, run.options.onHeartbeat);
|
|
736
738
|
const deleteScopeRoots = run.propagateDeletes === true
|
|
737
739
|
? resolveDeleteScopeRoots(run.paths, run.hqRoot, run.syncRoot, run.options.deleteScopeRoots)
|
|
738
740
|
: [];
|
|
@@ -839,6 +841,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
839
841
|
await primeObjectTransport(run.ctx, "get", uploadItems.map((it) => it.relativePath));
|
|
840
842
|
let aborted = false;
|
|
841
843
|
let abortFlightConflictPaths = [];
|
|
844
|
+
let hadUploadFailure = false;
|
|
842
845
|
const processUploadItem = async (item) => {
|
|
843
846
|
if (aborted)
|
|
844
847
|
return;
|
|
@@ -1015,6 +1018,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1015
1018
|
catch (retryErr) {
|
|
1016
1019
|
if (retryErr instanceof VaultAuthError)
|
|
1017
1020
|
throw retryErr;
|
|
1021
|
+
hadUploadFailure = true;
|
|
1018
1022
|
run.emit({
|
|
1019
1023
|
type: "error",
|
|
1020
1024
|
path: relativePath,
|
|
@@ -1028,9 +1032,11 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1028
1032
|
return;
|
|
1029
1033
|
}
|
|
1030
1034
|
if (isAccessDenied(err)) {
|
|
1035
|
+
hadUploadFailure = true;
|
|
1031
1036
|
run.scopeExcludedSet.add(relativePath);
|
|
1032
1037
|
return;
|
|
1033
1038
|
}
|
|
1039
|
+
hadUploadFailure = true;
|
|
1034
1040
|
run.emit({
|
|
1035
1041
|
type: "error",
|
|
1036
1042
|
path: relativePath,
|
|
@@ -1047,6 +1053,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1047
1053
|
const item = queue.shift();
|
|
1048
1054
|
const p = processUploadItem(item)
|
|
1049
1055
|
.catch((err) => {
|
|
1056
|
+
hadUploadFailure = true;
|
|
1050
1057
|
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1051
1058
|
})
|
|
1052
1059
|
.finally(() => {
|
|
@@ -1062,7 +1069,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1062
1069
|
}
|
|
1063
1070
|
}
|
|
1064
1071
|
}
|
|
1065
|
-
return { aborted, abortFlightConflictPaths, workerErrors };
|
|
1072
|
+
return { aborted, abortFlightConflictPaths, workerErrors, hadUploadFailure };
|
|
1066
1073
|
}
|
|
1067
1074
|
async function writePushConflictMirror(run, item, remoteHash) {
|
|
1068
1075
|
try {
|
|
@@ -1116,10 +1123,21 @@ async function writePushConflictMirror(run, item, remoteHash) {
|
|
|
1116
1123
|
}
|
|
1117
1124
|
}
|
|
1118
1125
|
}
|
|
1119
|
-
async function executeDeletes(run, deletePlan, decommissionPlan, counters, filesRefusedStalePaths, pathResults) {
|
|
1126
|
+
async function executeDeletes(run, deletePlan, decommissionPlan, counters, filesRefusedStalePaths, pathResults, allowMaterializedKnowledgeDeletes) {
|
|
1120
1127
|
const deleteItems = [
|
|
1121
|
-
...deletePlan.toDelete
|
|
1122
|
-
|
|
1128
|
+
...deletePlan.toDelete
|
|
1129
|
+
// A stale exact-key marker is removed only after every upload worker in
|
|
1130
|
+
// this pass settled successfully. If a directory child failed to PUT,
|
|
1131
|
+
// retain the marker + journal proof so the next sync retries the whole
|
|
1132
|
+
// representation transition instead of creating a remote visibility gap.
|
|
1133
|
+
.filter((item) => !item.materializedKnowledgeDir || allowMaterializedKnowledgeDeletes)
|
|
1134
|
+
.map((item) => ({ ...item, decommission: false })),
|
|
1135
|
+
...decommissionPlan.map((key) => ({
|
|
1136
|
+
key,
|
|
1137
|
+
intentVersion: null,
|
|
1138
|
+
decommission: true,
|
|
1139
|
+
materializedKnowledgeDir: false,
|
|
1140
|
+
})),
|
|
1123
1141
|
];
|
|
1124
1142
|
const deleteKeys = deleteItems.map((item) => item.key);
|
|
1125
1143
|
await primeObjectTransport(run.ctx, "delete", deleteKeys);
|
|
@@ -1143,8 +1161,11 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
1143
1161
|
// ENCODED local path or a win32 colon key always looks absent (a raw `:`
|
|
1144
1162
|
// name can never exist on disk) and the remote delete proceeds while the
|
|
1145
1163
|
// encoded local file is still present.
|
|
1146
|
-
const
|
|
1147
|
-
|
|
1164
|
+
const localPath = localPathForVaultKey(run.syncRoot, relativePath);
|
|
1165
|
+
const localStateStillSafe = item.materializedKnowledgeDir
|
|
1166
|
+
? isMaterializedKnowledgeDirectory(localPath, relativePath, entry, run.options.personalMode !== true)
|
|
1167
|
+
: isLocallyAbsent(localPath);
|
|
1168
|
+
const recreatedLocally = !item.decommission && !localStateStillSafe;
|
|
1148
1169
|
const intentChanged = !item.decommission &&
|
|
1149
1170
|
item.intentVersion !== null &&
|
|
1150
1171
|
(!hasCurrentLocalDeleteIntent(entry) ||
|
|
@@ -1519,6 +1540,10 @@ function defaultConsoleLogger(event) {
|
|
|
1519
1540
|
console.error(` ⚠ recreated-locally, kept on remote: ${event.path} ` +
|
|
1520
1541
|
`(the file exists locally again, so the delete was abandoned)`);
|
|
1521
1542
|
}
|
|
1543
|
+
else if (event.reason === "marker-mismatch") {
|
|
1544
|
+
console.error(` ⚠ marker-mismatch, kept on remote: ${event.path} ` +
|
|
1545
|
+
`(the remote object is not the symlink revision reindex replaced)`);
|
|
1546
|
+
}
|
|
1522
1547
|
else if (event.reason === "divergent-local") {
|
|
1523
1548
|
// Same trap as above — `remoteEtag` is `<not-checked>`. The entry's
|
|
1524
1549
|
// recorded etag was stamped only to silence a re-fired conflict; the
|
|
@@ -1835,6 +1860,7 @@ function collectFiles(paths, hqRoot, syncRoot, filter, hooks = {}) {
|
|
|
1835
1860
|
// boundary, so it deliberately does not.
|
|
1836
1861
|
const tenantRoots = new Map();
|
|
1837
1862
|
for (const p of paths) {
|
|
1863
|
+
hooks.onHeartbeat?.();
|
|
1838
1864
|
const absolutePath = resolveNamedPath(p, hqRoot, syncRoot);
|
|
1839
1865
|
// Ephemeral artifacts (conflict mirrors) — see EPHEMERAL_PATH_PATTERN doc.
|
|
1840
1866
|
// Caller may pass one explicitly; we still refuse to upload it. Basename
|
|
@@ -1939,6 +1965,7 @@ function walkDir(dir, syncRoot, filter, results, hooks = {}) {
|
|
|
1939
1965
|
};
|
|
1940
1966
|
enterDirectory(dir);
|
|
1941
1967
|
while (stack.length > 0) {
|
|
1968
|
+
hooks.onHeartbeat?.();
|
|
1942
1969
|
const frame = stack[stack.length - 1];
|
|
1943
1970
|
if (frame.index >= frame.entries.length) {
|
|
1944
1971
|
stack.pop();
|
|
@@ -2230,6 +2257,42 @@ function isLocallyAbsent(localPath) {
|
|
|
2230
2257
|
return code === "ENOENT";
|
|
2231
2258
|
}
|
|
2232
2259
|
}
|
|
2260
|
+
/** Exact canonical Knowledge keys scanned by reindex-knowledge.ts. */
|
|
2261
|
+
function isCanonicalKnowledgeMarkerKey(key, companyScoped) {
|
|
2262
|
+
const parts = key.split("/");
|
|
2263
|
+
if (companyScoped) {
|
|
2264
|
+
return parts[0] === "knowledge" && (parts.length === 1 || parts.length === 2);
|
|
2265
|
+
}
|
|
2266
|
+
if (parts[0] === "companies" &&
|
|
2267
|
+
parts[2] === "knowledge" &&
|
|
2268
|
+
(parts.length === 3 || parts.length === 4)) {
|
|
2269
|
+
return true;
|
|
2270
|
+
}
|
|
2271
|
+
if (parts[0] === "personal" &&
|
|
2272
|
+
parts[1] === "knowledge" &&
|
|
2273
|
+
(parts.length === 2 || parts.length === 3)) {
|
|
2274
|
+
return true;
|
|
2275
|
+
}
|
|
2276
|
+
return (parts[0] === "core" &&
|
|
2277
|
+
parts[1] === "knowledge" &&
|
|
2278
|
+
((parts.length === 3 && parts[2] !== "public" && parts[2] !== "private") ||
|
|
2279
|
+
(parts.length === 4 && (parts[2] === "public" || parts[2] === "private"))));
|
|
2280
|
+
}
|
|
2281
|
+
function isMaterializedKnowledgeDirectory(localPath, key, entry, companyScoped) {
|
|
2282
|
+
// `kind` is optional for journals written before representation tracking.
|
|
2283
|
+
// A known regular file is never eligible; an unknown legacy entry still is,
|
|
2284
|
+
// but only after the remote HEAD/GET proves the exact object is a symlink.
|
|
2285
|
+
if (!entry || entry.kind === "file" || !isCanonicalKnowledgeMarkerKey(key, companyScoped)) {
|
|
2286
|
+
return false;
|
|
2287
|
+
}
|
|
2288
|
+
try {
|
|
2289
|
+
const local = fs.lstatSync(localPath);
|
|
2290
|
+
return local.isDirectory() && !local.isSymbolicLink();
|
|
2291
|
+
}
|
|
2292
|
+
catch {
|
|
2293
|
+
return false;
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2233
2296
|
/**
|
|
2234
2297
|
* Concurrency cap for the per-file HEAD-O-meter (currency-gated). Sequential
|
|
2235
2298
|
* HEADs would add ~N×(50-200ms) to a sync — for the 261-mirror real-world
|
|
@@ -2424,8 +2487,49 @@ companyScoped = true) {
|
|
|
2424
2487
|
throw err;
|
|
2425
2488
|
}
|
|
2426
2489
|
}
|
|
2427
|
-
|
|
2490
|
+
const materializedKnowledgeDir = presentLocally &&
|
|
2491
|
+
isMaterializedKnowledgeDirectory(localPath, relativeKey, entry, companyScoped);
|
|
2492
|
+
if (presentLocally && !materializedKnowledgeDir)
|
|
2428
2493
|
continue;
|
|
2494
|
+
// Reindex's legacy Knowledge migration is a representation transition,
|
|
2495
|
+
// not a user-content deletion: the exact key changed from a symlink object
|
|
2496
|
+
// to a real directory whose children are uploaded separately. The normal
|
|
2497
|
+
// delete planner only accepts ENOENT, so without this explicit branch the
|
|
2498
|
+
// old exact-key marker survives forever. Require a current ETag here and
|
|
2499
|
+
// verify the remote marker body against the journal in the HEAD/GET pass
|
|
2500
|
+
// below before allowing the exact object to be removed. This branch runs
|
|
2501
|
+
// before personal-vault delete exclusions because canonical core Knowledge
|
|
2502
|
+
// markers are excluded from ordinary delete propagation; it does not add
|
|
2503
|
+
// to the bulk-asymmetry numerator because the verified marker carries no
|
|
2504
|
+
// file content.
|
|
2505
|
+
if (materializedKnowledgeDir) {
|
|
2506
|
+
if (entry.localDiverges) {
|
|
2507
|
+
plan.refusedStale.push({
|
|
2508
|
+
key: relativeKey,
|
|
2509
|
+
journalEtag: entry.remoteEtag ?? "<divergent-local>",
|
|
2510
|
+
remoteEtag: "<not-checked>",
|
|
2511
|
+
reason: "divergent-local",
|
|
2512
|
+
});
|
|
2513
|
+
continue;
|
|
2514
|
+
}
|
|
2515
|
+
if (!entry.remoteEtag) {
|
|
2516
|
+
plan.refusedStale.push({
|
|
2517
|
+
key: relativeKey,
|
|
2518
|
+
journalEtag: "<legacy-no-etag>",
|
|
2519
|
+
remoteEtag: "<unknown>",
|
|
2520
|
+
reason: "legacy-no-etag",
|
|
2521
|
+
});
|
|
2522
|
+
continue;
|
|
2523
|
+
}
|
|
2524
|
+
headCandidates.push({
|
|
2525
|
+
key: relativeKey,
|
|
2526
|
+
journalEtag: entry.remoteEtag,
|
|
2527
|
+
intentVersion: null,
|
|
2528
|
+
materializedKnowledgeDir: true,
|
|
2529
|
+
...(entry.kind === "symlink" ? { expectedSymlinkHash: entry.hash } : {}),
|
|
2530
|
+
});
|
|
2531
|
+
continue;
|
|
2532
|
+
}
|
|
2429
2533
|
if (!companyScoped && isPersonalVaultDeletionExcluded(relativeKey)) {
|
|
2430
2534
|
continue;
|
|
2431
2535
|
}
|
|
@@ -2685,9 +2789,26 @@ companyScoped = true) {
|
|
|
2685
2789
|
}
|
|
2686
2790
|
const currentEtag = normalizeEtag(remote.etag);
|
|
2687
2791
|
if (currentEtag === candidate.journalEtag) {
|
|
2792
|
+
if (candidate.materializedKnowledgeDir) {
|
|
2793
|
+
const target = await readVerifiedRemoteSymlinkTarget(ctx, candidate.key);
|
|
2794
|
+
if (target === null ||
|
|
2795
|
+
(candidate.expectedSymlinkHash !== undefined &&
|
|
2796
|
+
hashSymlinkTarget(target) !== candidate.expectedSymlinkHash)) {
|
|
2797
|
+
plan.refusedStale.push({
|
|
2798
|
+
key: candidate.key,
|
|
2799
|
+
journalEtag: candidate.journalEtag,
|
|
2800
|
+
remoteEtag: currentEtag,
|
|
2801
|
+
reason: "marker-mismatch",
|
|
2802
|
+
});
|
|
2803
|
+
continue;
|
|
2804
|
+
}
|
|
2805
|
+
}
|
|
2688
2806
|
plan.toDelete.push({
|
|
2689
2807
|
key: candidate.key,
|
|
2690
2808
|
intentVersion: candidate.intentVersion,
|
|
2809
|
+
...(candidate.materializedKnowledgeDir
|
|
2810
|
+
? { materializedKnowledgeDir: true }
|
|
2811
|
+
: {}),
|
|
2691
2812
|
});
|
|
2692
2813
|
}
|
|
2693
2814
|
else {
|