@indigoai-us/hq-cloud 6.16.53 → 6.16.54-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +1 -23
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +8 -92
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +0 -36
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +1 -2
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +6 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +30 -1
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +57 -126
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +12 -3
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +65 -3
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +108 -1
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +48 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +318 -120
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +33 -5
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +34 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +1079 -79
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js +300 -0
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +1 -0
- package/dist/cli/onepass-pull-planner.test.d.ts +2 -0
- package/dist/cli/onepass-pull-planner.test.d.ts.map +1 -0
- package/dist/cli/onepass-pull-planner.test.js +266 -0
- package/dist/cli/onepass-pull-planner.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +0 -40
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +162 -1454
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-rsync-preflight.test.js +4 -256
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue.d.ts +0 -4
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +1 -20
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +1 -16
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +0 -2
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +19 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +150 -467
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +95 -608
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +15 -0
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +1 -0
- package/dist/cli/streaming-pull-planner.bench.test.js +273 -0
- package/dist/cli/streaming-pull-planner.bench.test.js.map +1 -0
- package/dist/cli/sync-scope.test.js +8 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +13 -15
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +226 -118
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +215 -2
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +0 -2
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +0 -30
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +0 -175
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +40 -5
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +240 -21
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +54 -0
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +11 -6
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +259 -7
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +43 -15
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +297 -79
- package/dist/journal.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +0 -7
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/lib/describe-error.test.js +0 -11
- package/dist/lib/describe-error.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +15 -0
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +15 -0
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +0 -295
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +3 -18
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +10 -3
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +35 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +9 -2
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +59 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +1 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +24 -47
- package/dist/s3.js.map +1 -1
- package/dist/s3.list-cost.test.d.ts +2 -0
- package/dist/s3.list-cost.test.d.ts.map +1 -0
- package/dist/s3.list-cost.test.js +51 -0
- package/dist/s3.list-cost.test.js.map +1 -0
- package/dist/s3.test.js +1 -27
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +2 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +66 -34
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +90 -5
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +29 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -6
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +136 -16
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/remote-file-spool.d.ts +16 -0
- package/dist/sync/remote-file-spool.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.js +91 -0
- package/dist/sync/remote-file-spool.js.map +1 -0
- package/dist/sync/remote-file-spool.test.d.ts +2 -0
- package/dist/sync/remote-file-spool.test.d.ts.map +1 -0
- package/dist/sync/remote-file-spool.test.js +77 -0
- package/dist/sync/remote-file-spool.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +6 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +14 -2
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync-core.d.ts +4 -6
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +4 -10
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +1 -25
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +5 -26
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +20 -43
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +0 -32
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/cli/rescue-breaker.test.d.ts +0 -2
- package/dist/cli/rescue-breaker.test.d.ts.map +0 -1
- package/dist/cli/rescue-breaker.test.js +0 -334
- package/dist/cli/rescue-breaker.test.js.map +0 -1
- package/dist/cli/rescue-realpath.test.d.ts +0 -2
- package/dist/cli/rescue-realpath.test.d.ts.map +0 -1
- package/dist/cli/rescue-realpath.test.js +0 -109
- package/dist/cli/rescue-realpath.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-copy.test.js +0 -627
- package/dist/cli/rescue-snapshot-copy.test.js.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +0 -2
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +0 -1
- package/dist/cli/rescue-snapshot-recovery.test.js +0 -260
- package/dist/cli/rescue-snapshot-recovery.test.js.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.d.ts +0 -2
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +0 -1
- package/dist/cli/rescue-source-switch-floor.test.js +0 -210
- package/dist/cli/rescue-source-switch-floor.test.js.map +0 -1
package/dist/cli/share.js
CHANGED
|
@@ -11,7 +11,7 @@ import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../c
|
|
|
11
11
|
import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
12
12
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
13
13
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
|
|
14
|
-
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads,
|
|
14
|
+
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
15
15
|
import * as crypto from "crypto";
|
|
16
16
|
import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, timestampsTooCoarseToTrust, updateEntry, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
|
|
17
17
|
import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
@@ -24,7 +24,7 @@ import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransfer
|
|
|
24
24
|
import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
25
25
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
26
26
|
import { isCloudAuthoritative } from "../lib/cloud-authoritative.js";
|
|
27
|
-
import { VaultAuthError, VaultClient, VaultClientError,
|
|
27
|
+
import { VaultAuthError, VaultClient, VaultClientError, VaultPermissionDeniedError, } from "../vault-client.js";
|
|
28
28
|
import { describeError } from "../lib/describe-error.js";
|
|
29
29
|
import { isRetryableIdentityRefusal, isTerminalIdentityRefusal, } from "../cognito-auth.js";
|
|
30
30
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
@@ -628,10 +628,33 @@ export function wrapFilterWithIgnoreVisibility(underlying, hqRoot, onIgnoreExclu
|
|
|
628
628
|
return false;
|
|
629
629
|
};
|
|
630
630
|
}
|
|
631
|
+
class CooperativePushHandoff extends Error {
|
|
632
|
+
constructor() {
|
|
633
|
+
super("Realtime work ran at a settled full-push boundary");
|
|
634
|
+
this.name = "CooperativePushHandoff";
|
|
635
|
+
}
|
|
636
|
+
}
|
|
631
637
|
/**
|
|
632
638
|
* Share local file(s) to the entity vault.
|
|
639
|
+
*
|
|
640
|
+
* A cooperative hand-off discards the old plan instead of continuing it after
|
|
641
|
+
* another writer has run. Journal delta computation treats missing keys as
|
|
642
|
+
* deletes, so reusing a pre-handoff plan would be unsafe even though the
|
|
643
|
+
* hand-off itself only happens at a durable unit boundary.
|
|
633
644
|
*/
|
|
634
645
|
export async function share(options) {
|
|
646
|
+
for (;;) {
|
|
647
|
+
try {
|
|
648
|
+
return await shareOnce(options);
|
|
649
|
+
}
|
|
650
|
+
catch (error) {
|
|
651
|
+
if (error instanceof CooperativePushHandoff)
|
|
652
|
+
continue;
|
|
653
|
+
throw error;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
async function shareOnce(options) {
|
|
635
658
|
const run = await createPushRunContext(options);
|
|
636
659
|
const counters = createShareCounters();
|
|
637
660
|
const filesRefusedStalePaths = [];
|
|
@@ -666,11 +689,6 @@ export async function share(options) {
|
|
|
666
689
|
// checkpoint exists to prevent. `flushShareJournal` leaves `lastSync` at
|
|
667
690
|
// its pre-run value, so the pass does not look like a finished sync.
|
|
668
691
|
flushShareJournal(run);
|
|
669
|
-
// finalizeShareJournal never runs on the aborted return, so surface any
|
|
670
|
-
// recorded skill-registration conflict here too — otherwise a skill whose
|
|
671
|
-
// marker was already cleared in preflight would be skipped silently and no
|
|
672
|
-
// later pass could reproduce the warning.
|
|
673
|
-
emitSkillRegistrationConflicts(run);
|
|
674
692
|
return buildShareResult(run, counters, filesRefusedStalePaths, uploadResult.abortFlightConflictPaths, pathResults, true);
|
|
675
693
|
}
|
|
676
694
|
await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths, pathResults, !uploadResult.hadUploadFailure && uploadResult.workerErrors.length === 0);
|
|
@@ -830,7 +848,6 @@ async function createPushRunContext(options) {
|
|
|
830
848
|
};
|
|
831
849
|
const unreachablePaths = new Map();
|
|
832
850
|
const linkedSubtreeSet = new Set();
|
|
833
|
-
const skillRegistrationConflicts = new Map();
|
|
834
851
|
const baseFilter = options.personalMode === true
|
|
835
852
|
? wrapFilterWithPersonalVaultDefaults(recordedIgnoreFilter, syncRoot, onExcluded)
|
|
836
853
|
: recordedIgnoreFilter;
|
|
@@ -872,7 +889,6 @@ async function createPushRunContext(options) {
|
|
|
872
889
|
ignoreExcludedTotal,
|
|
873
890
|
unreachablePaths,
|
|
874
891
|
linkedSubtreeSet,
|
|
875
|
-
skillRegistrationConflicts,
|
|
876
892
|
hostLoadAtStart,
|
|
877
893
|
checkpoint: {
|
|
878
894
|
appliedSinceFlush: 0,
|
|
@@ -946,32 +962,14 @@ function assertValidSkillRegistration(result, expectedPath) {
|
|
|
946
962
|
throw new Error(`Skill registration returned an invalid identity for '${expectedPath}'; local bytes were not changed.`);
|
|
947
963
|
}
|
|
948
964
|
}
|
|
949
|
-
|
|
950
|
-
* Register one canonical company skill before its bytes upload.
|
|
951
|
-
*
|
|
952
|
-
* `mode: "unstamped"` is the creation backstop: a file with no `skill_uid`
|
|
953
|
-
* gets one minted and stamped; a file that already carries one is left alone.
|
|
954
|
-
*
|
|
955
|
-
* `mode: "stamped"` is the rename / copy backstop: the file already carries a
|
|
956
|
-
* `skill_uid` but this journal has never seen it at this path, so the vault
|
|
957
|
-
* may hold that identity at ANOTHER path (a folder rename or a physical copy).
|
|
958
|
-
* Registering with content lets the service classify it — `reused` (same
|
|
959
|
-
* path), `moved` (original gone → same uid, repointed), or `forked` (original
|
|
960
|
-
* still present → fresh uid) — and hand back the bytes to persist. Without
|
|
961
|
-
* this call the presign mint sees a new path, cannot read the bytes, and
|
|
962
|
-
* reserves a placeholder identity that the post-upload reconciliation can
|
|
963
|
-
* never satisfy (the undrdog 2026-09-11 outage: five days of aborted passes).
|
|
964
|
-
*/
|
|
965
|
-
async function registerOneCompanySkill(run, entry, registrar, mode = "unstamped") {
|
|
965
|
+
async function registerOneCompanySkill(run, entry, registrar) {
|
|
966
966
|
let content = fs.readFileSync(entry.absolutePath, "utf8");
|
|
967
967
|
// A live editor can save while the network request is in flight. Re-register
|
|
968
968
|
// the newest bytes instead of overwriting them with the response to an older
|
|
969
969
|
// body. Three attempts bound pathological save loops and fail visibly.
|
|
970
970
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
971
971
|
const currentUid = parseRegisteredCompanySkillUid(content);
|
|
972
|
-
if (
|
|
973
|
-
return;
|
|
974
|
-
if (mode === "stamped" && !currentUid)
|
|
972
|
+
if (currentUid)
|
|
975
973
|
return;
|
|
976
974
|
run.options.onHeartbeat?.();
|
|
977
975
|
let result;
|
|
@@ -998,11 +996,7 @@ async function registerOneCompanySkill(run, entry, registrar, mode = "unstamped"
|
|
|
998
996
|
content = current;
|
|
999
997
|
continue;
|
|
1000
998
|
}
|
|
1001
|
-
|
|
1002
|
-
// changes them. Skip the rewrite so an untouched file keeps its mtime.
|
|
1003
|
-
if (result.content !== content) {
|
|
1004
|
-
writeRegisteredSkillAtomically(entry.absolutePath, result.content);
|
|
1005
|
-
}
|
|
999
|
+
writeRegisteredSkillAtomically(entry.absolutePath, result.content);
|
|
1006
1000
|
return;
|
|
1007
1001
|
}
|
|
1008
1002
|
throw new Error(`Company skill '${entry.relativePath}' kept changing while HQ registered it; retry after the current edit finishes.`);
|
|
@@ -1012,129 +1006,54 @@ function isPermanentSkillRegistrationValidationError(err) {
|
|
|
1012
1006
|
(err.statusCode === 400 || err.statusCode === 422));
|
|
1013
1007
|
}
|
|
1014
1008
|
/**
|
|
1015
|
-
*
|
|
1016
|
-
*
|
|
1017
|
-
* must still abort the company leg. Everything else a registration call can
|
|
1018
|
-
* throw — a 409 identity refusal, a 5xx, a transport blip — is about ONE
|
|
1019
|
-
* file and is contained to that file.
|
|
1020
|
-
*/
|
|
1021
|
-
function isRunLevelSkillRegistrationError(err) {
|
|
1022
|
-
return isRunLevelRegistrarFailure(err) || err instanceof JournalCheckpointError;
|
|
1023
|
-
}
|
|
1024
|
-
function emptyPreflightResult() {
|
|
1025
|
-
return {
|
|
1026
|
-
registrationDenied: new Set(),
|
|
1027
|
-
registrationInvalid: new Set(),
|
|
1028
|
-
registrationFailed: new Set(),
|
|
1029
|
-
};
|
|
1030
|
-
}
|
|
1031
|
-
/**
|
|
1032
|
-
* Durable identity backstop. Hooks improve the authoring experience, but sync
|
|
1033
|
-
* is the invariant boundary:
|
|
1034
|
-
* - no unstamped canonical company skill reaches S3 (`unstamped`);
|
|
1035
|
-
* - a stamped skill at a path this journal has never seen (a folder rename
|
|
1036
|
-
* or a physical copy) reserves its identity for that path BEFORE its
|
|
1037
|
-
* bytes land (`relocated`);
|
|
1038
|
-
* - a skill whose post-upload metadata reconciliation is still pending is
|
|
1039
|
-
* retried (`pending`).
|
|
1040
|
-
*
|
|
1041
|
-
* Containment: a failure in any one skill excludes THAT skill and lets the
|
|
1042
|
-
* rest of the company leg run. Only run-level errors (auth, identity refusal,
|
|
1043
|
-
* journal checkpoint) propagate. Before this rule one skill's 409 escaped
|
|
1044
|
-
* `buildSharePlans` on every pass and wedged a whole tenant indefinitely.
|
|
1009
|
+
* Durable creation backstop. Hooks improve the authoring experience, but sync
|
|
1010
|
+
* is the invariant boundary: no unstamped canonical company skill reaches S3.
|
|
1045
1011
|
*/
|
|
1046
1012
|
async function preflightCompanySkillIdentities(run, entries) {
|
|
1047
1013
|
if (run.options.personalMode === true || run.ctx.uid.startsWith("prs_")) {
|
|
1048
|
-
return
|
|
1014
|
+
return { registrationDenied: new Set(), registrationInvalid: new Set() };
|
|
1049
1015
|
}
|
|
1050
1016
|
const canonical = entries.filter((entry) => entry.kind === "file" &&
|
|
1051
1017
|
CANONICAL_COMPANY_SKILL_RE.test(entry.relativePath));
|
|
1052
|
-
const unstamped =
|
|
1053
|
-
const
|
|
1054
|
-
const pending = [];
|
|
1055
|
-
for (const entry of canonical) {
|
|
1056
|
-
const stamped = Boolean(parseRegisteredCompanySkillUid(fs.readFileSync(entry.absolutePath, "utf8")));
|
|
1057
|
-
if (!stamped) {
|
|
1058
|
-
unstamped.push(entry);
|
|
1059
|
-
continue;
|
|
1060
|
-
}
|
|
1018
|
+
const unstamped = canonical.filter((entry) => !parseRegisteredCompanySkillUid(fs.readFileSync(entry.absolutePath, "utf8")));
|
|
1019
|
+
const pending = canonical.filter((entry) => {
|
|
1061
1020
|
const journalEntry = journalRows(run.journal).get(entry.relativePath);
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
}
|
|
1068
|
-
else if (journalEntry.skillMetadataPending === true &&
|
|
1069
|
-
journalEntry.hash === hashFile(entry.absolutePath)) {
|
|
1070
|
-
pending.push(entry);
|
|
1071
|
-
}
|
|
1072
|
-
}
|
|
1073
|
-
if (unstamped.length === 0 && relocated.length === 0 && pending.length === 0) {
|
|
1074
|
-
return emptyPreflightResult();
|
|
1021
|
+
return (journalEntry?.skillMetadataPending === true &&
|
|
1022
|
+
journalEntry.hash === hashFile(entry.absolutePath));
|
|
1023
|
+
});
|
|
1024
|
+
if (unstamped.length === 0 && pending.length === 0) {
|
|
1025
|
+
return { registrationDenied: new Set(), registrationInvalid: new Set() };
|
|
1075
1026
|
}
|
|
1076
1027
|
const registrar = skillRegistrarForRun(run);
|
|
1077
1028
|
if (!registrar) {
|
|
1078
1029
|
if (unstamped.length > 0) {
|
|
1079
1030
|
throw new UnregisteredCompanySkillError(unstamped.map((entry) => entry.relativePath));
|
|
1080
1031
|
}
|
|
1081
|
-
return
|
|
1032
|
+
return { registrationDenied: new Set(), registrationInvalid: new Set() };
|
|
1082
1033
|
}
|
|
1083
1034
|
const registrationDenied = new Set();
|
|
1084
1035
|
const registrationInvalid = new Set();
|
|
1085
|
-
const registrationFailed = new Set();
|
|
1086
1036
|
const registrationInvalidMessages = new Map();
|
|
1087
|
-
const registrationFailedMessages = new Map();
|
|
1088
|
-
const classifyRegistrationFailure = (relativePath, err, options) => {
|
|
1089
|
-
// A 403 has its existing permissions-specific report. Keep its priority
|
|
1090
|
-
// even for a caller-supplied generic VaultClientError rather than folding
|
|
1091
|
-
// it into the validation bucket below.
|
|
1092
|
-
if (err instanceof VaultPermissionDeniedError ||
|
|
1093
|
-
(err instanceof VaultClientError && err.statusCode === 403)) {
|
|
1094
|
-
registrationDenied.add(relativePath);
|
|
1095
|
-
return;
|
|
1096
|
-
}
|
|
1097
|
-
if (isPermanentSkillRegistrationValidationError(err)) {
|
|
1098
|
-
registrationInvalid.add(relativePath);
|
|
1099
|
-
if (err.message)
|
|
1100
|
-
registrationInvalidMessages.set(relativePath, err.message);
|
|
1101
|
-
return;
|
|
1102
|
-
}
|
|
1103
|
-
if (!options.containNonPermanent || isRunLevelSkillRegistrationError(err)) {
|
|
1104
|
-
throw err;
|
|
1105
|
-
}
|
|
1106
|
-
registrationFailed.add(relativePath);
|
|
1107
|
-
registrationFailedMessages.set(relativePath, describeError(err));
|
|
1108
|
-
};
|
|
1109
1037
|
for (const entry of unstamped) {
|
|
1110
1038
|
try {
|
|
1111
|
-
await registerOneCompanySkill(run, entry, registrar
|
|
1112
|
-
}
|
|
1113
|
-
catch (err) {
|
|
1114
|
-
// Unchanged contract for the creation backstop: a transport failure
|
|
1115
|
-
// while minting a brand-new identity still aborts the pass, because
|
|
1116
|
-
// the file has no identity at all and nothing else can proceed safely
|
|
1117
|
-
// with it. Denied / invalid are contained as before.
|
|
1118
|
-
classifyRegistrationFailure(entry.relativePath, err, {
|
|
1119
|
-
containNonPermanent: false,
|
|
1120
|
-
});
|
|
1121
|
-
}
|
|
1122
|
-
}
|
|
1123
|
-
for (const entry of relocated) {
|
|
1124
|
-
try {
|
|
1125
|
-
await registerOneCompanySkill(run, entry, registrar, "stamped");
|
|
1039
|
+
await registerOneCompanySkill(run, entry, registrar);
|
|
1126
1040
|
}
|
|
1127
1041
|
catch (err) {
|
|
1128
|
-
//
|
|
1129
|
-
//
|
|
1130
|
-
//
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1042
|
+
// A 403 has its existing permissions-specific report. Keep its priority
|
|
1043
|
+
// even for a caller-supplied generic VaultClientError rather than folding
|
|
1044
|
+
// it into the validation bucket below.
|
|
1045
|
+
if (err instanceof VaultPermissionDeniedError ||
|
|
1046
|
+
(err instanceof VaultClientError && err.statusCode === 403)) {
|
|
1047
|
+
registrationDenied.add(entry.relativePath);
|
|
1048
|
+
}
|
|
1049
|
+
else if (isPermanentSkillRegistrationValidationError(err)) {
|
|
1050
|
+
registrationInvalid.add(entry.relativePath);
|
|
1051
|
+
if (err.message)
|
|
1052
|
+
registrationInvalidMessages.set(entry.relativePath, err.message);
|
|
1053
|
+
}
|
|
1054
|
+
else {
|
|
1137
1055
|
throw err;
|
|
1056
|
+
}
|
|
1138
1057
|
}
|
|
1139
1058
|
}
|
|
1140
1059
|
if (registrationDenied.size > 0) {
|
|
@@ -1159,307 +1078,85 @@ async function preflightCompanySkillIdentities(run, entries) {
|
|
|
1159
1078
|
});
|
|
1160
1079
|
}
|
|
1161
1080
|
for (const entry of pending) {
|
|
1162
|
-
|
|
1163
|
-
// retries this one skill; nothing else in the company waits for it.
|
|
1164
|
-
try {
|
|
1165
|
-
await reconcileOnePendingCompanySkill(run, entry);
|
|
1166
|
-
}
|
|
1167
|
-
catch (err) {
|
|
1168
|
-
classifyRegistrationFailure(entry.relativePath, err, {
|
|
1169
|
-
containNonPermanent: true,
|
|
1170
|
-
});
|
|
1171
|
-
}
|
|
1172
|
-
}
|
|
1173
|
-
if (registrationFailed.size > 0) {
|
|
1174
|
-
const samplePaths = [...registrationFailed].slice(0, 10);
|
|
1175
|
-
const message = samplePaths
|
|
1176
|
-
.map((relativePath) => registrationFailedMessages.get(relativePath))
|
|
1177
|
-
.find((candidate) => candidate !== undefined);
|
|
1178
|
-
run.emit({
|
|
1179
|
-
type: "not-shipped",
|
|
1180
|
-
reason: "skill-registration-failed",
|
|
1181
|
-
count: registrationFailed.size,
|
|
1182
|
-
samplePaths,
|
|
1183
|
-
...(message ? { message } : {}),
|
|
1184
|
-
});
|
|
1185
|
-
}
|
|
1186
|
-
return { registrationDenied, registrationInvalid, registrationFailed };
|
|
1187
|
-
}
|
|
1188
|
-
/**
|
|
1189
|
-
* Retry the post-upload metadata reconciliation for one skill whose journal
|
|
1190
|
-
* row still carries `skillMetadataPending`. Pulled out of the preflight loop
|
|
1191
|
-
* so a throw from any step is contained to this entry by the caller.
|
|
1192
|
-
*/
|
|
1193
|
-
async function reconcileOnePendingCompanySkill(run, entry) {
|
|
1194
|
-
const journalEntry = journalRows(run.journal).get(entry.relativePath);
|
|
1195
|
-
run.options.onHeartbeat?.();
|
|
1196
|
-
const remote = await headRemoteFile(run.ctx, entry.relativePath);
|
|
1197
|
-
if (!remote) {
|
|
1198
|
-
// The upload this marker referred to no longer exists. Drop the stale
|
|
1199
|
-
// baseline so the normal planner cannot skip the local file as unchanged
|
|
1200
|
-
// and will upload it again on this pass.
|
|
1201
|
-
journalRows(run.journal).delete(entry.relativePath);
|
|
1202
|
-
flushShareJournal(run);
|
|
1203
|
-
return;
|
|
1204
|
-
}
|
|
1205
|
-
if (!journalEntry?.remoteEtag ||
|
|
1206
|
-
normalizeEtag(remote.etag) !== normalizeEtag(journalEntry.remoteEtag)) {
|
|
1207
|
-
// The successful PUT this marker describes is no longer the live remote
|
|
1208
|
-
// object. Reconcile the current remote bytes instead of stranding the old
|
|
1209
|
-
// marker behind skipUnchanged; the backend independently revalidates this
|
|
1210
|
-
// exact content against S3 before accepting its metadata.
|
|
1211
|
-
// Invalidate the stale baseline before any network work. If recovery
|
|
1212
|
-
// fails, the next pass must still see a real divergence rather than
|
|
1213
|
-
// classifying the local file as unchanged forever.
|
|
1214
|
-
journalRows(run.journal).delete(entry.relativePath);
|
|
1215
|
-
flushShareJournal(run);
|
|
1216
|
-
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-skill-metadata-"));
|
|
1217
|
-
const tempFile = path.join(tempDir, "SKILL.md");
|
|
1218
|
-
try {
|
|
1219
|
-
await downloadFile(run.ctx, entry.relativePath, tempFile);
|
|
1220
|
-
// The reconciled bytes are the REMOTE object (a peer replaced the one
|
|
1221
|
-
// this marker described), not the local file, so pass no absolutePath:
|
|
1222
|
-
// a 409 here is recorded, not forked (see reconcileLandedCompanySkill).
|
|
1223
|
-
// The journal row is already gone, so settling only records/drops the
|
|
1224
|
-
// conflict; clearing the (absent) marker is a no-op.
|
|
1225
|
-
const outcome = await reconcileLandedCompanySkill(run, entry.relativePath, fs.readFileSync(tempFile, "utf8"));
|
|
1226
|
-
settleLandedReconcile(run, entry.relativePath, outcome);
|
|
1227
|
-
}
|
|
1228
|
-
finally {
|
|
1229
|
-
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
1230
|
-
}
|
|
1231
|
-
return;
|
|
1232
|
-
}
|
|
1233
|
-
const landedBytes = fs.readFileSync(entry.absolutePath);
|
|
1234
|
-
const landedHash = crypto
|
|
1235
|
-
.createHash("sha256")
|
|
1236
|
-
.update(landedBytes)
|
|
1237
|
-
.digest("hex");
|
|
1238
|
-
if (landedHash !== journalEntry.hash) {
|
|
1239
|
-
// The editor saved after this entry was classified but before the
|
|
1240
|
-
// remote proof returned. Reconcile only the single local snapshot that
|
|
1241
|
-
// produced the journaled upload; the newer edit belongs to the planner.
|
|
1242
|
-
return;
|
|
1243
|
-
}
|
|
1244
|
-
const outcome = await reconcileLandedCompanySkill(run, entry.relativePath, landedBytes.toString("utf8"), entry.absolutePath);
|
|
1245
|
-
// A reconciled/conflict outcome clears the marker (the server has answered,
|
|
1246
|
-
// so carrying it is exactly what let a leg loop forever); a `deferred`
|
|
1247
|
-
// outcome keeps it so a later pass reconciles the metadata without looping.
|
|
1248
|
-
settleLandedReconcile(run, entry.relativePath, outcome);
|
|
1249
|
-
checkpointShareJournal(run);
|
|
1250
|
-
}
|
|
1251
|
-
/**
|
|
1252
|
-
* A 409 raised by the company-skill registrar seam: a landed identity mismatch
|
|
1253
|
-
* on a copied/renamed pre-stamped skill, or a sibling stale/concurrent race.
|
|
1254
|
-
* The error carries the service's `code` and `path` when the body named them
|
|
1255
|
-
* (see `VaultErrorDetails`), so a recorded conflict can name the file.
|
|
1256
|
-
*/
|
|
1257
|
-
function isSkillRegistrationConflict(err) {
|
|
1258
|
-
return err instanceof VaultConflictError;
|
|
1259
|
-
}
|
|
1260
|
-
/**
|
|
1261
|
-
* A registrar failure that is a RUN-level fact, never a per-object skip: a
|
|
1262
|
-
* session auth refusal or a Cognito identity refusal. These must propagate from
|
|
1263
|
-
* any registration attempt so one refusal aborts the pass exactly as before,
|
|
1264
|
-
* rather than being absorbed as a skill outcome.
|
|
1265
|
-
*/
|
|
1266
|
-
function isRunLevelRegistrarFailure(err) {
|
|
1267
|
-
return (err instanceof VaultAuthError ||
|
|
1268
|
-
isTerminalIdentityRefusal(err) ||
|
|
1269
|
-
isRetryableIdentityRefusal(err));
|
|
1270
|
-
}
|
|
1271
|
-
/** Record a company-skill path the client could not self-heal after a landed
|
|
1272
|
-
* 409 (first message wins), for a single end-of-pass not-shipped event. */
|
|
1273
|
-
function recordSkillRegistrationConflict(run, relativePath, message) {
|
|
1274
|
-
if (!run.skillRegistrationConflicts.has(relativePath)) {
|
|
1275
|
-
run.skillRegistrationConflicts.set(relativePath, message);
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
/**
|
|
1279
|
-
* Apply a landed-reconcile outcome at a call site that owns the pending marker:
|
|
1280
|
-
* clear it on a resolved (reconciled/conflict) outcome, keep it on `deferred`.
|
|
1281
|
-
* `reconciled` also drops any conflict this path accumulated earlier in the
|
|
1282
|
-
* pass (e.g. a stale-etag preflight record the later upload has now fixed), so
|
|
1283
|
-
* finalization does not report a conflict the pass actually resolved.
|
|
1284
|
-
*/
|
|
1285
|
-
function settleLandedReconcile(run, relativePath, outcome) {
|
|
1286
|
-
if (outcome.status === "deferred")
|
|
1287
|
-
return;
|
|
1288
|
-
if (outcome.status === "conflict") {
|
|
1289
|
-
recordSkillRegistrationConflict(run, relativePath, outcome.message);
|
|
1290
|
-
}
|
|
1291
|
-
else {
|
|
1292
|
-
run.skillRegistrationConflicts.delete(relativePath);
|
|
1293
|
-
}
|
|
1294
|
-
clearSkillMetadataPending(run, relativePath);
|
|
1295
|
-
}
|
|
1296
|
-
/** Emit the single end-of-pass not-shipped event for skills whose metadata was
|
|
1297
|
-
* not reconciled. Called from the normal finalizer AND before an early aborted
|
|
1298
|
-
* return, so a recorded conflict is never silently dropped. */
|
|
1299
|
-
function emitSkillRegistrationConflicts(run) {
|
|
1300
|
-
if (run.skillRegistrationConflicts.size === 0)
|
|
1301
|
-
return;
|
|
1302
|
-
const samplePaths = [...run.skillRegistrationConflicts.keys()].slice(0, 10);
|
|
1303
|
-
const message = samplePaths
|
|
1304
|
-
.map((relativePath) => run.skillRegistrationConflicts.get(relativePath))
|
|
1305
|
-
.find((candidate) => candidate !== undefined);
|
|
1306
|
-
run.emit({
|
|
1307
|
-
type: "not-shipped",
|
|
1308
|
-
reason: "skill-registration-conflict",
|
|
1309
|
-
count: run.skillRegistrationConflicts.size,
|
|
1310
|
-
samplePaths,
|
|
1311
|
-
...(message ? { message } : {}),
|
|
1312
|
-
});
|
|
1313
|
-
}
|
|
1314
|
-
/**
|
|
1315
|
-
* A landed reconcile got a 409: the stamped identity is not the one registered
|
|
1316
|
-
* for this path. Reserve identity with ONE non-landed register — the server
|
|
1317
|
-
* classifies a stamped copy/rename as fork (new uid, re-stamped bytes), move
|
|
1318
|
-
* (repoint), or reuse — then, when it forked and the local file is unchanged
|
|
1319
|
-
* since these bytes were read, rewrite it atomically so the planner re-uploads
|
|
1320
|
-
* the re-stamped bytes; then retry the landed reconcile once (best effort — its
|
|
1321
|
-
* result never gates success). If the non-landed fallback is itself refused (a
|
|
1322
|
-
* second 409, a 403, or a 400/422 validation) report a deliberate skill skip
|
|
1323
|
-
* rather than throwing, so the pending marker can be cleared and the company
|
|
1324
|
-
* leg can never loop on it.
|
|
1325
|
-
*/
|
|
1326
|
-
async function recoverLandedSkillConflict(run, relativePath, content, absolutePath, registrar) {
|
|
1327
|
-
let fallback;
|
|
1328
|
-
try {
|
|
1081
|
+
const journalEntry = journalRows(run.journal).get(entry.relativePath);
|
|
1329
1082
|
run.options.onHeartbeat?.();
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
// A refused fallback is a deliberate skill skip, not a leg abort: a second
|
|
1339
|
-
// 409, a permission denial (403, in either its typed or raw form, matching
|
|
1340
|
-
// the unstamped path), or a permanent 400/422 validation. Everything else
|
|
1341
|
-
// (auth, identity refusal, 404, 5xx, transport) propagates so the marker is
|
|
1342
|
-
// kept and the pass retries — unchanged from before.
|
|
1343
|
-
if (isSkillRegistrationConflict(err) ||
|
|
1344
|
-
err instanceof VaultPermissionDeniedError ||
|
|
1345
|
-
(err instanceof VaultClientError && err.statusCode === 403) ||
|
|
1346
|
-
isPermanentSkillRegistrationValidationError(err)) {
|
|
1347
|
-
return {
|
|
1348
|
-
status: "conflict",
|
|
1349
|
-
...(err instanceof Error && err.message ? { message: err.message } : {}),
|
|
1350
|
-
};
|
|
1351
|
-
}
|
|
1352
|
-
throw err;
|
|
1353
|
-
}
|
|
1354
|
-
// Apply the fork's re-stamped bytes only when the local file is unchanged
|
|
1355
|
-
// since `content` was read — the same guard registerOneCompanySkill uses so a
|
|
1356
|
-
// concurrent editor's newer body is never overwritten. The stale-etag path
|
|
1357
|
-
// reconciles REMOTE bytes and passes no absolutePath, so it never rewrites a
|
|
1358
|
-
// local file from a fork of bytes that are not that file.
|
|
1359
|
-
if (absolutePath !== undefined && fallback.content !== content) {
|
|
1360
|
-
let current;
|
|
1361
|
-
try {
|
|
1362
|
-
current = fs.readFileSync(absolutePath, "utf8");
|
|
1083
|
+
const remote = await headRemoteFile(run.ctx, entry.relativePath);
|
|
1084
|
+
if (!remote) {
|
|
1085
|
+
// The upload this marker referred to no longer exists. Drop the stale
|
|
1086
|
+
// baseline so the normal planner cannot skip the local file as unchanged
|
|
1087
|
+
// and will upload it again on this pass.
|
|
1088
|
+
journalRows(run.journal).delete(entry.relativePath);
|
|
1089
|
+
flushShareJournal(run);
|
|
1090
|
+
continue;
|
|
1363
1091
|
}
|
|
1364
|
-
|
|
1365
|
-
|
|
1092
|
+
if (!journalEntry?.remoteEtag ||
|
|
1093
|
+
normalizeEtag(remote.etag) !== normalizeEtag(journalEntry.remoteEtag)) {
|
|
1094
|
+
// The successful PUT this marker describes is no longer the live remote
|
|
1095
|
+
// object. Reconcile the current remote bytes instead of stranding the old
|
|
1096
|
+
// marker behind skipUnchanged; the backend independently revalidates this
|
|
1097
|
+
// exact content against S3 before accepting its metadata.
|
|
1098
|
+
// Invalidate the stale baseline before any network work. If recovery
|
|
1099
|
+
// fails, the next pass must still see a real divergence rather than
|
|
1100
|
+
// classifying the local file as unchanged forever.
|
|
1101
|
+
journalRows(run.journal).delete(entry.relativePath);
|
|
1102
|
+
flushShareJournal(run);
|
|
1103
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-skill-metadata-"));
|
|
1104
|
+
const tempFile = path.join(tempDir, "SKILL.md");
|
|
1105
|
+
try {
|
|
1106
|
+
await downloadFile(run.ctx, entry.relativePath, tempFile);
|
|
1107
|
+
await reconcileLandedCompanySkill(run, entry.relativePath, fs.readFileSync(tempFile, "utf8"));
|
|
1108
|
+
}
|
|
1109
|
+
finally {
|
|
1110
|
+
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
1111
|
+
}
|
|
1112
|
+
continue;
|
|
1366
1113
|
}
|
|
1367
|
-
|
|
1368
|
-
|
|
1114
|
+
const landedBytes = fs.readFileSync(entry.absolutePath);
|
|
1115
|
+
const landedHash = crypto
|
|
1116
|
+
.createHash("sha256")
|
|
1117
|
+
.update(landedBytes)
|
|
1118
|
+
.digest("hex");
|
|
1119
|
+
if (landedHash !== journalEntry.hash) {
|
|
1120
|
+
// The editor saved after this entry was classified but before the
|
|
1121
|
+
// remote proof returned. Reconcile only the single local snapshot that
|
|
1122
|
+
// produced the journaled upload; the newer edit belongs to the planner.
|
|
1123
|
+
continue;
|
|
1369
1124
|
}
|
|
1125
|
+
await reconcileLandedCompanySkill(run, entry.relativePath, landedBytes.toString("utf8"));
|
|
1126
|
+
clearSkillMetadataPending(run, entry.relativePath);
|
|
1127
|
+
checkpointShareJournal(run);
|
|
1370
1128
|
}
|
|
1371
|
-
|
|
1372
|
-
// metadata.
|
|
1373
|
-
try {
|
|
1374
|
-
run.options.onHeartbeat?.();
|
|
1375
|
-
const retry = await registrar({
|
|
1376
|
-
companySlug: run.ctx.slug,
|
|
1377
|
-
path: relativePath,
|
|
1378
|
-
content: fallback.content,
|
|
1379
|
-
landed: true,
|
|
1380
|
-
});
|
|
1381
|
-
assertValidSkillRegistration(retry, relativePath);
|
|
1382
|
-
}
|
|
1383
|
-
catch (retryErr) {
|
|
1384
|
-
// An auth/identity refusal is a run-level fact — propagate it.
|
|
1385
|
-
if (isRunLevelRegistrarFailure(retryErr))
|
|
1386
|
-
throw retryErr;
|
|
1387
|
-
// A repeat 409 means the fallback did not actually reserve identity for
|
|
1388
|
-
// this path (a pathological server state). Do not loop on it: record a
|
|
1389
|
-
// deliberate skip and let the marker clear.
|
|
1390
|
-
if (isSkillRegistrationConflict(retryErr)) {
|
|
1391
|
-
return {
|
|
1392
|
-
status: "conflict",
|
|
1393
|
-
...(retryErr instanceof Error && retryErr.message
|
|
1394
|
-
? { message: retryErr.message }
|
|
1395
|
-
: {}),
|
|
1396
|
-
};
|
|
1397
|
-
}
|
|
1398
|
-
// A transient failure (5xx/408/429/transport/404) after a successful
|
|
1399
|
-
// fallback: keep the pending marker so a later pass reconciles the
|
|
1400
|
-
// metadata. Identity is reserved, so the next landed call succeeds — this
|
|
1401
|
-
// cannot loop the way an unresolved 409 did, and it never strands tags on a
|
|
1402
|
-
// `moved`/`reused` fallback whose bytes did not change.
|
|
1403
|
-
return { status: "deferred" };
|
|
1404
|
-
}
|
|
1405
|
-
return { status: "reconciled" };
|
|
1129
|
+
return { registrationDenied, registrationInvalid };
|
|
1406
1130
|
}
|
|
1407
1131
|
/**
|
|
1408
1132
|
* Reconcile metadata only after the canonical bytes have landed. Identity
|
|
1409
1133
|
* reservation happens before upload; tags and other source-authored metadata
|
|
1410
|
-
* must not become authoritative until the S3 write succeeds.
|
|
1411
|
-
* not fatal: it is recovered (or recorded as a deliberate skip) by
|
|
1412
|
-
* {@link recoverLandedSkillConflict} instead of aborting the company leg.
|
|
1134
|
+
* must not become authoritative until the S3 write succeeds.
|
|
1413
1135
|
*/
|
|
1414
|
-
async function reconcileLandedCompanySkill(run, relativePath, content
|
|
1136
|
+
async function reconcileLandedCompanySkill(run, relativePath, content) {
|
|
1415
1137
|
if (run.options.personalMode === true ||
|
|
1416
1138
|
run.ctx.uid.startsWith("prs_") ||
|
|
1417
1139
|
!CANONICAL_COMPANY_SKILL_RE.test(relativePath)) {
|
|
1418
|
-
return
|
|
1140
|
+
return;
|
|
1419
1141
|
}
|
|
1420
1142
|
const registrar = skillRegistrarForRun(run);
|
|
1421
1143
|
if (!registrar) {
|
|
1422
|
-
return
|
|
1144
|
+
return;
|
|
1423
1145
|
}
|
|
1424
1146
|
const localUid = parseRegisteredCompanySkillUid(content);
|
|
1425
1147
|
if (!localUid) {
|
|
1426
1148
|
throw new UnregisteredCompanySkillError([relativePath]);
|
|
1427
1149
|
}
|
|
1428
1150
|
run.options.onHeartbeat?.();
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
throw new Error(`Skill registration returned identity '${result.skillUid}' for '${relativePath}', but the landed file is stamped '${localUid}'.`);
|
|
1439
|
-
}
|
|
1440
|
-
return { status: "reconciled" };
|
|
1441
|
-
}
|
|
1442
|
-
catch (err) {
|
|
1443
|
-
// Only a 409 from the registrar seam becomes a per-object outcome. Every
|
|
1444
|
-
// other failure (auth, 404, 5xx, transport, or an invalid-identity
|
|
1445
|
-
// response) propagates exactly as before, so a transient registry failure
|
|
1446
|
-
// still retries the pending marker next pass and an auth refusal still
|
|
1447
|
-
// aborts the pass.
|
|
1448
|
-
if (!isSkillRegistrationConflict(err))
|
|
1449
|
-
throw err;
|
|
1450
|
-
if (absolutePath === undefined) {
|
|
1451
|
-
// These bytes are a REPLACED REMOTE object, not a local file (the
|
|
1452
|
-
// stale-etag recovery path). A non-landed fork here would reserve a
|
|
1453
|
-
// server identity that no local file carries, and a later remote-adoption
|
|
1454
|
-
// could then strand that fork forever. Record the conflict and let the
|
|
1455
|
-
// local file self-heal through the upload path on a later pass, which
|
|
1456
|
-
// forks and re-stamps the actual local bytes.
|
|
1457
|
-
return {
|
|
1458
|
-
status: "conflict",
|
|
1459
|
-
...(err instanceof Error && err.message ? { message: err.message } : {}),
|
|
1460
|
-
};
|
|
1461
|
-
}
|
|
1462
|
-
return recoverLandedSkillConflict(run, relativePath, content, absolutePath, registrar);
|
|
1151
|
+
const result = await registrar({
|
|
1152
|
+
companySlug: run.ctx.slug,
|
|
1153
|
+
path: relativePath,
|
|
1154
|
+
content,
|
|
1155
|
+
landed: true,
|
|
1156
|
+
});
|
|
1157
|
+
assertValidSkillRegistration(result, relativePath);
|
|
1158
|
+
if (result.skillUid !== localUid) {
|
|
1159
|
+
throw new Error(`Skill registration returned identity '${result.skillUid}' for '${relativePath}', but the landed file is stamped '${localUid}'.`);
|
|
1463
1160
|
}
|
|
1464
1161
|
}
|
|
1465
1162
|
async function buildSharePlans(run) {
|
|
@@ -1546,12 +1243,8 @@ async function buildSharePlans(run) {
|
|
|
1546
1243
|
return false;
|
|
1547
1244
|
})
|
|
1548
1245
|
: controlCharFiltered;
|
|
1549
|
-
const { registrationDenied, registrationInvalid
|
|
1550
|
-
const registrationSkipped = new Set([
|
|
1551
|
-
...registrationDenied,
|
|
1552
|
-
...registrationInvalid,
|
|
1553
|
-
...registrationFailed,
|
|
1554
|
-
]);
|
|
1246
|
+
const { registrationDenied, registrationInvalid } = await preflightCompanySkillIdentities(run, filesToShare);
|
|
1247
|
+
const registrationSkipped = new Set([...registrationDenied, ...registrationInvalid]);
|
|
1555
1248
|
const uploadableFiles = registrationSkipped.size > 0
|
|
1556
1249
|
? filesToShare.filter((entry) => !registrationSkipped.has(entry.relativePath))
|
|
1557
1250
|
: filesToShare;
|
|
@@ -1663,7 +1356,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1663
1356
|
key: it.relativePath,
|
|
1664
1357
|
localPath: it.absolutePath,
|
|
1665
1358
|
isSymlink: it.kind === "symlink",
|
|
1666
|
-
...(it.kind === "file" ? { contentHash: it.localHash } : {}),
|
|
1667
1359
|
author: run.options.author,
|
|
1668
1360
|
})));
|
|
1669
1361
|
// Warm the GET presigns the per-item conflict HEAD (remoteMeta) reuses, so a
|
|
@@ -1835,39 +1527,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1835
1527
|
}
|
|
1836
1528
|
if (remoteMeta) {
|
|
1837
1529
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
1838
|
-
const
|
|
1839
|
-
const remoteChanged = !!journalEntry &&
|
|
1840
|
-
hasRemoteChanged({ ...remoteMeta, contentHash: remoteContentHash }, journalEntry);
|
|
1841
|
-
// A matching published digest is a content no-op even when SSE-KMS gave
|
|
1842
|
-
// this object a fresh ETag. Reconcile the journal to that version without
|
|
1843
|
-
// a needless PUT or a phantom two-sided conflict.
|
|
1844
|
-
if (remoteContentHash === localHash) {
|
|
1845
|
-
const lstat = fs.lstatSync(absolutePath);
|
|
1846
|
-
updateEntry(run.journal, relativePath, localHash, lstat.size, journalEntry?.direction ?? "up", absolutePath, {
|
|
1847
|
-
remoteEtag: remoteMeta.etag,
|
|
1848
|
-
mtimeMs: lstat.mtimeMs,
|
|
1849
|
-
ctimeMs: ctimeToStamp(item.kind === "file" ? item.statAtHash : undefined, lstat),
|
|
1850
|
-
kind: "file",
|
|
1851
|
-
});
|
|
1852
|
-
checkpointShareJournal(run);
|
|
1853
|
-
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
1854
|
-
counters.filesSkipped++;
|
|
1855
|
-
return;
|
|
1856
|
-
}
|
|
1857
|
-
// This machine's journal baseline is a superseded content generation.
|
|
1858
|
-
// Replaying that unchanged baseline is a stale revert, not an edit. A
|
|
1859
|
-
// user who deliberately restores some other historic content has a hash
|
|
1860
|
-
// different from the journal baseline and still reaches normal conflict
|
|
1861
|
-
// resolution below.
|
|
1862
|
-
if (journalEntry !== undefined &&
|
|
1863
|
-
remoteContentHash !== undefined &&
|
|
1864
|
-
journalEntry.hash === localHash &&
|
|
1865
|
-
remoteContentHash !== localHash) {
|
|
1866
|
-
conflictPaths.push(relativePath);
|
|
1867
|
-
run.emit({ type: "conflict", path: relativePath, direction: "push", resolution: "skip" });
|
|
1868
|
-
counters.filesSkipped++;
|
|
1869
|
-
return;
|
|
1870
|
-
}
|
|
1530
|
+
const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
|
|
1871
1531
|
// Every path below that needs the remote BODY — the no-row equality
|
|
1872
1532
|
// probe, and a `keep` that adopts the remote — is impossible against a
|
|
1873
1533
|
// DEEP_ARCHIVE session transcript: each GET fails with InvalidObjectState.
|
|
@@ -1946,8 +1606,8 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1946
1606
|
// journal row was persisted. Make the retry marker durable before the
|
|
1947
1607
|
// registry call so another interruption remains recoverable.
|
|
1948
1608
|
flushShareJournal(run);
|
|
1949
|
-
|
|
1950
|
-
|
|
1609
|
+
await reconcileLandedCompanySkill(run, relativePath, skillContent);
|
|
1610
|
+
clearSkillMetadataPending(run, relativePath);
|
|
1951
1611
|
}
|
|
1952
1612
|
checkpointShareJournal(run);
|
|
1953
1613
|
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
@@ -2100,8 +1760,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2100
1760
|
const skillContentAtPut = !isSymlinkUpload && CANONICAL_COMPANY_SKILL_RE.test(relativePath)
|
|
2101
1761
|
? fs.readFileSync(absolutePath, "utf8")
|
|
2102
1762
|
: undefined;
|
|
2103
|
-
if (!isSymlinkUpload)
|
|
2104
|
-
setUploadContentHash(absolutePath, localHash);
|
|
2105
1763
|
const { etag } = isSymlinkUpload
|
|
2106
1764
|
? await uploadSymlink(run.ctx, item.target, relativePath, run.options.author, pc)
|
|
2107
1765
|
: await uploadFile(run.ctx, absolutePath, relativePath, run.options.author, pc);
|
|
@@ -2136,8 +1794,8 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2136
1794
|
if (skillContentAtPut !== undefined &&
|
|
2137
1795
|
skillRegistrarForRun(run) !== undefined &&
|
|
2138
1796
|
fs.readFileSync(absolutePath, "utf8") === skillContentAtPut) {
|
|
2139
|
-
|
|
2140
|
-
|
|
1797
|
+
await reconcileLandedCompanySkill(run, relativePath, skillContentAtPut);
|
|
1798
|
+
clearSkillMetadataPending(run, relativePath);
|
|
2141
1799
|
checkpointShareJournal(run);
|
|
2142
1800
|
}
|
|
2143
1801
|
run.emit({
|
|
@@ -2345,9 +2003,16 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2345
2003
|
// this flag only halts admission, so the refusal still surfaces via
|
|
2346
2004
|
// throwUploadWorkerErrors below. In-flight legs still drain.
|
|
2347
2005
|
let sessionAuthAborted = false;
|
|
2006
|
+
// Cooperative work is admitted in bounded units, then fully drained before
|
|
2007
|
+
// the root lock can change owners. Skipped rows are not durable mutations
|
|
2008
|
+
// and therefore do not count toward the unit.
|
|
2009
|
+
let admittedSinceCooperativeCheckpoint = 0;
|
|
2348
2010
|
while (queue.length > 0 || inFlight.size > 0) {
|
|
2349
2011
|
while (!aborted &&
|
|
2350
2012
|
!sessionAuthAborted &&
|
|
2013
|
+
(!run.options.cooperativeCheckpoint ||
|
|
2014
|
+
admittedSinceCooperativeCheckpoint <
|
|
2015
|
+
FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE) &&
|
|
2351
2016
|
inFlight.size < TRANSFER_CONCURRENCY &&
|
|
2352
2017
|
queue.length > 0) {
|
|
2353
2018
|
const previewItem = queue[0];
|
|
@@ -2374,6 +2039,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2374
2039
|
if (inFlight.size > 0 && inFlightBytes + nextBytes > byteBudget)
|
|
2375
2040
|
break;
|
|
2376
2041
|
queue.shift();
|
|
2042
|
+
admittedSinceCooperativeCheckpoint++;
|
|
2377
2043
|
inFlightBytes += nextBytes;
|
|
2378
2044
|
const upload = () => processUploadItem(item);
|
|
2379
2045
|
const p = (run.options.transferSemaphore
|
|
@@ -2401,6 +2067,24 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2401
2067
|
if (inFlight.size > 0) {
|
|
2402
2068
|
await Promise.race(Array.from(inFlight));
|
|
2403
2069
|
}
|
|
2070
|
+
else if (run.options.cooperativeCheckpoint &&
|
|
2071
|
+
!aborted &&
|
|
2072
|
+
!sessionAuthAborted &&
|
|
2073
|
+
!hadUploadFailure &&
|
|
2074
|
+
admittedSinceCooperativeCheckpoint > 0 &&
|
|
2075
|
+
(admittedSinceCooperativeCheckpoint >=
|
|
2076
|
+
FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE ||
|
|
2077
|
+
queue.length === 0)) {
|
|
2078
|
+
// Every admitted upload has settled, so `processUploadItem` has
|
|
2079
|
+
// finished its mutable journal updates. Persist this durable unit
|
|
2080
|
+
// before yielding. A real hand-off restarts share() above and thereby
|
|
2081
|
+
// discards its now-stale upload/delete plan.
|
|
2082
|
+
flushShareJournal(run);
|
|
2083
|
+
admittedSinceCooperativeCheckpoint = 0;
|
|
2084
|
+
if ((await run.options.cooperativeCheckpoint()) === true) {
|
|
2085
|
+
throw new CooperativePushHandoff();
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2404
2088
|
else {
|
|
2405
2089
|
break;
|
|
2406
2090
|
}
|
|
@@ -2915,6 +2599,12 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
2915
2599
|
*/
|
|
2916
2600
|
const CHECKPOINT_MIN_APPLIED = 250;
|
|
2917
2601
|
const CHECKPOINT_MIN_INTERVAL_MS = 5_000;
|
|
2602
|
+
/**
|
|
2603
|
+
* Maximum completed upload mutations between cooperative full-push hand-offs.
|
|
2604
|
+
* The executor drains a unit before yielding, so realtime never overlaps an
|
|
2605
|
+
* upload body or the push's mutable journal state.
|
|
2606
|
+
*/
|
|
2607
|
+
export const FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
2918
2608
|
/**
|
|
2919
2609
|
* Fraction of the journal's current size that must be re-applied before another
|
|
2920
2610
|
* checkpoint is worth its write. `writeJournal` publishes a whole compact
|
|
@@ -3049,7 +2739,6 @@ function finalizeShareJournal(run) {
|
|
|
3049
2739
|
samplePaths: sampleSet(run.linkedSubtreeSet),
|
|
3050
2740
|
});
|
|
3051
2741
|
}
|
|
3052
|
-
emitSkillRegistrationConflicts(run);
|
|
3053
2742
|
}
|
|
3054
2743
|
/**
|
|
3055
2744
|
* Which unreachable named paths are FATAL under the run's policy.
|
|
@@ -3319,12 +3008,6 @@ function defaultConsoleLogger(event) {
|
|
|
3319
3008
|
else if (event.reason === "skill-registration-invalid") {
|
|
3320
3009
|
console.warn(` ! ${event.count} unstamped company skill${event.count === 1 ? "" : "s"} was rejected as invalid and was NOT uploaded:`);
|
|
3321
3010
|
}
|
|
3322
|
-
else if (event.reason === "skill-registration-conflict") {
|
|
3323
|
-
console.warn(` ! ${event.count} company skill${event.count === 1 ? "" : "s"} could NOT be re-identified after a copy or rename (registration conflict); its content synced but its skill metadata was not reconciled this pass:`);
|
|
3324
|
-
}
|
|
3325
|
-
else if (event.reason === "skill-registration-failed") {
|
|
3326
|
-
console.warn(` ! ${event.count} company skill${event.count === 1 ? "" : "s"} could NOT finish registering on this pass and will be retried next pass (everything else synced):`);
|
|
3327
|
-
}
|
|
3328
3011
|
else {
|
|
3329
3012
|
console.warn(` ! ${event.count} linked subtree${event.count === 1 ? "" : "s"} recorded but NOT uploaded — contents sync via their own repo, not the vault:`);
|
|
3330
3013
|
}
|