@indigoai-us/hq-cloud 6.16.54-alpha.0 → 6.16.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backup-prune.d.ts +23 -1
- package/dist/backup-prune.d.ts.map +1 -1
- package/dist/backup-prune.js +92 -8
- package/dist/backup-prune.js.map +1 -1
- package/dist/backup-prune.test.js +36 -0
- package/dist/backup-prune.test.js.map +1 -1
- package/dist/bin/rescue-runner.js +2 -1
- package/dist/bin/rescue-runner.js.map +1 -1
- package/dist/bin/sync-runner-company.d.ts +0 -6
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +18 -32
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +148 -57
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-planning.d.ts +3 -12
- package/dist/bin/sync-runner-planning.d.ts.map +1 -1
- package/dist/bin/sync-runner-planning.js +3 -65
- package/dist/bin/sync-runner-planning.js.map +1 -1
- package/dist/bin/sync-runner-planning.test.js +1 -108
- package/dist/bin/sync-runner-planning.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +2 -48
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +120 -318
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +5 -33
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +4 -34
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +79 -1079
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/rescue-breaker.test.d.ts +2 -0
- package/dist/cli/rescue-breaker.test.d.ts.map +1 -0
- package/dist/cli/rescue-breaker.test.js +334 -0
- package/dist/cli/rescue-breaker.test.js.map +1 -0
- package/dist/cli/rescue-core.d.ts +40 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +1454 -162
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/rescue-realpath.test.d.ts +2 -0
- package/dist/cli/rescue-realpath.test.d.ts.map +1 -0
- package/dist/cli/rescue-realpath.test.js +109 -0
- package/dist/cli/rescue-realpath.test.js.map +1 -0
- package/dist/cli/rescue-rsync-preflight.test.js +256 -4
- package/dist/cli/rescue-rsync-preflight.test.js.map +1 -1
- package/dist/cli/rescue-snapshot-copy.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-copy.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-copy.test.js +627 -0
- package/dist/cli/rescue-snapshot-copy.test.js.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts +2 -0
- package/dist/cli/rescue-snapshot-recovery.test.d.ts.map +1 -0
- package/dist/cli/rescue-snapshot-recovery.test.js +260 -0
- package/dist/cli/rescue-snapshot-recovery.test.js.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts +2 -0
- package/dist/cli/rescue-source-switch-floor.test.d.ts.map +1 -0
- package/dist/cli/rescue-source-switch-floor.test.js +210 -0
- package/dist/cli/rescue-source-switch-floor.test.js.map +1 -0
- package/dist/cli/rescue.d.ts +4 -0
- package/dist/cli/rescue.d.ts.map +1 -1
- package/dist/cli/rescue.js +20 -1
- package/dist/cli/rescue.js.map +1 -1
- package/dist/cli/rescue.test.js +16 -1
- package/dist/cli/rescue.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.js +2 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -1
- package/dist/cli/share.d.ts +0 -19
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +467 -150
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +608 -95
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +0 -8
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts +15 -13
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +118 -226
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +2 -215
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cli/watch-event-push-conflict.test.js +2 -0
- package/dist/cli/watch-event-push-conflict.test.js.map +1 -1
- package/dist/cognito-auth.d.ts +30 -0
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +175 -0
- package/dist/cognito-auth.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-row-store.d.ts +5 -40
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +21 -240
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-row-store.test.js +0 -54
- package/dist/journal-row-store.test.js.map +1 -1
- package/dist/journal-snapshot-v4.d.ts.map +1 -1
- package/dist/journal-snapshot-v4.js +6 -11
- package/dist/journal-snapshot-v4.js.map +1 -1
- package/dist/journal-store.test.js +7 -259
- package/dist/journal-store.test.js.map +1 -1
- package/dist/journal.d.ts +15 -43
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +79 -297
- package/dist/journal.js.map +1 -1
- package/dist/lib/describe-error.d.ts.map +1 -1
- package/dist/lib/describe-error.js +7 -0
- package/dist/lib/describe-error.js.map +1 -1
- package/dist/lib/describe-error.test.js +11 -0
- package/dist/lib/describe-error.test.js.map +1 -1
- package/dist/lib/exit-codes.d.ts +0 -15
- package/dist/lib/exit-codes.d.ts.map +1 -1
- package/dist/lib/exit-codes.js +0 -15
- package/dist/lib/exit-codes.js.map +1 -1
- package/dist/machine-auth.test.js +295 -0
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +2 -2
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +18 -3
- package/dist/object-io.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +3 -10
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +0 -35
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +2 -9
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +2 -59
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +18 -1
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +47 -24
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +27 -1
- package/dist/s3.test.js.map +1 -1
- package/dist/sync/area-ledger.d.ts +2 -2
- package/dist/sync/area-ledger.d.ts.map +1 -1
- package/dist/sync/area-ledger.js +34 -66
- package/dist/sync/area-ledger.js.map +1 -1
- package/dist/sync/area-ledger.test.js +5 -90
- package/dist/sync/area-ledger.test.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.test.js +0 -29
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +1 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +6 -14
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +16 -136
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +0 -6
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +2 -14
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync-core.d.ts +6 -4
- package/dist/sync-core.d.ts.map +1 -1
- package/dist/sync-core.js +10 -4
- package/dist/sync-core.js.map +1 -1
- package/dist/sync-core.test.js +25 -1
- package/dist/sync-core.test.js.map +1 -1
- package/dist/vault-client.d.ts +33 -5
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +69 -21
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +65 -0
- package/dist/vault-client.test.js.map +1 -1
- package/package.json +1 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner-regressions.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner-regressions.test.js +0 -300
- package/dist/cli/onepass-pull-planner-regressions.test.js.map +0 -1
- package/dist/cli/onepass-pull-planner.test.d.ts +0 -2
- package/dist/cli/onepass-pull-planner.test.d.ts.map +0 -1
- package/dist/cli/onepass-pull-planner.test.js +0 -266
- package/dist/cli/onepass-pull-planner.test.js.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.d.ts +0 -15
- package/dist/cli/streaming-pull-planner.bench.test.d.ts.map +0 -1
- package/dist/cli/streaming-pull-planner.bench.test.js +0 -273
- package/dist/cli/streaming-pull-planner.bench.test.js.map +0 -1
- package/dist/s3.list-cost.test.d.ts +0 -2
- package/dist/s3.list-cost.test.d.ts.map +0 -1
- package/dist/s3.list-cost.test.js +0 -51
- package/dist/s3.list-cost.test.js.map +0 -1
- package/dist/sync/remote-file-spool.d.ts +0 -16
- package/dist/sync/remote-file-spool.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.js +0 -91
- package/dist/sync/remote-file-spool.js.map +0 -1
- package/dist/sync/remote-file-spool.test.d.ts +0 -2
- package/dist/sync/remote-file-spool.test.d.ts.map +0 -1
- package/dist/sync/remote-file-spool.test.js +0 -77
- package/dist/sync/remote-file-spool.test.js.map +0 -1
package/dist/cli/share.js
CHANGED
|
@@ -11,7 +11,7 @@ import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../c
|
|
|
11
11
|
import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
|
|
12
12
|
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
13
13
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
|
|
14
|
-
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
14
|
+
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
15
15
|
import * as crypto from "crypto";
|
|
16
16
|
import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, timestampsTooCoarseToTrust, updateEntry, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
|
|
17
17
|
import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
@@ -24,7 +24,7 @@ import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransfer
|
|
|
24
24
|
import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
|
|
25
25
|
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
26
26
|
import { isCloudAuthoritative } from "../lib/cloud-authoritative.js";
|
|
27
|
-
import { VaultAuthError, VaultClient, VaultClientError, VaultPermissionDeniedError, } from "../vault-client.js";
|
|
27
|
+
import { VaultAuthError, VaultClient, VaultClientError, VaultConflictError, VaultPermissionDeniedError, } from "../vault-client.js";
|
|
28
28
|
import { describeError } from "../lib/describe-error.js";
|
|
29
29
|
import { isRetryableIdentityRefusal, isTerminalIdentityRefusal, } from "../cognito-auth.js";
|
|
30
30
|
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
@@ -628,33 +628,10 @@ export function wrapFilterWithIgnoreVisibility(underlying, hqRoot, onIgnoreExclu
|
|
|
628
628
|
return false;
|
|
629
629
|
};
|
|
630
630
|
}
|
|
631
|
-
class CooperativePushHandoff extends Error {
|
|
632
|
-
constructor() {
|
|
633
|
-
super("Realtime work ran at a settled full-push boundary");
|
|
634
|
-
this.name = "CooperativePushHandoff";
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
631
|
/**
|
|
638
632
|
* Share local file(s) to the entity vault.
|
|
639
|
-
*
|
|
640
|
-
* A cooperative hand-off discards the old plan instead of continuing it after
|
|
641
|
-
* another writer has run. Journal delta computation treats missing keys as
|
|
642
|
-
* deletes, so reusing a pre-handoff plan would be unsafe even though the
|
|
643
|
-
* hand-off itself only happens at a durable unit boundary.
|
|
644
633
|
*/
|
|
645
634
|
export async function share(options) {
|
|
646
|
-
for (;;) {
|
|
647
|
-
try {
|
|
648
|
-
return await shareOnce(options);
|
|
649
|
-
}
|
|
650
|
-
catch (error) {
|
|
651
|
-
if (error instanceof CooperativePushHandoff)
|
|
652
|
-
continue;
|
|
653
|
-
throw error;
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
async function shareOnce(options) {
|
|
658
635
|
const run = await createPushRunContext(options);
|
|
659
636
|
const counters = createShareCounters();
|
|
660
637
|
const filesRefusedStalePaths = [];
|
|
@@ -689,6 +666,11 @@ async function shareOnce(options) {
|
|
|
689
666
|
// checkpoint exists to prevent. `flushShareJournal` leaves `lastSync` at
|
|
690
667
|
// its pre-run value, so the pass does not look like a finished sync.
|
|
691
668
|
flushShareJournal(run);
|
|
669
|
+
// finalizeShareJournal never runs on the aborted return, so surface any
|
|
670
|
+
// recorded skill-registration conflict here too — otherwise a skill whose
|
|
671
|
+
// marker was already cleared in preflight would be skipped silently and no
|
|
672
|
+
// later pass could reproduce the warning.
|
|
673
|
+
emitSkillRegistrationConflicts(run);
|
|
692
674
|
return buildShareResult(run, counters, filesRefusedStalePaths, uploadResult.abortFlightConflictPaths, pathResults, true);
|
|
693
675
|
}
|
|
694
676
|
await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths, pathResults, !uploadResult.hadUploadFailure && uploadResult.workerErrors.length === 0);
|
|
@@ -848,6 +830,7 @@ async function createPushRunContext(options) {
|
|
|
848
830
|
};
|
|
849
831
|
const unreachablePaths = new Map();
|
|
850
832
|
const linkedSubtreeSet = new Set();
|
|
833
|
+
const skillRegistrationConflicts = new Map();
|
|
851
834
|
const baseFilter = options.personalMode === true
|
|
852
835
|
? wrapFilterWithPersonalVaultDefaults(recordedIgnoreFilter, syncRoot, onExcluded)
|
|
853
836
|
: recordedIgnoreFilter;
|
|
@@ -889,6 +872,7 @@ async function createPushRunContext(options) {
|
|
|
889
872
|
ignoreExcludedTotal,
|
|
890
873
|
unreachablePaths,
|
|
891
874
|
linkedSubtreeSet,
|
|
875
|
+
skillRegistrationConflicts,
|
|
892
876
|
hostLoadAtStart,
|
|
893
877
|
checkpoint: {
|
|
894
878
|
appliedSinceFlush: 0,
|
|
@@ -962,14 +946,32 @@ function assertValidSkillRegistration(result, expectedPath) {
|
|
|
962
946
|
throw new Error(`Skill registration returned an invalid identity for '${expectedPath}'; local bytes were not changed.`);
|
|
963
947
|
}
|
|
964
948
|
}
|
|
965
|
-
|
|
949
|
+
/**
|
|
950
|
+
* Register one canonical company skill before its bytes upload.
|
|
951
|
+
*
|
|
952
|
+
* `mode: "unstamped"` is the creation backstop: a file with no `skill_uid`
|
|
953
|
+
* gets one minted and stamped; a file that already carries one is left alone.
|
|
954
|
+
*
|
|
955
|
+
* `mode: "stamped"` is the rename / copy backstop: the file already carries a
|
|
956
|
+
* `skill_uid` but this journal has never seen it at this path, so the vault
|
|
957
|
+
* may hold that identity at ANOTHER path (a folder rename or a physical copy).
|
|
958
|
+
* Registering with content lets the service classify it — `reused` (same
|
|
959
|
+
* path), `moved` (original gone → same uid, repointed), or `forked` (original
|
|
960
|
+
* still present → fresh uid) — and hand back the bytes to persist. Without
|
|
961
|
+
* this call the presign mint sees a new path, cannot read the bytes, and
|
|
962
|
+
* reserves a placeholder identity that the post-upload reconciliation can
|
|
963
|
+
* never satisfy (the undrdog 2026-09-11 outage: five days of aborted passes).
|
|
964
|
+
*/
|
|
965
|
+
async function registerOneCompanySkill(run, entry, registrar, mode = "unstamped") {
|
|
966
966
|
let content = fs.readFileSync(entry.absolutePath, "utf8");
|
|
967
967
|
// A live editor can save while the network request is in flight. Re-register
|
|
968
968
|
// the newest bytes instead of overwriting them with the response to an older
|
|
969
969
|
// body. Three attempts bound pathological save loops and fail visibly.
|
|
970
970
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
971
971
|
const currentUid = parseRegisteredCompanySkillUid(content);
|
|
972
|
-
if (currentUid)
|
|
972
|
+
if (mode === "unstamped" && currentUid)
|
|
973
|
+
return;
|
|
974
|
+
if (mode === "stamped" && !currentUid)
|
|
973
975
|
return;
|
|
974
976
|
run.options.onHeartbeat?.();
|
|
975
977
|
let result;
|
|
@@ -996,7 +998,11 @@ async function registerOneCompanySkill(run, entry, registrar) {
|
|
|
996
998
|
content = current;
|
|
997
999
|
continue;
|
|
998
1000
|
}
|
|
999
|
-
|
|
1001
|
+
// `reused` / `moved` hand back the same bytes; only a mint or a fork
|
|
1002
|
+
// changes them. Skip the rewrite so an untouched file keeps its mtime.
|
|
1003
|
+
if (result.content !== content) {
|
|
1004
|
+
writeRegisteredSkillAtomically(entry.absolutePath, result.content);
|
|
1005
|
+
}
|
|
1000
1006
|
return;
|
|
1001
1007
|
}
|
|
1002
1008
|
throw new Error(`Company skill '${entry.relativePath}' kept changing while HQ registered it; retry after the current edit finishes.`);
|
|
@@ -1006,54 +1012,129 @@ function isPermanentSkillRegistrationValidationError(err) {
|
|
|
1006
1012
|
(err.statusCode === 400 || err.statusCode === 422));
|
|
1007
1013
|
}
|
|
1008
1014
|
/**
|
|
1009
|
-
*
|
|
1010
|
-
*
|
|
1015
|
+
* Errors that describe the RUN, not one skill: an expired session, a
|
|
1016
|
+
* mid-pass identity refusal, or a journal that cannot be checkpointed. These
|
|
1017
|
+
* must still abort the company leg. Everything else a registration call can
|
|
1018
|
+
* throw — a 409 identity refusal, a 5xx, a transport blip — is about ONE
|
|
1019
|
+
* file and is contained to that file.
|
|
1020
|
+
*/
|
|
1021
|
+
function isRunLevelSkillRegistrationError(err) {
|
|
1022
|
+
return isRunLevelRegistrarFailure(err) || err instanceof JournalCheckpointError;
|
|
1023
|
+
}
|
|
1024
|
+
function emptyPreflightResult() {
|
|
1025
|
+
return {
|
|
1026
|
+
registrationDenied: new Set(),
|
|
1027
|
+
registrationInvalid: new Set(),
|
|
1028
|
+
registrationFailed: new Set(),
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
/**
|
|
1032
|
+
* Durable identity backstop. Hooks improve the authoring experience, but sync
|
|
1033
|
+
* is the invariant boundary:
|
|
1034
|
+
* - no unstamped canonical company skill reaches S3 (`unstamped`);
|
|
1035
|
+
* - a stamped skill at a path this journal has never seen (a folder rename
|
|
1036
|
+
* or a physical copy) reserves its identity for that path BEFORE its
|
|
1037
|
+
* bytes land (`relocated`);
|
|
1038
|
+
* - a skill whose post-upload metadata reconciliation is still pending is
|
|
1039
|
+
* retried (`pending`).
|
|
1040
|
+
*
|
|
1041
|
+
* Containment: a failure in any one skill excludes THAT skill and lets the
|
|
1042
|
+
* rest of the company leg run. Only run-level errors (auth, identity refusal,
|
|
1043
|
+
* journal checkpoint) propagate. Before this rule one skill's 409 escaped
|
|
1044
|
+
* `buildSharePlans` on every pass and wedged a whole tenant indefinitely.
|
|
1011
1045
|
*/
|
|
1012
1046
|
async function preflightCompanySkillIdentities(run, entries) {
|
|
1013
1047
|
if (run.options.personalMode === true || run.ctx.uid.startsWith("prs_")) {
|
|
1014
|
-
return
|
|
1048
|
+
return emptyPreflightResult();
|
|
1015
1049
|
}
|
|
1016
1050
|
const canonical = entries.filter((entry) => entry.kind === "file" &&
|
|
1017
1051
|
CANONICAL_COMPANY_SKILL_RE.test(entry.relativePath));
|
|
1018
|
-
const unstamped =
|
|
1019
|
-
const
|
|
1052
|
+
const unstamped = [];
|
|
1053
|
+
const relocated = [];
|
|
1054
|
+
const pending = [];
|
|
1055
|
+
for (const entry of canonical) {
|
|
1056
|
+
const stamped = Boolean(parseRegisteredCompanySkillUid(fs.readFileSync(entry.absolutePath, "utf8")));
|
|
1057
|
+
if (!stamped) {
|
|
1058
|
+
unstamped.push(entry);
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1020
1061
|
const journalEntry = journalRows(run.journal).get(entry.relativePath);
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1062
|
+
if (!journalEntry) {
|
|
1063
|
+
// Stamped, but this journal has no row for the path: the file arrived
|
|
1064
|
+
// here by rename, copy, or `hq skill create --no-sync`. Reserve the
|
|
1065
|
+
// identity for THIS path with the real bytes before anything uploads.
|
|
1066
|
+
relocated.push(entry);
|
|
1067
|
+
}
|
|
1068
|
+
else if (journalEntry.skillMetadataPending === true &&
|
|
1069
|
+
journalEntry.hash === hashFile(entry.absolutePath)) {
|
|
1070
|
+
pending.push(entry);
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
if (unstamped.length === 0 && relocated.length === 0 && pending.length === 0) {
|
|
1074
|
+
return emptyPreflightResult();
|
|
1026
1075
|
}
|
|
1027
1076
|
const registrar = skillRegistrarForRun(run);
|
|
1028
1077
|
if (!registrar) {
|
|
1029
1078
|
if (unstamped.length > 0) {
|
|
1030
1079
|
throw new UnregisteredCompanySkillError(unstamped.map((entry) => entry.relativePath));
|
|
1031
1080
|
}
|
|
1032
|
-
return
|
|
1081
|
+
return emptyPreflightResult();
|
|
1033
1082
|
}
|
|
1034
1083
|
const registrationDenied = new Set();
|
|
1035
1084
|
const registrationInvalid = new Set();
|
|
1085
|
+
const registrationFailed = new Set();
|
|
1036
1086
|
const registrationInvalidMessages = new Map();
|
|
1087
|
+
const registrationFailedMessages = new Map();
|
|
1088
|
+
const classifyRegistrationFailure = (relativePath, err, options) => {
|
|
1089
|
+
// A 403 has its existing permissions-specific report. Keep its priority
|
|
1090
|
+
// even for a caller-supplied generic VaultClientError rather than folding
|
|
1091
|
+
// it into the validation bucket below.
|
|
1092
|
+
if (err instanceof VaultPermissionDeniedError ||
|
|
1093
|
+
(err instanceof VaultClientError && err.statusCode === 403)) {
|
|
1094
|
+
registrationDenied.add(relativePath);
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
if (isPermanentSkillRegistrationValidationError(err)) {
|
|
1098
|
+
registrationInvalid.add(relativePath);
|
|
1099
|
+
if (err.message)
|
|
1100
|
+
registrationInvalidMessages.set(relativePath, err.message);
|
|
1101
|
+
return;
|
|
1102
|
+
}
|
|
1103
|
+
if (!options.containNonPermanent || isRunLevelSkillRegistrationError(err)) {
|
|
1104
|
+
throw err;
|
|
1105
|
+
}
|
|
1106
|
+
registrationFailed.add(relativePath);
|
|
1107
|
+
registrationFailedMessages.set(relativePath, describeError(err));
|
|
1108
|
+
};
|
|
1037
1109
|
for (const entry of unstamped) {
|
|
1038
1110
|
try {
|
|
1039
|
-
await registerOneCompanySkill(run, entry, registrar);
|
|
1111
|
+
await registerOneCompanySkill(run, entry, registrar, "unstamped");
|
|
1040
1112
|
}
|
|
1041
1113
|
catch (err) {
|
|
1042
|
-
//
|
|
1043
|
-
//
|
|
1044
|
-
//
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
}
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1114
|
+
// Unchanged contract for the creation backstop: a transport failure
|
|
1115
|
+
// while minting a brand-new identity still aborts the pass, because
|
|
1116
|
+
// the file has no identity at all and nothing else can proceed safely
|
|
1117
|
+
// with it. Denied / invalid are contained as before.
|
|
1118
|
+
classifyRegistrationFailure(entry.relativePath, err, {
|
|
1119
|
+
containNonPermanent: false,
|
|
1120
|
+
});
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
for (const entry of relocated) {
|
|
1124
|
+
try {
|
|
1125
|
+
await registerOneCompanySkill(run, entry, registrar, "stamped");
|
|
1126
|
+
}
|
|
1127
|
+
catch (err) {
|
|
1128
|
+
// Reservation is the fast path, not a gate. A stamped skill uploaded
|
|
1129
|
+
// before this call existed and must keep doing so: a machine that cannot
|
|
1130
|
+
// register (403, e.g. an outpost principal), a validation refusal, a
|
|
1131
|
+
// transport blip, a 5xx — none of them hold the file back. The upload
|
|
1132
|
+
// proceeds and the post-upload reconciliation, with its own 409
|
|
1133
|
+
// recovery and per-skill containment, owns the outcome. Only a
|
|
1134
|
+
// run-level failure (auth, identity refusal, journal checkpoint) stops
|
|
1135
|
+
// the pass.
|
|
1136
|
+
if (isRunLevelSkillRegistrationError(err))
|
|
1055
1137
|
throw err;
|
|
1056
|
-
}
|
|
1057
1138
|
}
|
|
1058
1139
|
}
|
|
1059
1140
|
if (registrationDenied.size > 0) {
|
|
@@ -1078,85 +1159,307 @@ async function preflightCompanySkillIdentities(run, entries) {
|
|
|
1078
1159
|
});
|
|
1079
1160
|
}
|
|
1080
1161
|
for (const entry of pending) {
|
|
1081
|
-
|
|
1162
|
+
// Per-entry containment. The marker stays on the row, so the next pass
|
|
1163
|
+
// retries this one skill; nothing else in the company waits for it.
|
|
1164
|
+
try {
|
|
1165
|
+
await reconcileOnePendingCompanySkill(run, entry);
|
|
1166
|
+
}
|
|
1167
|
+
catch (err) {
|
|
1168
|
+
classifyRegistrationFailure(entry.relativePath, err, {
|
|
1169
|
+
containNonPermanent: true,
|
|
1170
|
+
});
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
if (registrationFailed.size > 0) {
|
|
1174
|
+
const samplePaths = [...registrationFailed].slice(0, 10);
|
|
1175
|
+
const message = samplePaths
|
|
1176
|
+
.map((relativePath) => registrationFailedMessages.get(relativePath))
|
|
1177
|
+
.find((candidate) => candidate !== undefined);
|
|
1178
|
+
run.emit({
|
|
1179
|
+
type: "not-shipped",
|
|
1180
|
+
reason: "skill-registration-failed",
|
|
1181
|
+
count: registrationFailed.size,
|
|
1182
|
+
samplePaths,
|
|
1183
|
+
...(message ? { message } : {}),
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
return { registrationDenied, registrationInvalid, registrationFailed };
|
|
1187
|
+
}
|
|
1188
|
+
/**
|
|
1189
|
+
* Retry the post-upload metadata reconciliation for one skill whose journal
|
|
1190
|
+
* row still carries `skillMetadataPending`. Pulled out of the preflight loop
|
|
1191
|
+
* so a throw from any step is contained to this entry by the caller.
|
|
1192
|
+
*/
|
|
1193
|
+
async function reconcileOnePendingCompanySkill(run, entry) {
|
|
1194
|
+
const journalEntry = journalRows(run.journal).get(entry.relativePath);
|
|
1195
|
+
run.options.onHeartbeat?.();
|
|
1196
|
+
const remote = await headRemoteFile(run.ctx, entry.relativePath);
|
|
1197
|
+
if (!remote) {
|
|
1198
|
+
// The upload this marker referred to no longer exists. Drop the stale
|
|
1199
|
+
// baseline so the normal planner cannot skip the local file as unchanged
|
|
1200
|
+
// and will upload it again on this pass.
|
|
1201
|
+
journalRows(run.journal).delete(entry.relativePath);
|
|
1202
|
+
flushShareJournal(run);
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
1205
|
+
if (!journalEntry?.remoteEtag ||
|
|
1206
|
+
normalizeEtag(remote.etag) !== normalizeEtag(journalEntry.remoteEtag)) {
|
|
1207
|
+
// The successful PUT this marker describes is no longer the live remote
|
|
1208
|
+
// object. Reconcile the current remote bytes instead of stranding the old
|
|
1209
|
+
// marker behind skipUnchanged; the backend independently revalidates this
|
|
1210
|
+
// exact content against S3 before accepting its metadata.
|
|
1211
|
+
// Invalidate the stale baseline before any network work. If recovery
|
|
1212
|
+
// fails, the next pass must still see a real divergence rather than
|
|
1213
|
+
// classifying the local file as unchanged forever.
|
|
1214
|
+
journalRows(run.journal).delete(entry.relativePath);
|
|
1215
|
+
flushShareJournal(run);
|
|
1216
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-skill-metadata-"));
|
|
1217
|
+
const tempFile = path.join(tempDir, "SKILL.md");
|
|
1218
|
+
try {
|
|
1219
|
+
await downloadFile(run.ctx, entry.relativePath, tempFile);
|
|
1220
|
+
// The reconciled bytes are the REMOTE object (a peer replaced the one
|
|
1221
|
+
// this marker described), not the local file, so pass no absolutePath:
|
|
1222
|
+
// a 409 here is recorded, not forked (see reconcileLandedCompanySkill).
|
|
1223
|
+
// The journal row is already gone, so settling only records/drops the
|
|
1224
|
+
// conflict; clearing the (absent) marker is a no-op.
|
|
1225
|
+
const outcome = await reconcileLandedCompanySkill(run, entry.relativePath, fs.readFileSync(tempFile, "utf8"));
|
|
1226
|
+
settleLandedReconcile(run, entry.relativePath, outcome);
|
|
1227
|
+
}
|
|
1228
|
+
finally {
|
|
1229
|
+
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
1230
|
+
}
|
|
1231
|
+
return;
|
|
1232
|
+
}
|
|
1233
|
+
const landedBytes = fs.readFileSync(entry.absolutePath);
|
|
1234
|
+
const landedHash = crypto
|
|
1235
|
+
.createHash("sha256")
|
|
1236
|
+
.update(landedBytes)
|
|
1237
|
+
.digest("hex");
|
|
1238
|
+
if (landedHash !== journalEntry.hash) {
|
|
1239
|
+
// The editor saved after this entry was classified but before the
|
|
1240
|
+
// remote proof returned. Reconcile only the single local snapshot that
|
|
1241
|
+
// produced the journaled upload; the newer edit belongs to the planner.
|
|
1242
|
+
return;
|
|
1243
|
+
}
|
|
1244
|
+
const outcome = await reconcileLandedCompanySkill(run, entry.relativePath, landedBytes.toString("utf8"), entry.absolutePath);
|
|
1245
|
+
// A reconciled/conflict outcome clears the marker (the server has answered,
|
|
1246
|
+
// so carrying it is exactly what let a leg loop forever); a `deferred`
|
|
1247
|
+
// outcome keeps it so a later pass reconciles the metadata without looping.
|
|
1248
|
+
settleLandedReconcile(run, entry.relativePath, outcome);
|
|
1249
|
+
checkpointShareJournal(run);
|
|
1250
|
+
}
|
|
1251
|
+
/**
|
|
1252
|
+
* A 409 raised by the company-skill registrar seam: a landed identity mismatch
|
|
1253
|
+
* on a copied/renamed pre-stamped skill, or a sibling stale/concurrent race.
|
|
1254
|
+
* The error carries the service's `code` and `path` when the body named them
|
|
1255
|
+
* (see `VaultErrorDetails`), so a recorded conflict can name the file.
|
|
1256
|
+
*/
|
|
1257
|
+
function isSkillRegistrationConflict(err) {
|
|
1258
|
+
return err instanceof VaultConflictError;
|
|
1259
|
+
}
|
|
1260
|
+
/**
|
|
1261
|
+
* A registrar failure that is a RUN-level fact, never a per-object skip: a
|
|
1262
|
+
* session auth refusal or a Cognito identity refusal. These must propagate from
|
|
1263
|
+
* any registration attempt so one refusal aborts the pass exactly as before,
|
|
1264
|
+
* rather than being absorbed as a skill outcome.
|
|
1265
|
+
*/
|
|
1266
|
+
function isRunLevelRegistrarFailure(err) {
|
|
1267
|
+
return (err instanceof VaultAuthError ||
|
|
1268
|
+
isTerminalIdentityRefusal(err) ||
|
|
1269
|
+
isRetryableIdentityRefusal(err));
|
|
1270
|
+
}
|
|
1271
|
+
/** Record a company-skill path the client could not self-heal after a landed
|
|
1272
|
+
* 409 (first message wins), for a single end-of-pass not-shipped event. */
|
|
1273
|
+
function recordSkillRegistrationConflict(run, relativePath, message) {
|
|
1274
|
+
if (!run.skillRegistrationConflicts.has(relativePath)) {
|
|
1275
|
+
run.skillRegistrationConflicts.set(relativePath, message);
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* Apply a landed-reconcile outcome at a call site that owns the pending marker:
|
|
1280
|
+
* clear it on a resolved (reconciled/conflict) outcome, keep it on `deferred`.
|
|
1281
|
+
* `reconciled` also drops any conflict this path accumulated earlier in the
|
|
1282
|
+
* pass (e.g. a stale-etag preflight record the later upload has now fixed), so
|
|
1283
|
+
* finalization does not report a conflict the pass actually resolved.
|
|
1284
|
+
*/
|
|
1285
|
+
function settleLandedReconcile(run, relativePath, outcome) {
|
|
1286
|
+
if (outcome.status === "deferred")
|
|
1287
|
+
return;
|
|
1288
|
+
if (outcome.status === "conflict") {
|
|
1289
|
+
recordSkillRegistrationConflict(run, relativePath, outcome.message);
|
|
1290
|
+
}
|
|
1291
|
+
else {
|
|
1292
|
+
run.skillRegistrationConflicts.delete(relativePath);
|
|
1293
|
+
}
|
|
1294
|
+
clearSkillMetadataPending(run, relativePath);
|
|
1295
|
+
}
|
|
1296
|
+
/** Emit the single end-of-pass not-shipped event for skills whose metadata was
|
|
1297
|
+
* not reconciled. Called from the normal finalizer AND before an early aborted
|
|
1298
|
+
* return, so a recorded conflict is never silently dropped. */
|
|
1299
|
+
function emitSkillRegistrationConflicts(run) {
|
|
1300
|
+
if (run.skillRegistrationConflicts.size === 0)
|
|
1301
|
+
return;
|
|
1302
|
+
const samplePaths = [...run.skillRegistrationConflicts.keys()].slice(0, 10);
|
|
1303
|
+
const message = samplePaths
|
|
1304
|
+
.map((relativePath) => run.skillRegistrationConflicts.get(relativePath))
|
|
1305
|
+
.find((candidate) => candidate !== undefined);
|
|
1306
|
+
run.emit({
|
|
1307
|
+
type: "not-shipped",
|
|
1308
|
+
reason: "skill-registration-conflict",
|
|
1309
|
+
count: run.skillRegistrationConflicts.size,
|
|
1310
|
+
samplePaths,
|
|
1311
|
+
...(message ? { message } : {}),
|
|
1312
|
+
});
|
|
1313
|
+
}
|
|
1314
|
+
/**
|
|
1315
|
+
* A landed reconcile got a 409: the stamped identity is not the one registered
|
|
1316
|
+
* for this path. Reserve identity with ONE non-landed register — the server
|
|
1317
|
+
* classifies a stamped copy/rename as fork (new uid, re-stamped bytes), move
|
|
1318
|
+
* (repoint), or reuse — then, when it forked and the local file is unchanged
|
|
1319
|
+
* since these bytes were read, rewrite it atomically so the planner re-uploads
|
|
1320
|
+
* the re-stamped bytes; then retry the landed reconcile once (best effort — its
|
|
1321
|
+
* result never gates success). If the non-landed fallback is itself refused (a
|
|
1322
|
+
* second 409, a 403, or a 400/422 validation) report a deliberate skill skip
|
|
1323
|
+
* rather than throwing, so the pending marker can be cleared and the company
|
|
1324
|
+
* leg can never loop on it.
|
|
1325
|
+
*/
|
|
1326
|
+
async function recoverLandedSkillConflict(run, relativePath, content, absolutePath, registrar) {
|
|
1327
|
+
let fallback;
|
|
1328
|
+
try {
|
|
1082
1329
|
run.options.onHeartbeat?.();
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1330
|
+
fallback = await registrar({
|
|
1331
|
+
companySlug: run.ctx.slug,
|
|
1332
|
+
path: relativePath,
|
|
1333
|
+
content,
|
|
1334
|
+
});
|
|
1335
|
+
assertValidSkillRegistration(fallback, relativePath);
|
|
1336
|
+
}
|
|
1337
|
+
catch (err) {
|
|
1338
|
+
// A refused fallback is a deliberate skill skip, not a leg abort: a second
|
|
1339
|
+
// 409, a permission denial (403, in either its typed or raw form, matching
|
|
1340
|
+
// the unstamped path), or a permanent 400/422 validation. Everything else
|
|
1341
|
+
// (auth, identity refusal, 404, 5xx, transport) propagates so the marker is
|
|
1342
|
+
// kept and the pass retries — unchanged from before.
|
|
1343
|
+
if (isSkillRegistrationConflict(err) ||
|
|
1344
|
+
err instanceof VaultPermissionDeniedError ||
|
|
1345
|
+
(err instanceof VaultClientError && err.statusCode === 403) ||
|
|
1346
|
+
isPermanentSkillRegistrationValidationError(err)) {
|
|
1347
|
+
return {
|
|
1348
|
+
status: "conflict",
|
|
1349
|
+
...(err instanceof Error && err.message ? { message: err.message } : {}),
|
|
1350
|
+
};
|
|
1091
1351
|
}
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-skill-metadata-"));
|
|
1104
|
-
const tempFile = path.join(tempDir, "SKILL.md");
|
|
1105
|
-
try {
|
|
1106
|
-
await downloadFile(run.ctx, entry.relativePath, tempFile);
|
|
1107
|
-
await reconcileLandedCompanySkill(run, entry.relativePath, fs.readFileSync(tempFile, "utf8"));
|
|
1108
|
-
}
|
|
1109
|
-
finally {
|
|
1110
|
-
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
1111
|
-
}
|
|
1112
|
-
continue;
|
|
1352
|
+
throw err;
|
|
1353
|
+
}
|
|
1354
|
+
// Apply the fork's re-stamped bytes only when the local file is unchanged
|
|
1355
|
+
// since `content` was read — the same guard registerOneCompanySkill uses so a
|
|
1356
|
+
// concurrent editor's newer body is never overwritten. The stale-etag path
|
|
1357
|
+
// reconciles REMOTE bytes and passes no absolutePath, so it never rewrites a
|
|
1358
|
+
// local file from a fork of bytes that are not that file.
|
|
1359
|
+
if (absolutePath !== undefined && fallback.content !== content) {
|
|
1360
|
+
let current;
|
|
1361
|
+
try {
|
|
1362
|
+
current = fs.readFileSync(absolutePath, "utf8");
|
|
1113
1363
|
}
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
.
|
|
1119
|
-
if (landedHash !== journalEntry.hash) {
|
|
1120
|
-
// The editor saved after this entry was classified but before the
|
|
1121
|
-
// remote proof returned. Reconcile only the single local snapshot that
|
|
1122
|
-
// produced the journaled upload; the newer edit belongs to the planner.
|
|
1123
|
-
continue;
|
|
1364
|
+
catch {
|
|
1365
|
+
current = undefined;
|
|
1366
|
+
}
|
|
1367
|
+
if (current === content) {
|
|
1368
|
+
writeRegisteredSkillAtomically(absolutePath, fallback.content);
|
|
1124
1369
|
}
|
|
1125
|
-
await reconcileLandedCompanySkill(run, entry.relativePath, landedBytes.toString("utf8"));
|
|
1126
|
-
clearSkillMetadataPending(run, entry.relativePath);
|
|
1127
|
-
checkpointShareJournal(run);
|
|
1128
1370
|
}
|
|
1129
|
-
|
|
1371
|
+
// Identity is reserved now, so retry the landed reconcile once for tag
|
|
1372
|
+
// metadata.
|
|
1373
|
+
try {
|
|
1374
|
+
run.options.onHeartbeat?.();
|
|
1375
|
+
const retry = await registrar({
|
|
1376
|
+
companySlug: run.ctx.slug,
|
|
1377
|
+
path: relativePath,
|
|
1378
|
+
content: fallback.content,
|
|
1379
|
+
landed: true,
|
|
1380
|
+
});
|
|
1381
|
+
assertValidSkillRegistration(retry, relativePath);
|
|
1382
|
+
}
|
|
1383
|
+
catch (retryErr) {
|
|
1384
|
+
// An auth/identity refusal is a run-level fact — propagate it.
|
|
1385
|
+
if (isRunLevelRegistrarFailure(retryErr))
|
|
1386
|
+
throw retryErr;
|
|
1387
|
+
// A repeat 409 means the fallback did not actually reserve identity for
|
|
1388
|
+
// this path (a pathological server state). Do not loop on it: record a
|
|
1389
|
+
// deliberate skip and let the marker clear.
|
|
1390
|
+
if (isSkillRegistrationConflict(retryErr)) {
|
|
1391
|
+
return {
|
|
1392
|
+
status: "conflict",
|
|
1393
|
+
...(retryErr instanceof Error && retryErr.message
|
|
1394
|
+
? { message: retryErr.message }
|
|
1395
|
+
: {}),
|
|
1396
|
+
};
|
|
1397
|
+
}
|
|
1398
|
+
// A transient failure (5xx/408/429/transport/404) after a successful
|
|
1399
|
+
// fallback: keep the pending marker so a later pass reconciles the
|
|
1400
|
+
// metadata. Identity is reserved, so the next landed call succeeds — this
|
|
1401
|
+
// cannot loop the way an unresolved 409 did, and it never strands tags on a
|
|
1402
|
+
// `moved`/`reused` fallback whose bytes did not change.
|
|
1403
|
+
return { status: "deferred" };
|
|
1404
|
+
}
|
|
1405
|
+
return { status: "reconciled" };
|
|
1130
1406
|
}
|
|
1131
1407
|
/**
|
|
1132
1408
|
* Reconcile metadata only after the canonical bytes have landed. Identity
|
|
1133
1409
|
* reservation happens before upload; tags and other source-authored metadata
|
|
1134
|
-
* must not become authoritative until the S3 write succeeds.
|
|
1410
|
+
* must not become authoritative until the S3 write succeeds. A landed 409 is
|
|
1411
|
+
* not fatal: it is recovered (or recorded as a deliberate skip) by
|
|
1412
|
+
* {@link recoverLandedSkillConflict} instead of aborting the company leg.
|
|
1135
1413
|
*/
|
|
1136
|
-
async function reconcileLandedCompanySkill(run, relativePath, content) {
|
|
1414
|
+
async function reconcileLandedCompanySkill(run, relativePath, content, absolutePath) {
|
|
1137
1415
|
if (run.options.personalMode === true ||
|
|
1138
1416
|
run.ctx.uid.startsWith("prs_") ||
|
|
1139
1417
|
!CANONICAL_COMPANY_SKILL_RE.test(relativePath)) {
|
|
1140
|
-
return;
|
|
1418
|
+
return { status: "reconciled" };
|
|
1141
1419
|
}
|
|
1142
1420
|
const registrar = skillRegistrarForRun(run);
|
|
1143
1421
|
if (!registrar) {
|
|
1144
|
-
return;
|
|
1422
|
+
return { status: "reconciled" };
|
|
1145
1423
|
}
|
|
1146
1424
|
const localUid = parseRegisteredCompanySkillUid(content);
|
|
1147
1425
|
if (!localUid) {
|
|
1148
1426
|
throw new UnregisteredCompanySkillError([relativePath]);
|
|
1149
1427
|
}
|
|
1150
1428
|
run.options.onHeartbeat?.();
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1429
|
+
try {
|
|
1430
|
+
const result = await registrar({
|
|
1431
|
+
companySlug: run.ctx.slug,
|
|
1432
|
+
path: relativePath,
|
|
1433
|
+
content,
|
|
1434
|
+
landed: true,
|
|
1435
|
+
});
|
|
1436
|
+
assertValidSkillRegistration(result, relativePath);
|
|
1437
|
+
if (result.skillUid !== localUid) {
|
|
1438
|
+
throw new Error(`Skill registration returned identity '${result.skillUid}' for '${relativePath}', but the landed file is stamped '${localUid}'.`);
|
|
1439
|
+
}
|
|
1440
|
+
return { status: "reconciled" };
|
|
1441
|
+
}
|
|
1442
|
+
catch (err) {
|
|
1443
|
+
// Only a 409 from the registrar seam becomes a per-object outcome. Every
|
|
1444
|
+
// other failure (auth, 404, 5xx, transport, or an invalid-identity
|
|
1445
|
+
// response) propagates exactly as before, so a transient registry failure
|
|
1446
|
+
// still retries the pending marker next pass and an auth refusal still
|
|
1447
|
+
// aborts the pass.
|
|
1448
|
+
if (!isSkillRegistrationConflict(err))
|
|
1449
|
+
throw err;
|
|
1450
|
+
if (absolutePath === undefined) {
|
|
1451
|
+
// These bytes are a REPLACED REMOTE object, not a local file (the
|
|
1452
|
+
// stale-etag recovery path). A non-landed fork here would reserve a
|
|
1453
|
+
// server identity that no local file carries, and a later remote-adoption
|
|
1454
|
+
// could then strand that fork forever. Record the conflict and let the
|
|
1455
|
+
// local file self-heal through the upload path on a later pass, which
|
|
1456
|
+
// forks and re-stamps the actual local bytes.
|
|
1457
|
+
return {
|
|
1458
|
+
status: "conflict",
|
|
1459
|
+
...(err instanceof Error && err.message ? { message: err.message } : {}),
|
|
1460
|
+
};
|
|
1461
|
+
}
|
|
1462
|
+
return recoverLandedSkillConflict(run, relativePath, content, absolutePath, registrar);
|
|
1160
1463
|
}
|
|
1161
1464
|
}
|
|
1162
1465
|
async function buildSharePlans(run) {
|
|
@@ -1243,8 +1546,12 @@ async function buildSharePlans(run) {
|
|
|
1243
1546
|
return false;
|
|
1244
1547
|
})
|
|
1245
1548
|
: controlCharFiltered;
|
|
1246
|
-
const { registrationDenied, registrationInvalid } = await preflightCompanySkillIdentities(run, filesToShare);
|
|
1247
|
-
const registrationSkipped = new Set([
|
|
1549
|
+
const { registrationDenied, registrationInvalid, registrationFailed } = await preflightCompanySkillIdentities(run, filesToShare);
|
|
1550
|
+
const registrationSkipped = new Set([
|
|
1551
|
+
...registrationDenied,
|
|
1552
|
+
...registrationInvalid,
|
|
1553
|
+
...registrationFailed,
|
|
1554
|
+
]);
|
|
1248
1555
|
const uploadableFiles = registrationSkipped.size > 0
|
|
1249
1556
|
? filesToShare.filter((entry) => !registrationSkipped.has(entry.relativePath))
|
|
1250
1557
|
: filesToShare;
|
|
@@ -1356,6 +1663,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1356
1663
|
key: it.relativePath,
|
|
1357
1664
|
localPath: it.absolutePath,
|
|
1358
1665
|
isSymlink: it.kind === "symlink",
|
|
1666
|
+
...(it.kind === "file" ? { contentHash: it.localHash } : {}),
|
|
1359
1667
|
author: run.options.author,
|
|
1360
1668
|
})));
|
|
1361
1669
|
// Warm the GET presigns the per-item conflict HEAD (remoteMeta) reuses, so a
|
|
@@ -1527,7 +1835,39 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1527
1835
|
}
|
|
1528
1836
|
if (remoteMeta) {
|
|
1529
1837
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
1530
|
-
const
|
|
1838
|
+
const remoteContentHash = item.kind === "file" ? publishedContentHash(remoteMeta.metadata) : undefined;
|
|
1839
|
+
const remoteChanged = !!journalEntry &&
|
|
1840
|
+
hasRemoteChanged({ ...remoteMeta, contentHash: remoteContentHash }, journalEntry);
|
|
1841
|
+
// A matching published digest is a content no-op even when SSE-KMS gave
|
|
1842
|
+
// this object a fresh ETag. Reconcile the journal to that version without
|
|
1843
|
+
// a needless PUT or a phantom two-sided conflict.
|
|
1844
|
+
if (remoteContentHash === localHash) {
|
|
1845
|
+
const lstat = fs.lstatSync(absolutePath);
|
|
1846
|
+
updateEntry(run.journal, relativePath, localHash, lstat.size, journalEntry?.direction ?? "up", absolutePath, {
|
|
1847
|
+
remoteEtag: remoteMeta.etag,
|
|
1848
|
+
mtimeMs: lstat.mtimeMs,
|
|
1849
|
+
ctimeMs: ctimeToStamp(item.kind === "file" ? item.statAtHash : undefined, lstat),
|
|
1850
|
+
kind: "file",
|
|
1851
|
+
});
|
|
1852
|
+
checkpointShareJournal(run);
|
|
1853
|
+
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
1854
|
+
counters.filesSkipped++;
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
// This machine's journal baseline is a superseded content generation.
|
|
1858
|
+
// Replaying that unchanged baseline is a stale revert, not an edit. A
|
|
1859
|
+
// user who deliberately restores some other historic content has a hash
|
|
1860
|
+
// different from the journal baseline and still reaches normal conflict
|
|
1861
|
+
// resolution below.
|
|
1862
|
+
if (journalEntry !== undefined &&
|
|
1863
|
+
remoteContentHash !== undefined &&
|
|
1864
|
+
journalEntry.hash === localHash &&
|
|
1865
|
+
remoteContentHash !== localHash) {
|
|
1866
|
+
conflictPaths.push(relativePath);
|
|
1867
|
+
run.emit({ type: "conflict", path: relativePath, direction: "push", resolution: "skip" });
|
|
1868
|
+
counters.filesSkipped++;
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1531
1871
|
// Every path below that needs the remote BODY — the no-row equality
|
|
1532
1872
|
// probe, and a `keep` that adopts the remote — is impossible against a
|
|
1533
1873
|
// DEEP_ARCHIVE session transcript: each GET fails with InvalidObjectState.
|
|
@@ -1606,8 +1946,8 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1606
1946
|
// journal row was persisted. Make the retry marker durable before the
|
|
1607
1947
|
// registry call so another interruption remains recoverable.
|
|
1608
1948
|
flushShareJournal(run);
|
|
1609
|
-
await reconcileLandedCompanySkill(run, relativePath, skillContent);
|
|
1610
|
-
|
|
1949
|
+
const outcome = await reconcileLandedCompanySkill(run, relativePath, skillContent, absolutePath);
|
|
1950
|
+
settleLandedReconcile(run, relativePath, outcome);
|
|
1611
1951
|
}
|
|
1612
1952
|
checkpointShareJournal(run);
|
|
1613
1953
|
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
@@ -1760,6 +2100,8 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1760
2100
|
const skillContentAtPut = !isSymlinkUpload && CANONICAL_COMPANY_SKILL_RE.test(relativePath)
|
|
1761
2101
|
? fs.readFileSync(absolutePath, "utf8")
|
|
1762
2102
|
: undefined;
|
|
2103
|
+
if (!isSymlinkUpload)
|
|
2104
|
+
setUploadContentHash(absolutePath, localHash);
|
|
1763
2105
|
const { etag } = isSymlinkUpload
|
|
1764
2106
|
? await uploadSymlink(run.ctx, item.target, relativePath, run.options.author, pc)
|
|
1765
2107
|
: await uploadFile(run.ctx, absolutePath, relativePath, run.options.author, pc);
|
|
@@ -1794,8 +2136,8 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
1794
2136
|
if (skillContentAtPut !== undefined &&
|
|
1795
2137
|
skillRegistrarForRun(run) !== undefined &&
|
|
1796
2138
|
fs.readFileSync(absolutePath, "utf8") === skillContentAtPut) {
|
|
1797
|
-
await reconcileLandedCompanySkill(run, relativePath, skillContentAtPut);
|
|
1798
|
-
|
|
2139
|
+
const outcome = await reconcileLandedCompanySkill(run, relativePath, skillContentAtPut, absolutePath);
|
|
2140
|
+
settleLandedReconcile(run, relativePath, outcome);
|
|
1799
2141
|
checkpointShareJournal(run);
|
|
1800
2142
|
}
|
|
1801
2143
|
run.emit({
|
|
@@ -2003,16 +2345,9 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2003
2345
|
// this flag only halts admission, so the refusal still surfaces via
|
|
2004
2346
|
// throwUploadWorkerErrors below. In-flight legs still drain.
|
|
2005
2347
|
let sessionAuthAborted = false;
|
|
2006
|
-
// Cooperative work is admitted in bounded units, then fully drained before
|
|
2007
|
-
// the root lock can change owners. Skipped rows are not durable mutations
|
|
2008
|
-
// and therefore do not count toward the unit.
|
|
2009
|
-
let admittedSinceCooperativeCheckpoint = 0;
|
|
2010
2348
|
while (queue.length > 0 || inFlight.size > 0) {
|
|
2011
2349
|
while (!aborted &&
|
|
2012
2350
|
!sessionAuthAborted &&
|
|
2013
|
-
(!run.options.cooperativeCheckpoint ||
|
|
2014
|
-
admittedSinceCooperativeCheckpoint <
|
|
2015
|
-
FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE) &&
|
|
2016
2351
|
inFlight.size < TRANSFER_CONCURRENCY &&
|
|
2017
2352
|
queue.length > 0) {
|
|
2018
2353
|
const previewItem = queue[0];
|
|
@@ -2039,7 +2374,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2039
2374
|
if (inFlight.size > 0 && inFlightBytes + nextBytes > byteBudget)
|
|
2040
2375
|
break;
|
|
2041
2376
|
queue.shift();
|
|
2042
|
-
admittedSinceCooperativeCheckpoint++;
|
|
2043
2377
|
inFlightBytes += nextBytes;
|
|
2044
2378
|
const upload = () => processUploadItem(item);
|
|
2045
2379
|
const p = (run.options.transferSemaphore
|
|
@@ -2067,24 +2401,6 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2067
2401
|
if (inFlight.size > 0) {
|
|
2068
2402
|
await Promise.race(Array.from(inFlight));
|
|
2069
2403
|
}
|
|
2070
|
-
else if (run.options.cooperativeCheckpoint &&
|
|
2071
|
-
!aborted &&
|
|
2072
|
-
!sessionAuthAborted &&
|
|
2073
|
-
!hadUploadFailure &&
|
|
2074
|
-
admittedSinceCooperativeCheckpoint > 0 &&
|
|
2075
|
-
(admittedSinceCooperativeCheckpoint >=
|
|
2076
|
-
FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE ||
|
|
2077
|
-
queue.length === 0)) {
|
|
2078
|
-
// Every admitted upload has settled, so `processUploadItem` has
|
|
2079
|
-
// finished its mutable journal updates. Persist this durable unit
|
|
2080
|
-
// before yielding. A real hand-off restarts share() above and thereby
|
|
2081
|
-
// discards its now-stale upload/delete plan.
|
|
2082
|
-
flushShareJournal(run);
|
|
2083
|
-
admittedSinceCooperativeCheckpoint = 0;
|
|
2084
|
-
if ((await run.options.cooperativeCheckpoint()) === true) {
|
|
2085
|
-
throw new CooperativePushHandoff();
|
|
2086
|
-
}
|
|
2087
|
-
}
|
|
2088
2404
|
else {
|
|
2089
2405
|
break;
|
|
2090
2406
|
}
|
|
@@ -2599,12 +2915,6 @@ async function executeDeletes(run, deletePlan, decommissionPlan, counters, files
|
|
|
2599
2915
|
*/
|
|
2600
2916
|
const CHECKPOINT_MIN_APPLIED = 250;
|
|
2601
2917
|
const CHECKPOINT_MIN_INTERVAL_MS = 5_000;
|
|
2602
|
-
/**
|
|
2603
|
-
* Maximum completed upload mutations between cooperative full-push hand-offs.
|
|
2604
|
-
* The executor drains a unit before yielding, so realtime never overlaps an
|
|
2605
|
-
* upload body or the push's mutable journal state.
|
|
2606
|
-
*/
|
|
2607
|
-
export const FULL_PUSH_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
2608
2918
|
/**
|
|
2609
2919
|
* Fraction of the journal's current size that must be re-applied before another
|
|
2610
2920
|
* checkpoint is worth its write. `writeJournal` publishes a whole compact
|
|
@@ -2739,6 +3049,7 @@ function finalizeShareJournal(run) {
|
|
|
2739
3049
|
samplePaths: sampleSet(run.linkedSubtreeSet),
|
|
2740
3050
|
});
|
|
2741
3051
|
}
|
|
3052
|
+
emitSkillRegistrationConflicts(run);
|
|
2742
3053
|
}
|
|
2743
3054
|
/**
|
|
2744
3055
|
* Which unreachable named paths are FATAL under the run's policy.
|
|
@@ -3008,6 +3319,12 @@ function defaultConsoleLogger(event) {
|
|
|
3008
3319
|
else if (event.reason === "skill-registration-invalid") {
|
|
3009
3320
|
console.warn(` ! ${event.count} unstamped company skill${event.count === 1 ? "" : "s"} was rejected as invalid and was NOT uploaded:`);
|
|
3010
3321
|
}
|
|
3322
|
+
else if (event.reason === "skill-registration-conflict") {
|
|
3323
|
+
console.warn(` ! ${event.count} company skill${event.count === 1 ? "" : "s"} could NOT be re-identified after a copy or rename (registration conflict); its content synced but its skill metadata was not reconciled this pass:`);
|
|
3324
|
+
}
|
|
3325
|
+
else if (event.reason === "skill-registration-failed") {
|
|
3326
|
+
console.warn(` ! ${event.count} company skill${event.count === 1 ? "" : "s"} could NOT finish registering on this pass and will be retried next pass (everything else synced):`);
|
|
3327
|
+
}
|
|
3011
3328
|
else {
|
|
3012
3329
|
console.warn(` ! ${event.count} linked subtree${event.count === 1 ? "" : "s"} recorded but NOT uploaded — contents sync via their own repo, not the vault:`);
|
|
3013
3330
|
}
|