harnery 0.45.1 → 0.45.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +6 -45
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +0 -18
- package/dist/core/agents/state/pidmap.js +2 -2
- package/dist/core/agents/state/proc-start.d.ts +20 -31
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +37 -70
- package/dist/core/agents/state/service-process.d.ts +61 -0
- package/dist/core/agents/state/service-process.d.ts.map +1 -0
- package/dist/core/agents/state/service-process.js +143 -0
- package/dist/core/context/index.d.ts +6 -4
- package/dist/core/context/index.d.ts.map +1 -1
- package/dist/core/context/index.js +194 -256
- package/dist/core/context/observation.d.ts +9 -9
- package/dist/core/context/observation.d.ts.map +1 -1
- package/dist/core/context/observation.js +37 -6
- package/dist/core/events/v3/runtime-build.d.ts +14 -0
- package/dist/core/events/v3/runtime-build.d.ts.map +1 -1
- package/dist/core/events/v3/runtime-build.js +44 -0
- package/dist/core/governor/index.d.ts +2 -1
- package/dist/core/governor/index.d.ts.map +1 -1
- package/dist/core/governor/index.js +2 -1
- package/dist/core/governor/service-read.d.ts +3 -10
- package/dist/core/governor/service-read.d.ts.map +1 -1
- package/dist/core/governor/service-read.js +10 -94
- package/dist/core/governor/service.d.ts +3 -7
- package/dist/core/governor/service.d.ts.map +1 -1
- package/dist/core/governor/service.js +17 -182
- package/dist/core/hooks/resolve/owner.js +2 -2
- package/dist/core/resources/sampler.js +2 -2
- package/dist/core/resources/service-status.d.ts +2 -14
- package/dist/core/resources/service-status.d.ts.map +1 -1
- package/dist/core/resources/service-status.js +3 -23
- package/dist/core/resources/service.d.ts +3 -7
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +12 -101
- package/dist/core/resources/status.d.ts.map +1 -1
- package/dist/core/resources/status.js +3 -2
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +9 -2
- package/dist/core/servers/index.d.ts +0 -2
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +3 -18
- package/dist/core/supervisor/service.d.ts +4 -7
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/supervisor/service.js +23 -122
- package/dist/core/supervisor/services.js +2 -2
- package/dist/core/supervisor/status.d.ts +3 -17
- package/dist/core/supervisor/status.d.ts.map +1 -1
- package/dist/core/supervisor/status.js +7 -70
- package/dist/core/workflow/engine.js +2 -1
- package/dist/core/workflow/index.d.ts +3 -4
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -2
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +30 -74
- package/dist/core/workflow/resume-lock.d.ts +17 -0
- package/dist/core/workflow/resume-lock.d.ts.map +1 -0
- package/dist/core/workflow/resume-lock.js +158 -0
- package/dist/core/workflow/run-state.d.ts +0 -21
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +3 -174
- package/dist/core/workflow/workspaces/leases.d.ts +4 -0
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +1 -1
- package/dist/core/workflow/workspaces/local-git.js +91 -210
- package/dist/core/workflow/workspaces/ownership.d.ts +0 -7
- package/dist/core/workflow/workspaces/ownership.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/ownership.js +3 -130
- package/dist/core/workflow/workspaces/reclaim.d.ts +4 -7
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/reclaim.js +0 -4
- package/dist/core/workflow/workspaces/recovery.d.ts +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +1 -1
- package/dist/core/workflow/workspaces/state.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/state.js +1 -4
- package/package.json +1 -1
- package/src/commands/workflow.ts +8 -58
- package/src/core/agents/coord-client.ts +0 -20
- package/src/core/agents/state/pidmap.ts +2 -2
- package/src/core/agents/state/proc-start.ts +41 -69
- package/src/core/agents/state/service-process.ts +217 -0
- package/src/core/context/index.ts +207 -272
- package/src/core/context/observation.ts +40 -9
- package/src/core/events/v3/runtime-build.ts +62 -0
- package/src/core/governor/index.ts +1 -2
- package/src/core/governor/service-read.ts +15 -114
- package/src/core/governor/service.ts +36 -220
- package/src/core/hooks/resolve/owner.ts +2 -2
- package/src/core/resources/sampler.ts +3 -2
- package/src/core/resources/service-status.ts +6 -35
- package/src/core/resources/service.ts +23 -133
- package/src/core/resources/status.ts +3 -2
- package/src/core/semantic/service.ts +8 -2
- package/src/core/servers/index.ts +3 -23
- package/src/core/supervisor/service.ts +33 -155
- package/src/core/supervisor/services.ts +2 -2
- package/src/core/supervisor/status.ts +10 -85
- package/src/core/workflow/engine.ts +1 -1
- package/src/core/workflow/index.ts +2 -13
- package/src/core/workflow/proof.ts +35 -83
- package/src/core/workflow/resume-lock.ts +179 -0
- package/src/core/workflow/run-state.ts +3 -207
- package/src/core/workflow/workspaces/leases.ts +1 -1
- package/src/core/workflow/workspaces/local-git.ts +120 -239
- package/src/core/workflow/workspaces/ownership.ts +3 -158
- package/src/core/workflow/workspaces/reclaim.ts +3 -9
- package/src/core/workflow/workspaces/recovery.ts +1 -1
- package/src/core/workflow/workspaces/state.ts +1 -5
- package/dist/core/agents/state/process-handle.d.ts +0 -26
- package/dist/core/agents/state/process-handle.d.ts.map +0 -1
- package/dist/core/agents/state/process-handle.js +0 -121
- package/dist/core/workflow/recovery-audit.d.ts +0 -30
- package/dist/core/workflow/recovery-audit.d.ts.map +0 -1
- package/dist/core/workflow/recovery-audit.js +0 -116
- package/src/core/agents/state/process-handle.ts +0 -165
- package/src/core/workflow/recovery-audit.ts +0 -181
|
@@ -7,10 +7,14 @@ import {
|
|
|
7
7
|
realpathSync,
|
|
8
8
|
writeFileSync,
|
|
9
9
|
} from "node:fs";
|
|
10
|
+
import { hostname } from "node:os";
|
|
10
11
|
import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
11
12
|
import { stateDirMode, stateFileMode } from "../../storage/modes.ts";
|
|
12
|
-
import {
|
|
13
|
-
|
|
13
|
+
import {
|
|
14
|
+
acquireWorkflowResumeLease,
|
|
15
|
+
readWorkflowResumeLock,
|
|
16
|
+
workflowResumeLockPath,
|
|
17
|
+
} from "../resume-lock.ts";
|
|
14
18
|
import { WorkspaceAttestationError } from "./attestation-error.ts";
|
|
15
19
|
import { cleanupWorkspace } from "./cleanup.ts";
|
|
16
20
|
import {
|
|
@@ -24,12 +28,8 @@ import {
|
|
|
24
28
|
parseWorktreeInventory,
|
|
25
29
|
worktreeInventory,
|
|
26
30
|
} from "./git.ts";
|
|
27
|
-
import { acquireNoClobberLease } from "./leases.ts";
|
|
28
|
-
import {
|
|
29
|
-
claimOverlapsWorkspace,
|
|
30
|
-
reserveWorkspaceReclaim,
|
|
31
|
-
type WorkspaceReclaimReservation,
|
|
32
|
-
} from "./ownership.ts";
|
|
31
|
+
import { acquireNoClobberLease, readOwner } from "./leases.ts";
|
|
32
|
+
import { claimOverlapsWorkspace } from "./ownership.ts";
|
|
33
33
|
import {
|
|
34
34
|
assertContainedExisting,
|
|
35
35
|
assertPathIdentity,
|
|
@@ -821,7 +821,6 @@ function reclaimPreview(
|
|
|
821
821
|
run_id: binding.run_id,
|
|
822
822
|
mode,
|
|
823
823
|
liveness: readWorkflowLiveness(coordRoot, binding.run_id),
|
|
824
|
-
evidence_sha256: reclaimEvidenceDigest(coordRoot, binding),
|
|
825
824
|
ownership: "unproven",
|
|
826
825
|
dirty_paths: null,
|
|
827
826
|
lease_holder: reclaimLeaseHolders(coordRoot, binding),
|
|
@@ -841,42 +840,15 @@ function reclaimPreview(
|
|
|
841
840
|
return preview;
|
|
842
841
|
}
|
|
843
842
|
|
|
844
|
-
function
|
|
845
|
-
const evidence = (name: string): string | null => {
|
|
846
|
-
try {
|
|
847
|
-
return readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, name), "utf8");
|
|
848
|
-
} catch (error) {
|
|
849
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
850
|
-
throw error;
|
|
851
|
-
}
|
|
852
|
-
};
|
|
853
|
-
return stableDigest({
|
|
854
|
-
binding,
|
|
855
|
-
proof: evidence("proof.json"),
|
|
856
|
-
transcript: evidence("transcript.jsonl"),
|
|
857
|
-
});
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
function assertReclaimLiveness(
|
|
861
|
-
coordRoot: string,
|
|
862
|
-
binding: WorkspaceBinding,
|
|
863
|
-
recovery?: WorkspaceReclaimOptions["unknownLiveness"],
|
|
864
|
-
): void {
|
|
843
|
+
function assertReclaimLiveness(coordRoot: string, binding: WorkspaceBinding, force = false): void {
|
|
865
844
|
const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
|
|
866
|
-
if (liveness === "active" || (
|
|
845
|
+
if (liveness === "active" || (force && hasActiveWorkflowEvidence(coordRoot, binding.run_id))) {
|
|
867
846
|
throw new WorkflowReclaimError(
|
|
868
847
|
"workflow_reclaim_run_live",
|
|
869
848
|
`run ${binding.run_id} has active evidence; its workspace was left untouched`,
|
|
870
849
|
);
|
|
871
850
|
}
|
|
872
|
-
if (
|
|
873
|
-
throw new WorkflowReclaimError(
|
|
874
|
-
"workflow_reclaim_recovery_mismatch",
|
|
875
|
-
"workflow reclaim requires the exact current preview evidence digest",
|
|
876
|
-
);
|
|
877
|
-
}
|
|
878
|
-
if (liveness === "inactive") return;
|
|
879
|
-
if (liveness === "unknown" && recovery) return;
|
|
851
|
+
if (liveness === "inactive" || force) return;
|
|
880
852
|
throw new WorkflowReclaimError(
|
|
881
853
|
"workflow_reclaim_liveness_unknown",
|
|
882
854
|
`run ${binding.run_id} has missing, corrupt, or ambiguous liveness evidence; its workspace was left untouched`,
|
|
@@ -907,7 +879,7 @@ function assertPeerWorkspaceAuthority(coordRoot: string, claim: WorkspaceClaim):
|
|
|
907
879
|
throw new Error("retained peer binding authority is missing");
|
|
908
880
|
}
|
|
909
881
|
if (!claimOverlapsWorkspace(peer, claim.workspace_root)) continue;
|
|
910
|
-
// Process termination, archival and an expired
|
|
882
|
+
// Process termination, archival and an expired lease do not release
|
|
911
883
|
// a resource. Require the provider's release record AND current absence.
|
|
912
884
|
if (binding && events.some((event) => event.event === "released")) {
|
|
913
885
|
const current = attestSync(coordRoot, binding);
|
|
@@ -920,24 +892,6 @@ function assertPeerWorkspaceAuthority(coordRoot: string, claim: WorkspaceClaim):
|
|
|
920
892
|
}
|
|
921
893
|
}
|
|
922
894
|
|
|
923
|
-
function reclaimAuthorityGuard(
|
|
924
|
-
coordRoot: string,
|
|
925
|
-
binding: WorkspaceBinding,
|
|
926
|
-
claim: WorkspaceClaim,
|
|
927
|
-
reservation: WorkspaceReclaimReservation,
|
|
928
|
-
recovery?: WorkspaceReclaimOptions["unknownLiveness"],
|
|
929
|
-
): void {
|
|
930
|
-
try {
|
|
931
|
-
reservation.assertCurrent();
|
|
932
|
-
assertReclaimLiveness(coordRoot, binding, recovery);
|
|
933
|
-
assertReclaimOwnership(coordRoot, binding);
|
|
934
|
-
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
935
|
-
} catch (error) {
|
|
936
|
-
if (error instanceof WorkflowReclaimError) throw error;
|
|
937
|
-
throw new WorkflowReclaimError("workflow_reclaim_ownership_unproven", (error as Error).message);
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
|
-
|
|
941
895
|
async function reclaim(
|
|
942
896
|
coordRoot: string,
|
|
943
897
|
provider: WorkspaceProvider,
|
|
@@ -948,179 +902,107 @@ async function reclaim(
|
|
|
948
902
|
throw new Error("unsupported workspace reclaim mode");
|
|
949
903
|
}
|
|
950
904
|
if (!options.yes) return reclaimPreview(coordRoot, binding, options.mode);
|
|
951
|
-
if (options.unknownLiveness) validateWorkflowRecoveryAuthorization(options.unknownLiveness);
|
|
952
905
|
|
|
953
|
-
// Liveness comes before terminal-proof cleanup helpers. A live run
|
|
954
|
-
// have a proof yet, and a missing proof must never be
|
|
955
|
-
assertReclaimLiveness(coordRoot, binding, options.
|
|
906
|
+
// Liveness comes before leases and terminal-proof cleanup helpers. A live run
|
|
907
|
+
// may not have a proof yet, and a missing proof must never be found after reset.
|
|
908
|
+
assertReclaimLiveness(coordRoot, binding, options.force);
|
|
956
909
|
const claim = assertReclaimOwnership(coordRoot, binding);
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
} catch (error) {
|
|
961
|
-
throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
|
|
962
|
-
}
|
|
963
|
-
try {
|
|
964
|
-
reclaimAuthorityGuard(coordRoot, binding, claim, reservation, options.unknownLiveness);
|
|
965
|
-
let releaseResume: () => void;
|
|
910
|
+
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
911
|
+
const releases: Array<() => void> = [];
|
|
912
|
+
const hold = (acquire: () => () => void) => {
|
|
966
913
|
try {
|
|
967
|
-
|
|
914
|
+
releases.unshift(acquire());
|
|
968
915
|
} catch (error) {
|
|
969
916
|
throw new WorkflowReclaimError("workflow_reclaim_lease_held", (error as Error).message);
|
|
970
917
|
}
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
let preparation: ReclaimPreparation | undefined;
|
|
1010
|
-
// Keep ordinary cleanup's terminal-evidence checks, intent and receipts.
|
|
1011
|
-
// Its provider callback uses the leases already held here instead of
|
|
1012
|
-
// trying to acquire them again. Preparation occurs only after all of
|
|
1013
|
-
// cleanup's path, ownership, ref and registration checks have succeeded.
|
|
1014
|
-
const cleanupResult = await cleanupWorkspace({
|
|
1015
|
-
coordRoot,
|
|
1016
|
-
runId: binding.run_id,
|
|
1017
|
-
provider: {
|
|
1018
|
-
...provider,
|
|
1019
|
-
cleanup: async (candidate, intent) => {
|
|
1020
|
-
assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
|
|
1021
|
-
assertReclaimOwnership(coordRoot, candidate);
|
|
1022
|
-
return cleanupUnderLease(coordRoot, candidate, claim, intent, {
|
|
1023
|
-
mode: options.mode,
|
|
1024
|
-
recovery: options.unknownLiveness,
|
|
1025
|
-
beforeEffect: () => {
|
|
1026
|
-
audit?.assertCurrent();
|
|
1027
|
-
reclaimAuthorityGuard(
|
|
1028
|
-
coordRoot,
|
|
1029
|
-
candidate,
|
|
1030
|
-
claim,
|
|
1031
|
-
reservation,
|
|
1032
|
-
options.unknownLiveness,
|
|
1033
|
-
);
|
|
1034
|
-
},
|
|
1035
|
-
beforeRefEffect: () => {
|
|
1036
|
-
audit?.assertCurrent();
|
|
1037
|
-
reservation.assertCurrent();
|
|
1038
|
-
assertReclaimLiveness(coordRoot, candidate, options.unknownLiveness);
|
|
1039
|
-
requiredClaim(coordRoot, candidate);
|
|
1040
|
-
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
1041
|
-
},
|
|
1042
|
-
prepared: (value) => {
|
|
1043
|
-
preparation = value;
|
|
1044
|
-
},
|
|
1045
|
-
});
|
|
1046
|
-
},
|
|
1047
|
-
},
|
|
1048
|
-
});
|
|
1049
|
-
if (
|
|
1050
|
-
!preparation &&
|
|
1051
|
-
cleanupResult.status !== "released" &&
|
|
1052
|
-
cleanupResult.status !== "already_released"
|
|
1053
|
-
) {
|
|
1054
|
-
throw new WorkflowReclaimError(
|
|
1055
|
-
"workflow_reclaim_ownership_unproven",
|
|
1056
|
-
("reason" in cleanupResult ? cleanupResult.reason : undefined) ??
|
|
1057
|
-
"workspace cleanup preconditions could not be proved",
|
|
1058
|
-
);
|
|
1059
|
-
}
|
|
1060
|
-
if (cleanupResult.status === "released" || cleanupResult.status === "already_released")
|
|
1061
|
-
audit?.complete();
|
|
1062
|
-
return {
|
|
1063
|
-
...preview,
|
|
1064
|
-
action: "reclaimed",
|
|
1065
|
-
lease_holder: [],
|
|
1066
|
-
preparation: preparation ?? {
|
|
1067
|
-
action: "already_gone",
|
|
1068
|
-
detail: "the workspace is already released",
|
|
918
|
+
};
|
|
919
|
+
try {
|
|
920
|
+
hold(() => acquireWorkflowResumeLease(coordRoot, binding.run_id));
|
|
921
|
+
hold(() =>
|
|
922
|
+
acquireLease(
|
|
923
|
+
coordRoot,
|
|
924
|
+
workspaceLockPath(coordRoot, PROVIDER_ID, binding.binding_id),
|
|
925
|
+
claim,
|
|
926
|
+
"binding",
|
|
927
|
+
),
|
|
928
|
+
);
|
|
929
|
+
hold(() => acquireRepositoryLease(coordRoot, claim));
|
|
930
|
+
const preview = reclaimPreview(coordRoot, binding, options.mode);
|
|
931
|
+
if (options.force && preview.liveness !== "inactive") {
|
|
932
|
+
appendWorkspaceEvent(coordRoot, claim, "reclaim_forced", {
|
|
933
|
+
run_id: binding.run_id,
|
|
934
|
+
mode: options.mode,
|
|
935
|
+
liveness: preview.liveness,
|
|
936
|
+
pid: process.pid,
|
|
937
|
+
host: hostname(),
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
let preparation: ReclaimPreparation | undefined;
|
|
941
|
+
// Keep ordinary cleanup's terminal-evidence checks, intent and receipts.
|
|
942
|
+
// Its provider callback uses the leases already held here. Preparation
|
|
943
|
+
// occurs only after cleanup's liveness, ownership, path, ref and
|
|
944
|
+
// registration checks have succeeded under those leases.
|
|
945
|
+
const cleanupResult = await cleanupWorkspace({
|
|
946
|
+
coordRoot,
|
|
947
|
+
runId: binding.run_id,
|
|
948
|
+
provider: {
|
|
949
|
+
...provider,
|
|
950
|
+
cleanup: async (candidate, intent) =>
|
|
951
|
+
cleanupUnderLease(coordRoot, candidate, claim, intent, {
|
|
952
|
+
mode: options.mode,
|
|
953
|
+
force: options.force,
|
|
954
|
+
prepared: (value) => {
|
|
955
|
+
preparation = value;
|
|
1069
956
|
},
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
957
|
+
}),
|
|
958
|
+
},
|
|
959
|
+
});
|
|
960
|
+
if (
|
|
961
|
+
!preparation &&
|
|
962
|
+
cleanupResult.status !== "released" &&
|
|
963
|
+
cleanupResult.status !== "already_released"
|
|
964
|
+
) {
|
|
965
|
+
throw new WorkflowReclaimError(
|
|
966
|
+
"workflow_reclaim_ownership_unproven",
|
|
967
|
+
("reason" in cleanupResult ? cleanupResult.reason : undefined) ??
|
|
968
|
+
"workspace cleanup preconditions could not be proved",
|
|
969
|
+
);
|
|
1080
970
|
}
|
|
971
|
+
return {
|
|
972
|
+
...preview,
|
|
973
|
+
action: "reclaimed",
|
|
974
|
+
lease_holder: [],
|
|
975
|
+
preparation: preparation ?? {
|
|
976
|
+
action: "already_gone",
|
|
977
|
+
detail: "the workspace is already released",
|
|
978
|
+
},
|
|
979
|
+
cleanup: cleanupResult,
|
|
980
|
+
};
|
|
1081
981
|
} finally {
|
|
1082
|
-
|
|
982
|
+
for (const release of releases) release();
|
|
1083
983
|
}
|
|
1084
984
|
}
|
|
1085
985
|
|
|
1086
986
|
function reclaimLeaseHolders(coordRoot: string, binding: WorkspaceBinding): ReclaimLeaseHolder[] {
|
|
1087
|
-
const
|
|
987
|
+
const holders: ReclaimLeaseHolder[] = [];
|
|
988
|
+
if (existsSync(workflowResumeLockPath(coordRoot, binding.run_id))) {
|
|
989
|
+
const owner = readWorkflowResumeLock(coordRoot, binding.run_id);
|
|
990
|
+
holders.push(
|
|
991
|
+
owner
|
|
992
|
+
? { scope: "resume", pid: owner.pid, host: owner.host }
|
|
993
|
+
: { scope: "resume", detail: "lease holder could not be read" },
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
const leases: Array<[ReclaimLeaseHolder["scope"], string]> = [
|
|
1088
997
|
["binding", workspaceLockPath(coordRoot, PROVIDER_ID, binding.binding_id)],
|
|
1089
998
|
];
|
|
1090
999
|
if (binding.repository) {
|
|
1091
|
-
|
|
1092
|
-
paths.push([
|
|
1093
|
-
"repository",
|
|
1094
|
-
join(coordRoot, ".harnery", "workspaces", PROVIDER_ID, ".leases", `repository-${key}.lock`),
|
|
1095
|
-
]);
|
|
1096
|
-
}
|
|
1097
|
-
const holders: ReclaimLeaseHolder[] = [];
|
|
1098
|
-
try {
|
|
1099
|
-
const owner = JSON.parse(
|
|
1100
|
-
readFileSync(join(coordRoot, ".harnery", "workflows", binding.run_id, "resume.lock"), "utf8"),
|
|
1101
|
-
) as Record<string, unknown>;
|
|
1102
|
-
holders.push({
|
|
1103
|
-
scope: "resume",
|
|
1104
|
-
pid: typeof owner.pid === "number" ? owner.pid : undefined,
|
|
1105
|
-
host: typeof owner.host === "string" ? owner.host : undefined,
|
|
1106
|
-
});
|
|
1107
|
-
} catch (error) {
|
|
1108
|
-
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
1109
|
-
holders.push({ scope: "resume", detail: "lease holder could not be read" });
|
|
1110
|
-
}
|
|
1000
|
+
leases.push(["repository", repositoryLeasePath(coordRoot, binding.repository)]);
|
|
1111
1001
|
}
|
|
1112
|
-
for (const [scope, path] of
|
|
1002
|
+
for (const [scope, path] of leases) {
|
|
1113
1003
|
try {
|
|
1114
|
-
const owner =
|
|
1115
|
-
|
|
1116
|
-
unknown
|
|
1117
|
-
>;
|
|
1118
|
-
holders.push({
|
|
1119
|
-
scope,
|
|
1120
|
-
owner_id: typeof owner.owner_id === "string" ? owner.owner_id : undefined,
|
|
1121
|
-
pid: typeof owner.pid === "number" ? owner.pid : undefined,
|
|
1122
|
-
host: typeof owner.host === "string" ? owner.host : undefined,
|
|
1123
|
-
});
|
|
1004
|
+
const { owner } = readOwner(join(path, "current"));
|
|
1005
|
+
holders.push({ scope, owner_id: owner.owner_id, pid: owner.pid, host: owner.host });
|
|
1124
1006
|
} catch (error) {
|
|
1125
1007
|
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
1126
1008
|
holders.push({ scope, detail: "lease holder could not be read" });
|
|
@@ -1146,17 +1028,7 @@ async function cleanup(
|
|
|
1146
1028
|
try {
|
|
1147
1029
|
const releaseRepository = acquireRepositoryLease(coordRoot, claim);
|
|
1148
1030
|
try {
|
|
1149
|
-
|
|
1150
|
-
try {
|
|
1151
|
-
const authorityGuard = () => {
|
|
1152
|
-
reservation.assertCurrent();
|
|
1153
|
-
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
1154
|
-
};
|
|
1155
|
-
authorityGuard();
|
|
1156
|
-
return cleanupUnderLease(coordRoot, binding, claim, intent, undefined, authorityGuard);
|
|
1157
|
-
} finally {
|
|
1158
|
-
reservation.release();
|
|
1159
|
-
}
|
|
1031
|
+
return cleanupUnderLease(coordRoot, binding, claim, intent);
|
|
1160
1032
|
} finally {
|
|
1161
1033
|
releaseRepository();
|
|
1162
1034
|
}
|
|
@@ -1172,12 +1044,9 @@ function cleanupUnderLease(
|
|
|
1172
1044
|
intent: WorkspaceCleanupIntent,
|
|
1173
1045
|
reclaimRequest?: {
|
|
1174
1046
|
mode: ReclaimMode;
|
|
1175
|
-
|
|
1176
|
-
beforeEffect: () => void;
|
|
1177
|
-
beforeRefEffect: () => void;
|
|
1047
|
+
force?: boolean;
|
|
1178
1048
|
prepared: (value: ReclaimPreparation) => void;
|
|
1179
1049
|
},
|
|
1180
|
-
beforeRemoval?: () => void,
|
|
1181
1050
|
): WorkspaceCleanupResult {
|
|
1182
1051
|
const repository = binding.repository;
|
|
1183
1052
|
if (
|
|
@@ -1222,7 +1091,7 @@ function cleanupUnderLease(
|
|
|
1222
1091
|
};
|
|
1223
1092
|
}
|
|
1224
1093
|
const liveness = readWorkflowLiveness(coordRoot, binding.run_id);
|
|
1225
|
-
if (liveness !== "inactive" && !(liveness === "unknown" && reclaimRequest?.
|
|
1094
|
+
if (liveness !== "inactive" && !(liveness === "unknown" && reclaimRequest?.force)) {
|
|
1226
1095
|
return {
|
|
1227
1096
|
...base,
|
|
1228
1097
|
status: "blocked",
|
|
@@ -1315,15 +1184,15 @@ function cleanupUnderLease(
|
|
|
1315
1184
|
}
|
|
1316
1185
|
|
|
1317
1186
|
if (reclaimRequest) {
|
|
1318
|
-
|
|
1319
|
-
|
|
1187
|
+
// The last liveness check before a destructive reset.
|
|
1188
|
+
assertReclaimLiveness(coordRoot, binding, reclaimRequest.force);
|
|
1320
1189
|
revalidateClaimResources(claim, root);
|
|
1321
1190
|
reclaimRequest.prepared(
|
|
1322
1191
|
prepareReclaim({
|
|
1323
1192
|
worktreePath: binding.workspace_root,
|
|
1324
1193
|
mode: reclaimRequest.mode,
|
|
1325
1194
|
runId: binding.run_id,
|
|
1326
|
-
beforeEffect:
|
|
1195
|
+
beforeEffect: () => assertPeerWorkspaceAuthority(coordRoot, claim),
|
|
1327
1196
|
}),
|
|
1328
1197
|
);
|
|
1329
1198
|
}
|
|
@@ -1336,8 +1205,7 @@ function cleanupUnderLease(
|
|
|
1336
1205
|
target_ref_oid: targetRef.out,
|
|
1337
1206
|
});
|
|
1338
1207
|
if (before.workspace_exists) {
|
|
1339
|
-
|
|
1340
|
-
beforeRemoval?.();
|
|
1208
|
+
assertPeerWorkspaceAuthority(coordRoot, claim);
|
|
1341
1209
|
revalidateClaimResources(claim, root);
|
|
1342
1210
|
assertContainedExisting(root, binding.workspace_root, "workspace cleanup path");
|
|
1343
1211
|
git(repository.source_root.realpath, ["worktree", "remove", binding.workspace_root]);
|
|
@@ -1363,7 +1231,6 @@ function cleanupUnderLease(
|
|
|
1363
1231
|
return { ...base, status: "blocked", reason: "workspace ref changed during cleanup" };
|
|
1364
1232
|
}
|
|
1365
1233
|
revalidateClaimResources(claim, root);
|
|
1366
|
-
reclaimRequest?.beforeRefEffect();
|
|
1367
1234
|
const deleted = gitMaybe(repository.source_root.realpath, [
|
|
1368
1235
|
"update-ref",
|
|
1369
1236
|
"-d",
|
|
@@ -2288,9 +2155,24 @@ function capabilityDigestForClaim(claim: WorkspaceClaim): string {
|
|
|
2288
2155
|
return stableDigest(basis);
|
|
2289
2156
|
}
|
|
2290
2157
|
|
|
2158
|
+
function repositoryLeasePath(
|
|
2159
|
+
coordRoot: string,
|
|
2160
|
+
repository: { common_dir: { identity: FilesystemIdentity } },
|
|
2161
|
+
): string {
|
|
2162
|
+
const key = stableDigest({ common: repository.common_dir.identity });
|
|
2163
|
+
return join(
|
|
2164
|
+
resolve(coordRoot),
|
|
2165
|
+
".harnery",
|
|
2166
|
+
"workspaces",
|
|
2167
|
+
PROVIDER_ID,
|
|
2168
|
+
".leases",
|
|
2169
|
+
`repository-${key}.lock`,
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
|
|
2291
2173
|
function acquireRepositoryLease(coordRoot: string, claim: WorkspaceClaim): () => void {
|
|
2292
|
-
const
|
|
2293
|
-
mkdirSync(
|
|
2174
|
+
const path = repositoryLeasePath(coordRoot, claim.repository);
|
|
2175
|
+
mkdirSync(dirname(path), { recursive: true, mode: stateDirMode() });
|
|
2294
2176
|
// Mutual exclusion keys on the Git common directory alone. `git worktree add`,
|
|
2295
2177
|
// `prune`, and the shared-`config` migration all write to the common directory's
|
|
2296
2178
|
// administrative area, and once worktrees and submodules are allowed, several
|
|
@@ -2299,8 +2181,7 @@ function acquireRepositoryLease(coordRoot: string, claim: WorkspaceClaim): () =>
|
|
|
2299
2181
|
// nested — writable roots. Including the writable root in the key would hand those
|
|
2300
2182
|
// agents different locks and let their admin-area writes race. The writable root
|
|
2301
2183
|
// stays in the lease metadata for diagnostics, not in the exclusion key.
|
|
2302
|
-
|
|
2303
|
-
return acquireLease(coordRoot, join(leaseDir, `repository-${key}.lock`), claim, "repository");
|
|
2184
|
+
return acquireLease(coordRoot, path, claim, "repository");
|
|
2304
2185
|
}
|
|
2305
2186
|
|
|
2306
2187
|
function acquireLease(
|
|
@@ -1,165 +1,10 @@
|
|
|
1
|
-
import { lstatSync,
|
|
2
|
-
import { basename, dirname, isAbsolute,
|
|
3
|
-
import {
|
|
4
|
-
import { acquireNoClobberLease, type NoClobberLease } from "./leases.ts";
|
|
1
|
+
import { lstatSync, realpathSync } from "node:fs";
|
|
2
|
+
import { basename, dirname, isAbsolute, resolve } from "node:path";
|
|
3
|
+
import { stableDigest } from "../durable-record.ts";
|
|
5
4
|
import { containsPath } from "./paths.ts";
|
|
6
5
|
import type { WorkspaceClaim } from "./state.ts";
|
|
7
6
|
import { isWorkspaceAllocationRequest } from "./validate.ts";
|
|
8
7
|
|
|
9
|
-
const STALE_MS = 5 * 60 * 1_000;
|
|
10
|
-
const RESERVATION_SCOPE = "workspace-reclaim";
|
|
11
|
-
|
|
12
|
-
function leaseRoot(coordRoot: string, providerId: string): string {
|
|
13
|
-
return join(resolve(coordRoot), ".harnery", "workspaces", providerId, ".leases");
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/** Serialize only publication and reservation registration, never Git work. */
|
|
17
|
-
function withPublicationGate<T>(coordRoot: string, providerId: string, operation: () => T): T {
|
|
18
|
-
const authority = stableDigest({ provider_id: providerId, scope: "workspace-publication" });
|
|
19
|
-
const gate = acquireNoClobberLease({
|
|
20
|
-
path: join(leaseRoot(coordRoot, providerId), "publication.lock"),
|
|
21
|
-
scope: "workspace-publication",
|
|
22
|
-
authoritySha256: authority,
|
|
23
|
-
staleAfterMs: STALE_MS,
|
|
24
|
-
validateStaleOwner: (owner) => owner.authority_sha256 === authority,
|
|
25
|
-
});
|
|
26
|
-
try {
|
|
27
|
-
return operation();
|
|
28
|
-
} finally {
|
|
29
|
-
gate.release();
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function reservationPath(coordRoot: string, claim: WorkspaceClaim): string {
|
|
34
|
-
return join(leaseRoot(coordRoot, claim.provider_id), `reclaim-${claim.binding_id}.lock`);
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
function acquireReservation(coordRoot: string, claim: WorkspaceClaim): NoClobberLease {
|
|
38
|
-
const authority = stableDigest(claim);
|
|
39
|
-
return acquireNoClobberLease({
|
|
40
|
-
path: reservationPath(coordRoot, claim),
|
|
41
|
-
scope: RESERVATION_SCOPE,
|
|
42
|
-
authoritySha256: authority,
|
|
43
|
-
staleAfterMs: STALE_MS,
|
|
44
|
-
metadata: { binding_id: claim.binding_id, request_sha256: claim.request_sha256 },
|
|
45
|
-
validateStaleOwner: (owner) =>
|
|
46
|
-
owner.authority_sha256 === authority &&
|
|
47
|
-
owner.metadata?.binding_id === claim.binding_id &&
|
|
48
|
-
owner.metadata?.request_sha256 === claim.request_sha256,
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export interface WorkspaceReclaimReservation {
|
|
53
|
-
assertCurrent(): void;
|
|
54
|
-
release(): void;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** The immutable claim is the authority; the lease reserves its destructive path. */
|
|
58
|
-
export function reserveWorkspaceReclaim(
|
|
59
|
-
coordRoot: string,
|
|
60
|
-
claim: WorkspaceClaim,
|
|
61
|
-
): WorkspaceReclaimReservation {
|
|
62
|
-
assertClaimAuthority(claim, claim.provider_id, claim.binding_id);
|
|
63
|
-
const lease = withPublicationGate(coordRoot, claim.provider_id, () =>
|
|
64
|
-
acquireReservation(coordRoot, claim),
|
|
65
|
-
);
|
|
66
|
-
return {
|
|
67
|
-
assertCurrent() {
|
|
68
|
-
const path = reservationPath(coordRoot, claim);
|
|
69
|
-
const current = join(path, "current");
|
|
70
|
-
const pin = join(path, lease.owner.owner_file);
|
|
71
|
-
const a = lstatSync(current, { bigint: true });
|
|
72
|
-
const b = lstatSync(pin, { bigint: true });
|
|
73
|
-
if (
|
|
74
|
-
!a.isFile() ||
|
|
75
|
-
!b.isFile() ||
|
|
76
|
-
a.dev !== b.dev ||
|
|
77
|
-
a.ino !== b.ino ||
|
|
78
|
-
readFileSync(current, "utf8") !== readFileSync(pin, "utf8") ||
|
|
79
|
-
stableDigest(readJsonRecord(current, "reclaim reservation")) !== stableDigest(lease.owner)
|
|
80
|
-
) {
|
|
81
|
-
throw new Error("workspace reclaim reservation owner changed");
|
|
82
|
-
}
|
|
83
|
-
const persisted = readJsonRecord<WorkspaceClaim>(
|
|
84
|
-
join(dirname(dirname(path)), claim.binding_id, "claim.json"),
|
|
85
|
-
"reserved workspace claim",
|
|
86
|
-
);
|
|
87
|
-
if (stableDigest(persisted) !== stableDigest(claim)) {
|
|
88
|
-
throw new Error("workspace reclaim reservation claim changed");
|
|
89
|
-
}
|
|
90
|
-
},
|
|
91
|
-
release: () => lease.release(),
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
export function publishWorkspaceClaim<T>(
|
|
96
|
-
coordRoot: string,
|
|
97
|
-
claim: WorkspaceClaim,
|
|
98
|
-
publish: () => T,
|
|
99
|
-
): T {
|
|
100
|
-
assertClaimAuthority(claim, claim.provider_id, claim.binding_id);
|
|
101
|
-
return withPublicationGate(coordRoot, claim.provider_id, () => {
|
|
102
|
-
for (const entry of readdirSync(leaseRoot(coordRoot, claim.provider_id), {
|
|
103
|
-
withFileTypes: true,
|
|
104
|
-
})) {
|
|
105
|
-
if (!entry.name.startsWith("reclaim-") || !entry.name.endsWith(".lock")) continue;
|
|
106
|
-
if (!entry.isDirectory()) throw new Error("workspace reclaim reservation is not a directory");
|
|
107
|
-
const path = join(leaseRoot(coordRoot, claim.provider_id), entry.name);
|
|
108
|
-
const current = join(path, "current");
|
|
109
|
-
try {
|
|
110
|
-
lstatSync(current);
|
|
111
|
-
} catch (error) {
|
|
112
|
-
if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
|
|
113
|
-
throw error;
|
|
114
|
-
}
|
|
115
|
-
const id = entry.name.slice("reclaim-".length, -".lock".length);
|
|
116
|
-
const reserved = readJsonRecord<WorkspaceClaim>(
|
|
117
|
-
join(dirname(leaseRoot(coordRoot, claim.provider_id)), id, "claim.json"),
|
|
118
|
-
"reserved workspace claim",
|
|
119
|
-
);
|
|
120
|
-
assertClaimAuthority(reserved, claim.provider_id, id);
|
|
121
|
-
// Unknown reservation authority is never interpreted as an unrelated owner.
|
|
122
|
-
const owner = readJsonRecord<NoClobberLease["owner"]>(current, "reclaim reservation");
|
|
123
|
-
if (
|
|
124
|
-
owner.schema_version !== 1 ||
|
|
125
|
-
typeof owner.owner_id !== "string" ||
|
|
126
|
-
!/^[a-f0-9-]{36}$/.test(owner.owner_id) ||
|
|
127
|
-
owner.owner_file !== `owner-${owner.owner_id}.json` ||
|
|
128
|
-
!Number.isSafeInteger(owner.pid) ||
|
|
129
|
-
owner.pid <= 0 ||
|
|
130
|
-
typeof owner.host !== "string" ||
|
|
131
|
-
owner.host.length === 0 ||
|
|
132
|
-
!Number.isFinite(Date.parse(owner.created_at)) ||
|
|
133
|
-
!Number.isFinite(Date.parse(owner.expires_at)) ||
|
|
134
|
-
Date.parse(owner.expires_at) < Date.parse(owner.created_at) ||
|
|
135
|
-
owner.scope !== RESERVATION_SCOPE ||
|
|
136
|
-
owner.authority_sha256 !== stableDigest(reserved) ||
|
|
137
|
-
owner.metadata?.binding_id !== id ||
|
|
138
|
-
owner.metadata?.request_sha256 !== reserved.request_sha256
|
|
139
|
-
) {
|
|
140
|
-
throw new Error("workspace reclaim reservation authority is unproven");
|
|
141
|
-
}
|
|
142
|
-
const a = lstatSync(current, { bigint: true });
|
|
143
|
-
const pin = join(path, owner.owner_file);
|
|
144
|
-
const b = lstatSync(pin, { bigint: true });
|
|
145
|
-
if (
|
|
146
|
-
!a.isFile() ||
|
|
147
|
-
!b.isFile() ||
|
|
148
|
-
a.dev !== b.dev ||
|
|
149
|
-
a.ino !== b.ino ||
|
|
150
|
-
readFileSync(current, "utf8") !== readFileSync(pin, "utf8")
|
|
151
|
-
)
|
|
152
|
-
throw new Error("workspace reclaim reservation owner is unproven");
|
|
153
|
-
if (!claimOverlapsWorkspace(claim, reserved.workspace_root)) continue;
|
|
154
|
-
// This succeeds only for an expired, provably dead, exact prior owner.
|
|
155
|
-
// It retires the reservation, never the immutable claim or resource.
|
|
156
|
-
const recovered = acquireReservation(coordRoot, reserved);
|
|
157
|
-
recovered.release();
|
|
158
|
-
}
|
|
159
|
-
return publish();
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
|
|
163
8
|
export function assertClaimAuthority(claim: WorkspaceClaim, providerId: string, id: string): void {
|
|
164
9
|
if (
|
|
165
10
|
claim?.schema_version !== 1 ||
|