harnery 0.45.0 → 0.45.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commander.js +0 -1
- package/dist/commands/files.d.ts +2 -2
- package/dist/commands/files.d.ts.map +1 -1
- package/dist/commands/files.js +4 -7
- package/dist/commands/servers.d.ts.map +1 -1
- package/dist/commands/servers.js +17 -11
- package/dist/commands/tunnel.d.ts +0 -15
- package/dist/commands/tunnel.d.ts.map +1 -1
- package/dist/commands/tunnel.js +153 -453
- package/dist/commands/workflow.d.ts.map +1 -1
- package/dist/commands/workflow.js +6 -45
- package/dist/core/agents/coord-client.d.ts.map +1 -1
- package/dist/core/agents/coord-client.js +0 -18
- package/dist/core/agents/live-authority-v3.d.ts +0 -1
- package/dist/core/agents/live-authority-v3.d.ts.map +1 -1
- package/dist/core/agents/live-authority-v3.js +0 -11
- package/dist/core/agents/live-lifecycle-v3.d.ts +2 -4
- package/dist/core/agents/live-lifecycle-v3.d.ts.map +1 -1
- package/dist/core/agents/live-lifecycle-v3.js +7 -28
- package/dist/core/agents/state/heartbeat-writer.d.ts.map +1 -1
- package/dist/core/agents/state/heartbeat-writer.js +14 -25
- package/dist/core/agents/state/live-coordination-writer.d.ts.map +1 -1
- package/dist/core/agents/state/live-coordination-writer.js +0 -5
- package/dist/core/agents/state/pidmap.d.ts.map +1 -1
- package/dist/core/agents/state/pidmap.js +2 -7
- package/dist/core/agents/state/proc-start.d.ts +20 -31
- package/dist/core/agents/state/proc-start.d.ts.map +1 -1
- package/dist/core/agents/state/proc-start.js +37 -70
- package/dist/core/agents/state/service-process.d.ts +61 -0
- package/dist/core/agents/state/service-process.d.ts.map +1 -0
- package/dist/core/agents/state/service-process.js +143 -0
- package/dist/core/agents/state/stale-sweep.d.ts.map +1 -1
- package/dist/core/agents/state/stale-sweep.js +70 -180
- package/dist/core/artifacts/activity.d.ts +4 -1
- package/dist/core/artifacts/activity.d.ts.map +1 -1
- package/dist/core/artifacts/activity.js +8 -2
- package/dist/core/artifacts/delivery-card.d.ts.map +1 -1
- package/dist/core/artifacts/delivery-card.js +1 -2
- package/dist/core/artifacts/index.d.ts.map +1 -1
- package/dist/core/artifacts/index.js +38 -12
- package/dist/core/context/index.d.ts +6 -4
- package/dist/core/context/index.d.ts.map +1 -1
- package/dist/core/context/index.js +194 -256
- package/dist/core/context/observation.d.ts +9 -9
- package/dist/core/context/observation.d.ts.map +1 -1
- package/dist/core/context/observation.js +37 -6
- package/dist/core/events/v3/authority-outbox.d.ts.map +1 -1
- package/dist/core/events/v3/authority-outbox.js +0 -29
- package/dist/core/events/v3/producers/coordination-recorder.d.ts +0 -1
- package/dist/core/events/v3/producers/coordination-recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/coordination-recorder.js +0 -7
- package/dist/core/events/v3/producers/recorder.d.ts.map +1 -1
- package/dist/core/events/v3/producers/recorder.js +64 -77
- package/dist/core/events/v3/runtime-build.d.ts +14 -0
- package/dist/core/events/v3/runtime-build.d.ts.map +1 -1
- package/dist/core/events/v3/runtime-build.js +44 -0
- package/dist/core/governor/index.d.ts +2 -1
- package/dist/core/governor/index.d.ts.map +1 -1
- package/dist/core/governor/index.js +2 -1
- package/dist/core/governor/service-read.d.ts +3 -10
- package/dist/core/governor/service-read.d.ts.map +1 -1
- package/dist/core/governor/service-read.js +10 -94
- package/dist/core/governor/service.d.ts +3 -7
- package/dist/core/governor/service.d.ts.map +1 -1
- package/dist/core/governor/service.js +17 -182
- package/dist/core/hooks/cli.js +78 -48
- package/dist/core/hooks/resolve/owner.js +2 -2
- package/dist/core/resources/sampler.js +2 -2
- package/dist/core/resources/service-status.d.ts +2 -14
- package/dist/core/resources/service-status.d.ts.map +1 -1
- package/dist/core/resources/service-status.js +3 -23
- package/dist/core/resources/service.d.ts +3 -7
- package/dist/core/resources/service.d.ts.map +1 -1
- package/dist/core/resources/service.js +12 -101
- package/dist/core/resources/status.d.ts.map +1 -1
- package/dist/core/resources/status.js +3 -2
- package/dist/core/semantic/service.d.ts.map +1 -1
- package/dist/core/semantic/service.js +9 -2
- package/dist/core/servers/index.d.ts +11 -10
- package/dist/core/servers/index.d.ts.map +1 -1
- package/dist/core/servers/index.js +82 -192
- package/dist/core/servers/tunnels.d.ts +3 -5
- package/dist/core/servers/tunnels.d.ts.map +1 -1
- package/dist/core/servers/tunnels.js +2 -45
- package/dist/core/storage/durable-history.js +56 -22
- package/dist/core/storage/segments.d.ts.map +1 -1
- package/dist/core/storage/segments.js +73 -17
- package/dist/core/supervisor/service.d.ts +4 -7
- package/dist/core/supervisor/service.d.ts.map +1 -1
- package/dist/core/supervisor/service.js +23 -122
- package/dist/core/supervisor/services.js +2 -2
- package/dist/core/supervisor/status.d.ts +3 -17
- package/dist/core/supervisor/status.d.ts.map +1 -1
- package/dist/core/supervisor/status.js +7 -70
- package/dist/core/work/state.js +7 -7
- package/dist/core/workflow/engine.d.ts.map +1 -1
- package/dist/core/workflow/engine.js +23 -66
- package/dist/core/workflow/index.d.ts +3 -4
- package/dist/core/workflow/index.d.ts.map +1 -1
- package/dist/core/workflow/index.js +2 -2
- package/dist/core/workflow/live-session-v3.d.ts +0 -6
- package/dist/core/workflow/live-session-v3.d.ts.map +1 -1
- package/dist/core/workflow/live-session-v3.js +5 -126
- package/dist/core/workflow/proof.d.ts +4 -4
- package/dist/core/workflow/proof.d.ts.map +1 -1
- package/dist/core/workflow/proof.js +34 -86
- package/dist/core/workflow/resume-lock.d.ts +17 -0
- package/dist/core/workflow/resume-lock.d.ts.map +1 -0
- package/dist/core/workflow/resume-lock.js +158 -0
- package/dist/core/workflow/run-state.d.ts +0 -21
- package/dist/core/workflow/run-state.d.ts.map +1 -1
- package/dist/core/workflow/run-state.js +3 -174
- package/dist/core/workflow/spawn-claude.d.ts.map +1 -1
- package/dist/core/workflow/spawn-claude.js +0 -19
- package/dist/core/workflow/spawn-codex.d.ts.map +1 -1
- package/dist/core/workflow/spawn-codex.js +0 -13
- package/dist/core/workflow/spawn-cursor.d.ts.map +1 -1
- package/dist/core/workflow/spawn-cursor.js +0 -19
- package/dist/core/workflow/spawn-opencode.d.ts.map +1 -1
- package/dist/core/workflow/spawn-opencode.js +0 -14
- package/dist/core/workflow/types.d.ts +11 -54
- package/dist/core/workflow/types.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.d.ts +4 -0
- package/dist/core/workflow/workspaces/leases.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/leases.js +1 -1
- package/dist/core/workflow/workspaces/local-git.js +91 -210
- package/dist/core/workflow/workspaces/ownership.d.ts +0 -7
- package/dist/core/workflow/workspaces/ownership.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/ownership.js +3 -130
- package/dist/core/workflow/workspaces/reclaim.d.ts +4 -7
- package/dist/core/workflow/workspaces/reclaim.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/reclaim.js +0 -4
- package/dist/core/workflow/workspaces/recovery.d.ts +1 -1
- package/dist/core/workflow/workspaces/recovery.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/recovery.js +1 -1
- package/dist/core/workflow/workspaces/state.d.ts.map +1 -1
- package/dist/core/workflow/workspaces/state.js +1 -4
- package/dist/lib/identities/assume.d.ts.map +1 -1
- package/dist/lib/identities/assume.js +0 -4
- package/dist/lib/tunnel/state.d.ts +6 -13
- package/dist/lib/tunnel/state.d.ts.map +1 -1
- package/dist/lib/tunnel/state.js +12 -58
- package/package.json +1 -6
- package/src/commander.ts +0 -10
- package/src/commands/files.ts +5 -13
- package/src/commands/servers.ts +23 -12
- package/src/commands/tunnel.ts +166 -533
- package/src/commands/workflow.ts +8 -58
- package/src/core/agents/coord-client.ts +0 -20
- package/src/core/agents/live-authority-v3.ts +0 -17
- package/src/core/agents/live-lifecycle-v3.ts +14 -52
- package/src/core/agents/state/heartbeat-writer.ts +12 -29
- package/src/core/agents/state/live-coordination-writer.ts +0 -29
- package/src/core/agents/state/pidmap.ts +2 -15
- package/src/core/agents/state/proc-start.ts +41 -69
- package/src/core/agents/state/service-process.ts +217 -0
- package/src/core/agents/state/stale-sweep.ts +82 -247
- package/src/core/artifacts/activity.ts +9 -2
- package/src/core/artifacts/delivery-card.ts +1 -7
- package/src/core/artifacts/index.ts +45 -12
- package/src/core/context/index.ts +207 -272
- package/src/core/context/observation.ts +40 -9
- package/src/core/events/v3/authority-outbox.ts +0 -56
- package/src/core/events/v3/producers/coordination-recorder.ts +0 -15
- package/src/core/events/v3/producers/recorder.ts +68 -94
- package/src/core/events/v3/runtime-build.ts +62 -0
- package/src/core/governor/index.ts +1 -2
- package/src/core/governor/service-read.ts +15 -114
- package/src/core/governor/service.ts +36 -220
- package/src/core/hooks/cli.ts +85 -47
- package/src/core/hooks/resolve/owner.ts +2 -2
- package/src/core/resources/sampler.ts +3 -2
- package/src/core/resources/service-status.ts +6 -35
- package/src/core/resources/service.ts +23 -133
- package/src/core/resources/status.ts +3 -2
- package/src/core/semantic/service.ts +8 -2
- package/src/core/servers/index.ts +86 -228
- package/src/core/servers/tunnels.ts +3 -55
- package/src/core/storage/durable-history.ts +52 -20
- package/src/core/storage/segments.ts +79 -17
- package/src/core/supervisor/service.ts +33 -155
- package/src/core/supervisor/services.ts +2 -2
- package/src/core/supervisor/status.ts +10 -85
- package/src/core/work/state.ts +7 -7
- package/src/core/workflow/engine.ts +22 -74
- package/src/core/workflow/index.ts +2 -13
- package/src/core/workflow/live-session-v3.ts +5 -146
- package/src/core/workflow/proof.ts +40 -104
- package/src/core/workflow/resume-lock.ts +179 -0
- package/src/core/workflow/run-state.ts +3 -207
- package/src/core/workflow/spawn-claude.ts +0 -18
- package/src/core/workflow/spawn-codex.ts +0 -13
- package/src/core/workflow/spawn-cursor.ts +0 -18
- package/src/core/workflow/spawn-opencode.ts +0 -14
- package/src/core/workflow/types.ts +11 -41
- package/src/core/workflow/workspaces/leases.ts +1 -1
- package/src/core/workflow/workspaces/local-git.ts +120 -239
- package/src/core/workflow/workspaces/ownership.ts +3 -158
- package/src/core/workflow/workspaces/reclaim.ts +3 -9
- package/src/core/workflow/workspaces/recovery.ts +1 -1
- package/src/core/workflow/workspaces/state.ts +1 -5
- package/src/lib/identities/assume.ts +0 -4
- package/src/lib/tunnel/state.ts +15 -70
- package/dist/commands/turn-rollback.d.ts +0 -5
- package/dist/commands/turn-rollback.d.ts.map +0 -1
- package/dist/commands/turn-rollback.js +0 -76
- package/dist/core/agents/state/process-handle.d.ts +0 -26
- package/dist/core/agents/state/process-handle.d.ts.map +0 -1
- package/dist/core/agents/state/process-handle.js +0 -121
- package/dist/core/agents/state/unchanged-file.d.ts +0 -44
- package/dist/core/agents/state/unchanged-file.d.ts.map +0 -1
- package/dist/core/agents/state/unchanged-file.js +0 -390
- package/dist/core/events/v3/generation-custody.d.ts +0 -5
- package/dist/core/events/v3/generation-custody.d.ts.map +0 -1
- package/dist/core/events/v3/generation-custody.js +0 -50
- package/dist/core/turn-rollback/content.d.ts +0 -53
- package/dist/core/turn-rollback/content.d.ts.map +0 -1
- package/dist/core/turn-rollback/content.js +0 -396
- package/dist/core/turn-rollback/index.d.ts +0 -72
- package/dist/core/turn-rollback/index.d.ts.map +0 -1
- package/dist/core/turn-rollback/index.js +0 -443
- package/dist/core/workflow/failure-evidence.d.ts +0 -16
- package/dist/core/workflow/failure-evidence.d.ts.map +0 -1
- package/dist/core/workflow/failure-evidence.js +0 -114
- package/dist/core/workflow/recovery-audit.d.ts +0 -30
- package/dist/core/workflow/recovery-audit.d.ts.map +0 -1
- package/dist/core/workflow/recovery-audit.js +0 -116
- package/src/commands/turn-rollback.ts +0 -111
- package/src/core/agents/state/process-handle.ts +0 -165
- package/src/core/agents/state/unchanged-file.ts +0 -456
- package/src/core/events/v3/generation-custody.ts +0 -59
- package/src/core/turn-rollback/content.ts +0 -499
- package/src/core/turn-rollback/index.ts +0 -580
- package/src/core/workflow/failure-evidence.ts +0 -141
- package/src/core/workflow/recovery-audit.ts +0 -181
|
@@ -338,23 +338,20 @@ export async function withFamilyLease<T>(
|
|
|
338
338
|
let owner: LeaseOwner | undefined;
|
|
339
339
|
while (true) {
|
|
340
340
|
try {
|
|
341
|
-
|
|
342
|
-
const leaseIdentity = captureDirectoryIdentity(lease);
|
|
343
|
-
owner = {
|
|
344
|
-
owner_id: randomUUID(),
|
|
345
|
-
pid: process.pid,
|
|
346
|
-
acquired_at: new Date().toISOString(),
|
|
347
|
-
};
|
|
348
|
-
writeNewFile(join(lease, "owner.json"), Buffer.from(`${JSON.stringify(owner)}\n`));
|
|
349
|
-
assertDirectoryIdentity(leaseIdentity);
|
|
341
|
+
owner = claimLease(lease);
|
|
350
342
|
break;
|
|
351
343
|
} catch (error) {
|
|
352
|
-
if (
|
|
344
|
+
if (leaseHasWrongType(lease)) {
|
|
353
345
|
throw new HarneryLogLeaseError("wrong_type", `log lease is not a directory: ${lease}`);
|
|
354
346
|
}
|
|
355
347
|
if (recoverStaleLease(lease, policy.lease_stale_ms)) continue;
|
|
356
348
|
if (Date.now() - started >= policy.lease_timeout_ms) {
|
|
357
|
-
throw new HarneryLogLeaseError(
|
|
349
|
+
throw new HarneryLogLeaseError(
|
|
350
|
+
"timeout",
|
|
351
|
+
leaseHasValidOwner(lease)
|
|
352
|
+
? `timed out acquiring log lease: ${lease}`
|
|
353
|
+
: `timed out acquiring log lease with no valid owner; an interrupted writer left it, and it is never recovered automatically: ${lease}`,
|
|
354
|
+
);
|
|
358
355
|
}
|
|
359
356
|
await new Promise((resolve) => setTimeout(resolve, policy.lease_retry_ms));
|
|
360
357
|
void error;
|
|
@@ -367,6 +364,41 @@ export async function withFamilyLease<T>(
|
|
|
367
364
|
}
|
|
368
365
|
}
|
|
369
366
|
|
|
367
|
+
/**
|
|
368
|
+
* Publish a complete lease in one rename. The owner record is written and synced
|
|
369
|
+
* in a private claim directory first, so a writer stopped at any point leaves
|
|
370
|
+
* either no lease or one with an owner. A held lease always has an owner record,
|
|
371
|
+
* here and in the earlier create-then-write protocol, so a rename that replaces
|
|
372
|
+
* an empty lease directory never takes a held lease. A lease with any entry makes
|
|
373
|
+
* the rename fail, and the caller waits or recovers as before.
|
|
374
|
+
*/
|
|
375
|
+
function claimLease(lease: string): LeaseOwner {
|
|
376
|
+
const owner: LeaseOwner = {
|
|
377
|
+
owner_id: randomUUID(),
|
|
378
|
+
pid: process.pid,
|
|
379
|
+
acquired_at: new Date().toISOString(),
|
|
380
|
+
};
|
|
381
|
+
const content = `${JSON.stringify(owner)}\n`;
|
|
382
|
+
const claim = `${lease}.claim-${process.pid}-${owner.owner_id}`;
|
|
383
|
+
try {
|
|
384
|
+
mkdirSync(claim, { mode: stateDirMode() });
|
|
385
|
+
writeNewFile(join(claim, "owner.json"), Buffer.from(content));
|
|
386
|
+
renameSync(claim, lease);
|
|
387
|
+
} finally {
|
|
388
|
+
rmSync(claim, { recursive: true, force: true });
|
|
389
|
+
}
|
|
390
|
+
try {
|
|
391
|
+
captureDirectoryIdentity(lease);
|
|
392
|
+
if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== content) {
|
|
393
|
+
throw new Error(`log lease owner changed during claim: ${lease}`);
|
|
394
|
+
}
|
|
395
|
+
} catch (error) {
|
|
396
|
+
releaseOwnedLease(lease, owner);
|
|
397
|
+
throw error;
|
|
398
|
+
}
|
|
399
|
+
return owner;
|
|
400
|
+
}
|
|
401
|
+
|
|
370
402
|
function shouldRotate(
|
|
371
403
|
active: string,
|
|
372
404
|
incoming: readonly Buffer[],
|
|
@@ -476,6 +508,22 @@ interface LeaseOwner {
|
|
|
476
508
|
acquired_at: string;
|
|
477
509
|
}
|
|
478
510
|
|
|
511
|
+
function leaseHasValidOwner(lease: string): boolean {
|
|
512
|
+
try {
|
|
513
|
+
const owner = JSON.parse(
|
|
514
|
+
readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
|
|
515
|
+
) as LeaseOwner;
|
|
516
|
+
return (
|
|
517
|
+
typeof owner?.owner_id === "string" &&
|
|
518
|
+
owner.owner_id.length > 0 &&
|
|
519
|
+
Number.isSafeInteger(owner.pid) &&
|
|
520
|
+
Number.isFinite(Date.parse(owner.acquired_at))
|
|
521
|
+
);
|
|
522
|
+
} catch {
|
|
523
|
+
return false;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
479
527
|
function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
|
|
480
528
|
let identity: DirectoryIdentity;
|
|
481
529
|
let raw: string;
|
|
@@ -504,7 +552,7 @@ function recoverStaleLease(lease: string, maximumAgeMs: number): boolean {
|
|
|
504
552
|
try {
|
|
505
553
|
assertDirectoryIdentity(identity);
|
|
506
554
|
if (readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8") !== raw) return false;
|
|
507
|
-
|
|
555
|
+
retireLease(lease, randomUUID());
|
|
508
556
|
return true;
|
|
509
557
|
} catch {
|
|
510
558
|
return false;
|
|
@@ -517,12 +565,23 @@ function releaseOwnedLease(lease: string, owner: LeaseOwner): void {
|
|
|
517
565
|
readRegularFile(join(lease, "owner.json"), 4_096).toString("utf8"),
|
|
518
566
|
) as LeaseOwner;
|
|
519
567
|
if (current.owner_id !== owner.owner_id) return;
|
|
520
|
-
|
|
568
|
+
retireLease(lease, owner.owner_id);
|
|
521
569
|
} catch {
|
|
522
570
|
// A missing or replaced lease does not belong to this operation.
|
|
523
571
|
}
|
|
524
572
|
}
|
|
525
573
|
|
|
574
|
+
/**
|
|
575
|
+
* Move a lease aside before deleting it. Its path goes from held to absent in
|
|
576
|
+
* one rename and is never an empty directory that a concurrent claim could
|
|
577
|
+
* replace while a recursive removal is still running.
|
|
578
|
+
*/
|
|
579
|
+
function retireLease(lease: string, tag: string): void {
|
|
580
|
+
const retired = `${lease}.retired-${process.pid}-${tag}`;
|
|
581
|
+
renameSync(lease, retired);
|
|
582
|
+
rmSync(retired, { recursive: true, force: true });
|
|
583
|
+
}
|
|
584
|
+
|
|
526
585
|
function processIsAlive(pid: number): boolean {
|
|
527
586
|
try {
|
|
528
587
|
process.kill(pid, 0);
|
|
@@ -699,11 +758,14 @@ function nativeWriteSync(fd: number, buffer: Buffer, offset: number, length: num
|
|
|
699
758
|
return writeSync(fd, buffer, offset, length);
|
|
700
759
|
}
|
|
701
760
|
|
|
702
|
-
|
|
761
|
+
/** One lstat, so a lease released between checks reads as absent, not as a wrong type. */
|
|
762
|
+
function leaseHasWrongType(path: string): boolean {
|
|
703
763
|
try {
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
764
|
+
const stat = lstatSync(path);
|
|
765
|
+
return stat.isSymbolicLink() || !stat.isDirectory();
|
|
766
|
+
} catch (error) {
|
|
767
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
768
|
+
throw error;
|
|
707
769
|
}
|
|
708
770
|
}
|
|
709
771
|
|
|
@@ -1,32 +1,26 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
closeSync,
|
|
5
|
-
existsSync,
|
|
6
|
-
mkdirSync,
|
|
7
|
-
openSync,
|
|
8
|
-
readdirSync,
|
|
9
|
-
readFileSync,
|
|
10
|
-
rmSync,
|
|
11
|
-
unlinkSync,
|
|
12
|
-
writeFileSync,
|
|
13
|
-
} from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
|
|
14
3
|
import { hostname } from "node:os";
|
|
15
4
|
import { resolve } from "node:path";
|
|
16
5
|
import { performance } from "node:perf_hooks";
|
|
17
6
|
import { fileURLToPath } from "node:url";
|
|
18
7
|
import { collectCoordinationHealthSnapshot } from "../agents/health.ts";
|
|
8
|
+
import { processStartToken } from "../agents/state/proc-start.ts";
|
|
19
9
|
import {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
10
|
+
acquireServiceLease,
|
|
11
|
+
requestServiceStop,
|
|
12
|
+
type ServiceLiveness,
|
|
13
|
+
type ServiceProcessDependencies,
|
|
14
|
+
type ServiceStopFields,
|
|
15
|
+
serviceLeaseOwner,
|
|
16
|
+
serviceStartRefusal,
|
|
17
|
+
} from "../agents/state/service-process.ts";
|
|
25
18
|
import { resolveBinName } from "../config.ts";
|
|
26
19
|
import { pressureHistoryFromSupervisor } from "../diagnostics/advice.ts";
|
|
27
20
|
import { assessPressure } from "../diagnostics/pressure.ts";
|
|
28
21
|
import { readEventV3ControlState } from "../events/v3/control.ts";
|
|
29
22
|
import { janitorLiveDisplayV3 } from "../events/v3/live-feed.ts";
|
|
23
|
+
import { installedHarneryBuild } from "../events/v3/runtime-build.ts";
|
|
30
24
|
import type { ResourceSamplerState } from "../resources/contract.ts";
|
|
31
25
|
import { sampleResources } from "../resources/sampler.ts";
|
|
32
26
|
import { requestResourceServiceStop } from "../resources/service.ts";
|
|
@@ -34,7 +28,7 @@ import { readResourceServiceStatus } from "../resources/service-status.ts";
|
|
|
34
28
|
import { resourcePaths } from "../resources/storage.ts";
|
|
35
29
|
import { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
|
|
36
30
|
import { closeProcessLoggers, legacyLogFields, processLogger } from "../storage/logger.ts";
|
|
37
|
-
import { stateDirMode
|
|
31
|
+
import { stateDirMode } from "../storage/modes.ts";
|
|
38
32
|
import { collectSupervisorActivitySnapshot } from "./activity.ts";
|
|
39
33
|
import {
|
|
40
34
|
SUPERVISOR_PRESSURE_SCHEMA_VERSION,
|
|
@@ -58,12 +52,7 @@ import type {
|
|
|
58
52
|
} from "./ledger-state.ts";
|
|
59
53
|
import { SupervisorLogCollector } from "./log-feed.ts";
|
|
60
54
|
import { collectServiceHealth } from "./services.ts";
|
|
61
|
-
import {
|
|
62
|
-
installedSupervisorBuild,
|
|
63
|
-
readSupervisorStatus,
|
|
64
|
-
type SupervisorProcessDependencies,
|
|
65
|
-
supervisorProcessLiveness,
|
|
66
|
-
} from "./status.ts";
|
|
55
|
+
import { readSupervisorStatus } from "./status.ts";
|
|
67
56
|
import {
|
|
68
57
|
readSupervisorFindings,
|
|
69
58
|
readSupervisorHistory,
|
|
@@ -80,14 +69,6 @@ const SUPERVISOR_MIN_IDLE_EXIT_MS = 5_000;
|
|
|
80
69
|
const SUPERVISOR_MAX_IDLE_EXIT_MS = 24 * 60 * 60_000;
|
|
81
70
|
const LIVE_DISPLAY_SWEEP_INTERVAL_MS = 5 * 60_000;
|
|
82
71
|
|
|
83
|
-
interface SupervisorLease {
|
|
84
|
-
pid: number;
|
|
85
|
-
start_token?: string;
|
|
86
|
-
host: string;
|
|
87
|
-
nonce: string;
|
|
88
|
-
created_at: string;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
72
|
export interface RunSupervisorInput {
|
|
92
73
|
coordRoot: string;
|
|
93
74
|
intervalMs?: number;
|
|
@@ -108,12 +89,10 @@ export async function spawnSupervisor(
|
|
|
108
89
|
): Promise<SupervisorStatus> {
|
|
109
90
|
const coordRoot = resolve(coordRootRaw);
|
|
110
91
|
const current = readSupervisorStatus(coordRoot);
|
|
111
|
-
if (current.liveness
|
|
112
|
-
if (current.liveness === "running") {
|
|
113
|
-
throw new Error(`local supervisor is already running under pid ${current.record?.pid}`);
|
|
114
|
-
}
|
|
92
|
+
if (current.liveness !== "stopped") throw startRefusal(current.record?.pid, current.liveness);
|
|
115
93
|
const paths = supervisorPaths(coordRoot);
|
|
116
|
-
|
|
94
|
+
const lease = serviceLeaseOwner(paths.lease);
|
|
95
|
+
if (lease.liveness !== "stopped") throw startRefusal(lease.lease?.pid, lease.liveness);
|
|
117
96
|
await stopLegacyResourceObserver(coordRoot);
|
|
118
97
|
const intervalMs = normalizeInterval(options.intervalMs);
|
|
119
98
|
const idleExitMs = normalizeIdleExit(options.idleExitMs);
|
|
@@ -179,44 +158,21 @@ export async function ensureSupervisorRunning(coordRoot: string): Promise<Ensure
|
|
|
179
158
|
}
|
|
180
159
|
}
|
|
181
160
|
|
|
182
|
-
export
|
|
183
|
-
signal_sent: boolean;
|
|
184
|
-
signal_verdict: ProcessGenerationVerdict;
|
|
185
|
-
}
|
|
161
|
+
export type SupervisorStopResult = SupervisorStatus & ServiceStopFields;
|
|
186
162
|
|
|
163
|
+
/** Windows has no graceful SIGTERM, so the durable request alone stops it there. */
|
|
187
164
|
export function requestSupervisorStop(
|
|
188
165
|
coordRootRaw: string,
|
|
189
|
-
dependencies:
|
|
166
|
+
dependencies: ServiceProcessDependencies = {},
|
|
190
167
|
): SupervisorStopResult {
|
|
191
168
|
const coordRoot = resolve(coordRootRaw);
|
|
192
|
-
const paths = supervisorPaths(coordRoot);
|
|
193
|
-
const status = readSupervisorStatus(coordRoot, Date.now(), dependencies);
|
|
194
|
-
mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
|
|
195
|
-
writePrivateJsonAtomic(paths.stop, {
|
|
196
|
-
requested_at: new Date().toISOString(),
|
|
197
|
-
requested_by_pid: process.pid,
|
|
198
|
-
});
|
|
199
169
|
const legacyPaths = resourcePaths(coordRoot);
|
|
200
170
|
if (existsSync(legacyPaths.service) || existsSync(legacyPaths.lease))
|
|
201
171
|
requestResourceServiceStop(coordRoot, dependencies);
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
process.platform !== "win32"
|
|
207
|
-
) {
|
|
208
|
-
signal = signalIfSameGeneration(
|
|
209
|
-
status.record.pid,
|
|
210
|
-
status.record.start_token,
|
|
211
|
-
"SIGTERM",
|
|
212
|
-
dependencies,
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
return {
|
|
216
|
-
...readSupervisorStatus(coordRoot, Date.now(), dependencies),
|
|
217
|
-
signal_sent: signal.sent,
|
|
218
|
-
signal_verdict: signal.verdict,
|
|
219
|
-
};
|
|
172
|
+
const read = () => readSupervisorStatus(coordRoot, Date.now(), dependencies);
|
|
173
|
+
return requestServiceStop(read(), read, supervisorPaths(coordRoot).stop, dependencies, {
|
|
174
|
+
signal: process.platform !== "win32",
|
|
175
|
+
});
|
|
220
176
|
}
|
|
221
177
|
|
|
222
178
|
export async function runSupervisor(
|
|
@@ -224,17 +180,14 @@ export async function runSupervisor(
|
|
|
224
180
|
): Promise<SupervisorServiceStatusRecord> {
|
|
225
181
|
const coordRoot = resolve(input.coordRoot);
|
|
226
182
|
const current = readSupervisorStatus(coordRoot);
|
|
227
|
-
if (current.liveness
|
|
228
|
-
if (current.liveness === "running") {
|
|
229
|
-
throw new Error(`local supervisor is already running under pid ${current.record?.pid}`);
|
|
230
|
-
}
|
|
183
|
+
if (current.liveness !== "stopped") throw startRefusal(current.record?.pid, current.liveness);
|
|
231
184
|
const paths = supervisorPaths(coordRoot);
|
|
232
185
|
const intervalMs = normalizeInterval(input.intervalMs);
|
|
233
186
|
const idleExitMs = normalizeIdleExit(input.idleExitMs);
|
|
234
187
|
const keepAlive = input.keepAlive ?? false;
|
|
235
188
|
const now = input.now ?? (() => new Date());
|
|
236
189
|
const wait = input.wait ?? delay;
|
|
237
|
-
const lease =
|
|
190
|
+
const lease = acquireServiceLease(paths.lease, startRefusal);
|
|
238
191
|
rmSync(paths.stop, { force: true });
|
|
239
192
|
const startedAt = now().toISOString();
|
|
240
193
|
const startToken = processStartToken(process.pid);
|
|
@@ -242,7 +195,7 @@ export async function runSupervisor(
|
|
|
242
195
|
schema_version: SUPERVISOR_STATUS_SCHEMA_VERSION,
|
|
243
196
|
pid: process.pid,
|
|
244
197
|
...(startToken ? { start_token: startToken } : {}),
|
|
245
|
-
running_build:
|
|
198
|
+
running_build: installedHarneryBuild(),
|
|
246
199
|
host: hostname(),
|
|
247
200
|
nonce: lease.nonce,
|
|
248
201
|
state: "starting",
|
|
@@ -488,7 +441,7 @@ export async function runSupervisor(
|
|
|
488
441
|
try {
|
|
489
442
|
await closeProcessLoggers();
|
|
490
443
|
} finally {
|
|
491
|
-
|
|
444
|
+
lease.release();
|
|
492
445
|
}
|
|
493
446
|
}
|
|
494
447
|
return status;
|
|
@@ -533,44 +486,6 @@ function observeLedgerControlState(
|
|
|
533
486
|
}
|
|
534
487
|
}
|
|
535
488
|
|
|
536
|
-
function acquireLease(coordRoot: string, now: Date): SupervisorLease {
|
|
537
|
-
const paths = supervisorPaths(coordRoot);
|
|
538
|
-
mkdirSync(paths.root, { recursive: true, mode: stateDirMode() });
|
|
539
|
-
for (let attempt = 0; attempt < 2; attempt++) {
|
|
540
|
-
const startToken = processStartToken(process.pid);
|
|
541
|
-
const lease: SupervisorLease = {
|
|
542
|
-
pid: process.pid,
|
|
543
|
-
...(startToken ? { start_token: startToken } : {}),
|
|
544
|
-
host: hostname(),
|
|
545
|
-
nonce: randomUUID(),
|
|
546
|
-
created_at: now.toISOString(),
|
|
547
|
-
};
|
|
548
|
-
try {
|
|
549
|
-
const fd = openSync(paths.lease, "wx", stateFileMode());
|
|
550
|
-
try {
|
|
551
|
-
writeFileSync(fd, `${JSON.stringify(lease)}\n`, "utf8");
|
|
552
|
-
} finally {
|
|
553
|
-
closeSync(fd);
|
|
554
|
-
}
|
|
555
|
-
return lease;
|
|
556
|
-
} catch (error) {
|
|
557
|
-
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
558
|
-
const existing = readLease(paths.lease);
|
|
559
|
-
const liveness = existing ? leaseIsLive(existing) : "unknown";
|
|
560
|
-
if (liveness === "unknown") throw unknownOwnerError();
|
|
561
|
-
if (existing && liveness === "running") {
|
|
562
|
-
throw new Error(`local supervisor lease is held by pid ${existing.pid}`);
|
|
563
|
-
}
|
|
564
|
-
try {
|
|
565
|
-
unlinkSync(paths.lease);
|
|
566
|
-
} catch {
|
|
567
|
-
// A concurrent starter may have replaced or removed the stale lease.
|
|
568
|
-
}
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
throw new Error("local supervisor could not acquire its singleton lease");
|
|
572
|
-
}
|
|
573
|
-
|
|
574
489
|
function pruneProjectionDirectory(
|
|
575
490
|
directory: string,
|
|
576
491
|
retainedFindingIds: ReadonlySet<string>,
|
|
@@ -582,49 +497,12 @@ function pruneProjectionDirectory(
|
|
|
582
497
|
}
|
|
583
498
|
}
|
|
584
499
|
|
|
585
|
-
function
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
// A missing lease is already released.
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
function readLease(path: string): SupervisorLease | undefined {
|
|
596
|
-
try {
|
|
597
|
-
const value = JSON.parse(readFileSync(path, "utf8")) as SupervisorLease;
|
|
598
|
-
return value &&
|
|
599
|
-
Number.isSafeInteger(value.pid) &&
|
|
600
|
-
value.pid > 0 &&
|
|
601
|
-
typeof value.host === "string" &&
|
|
602
|
-
value.host.length > 0 &&
|
|
603
|
-
typeof value.nonce === "string" &&
|
|
604
|
-
value.nonce.length > 0
|
|
605
|
-
? value
|
|
606
|
-
: undefined;
|
|
607
|
-
} catch {
|
|
608
|
-
return undefined;
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
function leaseIsLive(lease: SupervisorLease): SupervisorStatus["liveness"] {
|
|
613
|
-
return supervisorProcessLiveness(lease).liveness;
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
function assertLeaseAvailable(path: string): void {
|
|
617
|
-
if (!existsSync(path)) return;
|
|
618
|
-
const lease = readLease(path);
|
|
619
|
-
const liveness = lease ? leaseIsLive(lease) : "unknown";
|
|
620
|
-
if (liveness === "unknown") throw unknownOwnerError();
|
|
621
|
-
if (liveness === "running")
|
|
622
|
-
throw new Error(`local supervisor lease is held by pid ${lease?.pid}`);
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
function unknownOwnerError(): Error {
|
|
626
|
-
return new Error(
|
|
627
|
-
`local supervisor process identity is unknown; run ${resolveBinName()} supervisor stop to write the durable stop request before starting again`,
|
|
500
|
+
function startRefusal(pid: number | undefined, liveness: ServiceLiveness): Error {
|
|
501
|
+
return serviceStartRefusal(
|
|
502
|
+
"local supervisor",
|
|
503
|
+
`${resolveBinName()} supervisor stop`,
|
|
504
|
+
pid,
|
|
505
|
+
liveness,
|
|
628
506
|
);
|
|
629
507
|
}
|
|
630
508
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
|
|
2
2
|
import { hostname } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
|
-
import {
|
|
4
|
+
import { processStartToken, verifyProcessGeneration } from "../agents/state/proc-start.ts";
|
|
5
5
|
import { readGovernorServiceStatus } from "../governor/service-read.ts";
|
|
6
6
|
import { writePrivateJsonAtomic } from "../storage/atomic-json.ts";
|
|
7
7
|
import {
|
|
@@ -105,7 +105,7 @@ function consumerAlive(record: SupervisorConsumerRecord): boolean {
|
|
|
105
105
|
} catch (error) {
|
|
106
106
|
if ((error as NodeJS.ErrnoException).code !== "EPERM") return false;
|
|
107
107
|
}
|
|
108
|
-
return
|
|
108
|
+
return verifyProcessGeneration(record.pid, record.start_token) !== "different";
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
interface DetachedServiceRecord {
|
|
@@ -1,14 +1,11 @@
|
|
|
1
|
-
import { existsSync
|
|
2
|
-
import { hostname } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
4
2
|
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} from "../agents/state/
|
|
8
|
-
import {
|
|
3
|
+
observeServiceProcess,
|
|
4
|
+
type ServiceProcessDependencies,
|
|
5
|
+
} from "../agents/state/service-process.ts";
|
|
6
|
+
import { serviceBuildStatus } from "../events/v3/runtime-build.ts";
|
|
9
7
|
import {
|
|
10
8
|
SUPERVISOR_STATUS_SCHEMA_VERSION,
|
|
11
|
-
type SupervisorBuild,
|
|
12
9
|
type SupervisorServiceStatusRecord,
|
|
13
10
|
type SupervisorStatus,
|
|
14
11
|
} from "./contract.ts";
|
|
@@ -16,49 +13,14 @@ import { readSupervisorServiceRecord, supervisorPaths } from "./storage.ts";
|
|
|
16
13
|
|
|
17
14
|
const STALE_HEARTBEAT_MS = 15_000;
|
|
18
15
|
|
|
19
|
-
export interface SupervisorProcessDependencies {
|
|
20
|
-
readToken?: (pid: number) => string | null;
|
|
21
|
-
probePid?: (pid: number) => unknown;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export function installedSupervisorBuild(): SupervisorBuild {
|
|
25
|
-
let version = "unknown";
|
|
26
|
-
let revision = "unknown";
|
|
27
|
-
try {
|
|
28
|
-
const pkg = JSON.parse(readFileSync(join(harneryPackageRoot(), "package.json"), "utf8"));
|
|
29
|
-
if (typeof pkg.version === "string") version = pkg.version;
|
|
30
|
-
revision = currentHarneryRuntimeBuild();
|
|
31
|
-
} catch {
|
|
32
|
-
// Missing build evidence must still produce an inspectable status.
|
|
33
|
-
}
|
|
34
|
-
return { version, revision };
|
|
35
|
-
}
|
|
36
|
-
|
|
37
16
|
export function readSupervisorStatus(
|
|
38
17
|
coordRoot: string,
|
|
39
18
|
nowMs = Date.now(),
|
|
40
|
-
dependencies:
|
|
19
|
+
dependencies: ServiceProcessDependencies = {},
|
|
41
20
|
): SupervisorStatus {
|
|
42
21
|
const paths = supervisorPaths(coordRoot);
|
|
43
22
|
const record = readSupervisorServiceRecord(coordRoot);
|
|
44
|
-
const
|
|
45
|
-
const runningBuild = validBuild(record?.running_build) ? record.running_build : null;
|
|
46
|
-
const restartNeeded =
|
|
47
|
-
!runningBuild ||
|
|
48
|
-
installedBuild.version === "unknown" ||
|
|
49
|
-
installedBuild.revision === "unknown" ||
|
|
50
|
-
runningBuild.version !== installedBuild.version ||
|
|
51
|
-
runningBuild.revision !== installedBuild.revision;
|
|
52
|
-
const build = {
|
|
53
|
-
running_build: runningBuild,
|
|
54
|
-
installed_build: installedBuild,
|
|
55
|
-
restart_needed: restartNeeded,
|
|
56
|
-
build_status: !runningBuild
|
|
57
|
-
? "running build unknown; restart needed"
|
|
58
|
-
: restartNeeded
|
|
59
|
-
? "restart needed"
|
|
60
|
-
: "current",
|
|
61
|
-
};
|
|
23
|
+
const build = serviceBuildStatus(record?.running_build);
|
|
62
24
|
if (!validRecord(record)) {
|
|
63
25
|
return {
|
|
64
26
|
running: false,
|
|
@@ -72,13 +34,11 @@ export function readSupervisorStatus(
|
|
|
72
34
|
}
|
|
73
35
|
const heartbeatAt = Date.parse(record.heartbeat_at);
|
|
74
36
|
const stale = !Number.isFinite(heartbeatAt) || nowMs - heartbeatAt > STALE_HEARTBEAT_MS;
|
|
75
|
-
const
|
|
76
|
-
const liveness =
|
|
77
|
-
owner.generation === "same" && record.state === "stopped" ? "stopped" : owner.liveness;
|
|
37
|
+
const { generation, liveness } = observeServiceProcess(record, dependencies);
|
|
78
38
|
return {
|
|
79
|
-
running: liveness === "running" && !stale
|
|
39
|
+
running: liveness === "running" && !stale,
|
|
80
40
|
stale,
|
|
81
|
-
generation
|
|
41
|
+
generation,
|
|
82
42
|
liveness,
|
|
83
43
|
...build,
|
|
84
44
|
record,
|
|
@@ -99,38 +59,3 @@ function validRecord(
|
|
|
99
59
|
typeof value.keep_alive === "boolean"
|
|
100
60
|
);
|
|
101
61
|
}
|
|
102
|
-
|
|
103
|
-
/** A stale heartbeat does not prove that a process has exited. */
|
|
104
|
-
export function supervisorProcessLiveness(
|
|
105
|
-
owner: { pid: number; host: string; start_token?: string },
|
|
106
|
-
dependencies: SupervisorProcessDependencies = {},
|
|
107
|
-
): { liveness: SupervisorStatus["liveness"]; generation: ProcessGenerationVerdict } {
|
|
108
|
-
if (owner.host !== hostname()) return { liveness: "unknown", generation: "unknown" };
|
|
109
|
-
try {
|
|
110
|
-
(dependencies.probePid ?? ((pid) => process.kill(pid, 0)))(owner.pid);
|
|
111
|
-
} catch (error) {
|
|
112
|
-
return {
|
|
113
|
-
liveness: (error as NodeJS.ErrnoException).code === "ESRCH" ? "stopped" : "unknown",
|
|
114
|
-
generation: "unknown",
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
const generation = verifyProcessGeneration(owner.pid, owner.start_token, dependencies.readToken);
|
|
118
|
-
return {
|
|
119
|
-
generation,
|
|
120
|
-
liveness:
|
|
121
|
-
generation === "same" ? "running" : generation === "different" ? "stopped" : "unknown",
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function validBuild(value: unknown): value is SupervisorBuild {
|
|
126
|
-
if (!value || typeof value !== "object") return false;
|
|
127
|
-
const build = value as Partial<SupervisorBuild>;
|
|
128
|
-
return (
|
|
129
|
-
typeof build.version === "string" &&
|
|
130
|
-
!!build.version &&
|
|
131
|
-
build.version !== "unknown" &&
|
|
132
|
-
typeof build.revision === "string" &&
|
|
133
|
-
!!build.revision &&
|
|
134
|
-
build.revision !== "unknown"
|
|
135
|
-
);
|
|
136
|
-
}
|
package/src/core/work/state.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
import { stateDirMode, stateFileMode } from "../storage/modes.ts";
|
|
21
21
|
import { readWorkflowApproval } from "../workflow/approvals.ts";
|
|
22
22
|
import { stableDigest } from "../workflow/durable-record.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { readWorkflowProof } from "../workflow/proof.ts";
|
|
24
24
|
import { readWorkflowRunManifest, workflowScriptDigest } from "../workflow/run-state.ts";
|
|
25
25
|
import { WORKFLOW_TRANSCRIPT_EVENT_BYTES, workflowTranscriptPath } from "../workflow/transcript.ts";
|
|
26
26
|
import { isWorkspaceBoundExecutionEvidence } from "../workflow/workspaces/validate.ts";
|
|
@@ -718,14 +718,14 @@ function inspectAttempt(
|
|
|
718
718
|
proof.acceptance.summary.unknown === 0
|
|
719
719
|
? "succeeded"
|
|
720
720
|
: "failed";
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
//
|
|
721
|
+
// A failed attempt the run classified as uninformative about the work is
|
|
722
|
+
// uncharged (ADR 0046). Absent class ⇒ charged, as before. Read only for a
|
|
723
|
+
// failed attempt: a succeeded run never carries a class.
|
|
724
724
|
if (
|
|
725
725
|
attempt.status === "failed" &&
|
|
726
|
-
(proof.run.class === "
|
|
727
|
-
|
|
728
|
-
|
|
726
|
+
(proof.run.class === "environment" ||
|
|
727
|
+
proof.run.class === "upstream" ||
|
|
728
|
+
proof.run.class === "decision")
|
|
729
729
|
) {
|
|
730
730
|
attempt.uncharged = proof.run.class;
|
|
731
731
|
if (proof.run.class === "decision" && typeof proof.run.decision_id === "string") {
|