@codedology/code-forge 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +17 -1
- package/docs/reference/blocks.json +84 -1
- package/docs/reference/config.md +5 -0
- package/package.json +4 -2
- package/schema/code-forge.schema.json +12 -1
- package/skill/SKILL.md +1 -0
- package/skill/references/adapters/claude-code.md +4 -0
- package/skill/references/ledger.md +3 -1
- package/skill/references/plan.md +9 -0
- package/skill/references/review.md +5 -1
- package/skill/scripts/forge +1 -1
- package/src/cli/facts.mjs +16 -1
- package/src/cli/review.mjs +40 -9
- package/src/doctor/index.mjs +2 -1
- package/src/doctor/local.mjs +15 -0
- package/src/ledger/spend.mjs +123 -6
- package/src/session/facts.mjs +218 -53
- package/src/session/spawn.mjs +164 -20
- package/src/util/locks.mjs +104 -0
- package/src/worker/engine.mjs +109 -32
- package/src/worker/loop.mjs +197 -45
package/src/session/spawn.mjs
CHANGED
|
@@ -16,7 +16,15 @@
|
|
|
16
16
|
* Retry ladder (§5.6, V9): step 0 is the level's own model; step `k + 1` is `fallback[k]`. For a
|
|
17
17
|
* Claude coder, `fallback[0]` (same provider) rides on `--fallback-model` inside step 0 and is not
|
|
18
18
|
* spawned again. A step that reports unavailability (CLI missing, login expired, HTTP 402, or
|
|
19
|
-
* rate-limited
|
|
19
|
+
* rate-limited three times in a row) hands over to the next step; any other failure is final.
|
|
20
|
+
*
|
|
21
|
+
* Provider slots and backoff (B40): every attempt holds one slot of the module-level semaphore of
|
|
22
|
+
* its step's provider (`review.provider_concurrency[provider]`, defaults anthropic 4, openai 2,
|
|
23
|
+
* xai 2) for as long as the attempt runs, and frees it on every exit path. B41: the slot is taken
|
|
24
|
+
* FIRST, then the budget gate (check + reservation under the run's lock) runs; a refused session
|
|
25
|
+
* frees its slot at once, and a session waiting for a slot holds no reservation. A rate-limited attempt frees its slot, waits
|
|
26
|
+
* `RATE_LIMIT_BACKOFF_MS` (2 s, then 6 s, each ±30 % jitter) and tries the same step again; after
|
|
27
|
+
* the second wait a third rate limit moves the ladder to the next step.
|
|
20
28
|
*
|
|
21
29
|
* Every spawn prints `level=<Lx> provider=<p> model=<id> effort=<e> fallback_step=<n>` on stderr,
|
|
22
30
|
* and every attempt writes one ledger row (`event: session`) with `tokens_source` `reported` when
|
|
@@ -47,10 +55,11 @@ import { removeCodexHome } from '../engines/codex-home.mjs';
|
|
|
47
55
|
import { cliNameForProvider } from '../engines/provider-cli.mjs';
|
|
48
56
|
import { parseSentinel } from '../engines/sentinel.mjs';
|
|
49
57
|
import { parseUsage } from '../engines/usage-parse.mjs';
|
|
50
|
-
import { budgetUsdOf, checkBudget, priceSession, SPEND_UNREADABLE_MESSAGE } from '../ledger/spend.mjs';
|
|
58
|
+
import { budgetUsdOf, checkBudget, estimateSessionUsd, priceSession, reservedUsd, reserveBudget, SPEND_UNREADABLE_MESSAGE, withBudgetLock } from '../ledger/spend.mjs';
|
|
51
59
|
import { appendRow, readAllRows } from '../ledger/write.mjs';
|
|
52
60
|
import { exec } from '../util/exec.mjs';
|
|
53
61
|
import { isForbidden, mergeForbidden } from '../util/forbidden.mjs';
|
|
62
|
+
import { semaphore } from '../util/locks.mjs';
|
|
54
63
|
import { readStartTime, registerPid, UNKNOWN_START_TIME } from '../util/reaper.mjs';
|
|
55
64
|
import { redact, writeSafe } from '../util/redact.mjs';
|
|
56
65
|
import { currentRunRoot, pidsDir, runRoot, tmpBase, untrustedReason } from '../util/tmp.mjs';
|
|
@@ -64,6 +73,57 @@ export const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000;
|
|
|
64
73
|
/** Unavailability reasons that move the ladder to the next step (§5.6). */
|
|
65
74
|
export const UNAVAILABLE_REASONS = Object.freeze(['cli-missing', 'login-expired', 'http-402', 'rate-limited']);
|
|
66
75
|
|
|
76
|
+
/** B40: concurrent sessions per provider when `review.provider_concurrency` does not say. */
|
|
77
|
+
export const DEFAULT_PROVIDER_CONCURRENCY = Object.freeze({ anthropic: 4, openai: 2, xai: 2 });
|
|
78
|
+
|
|
79
|
+
/** B40: the waits before each retry of a rate-limited step (ms, before jitter); its length is the retry count. */
|
|
80
|
+
export const RATE_LIMIT_BACKOFF_MS = Object.freeze([2000, 6000]);
|
|
81
|
+
|
|
82
|
+
/** B40: each backoff wait is its base ±30 %. */
|
|
83
|
+
export const RATE_LIMIT_JITTER = 0.3;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* B40: the session limit of `provider` — `review.provider_concurrency[provider]` when it is an
|
|
87
|
+
* integer ≥ 1, else the default (1 for a provider with no default).
|
|
88
|
+
* @param {Record<string, any>} cfg @param {string} provider @returns {number}
|
|
89
|
+
*/
|
|
90
|
+
export function providerLimit(cfg, provider) {
|
|
91
|
+
const set = /** @type {any} */ (cfg)?.review?.provider_concurrency?.[provider];
|
|
92
|
+
if (Number.isInteger(set) && set >= 1) return set;
|
|
93
|
+
return Object.hasOwn(DEFAULT_PROVIDER_CONCURRENCY, provider) ? DEFAULT_PROVIDER_CONCURRENCY[/** @type {"anthropic"|"openai"|"xai"} */ (provider)] : 1;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** @type {Map<string, import('../util/locks.mjs').Semaphore>} one semaphore per (provider, limit), created on first use. */
|
|
97
|
+
const providerSemaphores = new Map();
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* B40: the in-process semaphore holding `provider`'s session slots at `limit`.
|
|
101
|
+
* @param {string} provider @param {number} limit
|
|
102
|
+
* @returns {import('../util/locks.mjs').Semaphore}
|
|
103
|
+
*/
|
|
104
|
+
export function providerSemaphore(provider, limit) {
|
|
105
|
+
const key = `${provider}:${limit}`;
|
|
106
|
+
let sem = providerSemaphores.get(key);
|
|
107
|
+
if (sem === undefined) {
|
|
108
|
+
sem = semaphore(limit);
|
|
109
|
+
providerSemaphores.set(key, sem);
|
|
110
|
+
}
|
|
111
|
+
return sem;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* B40: the wait before retry number `n` (0-based) of a rate-limited step: the base ±30 %, from
|
|
116
|
+
* `random` in [0, 1) (0.5 gives the base exactly).
|
|
117
|
+
* @param {number} n @param {() => number} random @returns {number}
|
|
118
|
+
*/
|
|
119
|
+
export function backoffMs(n, random) {
|
|
120
|
+
const base = RATE_LIMIT_BACKOFF_MS[n];
|
|
121
|
+
return Math.round(base * (1 + RATE_LIMIT_JITTER * (2 * random() - 1)));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** @param {number} ms @returns {Promise<void>} */
|
|
125
|
+
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
126
|
+
|
|
67
127
|
/** An error the spawner raises before anything runs (`code`: `usage`, `forbidden`, `closed-book`). */
|
|
68
128
|
export class SessionError extends Error {
|
|
69
129
|
/** @param {string} code @param {string} message */
|
|
@@ -445,6 +505,8 @@ function childEnv(/** @type {NodeJS.ProcessEnv} */ env) {
|
|
|
445
505
|
* `budget.usd` check reads (default: `readAllRows(opts.slug)`).
|
|
446
506
|
* @property {{write: (s: string) => unknown}} [stderr]
|
|
447
507
|
* @property {NodeJS.ProcessEnv} [env]
|
|
508
|
+
* @property {(ms: number) => Promise<unknown>} [sleep] - B40: the rate-limit backoff wait (tests: a recorder).
|
|
509
|
+
* @property {() => number} [random] - B40: the backoff jitter source, in [0, 1) (default `Math.random`).
|
|
448
510
|
*/
|
|
449
511
|
|
|
450
512
|
/**
|
|
@@ -569,11 +631,51 @@ export async function spawnSession(opts, deps = {}) {
|
|
|
569
631
|
if (opts.background) steps.length = 1;
|
|
570
632
|
|
|
571
633
|
const attempts = [];
|
|
572
|
-
|
|
634
|
+
const sleep = deps.sleep ?? realSleep;
|
|
635
|
+
const random = deps.random ?? Math.random;
|
|
636
|
+
let rateLimitWaits = 0; // backoff waits taken on the current step
|
|
573
637
|
for (let i = 0; i < steps.length; i += 1) {
|
|
574
638
|
const step = steps[i];
|
|
575
|
-
|
|
576
|
-
|
|
639
|
+
// B40/B41: the provider slot first, then the budget gate — a session queued for a slot never
|
|
640
|
+
// holds a reservation; a refusal frees the slot at once (the `finally` below)
|
|
641
|
+
const releaseSlot = await providerSemaphore(step.provider, providerLimit(cfg, step.provider)).acquire();
|
|
642
|
+
/** @type {number | null} */
|
|
643
|
+
let waitMs = null;
|
|
644
|
+
try {
|
|
645
|
+
const gate = await budgetGate(opts, deps, writeRow, stderr, step);
|
|
646
|
+
if (gate.refused) return { ...gate.refused, provider: step.provider, model: step.model, effort: step.effort ?? null, fallback_step: step.fallback_step, attempts };
|
|
647
|
+
// B41: the reservation is released once the row is written (inside runAttempt) and, as a net,
|
|
648
|
+
// in the `finally` below on every other exit path (a throw, a background start)
|
|
649
|
+
const releaseReservation = gate.release;
|
|
650
|
+
try {
|
|
651
|
+
const outcome = await runAttempt(step, i + 1 === steps.length, releaseReservation);
|
|
652
|
+
if (outcome.result) return outcome.result;
|
|
653
|
+
waitMs = outcome.waitMs;
|
|
654
|
+
} finally {
|
|
655
|
+
releaseReservation();
|
|
656
|
+
}
|
|
657
|
+
} finally {
|
|
658
|
+
releaseSlot();
|
|
659
|
+
}
|
|
660
|
+
if (waitMs !== null) {
|
|
661
|
+
// B40: the slot is free while we wait; the same step is tried again (through the budget gate)
|
|
662
|
+
writeSafe(stderr, `rate-limited: provider=${step.provider} model=${step.model} retry in ${(waitMs / 1000).toFixed(1)} s\n`);
|
|
663
|
+
await sleep(waitMs);
|
|
664
|
+
i -= 1;
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
rateLimitWaits = 0;
|
|
668
|
+
}
|
|
669
|
+
throw new Error('spawnSession: unreachable — the ladder always has a step');
|
|
670
|
+
|
|
671
|
+
/**
|
|
672
|
+
* One attempt of `step` (the caller holds its provider slot).
|
|
673
|
+
* @param {ReturnType<typeof ladderFor>[number]} step @param {boolean} isLast - no step after this one.
|
|
674
|
+
* @param {() => void} releaseReservation - B41: called as soon as the session row is written.
|
|
675
|
+
* @returns {Promise<{result: SessionResult | null, waitMs: number | null}>} a `result` ends the
|
|
676
|
+
* session; otherwise `waitMs` = back off and retry the step, null = go to the next step.
|
|
677
|
+
*/
|
|
678
|
+
async function runAttempt(step, isLast, releaseReservation) {
|
|
577
679
|
const sessionDir = path.join(runRootDir, 'sessions', `${role}-${Date.now()}-${randomBytes(4).toString('hex')}`);
|
|
578
680
|
mkdirSync(sessionDir, { recursive: true, mode: 0o700 });
|
|
579
681
|
let keep = false;
|
|
@@ -594,8 +696,12 @@ export async function spawnSession(opts, deps = {}) {
|
|
|
594
696
|
// background session's home stays until the run root is swept.
|
|
595
697
|
const bg = startBackground(built, sessionDir, runRootDir, { ...childEnv(deps.env ?? process.env), ...(built.env ?? {}) });
|
|
596
698
|
// a detached child's tokens are never read back: its price is unknown (use `ledger add coder`)
|
|
597
|
-
|
|
598
|
-
|
|
699
|
+
try {
|
|
700
|
+
if (writeRow) await writeRow({ event: 'session.background', ...base, pid: bg.pid, status: 'started', usd: null, usd_unknown: true });
|
|
701
|
+
} finally {
|
|
702
|
+
releaseReservation();
|
|
703
|
+
}
|
|
704
|
+
return { waitMs: null, result: { status: 'started', ...bg, provider: step.provider, model: step.model, effort: step.effort ?? null, fallback_step: step.fallback_step, attempts } };
|
|
599
705
|
}
|
|
600
706
|
|
|
601
707
|
const attempt = await runForeground(built, opts, deps);
|
|
@@ -611,21 +717,24 @@ export async function spawnSession(opts, deps = {}) {
|
|
|
611
717
|
duration_ms: attempt.duration_ms,
|
|
612
718
|
...(level === 'L3' && step.fallback_step > 0 ? { l3_fallback: true } : {}),
|
|
613
719
|
};
|
|
614
|
-
|
|
720
|
+
try {
|
|
721
|
+
if (writeRow) await writeRow(row);
|
|
722
|
+
} finally {
|
|
723
|
+
releaseReservation(); // B41: the row now carries the spend (or the write failed)
|
|
724
|
+
}
|
|
615
725
|
attempts.push({ fallback_step: step.fallback_step, provider: step.provider, model: step.model, status: attempt.status, reason: attempt.reason ?? null });
|
|
616
|
-
if (attempt.status === 'unavailable' && attempt.reason === 'rate-limited' &&
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
726
|
+
if (attempt.status === 'unavailable' && attempt.reason === 'rate-limited' && rateLimitWaits < RATE_LIMIT_BACKOFF_MS.length) {
|
|
727
|
+
// B40: back off (2 s, then 6 s, ±30 %) and try the same step again — never at once
|
|
728
|
+
const waitMs = backoffMs(rateLimitWaits, random);
|
|
729
|
+
rateLimitWaits += 1;
|
|
730
|
+
return { waitMs, result: null };
|
|
620
731
|
}
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
return { ...attempt, provider: step.provider, model: step.model, effort: step.effort ?? null, fallback_step: step.fallback_step, row, attempts };
|
|
732
|
+
if (attempt.status === 'unavailable' && !isLast) return { waitMs: null, result: null };
|
|
733
|
+
return { waitMs: null, result: { ...attempt, provider: step.provider, model: step.model, effort: step.effort ?? null, fallback_step: step.fallback_step, row, attempts } };
|
|
624
734
|
} finally {
|
|
625
735
|
if (!keep) rmSync(sessionDir, { recursive: true, force: true });
|
|
626
736
|
}
|
|
627
737
|
}
|
|
628
|
-
throw new Error('spawnSession: unreachable — the ladder always has a step');
|
|
629
738
|
}
|
|
630
739
|
|
|
631
740
|
/**
|
|
@@ -647,14 +756,49 @@ function sessionUsd(provider, level, usage) {
|
|
|
647
756
|
* read throws) the session is refused too. It sits at the top of the ladder loop, so every attempt
|
|
648
757
|
* — each fallback step, the rate-limit retry of a step, and every `spawnSession` call a caller
|
|
649
758
|
* makes again (B30's timeout retry) — passes it.
|
|
759
|
+
*
|
|
760
|
+
* B41: with a budget, the ledger read, the check and the reservation of this attempt's estimated
|
|
761
|
+
* cost (`estimateSessionUsd`: the packet's bytes / 4 in, a fixed tokens_out per role and level)
|
|
762
|
+
* run under the run's budget lock, so concurrent sessions of one run see each other's holds. No
|
|
763
|
+
* budget (or no run) ⇒ no lock, no reservation.
|
|
650
764
|
* @param {SessionOpts} opts @param {SessionDeps} deps
|
|
651
765
|
* @param {((row: Record<string, any>) => Promise<unknown>) | null} writeRow
|
|
652
766
|
* @param {{write: (s: string) => unknown}} stderr
|
|
653
|
-
* @
|
|
767
|
+
* @param {{provider: string}} step - the ladder step about to run (its provider prices the estimate).
|
|
768
|
+
* @returns {Promise<{refused: SessionResult, release?: undefined} | {refused: null, release: () => void}>}
|
|
769
|
+
* the refusal result, or the (idempotent) release of this attempt's reservation.
|
|
654
770
|
*/
|
|
655
|
-
async function budgetGate(opts, deps, writeRow, stderr) {
|
|
771
|
+
async function budgetGate(opts, deps, writeRow, stderr, step) {
|
|
656
772
|
const budget = budgetUsdOf(opts.cfg);
|
|
657
|
-
if (budget === null || typeof opts.run !== 'string' || opts.run.length === 0) return null;
|
|
773
|
+
if (budget === null || typeof opts.run !== 'string' || opts.run.length === 0) return { refused: null, release: () => {} };
|
|
774
|
+
const run = opts.run;
|
|
775
|
+
const estimate = estimateSessionUsd({ provider: step.provider, level: opts.level, role: opts.role, tokensIn: packetTokens(opts) });
|
|
776
|
+
return withBudgetLock(run, async () => {
|
|
777
|
+
const refused = await checkGate(opts, deps, writeRow, stderr, budget, estimate);
|
|
778
|
+
if (refused) return { refused };
|
|
779
|
+
return { refused: null, release: reserveBudget(run, estimate) };
|
|
780
|
+
});
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* The packet's (or brief's) tokens plus the system prompt's, by the ledger's bytes / 4 rule (an
|
|
785
|
+
* unreadable packet counts 0 bytes; tokens_out is still reserved).
|
|
786
|
+
* @param {SessionOpts} opts @returns {number}
|
|
787
|
+
*/
|
|
788
|
+
function packetTokens(opts) {
|
|
789
|
+
return estimateTokens(fileSize(opts.promptPath) + (typeof opts.systemPromptText === 'string' ? Buffer.byteLength(opts.systemPromptText) : 0));
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
/**
|
|
793
|
+
* The B33 check proper (called under the budget lock): read the run's spend, fail closed, refuse
|
|
794
|
+
* when spent + reserved + `estimate` passes the budget.
|
|
795
|
+
* @param {SessionOpts} opts - `opts.run` is a non-empty string (checked by `budgetGate`).
|
|
796
|
+
* @param {SessionDeps} deps
|
|
797
|
+
* @param {((row: Record<string, any>) => Promise<unknown>) | null} writeRow
|
|
798
|
+
* @param {{write: (s: string) => unknown}} stderr @param {number} budget @param {number} estimate
|
|
799
|
+
* @returns {Promise<SessionResult | null>} the refusal result, or null to go on.
|
|
800
|
+
*/
|
|
801
|
+
async function checkGate(opts, deps, writeRow, stderr, budget, estimate) {
|
|
658
802
|
/** @type {Array<Record<string, any>> | null} */
|
|
659
803
|
let rows = null;
|
|
660
804
|
try {
|
|
@@ -677,7 +821,7 @@ async function budgetGate(opts, deps, writeRow, stderr) {
|
|
|
677
821
|
}
|
|
678
822
|
return { status: 'unavailable', reason: 'budget', message: SPEND_UNREADABLE_MESSAGE, answer: null, spent_usd: null, budget_usd: budget };
|
|
679
823
|
}
|
|
680
|
-
const verdict = await checkBudget({ budget, run: opts.run, rows, block: opts.block ?? null, role: opts.role, writeRow, stderr });
|
|
824
|
+
const verdict = await checkBudget({ budget, run: /** @type {string} */ (opts.run), rows, block: opts.block ?? null, role: opts.role, writeRow, stderr, reserved: reservedUsd(/** @type {string} */ (opts.run)), estimate });
|
|
681
825
|
if (!verdict.refuse) return null;
|
|
682
826
|
return { status: 'unavailable', reason: 'budget', message: verdict.message, answer: null, spent_usd: verdict.spent, budget_usd: budget };
|
|
683
827
|
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process locks (parallel review, block B40): a keyed mutex and a counting semaphore.
|
|
3
|
+
*
|
|
4
|
+
* Both live in this process only — no files, no cross-process coordination — and use no timers,
|
|
5
|
+
* so a waiting lock never keeps the event loop alive on its own.
|
|
6
|
+
*
|
|
7
|
+
* `keyedLock(key, fn)`: calls for the same key run one at a time, in call order (FIFO); calls for
|
|
8
|
+
* different keys run together. A throw (or rejection) in `fn` releases the key and is passed on.
|
|
9
|
+
*
|
|
10
|
+
* `semaphore(n)`: `acquire()` resolves to a `release` function once fewer than `n` holders are
|
|
11
|
+
* active; waiters are served in order. `release` is idempotent (a second call does nothing).
|
|
12
|
+
* `run(fn)` acquires, runs `fn`, and releases on every exit path.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** @type {Map<unknown, Promise<void>>} the tail of each key's queue. */
|
|
16
|
+
const tails = new Map();
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Run `fn` once every earlier call for `key` has finished.
|
|
20
|
+
* @template T
|
|
21
|
+
* @param {unknown} key
|
|
22
|
+
* @param {() => T | Promise<T>} fn
|
|
23
|
+
* @returns {Promise<T>}
|
|
24
|
+
*/
|
|
25
|
+
export function keyedLock(key, fn) {
|
|
26
|
+
const prev = tails.get(key) ?? Promise.resolve();
|
|
27
|
+
/** @type {() => void} */
|
|
28
|
+
let release = () => {};
|
|
29
|
+
const done = new Promise((resolve) => {
|
|
30
|
+
release = () => resolve(undefined);
|
|
31
|
+
});
|
|
32
|
+
const tail = prev.then(() => done);
|
|
33
|
+
tails.set(key, tail);
|
|
34
|
+
return prev.then(async () => {
|
|
35
|
+
try {
|
|
36
|
+
return await fn();
|
|
37
|
+
} finally {
|
|
38
|
+
release();
|
|
39
|
+
if (tails.get(key) === tail) tails.delete(key);
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @typedef {object} Semaphore
|
|
46
|
+
* @property {() => Promise<() => void>} acquire - resolves to an idempotent `release`.
|
|
47
|
+
* @property {<T>(fn: () => T | Promise<T>) => Promise<T>} run - `fn` under one slot, released on every exit path.
|
|
48
|
+
* @property {number} active - holders right now.
|
|
49
|
+
* @property {number} pending - waiters right now.
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A counting semaphore: never more than `n` holders at once.
|
|
54
|
+
* @param {number} n - an integer ≥ 1.
|
|
55
|
+
* @returns {Semaphore}
|
|
56
|
+
*/
|
|
57
|
+
export function semaphore(n) {
|
|
58
|
+
if (!Number.isInteger(n) || n < 1) throw new TypeError('semaphore: n must be an integer >= 1');
|
|
59
|
+
let active = 0;
|
|
60
|
+
/** @type {Array<(release: () => void) => void>} */
|
|
61
|
+
const waiters = [];
|
|
62
|
+
|
|
63
|
+
/** @returns {() => void} */
|
|
64
|
+
function grant() {
|
|
65
|
+
active += 1;
|
|
66
|
+
let released = false;
|
|
67
|
+
return () => {
|
|
68
|
+
if (released) return;
|
|
69
|
+
released = true;
|
|
70
|
+
active -= 1;
|
|
71
|
+
const next = waiters.shift();
|
|
72
|
+
if (next) next(grant());
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** @returns {Promise<() => void>} */
|
|
77
|
+
function acquire() {
|
|
78
|
+
if (active < n && waiters.length === 0) return Promise.resolve(grant());
|
|
79
|
+
return new Promise((resolve) => {
|
|
80
|
+
waiters.push(resolve);
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** @template T @param {() => T | Promise<T>} fn @returns {Promise<T>} */
|
|
85
|
+
async function run(fn) {
|
|
86
|
+
const release = await acquire();
|
|
87
|
+
try {
|
|
88
|
+
return await fn();
|
|
89
|
+
} finally {
|
|
90
|
+
release();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return {
|
|
95
|
+
acquire,
|
|
96
|
+
run,
|
|
97
|
+
get active() {
|
|
98
|
+
return active;
|
|
99
|
+
},
|
|
100
|
+
get pending() {
|
|
101
|
+
return waiters.length;
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
package/src/worker/engine.mjs
CHANGED
|
@@ -45,7 +45,8 @@ import { computeFileSet, ownsFile } from '../gates/scope.mjs';
|
|
|
45
45
|
import { newFileState, runRound } from '../review/fixloop.mjs';
|
|
46
46
|
import { readRun } from '../state/run.mjs';
|
|
47
47
|
import { redact } from '../util/redact.mjs';
|
|
48
|
-
import {
|
|
48
|
+
import { keyedLock } from '../util/locks.mjs';
|
|
49
|
+
import { blockAnchors, blockRungUsed, loadState, saveState } from './review-state.mjs';
|
|
49
50
|
import { assertTicketId } from './ticket.mjs';
|
|
50
51
|
|
|
51
52
|
/**
|
|
@@ -207,47 +208,123 @@ export async function reviewTicket(ticket, ctx) {
|
|
|
207
208
|
}
|
|
208
209
|
}
|
|
209
210
|
|
|
211
|
+
/**
|
|
212
|
+
* The key of a block's lock (B42): the block's budget row and its L3 rung are taken under it.
|
|
213
|
+
* It carries the run, so two runs in one process never share a lock.
|
|
214
|
+
* @param {string} runId @param {string} block @returns {string}
|
|
215
|
+
*/
|
|
216
|
+
export const blockLockKey = (runId, block) => `block:${runId}\0${block}`;
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Whether the block's one L3 rung is taken: a verified anchor says so (`blockRungUsed`), OR another
|
|
220
|
+
* file of the block has a saved state whose round decided `next: patch` — that round took the rung,
|
|
221
|
+
* but its anchor records `l3_rung_used` only once the patch_check ticket saves. A sibling state that
|
|
222
|
+
* does not load is counted as taken (fail closed; its own ticket refuses it).
|
|
223
|
+
* @param {import('./review-state.mjs').Where} where
|
|
224
|
+
* @returns {{status: 'ok' | 'tampered', used: boolean}}
|
|
225
|
+
*/
|
|
226
|
+
export function rungTaken(where) {
|
|
227
|
+
const rung = blockRungUsed(where);
|
|
228
|
+
if (rung.status === 'tampered' || rung.used) return rung;
|
|
229
|
+
const anchors = blockAnchors(where) ?? [];
|
|
230
|
+
const siblings = new Set(anchors.map((a) => a.file).filter((f) => typeof f === 'string' && f !== where.file));
|
|
231
|
+
for (const file of siblings) {
|
|
232
|
+
let other;
|
|
233
|
+
try {
|
|
234
|
+
other = loadState({ ...where, file });
|
|
235
|
+
} catch {
|
|
236
|
+
return { status: 'ok', used: true };
|
|
237
|
+
}
|
|
238
|
+
if (other.status !== 'ok' || other.state?.next?.action === 'patch') return { status: 'ok', used: true };
|
|
239
|
+
}
|
|
240
|
+
return rung;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Whether this round could take the block's L3 rung (so it must run under the block lock): never
|
|
245
|
+
* when the rung is taken or the round is the patch_check (the rung is already this file's); never
|
|
246
|
+
* for a file's first full round while the cap and the per-level ladder are both above 1 (round 1
|
|
247
|
+
* cannot stall, reach the cap or exhaust the ladder); else conservatively yes.
|
|
248
|
+
* @param {import('../review/fixloop.mjs').FileState} state @param {'full' | 'recheck' | 'patch_check'} kind
|
|
249
|
+
* @param {Record<string, any>} cfg @param {boolean} rungUsed
|
|
250
|
+
* @returns {boolean}
|
|
251
|
+
*/
|
|
252
|
+
export function mayTakeRung(state, kind, cfg, rungUsed) {
|
|
253
|
+
if (rungUsed || kind === 'patch_check') return false;
|
|
254
|
+
const int = (/** @type {unknown} */ v, /** @type {number} */ d) => (Number.isInteger(v) && /** @type {number} */ (v) >= 0 ? /** @type {number} */ (v) : d);
|
|
255
|
+
const maxRounds = int(cfg?.review?.max_rounds_per_file, 4);
|
|
256
|
+
const perLevel = int(cfg?.escalation?.review_rounds_per_level, 2);
|
|
257
|
+
if (kind === 'full' && state.round === 0 && state.rounds_at_level === 0 && maxRounds > 1 && perLevel > 1) return false;
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
|
|
210
261
|
/**
|
|
211
262
|
* One fix-loop round for the ticket (see the module doc).
|
|
263
|
+
*
|
|
264
|
+
* Concurrency (B42): the worker runs several tickets at once but never two for the same
|
|
265
|
+
* (block, file) (its file lock, `loop.mjs`), so this file's state `seq` is read and saved by one
|
|
266
|
+
* ticket at a time. What the files of a BLOCK share is taken under the block lock
|
|
267
|
+
* (`blockLockKey`): the budget row's check-then-write (written once per block) and the L3 rung —
|
|
268
|
+
* the rung read and, for a round that could take the rung (`mayTakeRung`), the whole round and its
|
|
269
|
+
* state save, so two files of one block never both take it. A round that cannot take the rung (a
|
|
270
|
+
* first full round, or any round once the rung is gone) runs outside the block lock, in parallel
|
|
271
|
+
* with the block's other files.
|
|
212
272
|
* @param {import('./queue.mjs').Ticket} ticket @param {ReviewContext} ctx
|
|
213
273
|
* @param {{base: string | null, level: string, owned: string[], blockKindOf: string, key: Buffer, workDir: string, factsExcerpt: string}} opts
|
|
214
274
|
* @returns {Promise<ReviewOutcome>}
|
|
215
275
|
*/
|
|
216
276
|
async function fixLoopRound(ticket, ctx, { base, level, owned, blockKindOf, key, workDir, factsExcerpt }) {
|
|
217
277
|
if (!ctx.readRows || !ctx.writeRow) return { status: 'unavailable', reason: 'no-ledger', approved: false, engine: 'adaptive', sessions: [] };
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
278
|
+
const readRows = ctx.readRows;
|
|
279
|
+
/** @type {{outcome: ReviewOutcome} | {round: () => Promise<ReviewOutcome>}} */
|
|
280
|
+
const prepared = await keyedLock(blockLockKey(ctx.runId, ticket.block), async () => {
|
|
281
|
+
let rows;
|
|
282
|
+
try {
|
|
283
|
+
rows = (await readRows()).filter((r) => r?.run === ctx.runId);
|
|
284
|
+
} catch (err) {
|
|
285
|
+
return { outcome: /** @type {ReviewOutcome} */ ({ status: 'unavailable', reason: failureReason(err), approved: false, engine: 'adaptive', sessions: [] }) };
|
|
286
|
+
}
|
|
287
|
+
// Check-then-write under the block lock: no other ticket of this block reads the rows between
|
|
288
|
+
// this check and the budget row's write, so the row is written once per block.
|
|
289
|
+
if (!rows.some((r) => r?.event === 'review.budget' && r.block === ticket.block)) {
|
|
290
|
+
await recordBlockBudget(ticket, ctx, { base, owned });
|
|
291
|
+
}
|
|
292
|
+
const where = { runRootDir: ctx.runRootDir, runId: ctx.runId, block: ticket.block, file: ticket.file, key, rows };
|
|
293
|
+
const loaded = loadState(where);
|
|
294
|
+
const rung = rungTaken(where);
|
|
295
|
+
if (loaded.status === 'tampered' || rung.status === 'tampered') return { outcome: /** @type {ReviewOutcome} */ ({ status: 'refused', reason: 'review-state-tampered', approved: false, engine: 'adaptive', sessions: [] }) };
|
|
296
|
+
const rungUsed = rung.used;
|
|
297
|
+
let state = loaded.state;
|
|
298
|
+
if (state === null || state.status === 'complete') state = newFileState({ file: ticket.file, level, l3RungUsed: rungUsed });
|
|
299
|
+
state.l3_rung_used = state.l3_rung_used || rungUsed;
|
|
238
300
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
301
|
+
if (state.status === 'stopped') {
|
|
302
|
+
return { outcome: /** @type {ReviewOutcome} */ ({ ...loopFields(state, null), status: 'stopped', stopped: state.next?.reason ?? 'stopped', approved: false, engine: 'adaptive', sessions: [], findings: state.open }) };
|
|
303
|
+
}
|
|
242
304
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
305
|
+
/** @type {'full' | 'recheck' | 'patch_check'} */
|
|
306
|
+
let kind = state.round === 0 ? 'full' : 'recheck';
|
|
307
|
+
if (state.next?.action === 'retry') kind = state.pending_kind ?? kind;
|
|
308
|
+
else if (state.next?.action === 'patch') {
|
|
309
|
+
kind = 'patch_check';
|
|
310
|
+
state.l3_rung_used = true; // the orchestrator ran the block's one L3 patch before this ticket
|
|
311
|
+
}
|
|
312
|
+
const fixed = state;
|
|
313
|
+
const round = () => playRound(ticket, ctx, { state: fixed, kind, where, seq: loaded.seq, base, blockKindOf, workDir, factsExcerpt });
|
|
314
|
+
// a round that could take the rung keeps the block lock until its state is saved
|
|
315
|
+
if (mayTakeRung(fixed, kind, ctx.cfg, rungUsed)) return { outcome: await round() };
|
|
316
|
+
return { round };
|
|
317
|
+
});
|
|
318
|
+
return 'outcome' in prepared ? prepared.outcome : prepared.round();
|
|
319
|
+
}
|
|
250
320
|
|
|
321
|
+
/**
|
|
322
|
+
* Run the prepared round and save the file's state: the ticket's outcome.
|
|
323
|
+
* @param {import('./queue.mjs').Ticket} ticket @param {ReviewContext} ctx
|
|
324
|
+
* @param {{state: import('../review/fixloop.mjs').FileState, kind: 'full' | 'recheck' | 'patch_check', where: import('./review-state.mjs').Where, seq: number, base: string | null, blockKindOf: string, workDir: string, factsExcerpt: string}} opts
|
|
325
|
+
* @returns {Promise<ReviewOutcome>}
|
|
326
|
+
*/
|
|
327
|
+
async function playRound(ticket, ctx, { state, kind, where, seq, base, blockKindOf, workDir, factsExcerpt }) {
|
|
251
328
|
/** @type {Record<string, any> | null} */
|
|
252
329
|
let engineOutcome = null;
|
|
253
330
|
let approvalWritten = false;
|
|
@@ -307,7 +384,7 @@ async function fixLoopRound(ticket, ctx, { base, level, owned, blockKindOf, key,
|
|
|
307
384
|
state.pending_kind = kind;
|
|
308
385
|
}
|
|
309
386
|
}
|
|
310
|
-
await saveState({ ...where, state, seq
|
|
387
|
+
await saveState({ ...where, state, seq, writeRow: /** @type {(row: Record<string, any>) => Promise<unknown>} */ (ctx.writeRow) });
|
|
311
388
|
|
|
312
389
|
const eng = /** @type {Record<string, any> | null} */ (engineOutcome);
|
|
313
390
|
const head = { ...loopFields(state, kind), engine: eng?.engine ?? 'adaptive', sessions: eng?.sessions ?? rechecks };
|