@remnic/bench 9.7.4 → 9.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +27 -22
- package/dist/index.js +631 -206
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5886,8 +5886,49 @@ import { createHash as createHash4 } from "crypto";
|
|
|
5886
5886
|
import { mkdir as mkdir2, open, readFile as readFile3, rename as rename2, rmdir, unlink as unlink2, writeFile as writeFile2 } from "fs/promises";
|
|
5887
5887
|
import os from "os";
|
|
5888
5888
|
import path3 from "path";
|
|
5889
|
-
|
|
5890
|
-
|
|
5889
|
+
|
|
5890
|
+
// src/benchmark-run-blocked-error.ts
|
|
5891
|
+
var BenchmarkRunBlockReason = {
|
|
5892
|
+
InfrastructureUnavailable: "infrastructure_unavailable",
|
|
5893
|
+
ManualReconciliationRequired: "manual_reconciliation_required",
|
|
5894
|
+
SpendHeadroomExhausted: "spend_headroom_exhausted",
|
|
5895
|
+
SpendCeilingExceeded: "spend_ceiling_exceeded",
|
|
5896
|
+
ResourceLocked: "resource_locked"
|
|
5897
|
+
};
|
|
5898
|
+
var BENCHMARK_RUN_BLOCKED_ERROR_CODE = "REMNIC_BENCHMARK_RUN_BLOCKED";
|
|
5899
|
+
var BLOCK_REASONS = new Set(
|
|
5900
|
+
Object.values(BenchmarkRunBlockReason)
|
|
5901
|
+
);
|
|
5902
|
+
var BenchmarkRunBlockedError = class extends Error {
|
|
5903
|
+
code = BENCHMARK_RUN_BLOCKED_ERROR_CODE;
|
|
5904
|
+
reason;
|
|
5905
|
+
constructor(reason, message, options) {
|
|
5906
|
+
super(message, options);
|
|
5907
|
+
this.name = "BenchmarkRunBlockedError";
|
|
5908
|
+
this.reason = reason;
|
|
5909
|
+
}
|
|
5910
|
+
};
|
|
5911
|
+
function findBenchmarkRunBlockedError(error) {
|
|
5912
|
+
const visited = /* @__PURE__ */ new Set();
|
|
5913
|
+
let current = error;
|
|
5914
|
+
while (typeof current === "object" && current !== null) {
|
|
5915
|
+
if (visited.has(current)) {
|
|
5916
|
+
return void 0;
|
|
5917
|
+
}
|
|
5918
|
+
visited.add(current);
|
|
5919
|
+
const candidate = current;
|
|
5920
|
+
if (candidate.code === BENCHMARK_RUN_BLOCKED_ERROR_CODE && typeof candidate.reason === "string" && BLOCK_REASONS.has(candidate.reason)) {
|
|
5921
|
+
return current;
|
|
5922
|
+
}
|
|
5923
|
+
current = candidate.cause;
|
|
5924
|
+
}
|
|
5925
|
+
return void 0;
|
|
5926
|
+
}
|
|
5927
|
+
function isBenchmarkRunBlockedError(error) {
|
|
5928
|
+
return findBenchmarkRunBlockedError(error) !== void 0;
|
|
5929
|
+
}
|
|
5930
|
+
|
|
5931
|
+
// src/providers/codex-credit-budget.ts
|
|
5891
5932
|
var MAX_BOUNDED_CALL_CREDITS = 300;
|
|
5892
5933
|
var SOL_MODEL = /^gpt-5\.6-sol$/i;
|
|
5893
5934
|
var CREDIT_RATES = [
|
|
@@ -5901,6 +5942,10 @@ var CREDIT_RATES = [
|
|
|
5901
5942
|
[/^gpt-5\.2$/i, { input: 43.75, cachedInput: 4.375, output: 350 }]
|
|
5902
5943
|
];
|
|
5903
5944
|
var completionQueue = Promise.resolve();
|
|
5945
|
+
var failNextSettledLockWriteForTest = false;
|
|
5946
|
+
var failNextOwnedLockRemovalForTest = false;
|
|
5947
|
+
var failNextLedgerSetupForTest = false;
|
|
5948
|
+
var failNextLedgerWriteForTest = false;
|
|
5904
5949
|
var CodexCreditAccountingError = class extends Error {
|
|
5905
5950
|
constructor(message) {
|
|
5906
5951
|
super(message);
|
|
@@ -5914,26 +5959,34 @@ var CodexCreditDispatchError = class extends Error {
|
|
|
5914
5959
|
}
|
|
5915
5960
|
};
|
|
5916
5961
|
async function reconcileCodexCreditLedger(args) {
|
|
5917
|
-
if (args.
|
|
5918
|
-
throw new Error(
|
|
5919
|
-
"Codex credit reconciliation requires confirmation that the observed balance belongs to the ledger's original budget"
|
|
5920
|
-
);
|
|
5962
|
+
if (args.sameAccountConfirmed !== true) {
|
|
5963
|
+
throw new Error("Codex credit reconciliation requires confirmation that both balances belong to the same account");
|
|
5921
5964
|
}
|
|
5922
|
-
if (args.
|
|
5923
|
-
throw new Error(
|
|
5924
|
-
"Codex credit reconciliation requires confirmation that no credits were added or refunded after the original budget was established"
|
|
5925
|
-
);
|
|
5965
|
+
if (args.snapshotsBracketBlockedEventConfirmed !== true) {
|
|
5966
|
+
throw new Error("Codex credit reconciliation requires confirmation that the snapshots bracket the blocked event");
|
|
5926
5967
|
}
|
|
5927
|
-
if (args.
|
|
5968
|
+
if (args.balanceSettledConfirmed !== true) {
|
|
5969
|
+
throw new Error("Codex credit reconciliation requires confirmation that the displayed balances are settled");
|
|
5970
|
+
}
|
|
5971
|
+
if (args.noCreditsAddedOrRefundedConfirmed !== true) {
|
|
5928
5972
|
throw new Error(
|
|
5929
|
-
"Codex credit reconciliation requires
|
|
5973
|
+
"Codex credit reconciliation requires confirmation that no credits were added or refunded between snapshots"
|
|
5930
5974
|
);
|
|
5931
5975
|
}
|
|
5932
5976
|
if (!isSha256(args.priorLedgerSha256)) {
|
|
5933
5977
|
throw new Error("priorLedgerSha256 must be a lowercase SHA-256 digest");
|
|
5934
5978
|
}
|
|
5935
|
-
|
|
5936
|
-
|
|
5979
|
+
const beforeAccountBalance = parseExactDecimal(args.beforeAccountBalance, "beforeAccountBalance");
|
|
5980
|
+
const afterAccountBalance = parseExactDecimal(args.afterAccountBalance, "afterAccountBalance");
|
|
5981
|
+
const observedAccountDebit = subtractExactDecimals(beforeAccountBalance, afterAccountBalance);
|
|
5982
|
+
if (observedAccountDebit.startsWith("-")) {
|
|
5983
|
+
throw new Error("afterAccountBalance exceeds beforeAccountBalance; reconciliation refused");
|
|
5984
|
+
}
|
|
5985
|
+
const localBudgetChargeUnits = observedAccountDebit === "0" ? 0 : MAX_BOUNDED_CALL_CREDITS;
|
|
5986
|
+
if (localBudgetChargeUnits > 0 && args.noInterveningCodexActivityConfirmed !== true) {
|
|
5987
|
+
throw new Error(
|
|
5988
|
+
"a positive account-wide debit requires confirmation that no other Codex activity occurred between snapshots"
|
|
5989
|
+
);
|
|
5937
5990
|
}
|
|
5938
5991
|
const affectedRunId = parseRequiredRunId(args.affectedRunId, "affectedRunId");
|
|
5939
5992
|
const ledgerPath = path3.resolve(expandHomeRelativePath(args.ledgerPath));
|
|
@@ -5946,7 +5999,11 @@ async function reconcileCodexCreditLedger(args) {
|
|
|
5946
5999
|
const lockPath = `${ledgerPath}.lock`;
|
|
5947
6000
|
let lock;
|
|
5948
6001
|
try {
|
|
5949
|
-
|
|
6002
|
+
try {
|
|
6003
|
+
await prepareLedgerDirectory(lockPath);
|
|
6004
|
+
} catch (error) {
|
|
6005
|
+
throw infrastructureUnavailableError(error);
|
|
6006
|
+
}
|
|
5950
6007
|
lock = await acquireLedgerLock(lockPath);
|
|
5951
6008
|
const contents = await readFile3(ledgerPath);
|
|
5952
6009
|
const currentSha256 = sha256(contents);
|
|
@@ -5955,74 +6012,81 @@ async function reconcileCodexCreditLedger(args) {
|
|
|
5955
6012
|
`Codex credit ledger changed since operator observation: expected ${args.priorLedgerSha256}, found ${currentSha256}; obtain a fresh ledger hash and remaining balance`
|
|
5956
6013
|
);
|
|
5957
6014
|
}
|
|
5958
|
-
const ledger = parseLedger(JSON.parse(contents.toString("utf8")));
|
|
5959
|
-
if (!ledger.
|
|
6015
|
+
const ledger = parseLedger(JSON.parse(contents.toString("utf8")), contents);
|
|
6016
|
+
if (!ledger.blockedEvent) {
|
|
5960
6017
|
throw new Error("Codex credit ledger is not blocked; reconciliation is not permitted");
|
|
5961
6018
|
}
|
|
5962
|
-
if (
|
|
5963
|
-
throw new Error(
|
|
6019
|
+
if (ledger.blockedEvent.runId && ledger.blockedEvent.runId !== affectedRunId) {
|
|
6020
|
+
throw new Error(
|
|
6021
|
+
`affectedRunId does not match the blocked event run ID ${JSON.stringify(ledger.blockedEvent.runId)}`
|
|
6022
|
+
);
|
|
5964
6023
|
}
|
|
5965
|
-
const
|
|
5966
|
-
|
|
6024
|
+
const plannedSpendCeilingNanounits = budgetUnitsToNanounits(ledger.budgetUnits) - budgetUnitsToNanounits(ledger.reserveUnits);
|
|
6025
|
+
const spentNanounits = ledgerSpentNanounits(ledger);
|
|
6026
|
+
const localBudgetChargeNanounits = budgetUnitsToNanounits(localBudgetChargeUnits);
|
|
6027
|
+
if (plannedSpendCeilingNanounits - spentNanounits < localBudgetChargeNanounits) {
|
|
5967
6028
|
throw new Error(
|
|
5968
|
-
|
|
6029
|
+
`reconciliation requires ${localBudgetChargeUnits} local budget units but only ${nanounitsToBudgetUnits(
|
|
6030
|
+
plannedSpendCeilingNanounits > spentNanounits ? plannedSpendCeilingNanounits - spentNanounits : 0n
|
|
6031
|
+
)} remain below the planned-spend ceiling`
|
|
5969
6032
|
);
|
|
5970
6033
|
}
|
|
5971
|
-
const
|
|
5972
|
-
const
|
|
6034
|
+
const totalSpentNanounits = spentNanounits + localBudgetChargeNanounits;
|
|
6035
|
+
const totalSpentUnits = nanounitsToBudgetUnits(totalSpentNanounits);
|
|
5973
6036
|
const at = (/* @__PURE__ */ new Date()).toISOString();
|
|
5974
|
-
const
|
|
6037
|
+
const affectedBlockedEvent = ledger.blockedEvent;
|
|
6038
|
+
const resolution = {
|
|
5975
6039
|
at,
|
|
5976
|
-
basis: "operator-observed-
|
|
5977
|
-
attribution: "account-wide-
|
|
6040
|
+
basis: "operator-observed-account-balance-delta",
|
|
6041
|
+
attribution: "account-wide-observation-window",
|
|
5978
6042
|
priorLedgerSha256: currentSha256,
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
6043
|
+
priorRecordedSpentUnits: nanounitsToBudgetUnits(spentNanounits),
|
|
6044
|
+
priorEntryCount: ledger.entries.length,
|
|
6045
|
+
beforeAccountBalance,
|
|
6046
|
+
afterAccountBalance,
|
|
6047
|
+
observedAccountDebit,
|
|
6048
|
+
localBudgetChargeUnits,
|
|
5983
6049
|
confirmations: {
|
|
5984
|
-
|
|
6050
|
+
sameAccount: true,
|
|
6051
|
+
snapshotsBracketBlockedEvent: true,
|
|
6052
|
+
balanceSettled: true,
|
|
5985
6053
|
noCreditsAddedOrRefunded: true,
|
|
5986
|
-
|
|
6054
|
+
...args.noInterveningCodexActivityConfirmed === true ? { noInterveningCodexActivity: true } : {}
|
|
5987
6055
|
},
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
blockedReason: ledger.blockedReason
|
|
5991
|
-
}
|
|
6056
|
+
affectedRunId,
|
|
6057
|
+
affectedBlockedEvent
|
|
5992
6058
|
};
|
|
5993
6059
|
const nextLedger = {
|
|
5994
6060
|
...ledger,
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
6061
|
+
spentUnits: totalSpentUnits,
|
|
6062
|
+
resolutions: [...ledger.resolutions ?? [], resolution],
|
|
6063
|
+
blockedEvent: void 0
|
|
5998
6064
|
};
|
|
5999
|
-
await writeLedger(ledgerPath, nextLedger);
|
|
6000
|
-
await
|
|
6001
|
-
const nextContents = await readFile3(ledgerPath);
|
|
6065
|
+
const committed = await writeLedger(ledgerPath, nextLedger);
|
|
6066
|
+
await bestEffortSettleLock(lock);
|
|
6002
6067
|
return {
|
|
6003
|
-
schemaVersion:
|
|
6068
|
+
schemaVersion: 2,
|
|
6004
6069
|
priorLedgerSha256: currentSha256,
|
|
6005
|
-
ledgerSha256: sha256
|
|
6070
|
+
ledgerSha256: committed.sha256,
|
|
6006
6071
|
at,
|
|
6007
|
-
attribution: "account-wide-
|
|
6072
|
+
attribution: "account-wide-observation-window",
|
|
6008
6073
|
affectedRunId,
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
)
|
|
6074
|
+
priorRecordedSpentUnits: nanounitsToBudgetUnits(spentNanounits),
|
|
6075
|
+
beforeAccountBalance,
|
|
6076
|
+
afterAccountBalance,
|
|
6077
|
+
observedAccountDebit,
|
|
6078
|
+
localBudgetChargeUnits,
|
|
6079
|
+
totalSpentUnits,
|
|
6080
|
+
remainingPlannedSpendUnits: nanounitsToBudgetUnits(plannedSpendCeilingNanounits - totalSpentNanounits),
|
|
6081
|
+
affectedBlockedEventSha256: sha256(JSON.stringify(affectedBlockedEvent))
|
|
6018
6082
|
};
|
|
6019
6083
|
} finally {
|
|
6020
6084
|
try {
|
|
6021
6085
|
if (lock) {
|
|
6022
6086
|
try {
|
|
6023
|
-
await lock
|
|
6087
|
+
await bestEffortCloseLock(lock);
|
|
6024
6088
|
} finally {
|
|
6025
|
-
await
|
|
6089
|
+
await bestEffortRemoveOwnedLedgerLock(lockPath);
|
|
6026
6090
|
}
|
|
6027
6091
|
}
|
|
6028
6092
|
} finally {
|
|
@@ -6033,18 +6097,13 @@ async function reconcileCodexCreditLedger(args) {
|
|
|
6033
6097
|
function resolveCodexCreditBudgetConfig(env = process.env, fallbackRunId) {
|
|
6034
6098
|
const rawBudget = env.REMNIC_BENCH_CODEX_CREDIT_BUDGET?.trim();
|
|
6035
6099
|
if (!rawBudget) return void 0;
|
|
6036
|
-
const budgetCredits = parsePositiveNumber(
|
|
6037
|
-
rawBudget,
|
|
6038
|
-
"REMNIC_BENCH_CODEX_CREDIT_BUDGET"
|
|
6039
|
-
);
|
|
6100
|
+
const budgetCredits = parsePositiveNumber(rawBudget, "REMNIC_BENCH_CODEX_CREDIT_BUDGET");
|
|
6040
6101
|
const reserveCredits = parseNonNegativeNumber(
|
|
6041
6102
|
env.REMNIC_BENCH_CODEX_CREDIT_RESERVE?.trim() ?? "473",
|
|
6042
6103
|
"REMNIC_BENCH_CODEX_CREDIT_RESERVE"
|
|
6043
6104
|
);
|
|
6044
6105
|
if (reserveCredits >= budgetCredits) {
|
|
6045
|
-
throw new Error(
|
|
6046
|
-
"REMNIC_BENCH_CODEX_CREDIT_RESERVE must be smaller than REMNIC_BENCH_CODEX_CREDIT_BUDGET"
|
|
6047
|
-
);
|
|
6106
|
+
throw new Error("REMNIC_BENCH_CODEX_CREDIT_RESERVE must be smaller than REMNIC_BENCH_CODEX_CREDIT_BUDGET");
|
|
6048
6107
|
}
|
|
6049
6108
|
if (reserveCredits < MAX_BOUNDED_CALL_CREDITS) {
|
|
6050
6109
|
throw new Error(
|
|
@@ -6052,18 +6111,14 @@ function resolveCodexCreditBudgetConfig(env = process.env, fallbackRunId) {
|
|
|
6052
6111
|
);
|
|
6053
6112
|
}
|
|
6054
6113
|
const ledgerPath = path3.resolve(
|
|
6055
|
-
expandHomeRelativePath(
|
|
6056
|
-
env.REMNIC_BENCH_CODEX_CREDIT_LEDGER?.trim() || ".remnic/bench/codex-credit-ledger.json"
|
|
6057
|
-
)
|
|
6114
|
+
expandHomeRelativePath(env.REMNIC_BENCH_CODEX_CREDIT_LEDGER?.trim() || ".remnic/bench/codex-credit-ledger.json")
|
|
6058
6115
|
);
|
|
6059
6116
|
const runId = parseOptionalRunId(env.REMNIC_BENCH_RUN_ID) ?? parseOptionalRunId(fallbackRunId);
|
|
6060
6117
|
return {
|
|
6061
6118
|
budgetCredits,
|
|
6062
6119
|
reserveCredits,
|
|
6063
6120
|
ledgerPath,
|
|
6064
|
-
allowSol: /^(?:1|true|yes|on)$/i.test(
|
|
6065
|
-
env.REMNIC_BENCH_CODEX_ALLOW_SOL?.trim() ?? ""
|
|
6066
|
-
),
|
|
6121
|
+
allowSol: /^(?:1|true|yes|on)$/i.test(env.REMNIC_BENCH_CODEX_ALLOW_SOL?.trim() ?? ""),
|
|
6067
6122
|
...runId ? { runId } : {}
|
|
6068
6123
|
};
|
|
6069
6124
|
}
|
|
@@ -6071,6 +6126,12 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6071
6126
|
if (!args.config) {
|
|
6072
6127
|
return (await args.run()).value;
|
|
6073
6128
|
}
|
|
6129
|
+
try {
|
|
6130
|
+
budgetUnitsToNanounits(args.config.budgetCredits);
|
|
6131
|
+
budgetUnitsToNanounits(args.config.reserveCredits);
|
|
6132
|
+
} catch (error) {
|
|
6133
|
+
throw infrastructureUnavailableError(error);
|
|
6134
|
+
}
|
|
6074
6135
|
const previous = completionQueue;
|
|
6075
6136
|
let release;
|
|
6076
6137
|
completionQueue = new Promise((resolve) => {
|
|
@@ -6081,21 +6142,36 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6081
6142
|
let lock;
|
|
6082
6143
|
let dispatchStarted = false;
|
|
6083
6144
|
let accountingSettled = false;
|
|
6145
|
+
let ledgerCommitted = false;
|
|
6084
6146
|
try {
|
|
6085
|
-
|
|
6147
|
+
try {
|
|
6148
|
+
await prepareLedgerDirectory(lockPath);
|
|
6149
|
+
} catch (error) {
|
|
6150
|
+
throw infrastructureUnavailableError(error);
|
|
6151
|
+
}
|
|
6086
6152
|
lock = await acquireLedgerLock(lockPath);
|
|
6087
6153
|
assertModelAllowed(args.model, args.config);
|
|
6088
6154
|
const ledger = await readLedger(args.config);
|
|
6089
|
-
if (ledger.
|
|
6090
|
-
throw new
|
|
6091
|
-
|
|
6155
|
+
if (ledger.blockedEvent) {
|
|
6156
|
+
throw new BenchmarkRunBlockedError(
|
|
6157
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6158
|
+
"Codex credit ledger requires manual reconciliation.",
|
|
6159
|
+
{ cause: new Error(`Private ledger block: ${ledger.blockedEvent.reason}`) }
|
|
6092
6160
|
);
|
|
6093
6161
|
}
|
|
6094
|
-
const
|
|
6095
|
-
const
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6162
|
+
const usableNanounits = budgetUnitsToNanounits(args.config.budgetCredits) - budgetUnitsToNanounits(args.config.reserveCredits);
|
|
6163
|
+
const usableCredits = nanounitsToBudgetUnits(usableNanounits);
|
|
6164
|
+
const spentNanounits = ledgerSpentNanounits(ledger);
|
|
6165
|
+
const dispatchHeadroomNanounits = usableNanounits - spentNanounits;
|
|
6166
|
+
if (dispatchHeadroomNanounits < budgetUnitsToNanounits(MAX_BOUNDED_CALL_CREDITS)) {
|
|
6167
|
+
throw new BenchmarkRunBlockedError(
|
|
6168
|
+
BenchmarkRunBlockReason.SpendHeadroomExhausted,
|
|
6169
|
+
"Codex credit budget lacks conservative dispatch headroom.",
|
|
6170
|
+
{
|
|
6171
|
+
cause: new Error(
|
|
6172
|
+
`${nanounitsToBudgetUnits(spentNanounits)} local units spent; ${nanounitsToBudgetUnits(dispatchHeadroomNanounits)} available; ${MAX_BOUNDED_CALL_CREDITS} required.`
|
|
6173
|
+
)
|
|
6174
|
+
}
|
|
6099
6175
|
);
|
|
6100
6176
|
}
|
|
6101
6177
|
await writeLockState(lock, "in-flight");
|
|
@@ -6105,42 +6181,74 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6105
6181
|
result = await args.run();
|
|
6106
6182
|
} catch (error) {
|
|
6107
6183
|
if (error instanceof CodexCreditDispatchError) {
|
|
6108
|
-
await writeLockState(lock, "settled");
|
|
6109
|
-
accountingSettled = true;
|
|
6110
|
-
} else {
|
|
6111
|
-
const blockedLedger = {
|
|
6112
|
-
...ledger,
|
|
6113
|
-
blockedReason: error instanceof CodexCreditAccountingError ? error.message : `Codex dispatch outcome is unknown after an unexpected error: ${safeErrorMessage(error)}`
|
|
6114
|
-
};
|
|
6115
|
-
await writeLedger(args.config.ledgerPath, blockedLedger);
|
|
6116
|
-
await writeLockState(lock, "settled");
|
|
6117
6184
|
accountingSettled = true;
|
|
6185
|
+
await bestEffortSettleLock(lock);
|
|
6186
|
+
throw new BenchmarkRunBlockedError(
|
|
6187
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6188
|
+
"Codex CLI infrastructure was unavailable before dispatch.",
|
|
6189
|
+
{ cause: error }
|
|
6190
|
+
);
|
|
6118
6191
|
}
|
|
6119
|
-
|
|
6192
|
+
const blockedLedger = {
|
|
6193
|
+
...ledger,
|
|
6194
|
+
blockedEvent: {
|
|
6195
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6196
|
+
...args.config.runId ? { runId: args.config.runId } : {},
|
|
6197
|
+
model: args.model,
|
|
6198
|
+
reason: error instanceof CodexCreditAccountingError ? error.message : `Codex dispatch outcome is unknown after an unexpected error: ${safeErrorMessage(error)}`
|
|
6199
|
+
}
|
|
6200
|
+
};
|
|
6201
|
+
try {
|
|
6202
|
+
await writeLedger(args.config.ledgerPath, blockedLedger);
|
|
6203
|
+
} catch (persistenceError) {
|
|
6204
|
+
throw accountingPersistenceBlockedError(persistenceError, error);
|
|
6205
|
+
}
|
|
6206
|
+
ledgerCommitted = true;
|
|
6207
|
+
accountingSettled = true;
|
|
6208
|
+
await bestEffortSettleLock(lock);
|
|
6209
|
+
throw new BenchmarkRunBlockedError(
|
|
6210
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6211
|
+
"Codex usage accounting is uncertain; manual reconciliation is required.",
|
|
6212
|
+
{
|
|
6213
|
+
cause: error
|
|
6214
|
+
}
|
|
6215
|
+
);
|
|
6120
6216
|
}
|
|
6121
|
-
const
|
|
6122
|
-
const
|
|
6217
|
+
const creditNanounits = calculateCodexBudgetNanounits(args.model, result.usage);
|
|
6218
|
+
const credits = nanounitsToBudgetUnits(creditNanounits);
|
|
6219
|
+
const nextSpentNanounits = spentNanounits + creditNanounits;
|
|
6220
|
+
const nextSpent = nanounitsToBudgetUnits(nextSpentNanounits);
|
|
6123
6221
|
const nextLedger = {
|
|
6124
6222
|
...ledger,
|
|
6125
|
-
|
|
6223
|
+
spentUnits: nextSpent,
|
|
6126
6224
|
entries: [
|
|
6127
6225
|
...ledger.entries,
|
|
6128
6226
|
{
|
|
6129
6227
|
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6130
6228
|
model: args.model,
|
|
6131
|
-
credits,
|
|
6229
|
+
budgetUnits: credits,
|
|
6132
6230
|
...args.config.runId ? { runId: args.config.runId } : {},
|
|
6133
6231
|
...result.usage
|
|
6134
6232
|
}
|
|
6135
6233
|
]
|
|
6136
6234
|
};
|
|
6137
|
-
|
|
6138
|
-
|
|
6235
|
+
try {
|
|
6236
|
+
await writeLedger(args.config.ledgerPath, nextLedger);
|
|
6237
|
+
} catch (error) {
|
|
6238
|
+
throw accountingPersistenceBlockedError(error);
|
|
6239
|
+
}
|
|
6240
|
+
ledgerCommitted = true;
|
|
6139
6241
|
accountingSettled = true;
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6242
|
+
await bestEffortSettleLock(lock);
|
|
6243
|
+
try {
|
|
6244
|
+
args.onUsagePersisted?.(result.usage);
|
|
6245
|
+
} catch {
|
|
6246
|
+
}
|
|
6247
|
+
if (nextSpentNanounits > usableNanounits) {
|
|
6248
|
+
throw new BenchmarkRunBlockedError(
|
|
6249
|
+
BenchmarkRunBlockReason.SpendCeilingExceeded,
|
|
6250
|
+
"Codex planned-spend ceiling was exceeded by committed usage.",
|
|
6251
|
+
{ cause: new Error(`${nextSpent} local units spent exceeds the ${usableCredits} planned ceiling.`) }
|
|
6144
6252
|
);
|
|
6145
6253
|
}
|
|
6146
6254
|
return result.value;
|
|
@@ -6148,10 +6256,10 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6148
6256
|
try {
|
|
6149
6257
|
if (lock) {
|
|
6150
6258
|
try {
|
|
6151
|
-
await lock
|
|
6259
|
+
await bestEffortCloseLock(lock);
|
|
6152
6260
|
} finally {
|
|
6153
|
-
if (!dispatchStarted || accountingSettled) {
|
|
6154
|
-
await
|
|
6261
|
+
if (!dispatchStarted || accountingSettled || ledgerCommitted) {
|
|
6262
|
+
await bestEffortRemoveOwnedLedgerLock(lockPath);
|
|
6155
6263
|
}
|
|
6156
6264
|
}
|
|
6157
6265
|
}
|
|
@@ -6165,11 +6273,13 @@ async function acquireLedgerLock(lockPath) {
|
|
|
6165
6273
|
try {
|
|
6166
6274
|
await mkdir2(lockPath, { mode: 448 });
|
|
6167
6275
|
} catch (error) {
|
|
6168
|
-
if (error.code !== "EEXIST") throw error;
|
|
6276
|
+
if (error.code !== "EEXIST") throw infrastructureUnavailableError(error);
|
|
6169
6277
|
const owner = await readLockOwner(lockPath);
|
|
6170
6278
|
if (!owner || isProcessAlive(owner.pid) || owner.phase === "in-flight") {
|
|
6171
|
-
throw
|
|
6172
|
-
|
|
6279
|
+
throw resourceLockedError(
|
|
6280
|
+
new Error(
|
|
6281
|
+
`Codex credit ledger is locked${owner?.phase === "in-flight" ? " with unreconciled in-flight usage" : " by another benchmark process"} (${lockPath}); refusing credit spend.`
|
|
6282
|
+
)
|
|
6173
6283
|
);
|
|
6174
6284
|
}
|
|
6175
6285
|
await reclaimStaleLedgerLock(lockPath, owner);
|
|
@@ -6186,10 +6296,10 @@ async function acquireLedgerLock(lockPath) {
|
|
|
6186
6296
|
await unlink2(lockOwnerPath(lockPath)).catch(() => void 0);
|
|
6187
6297
|
await rmdir(lockHeldPath(lockPath)).catch(() => void 0);
|
|
6188
6298
|
await rmdir(lockPath).catch(() => void 0);
|
|
6189
|
-
throw error;
|
|
6299
|
+
throw infrastructureUnavailableError(error);
|
|
6190
6300
|
}
|
|
6191
6301
|
}
|
|
6192
|
-
throw new Error(`Unable to acquire Codex credit ledger lock (${lockPath})`);
|
|
6302
|
+
throw resourceLockedError(new Error(`Unable to acquire Codex credit ledger lock (${lockPath})`));
|
|
6193
6303
|
}
|
|
6194
6304
|
async function readLockOwner(lockPath) {
|
|
6195
6305
|
try {
|
|
@@ -6206,14 +6316,17 @@ async function reclaimStaleLedgerLock(lockPath, expectedOwner) {
|
|
|
6206
6316
|
try {
|
|
6207
6317
|
await rmdir(lockHeldPath(lockPath));
|
|
6208
6318
|
} catch (error) {
|
|
6209
|
-
throw
|
|
6210
|
-
|
|
6319
|
+
throw resourceLockedError(
|
|
6320
|
+
new Error(
|
|
6321
|
+
`Codex credit ledger stale-lock reclamation is already claimed or incomplete (${lockPath}); refusing credit spend: ${safeErrorMessage(error)}`,
|
|
6322
|
+
{ cause: error }
|
|
6323
|
+
)
|
|
6211
6324
|
);
|
|
6212
6325
|
}
|
|
6213
6326
|
const currentOwner = await readLockOwner(lockPath);
|
|
6214
6327
|
if (!currentOwner || currentOwner.pid !== expectedOwner.pid || currentOwner.phase !== expectedOwner.phase || isProcessAlive(currentOwner.pid) || currentOwner.phase === "in-flight") {
|
|
6215
|
-
throw
|
|
6216
|
-
`Codex credit ledger owner changed during stale-lock reclamation (${lockPath}); refusing credit spend.`
|
|
6328
|
+
throw resourceLockedError(
|
|
6329
|
+
new Error(`Codex credit ledger owner changed during stale-lock reclamation (${lockPath}); refusing credit spend.`)
|
|
6217
6330
|
);
|
|
6218
6331
|
}
|
|
6219
6332
|
await unlink2(lockOwnerPath(lockPath));
|
|
@@ -6231,6 +6344,10 @@ function lockHeldPath(lockPath) {
|
|
|
6231
6344
|
return path3.join(lockPath, "held");
|
|
6232
6345
|
}
|
|
6233
6346
|
async function writeLockState(lock, phase) {
|
|
6347
|
+
if (phase === "settled" && failNextSettledLockWriteForTest) {
|
|
6348
|
+
failNextSettledLockWriteForTest = false;
|
|
6349
|
+
throw new Error("injected settled lock-state failure");
|
|
6350
|
+
}
|
|
6234
6351
|
const contents = `${JSON.stringify({
|
|
6235
6352
|
pid: process.pid,
|
|
6236
6353
|
phase,
|
|
@@ -6259,12 +6376,8 @@ function parseCodexJsonlUsage(output) {
|
|
|
6259
6376
|
if (event.type !== "turn.completed" || !event.usage) continue;
|
|
6260
6377
|
const inputTokens = readCounter(event.usage.input_tokens);
|
|
6261
6378
|
const outputTokens = readCounter(event.usage.output_tokens);
|
|
6262
|
-
const cachedInputTokens = readOptionalCounter(
|
|
6263
|
-
|
|
6264
|
-
);
|
|
6265
|
-
const reasoningOutputTokens = readOptionalCounter(
|
|
6266
|
-
event.usage.reasoning_output_tokens
|
|
6267
|
-
);
|
|
6379
|
+
const cachedInputTokens = readOptionalCounter(event.usage.cached_input_tokens);
|
|
6380
|
+
const reasoningOutputTokens = readOptionalCounter(event.usage.reasoning_output_tokens);
|
|
6268
6381
|
if (inputTokens !== void 0 && outputTokens !== void 0 && cachedInputTokens !== void 0 && reasoningOutputTokens !== void 0) {
|
|
6269
6382
|
usage = {
|
|
6270
6383
|
inputTokens,
|
|
@@ -6278,29 +6391,38 @@ function parseCodexJsonlUsage(output) {
|
|
|
6278
6391
|
}
|
|
6279
6392
|
return usage;
|
|
6280
6393
|
}
|
|
6281
|
-
function
|
|
6394
|
+
function calculateCodexBudgetUnits(model, usage) {
|
|
6395
|
+
return nanounitsToBudgetUnits(calculateCodexBudgetNanounits(model, usage));
|
|
6396
|
+
}
|
|
6397
|
+
function calculateCodexBudgetNanounits(model, usage) {
|
|
6282
6398
|
const rate = resolveRate(model);
|
|
6283
6399
|
const cached = Math.min(usage.inputTokens, usage.cachedInputTokens);
|
|
6284
6400
|
const uncached = usage.inputTokens - cached;
|
|
6285
|
-
return (uncached * rate.input + cached * rate.cachedInput + usage.outputTokens * rate.output)
|
|
6401
|
+
return BigInt(uncached) * rateNanounitsPerToken(rate.input) + BigInt(cached) * rateNanounitsPerToken(rate.cachedInput) + BigInt(usage.outputTokens) * rateNanounitsPerToken(rate.output);
|
|
6286
6402
|
}
|
|
6287
6403
|
async function buildCodexCreditReceipt(ledgerPath, runId) {
|
|
6288
6404
|
const resolvedPath = path3.resolve(expandHomeRelativePath(ledgerPath));
|
|
6289
6405
|
const contents = await readFile3(resolvedPath);
|
|
6290
|
-
const ledger = parseLedger(JSON.parse(contents.toString("utf8")));
|
|
6406
|
+
const ledger = parseLedger(JSON.parse(contents.toString("utf8")), contents);
|
|
6291
6407
|
const normalizedRunId = parseOptionalRunId(runId);
|
|
6292
|
-
const
|
|
6293
|
-
const
|
|
6408
|
+
const spentNanounits = ledgerSpentNanounits(ledger);
|
|
6409
|
+
const budgetNanounits = budgetUnitsToNanounits(ledger.budgetUnits);
|
|
6410
|
+
const reserveNanounits = budgetUnitsToNanounits(ledger.reserveUnits);
|
|
6411
|
+
const cumulative = summarizeLedgerEntries(
|
|
6412
|
+
ledger.entries,
|
|
6413
|
+
ledger.resolutions ?? [],
|
|
6414
|
+
ledger.legacyReconciliations ?? []
|
|
6415
|
+
);
|
|
6294
6416
|
const runEntries = normalizedRunId ? ledger.entries.filter((entry) => entry.runId === normalizedRunId) : [];
|
|
6295
6417
|
return {
|
|
6296
|
-
schemaVersion:
|
|
6418
|
+
schemaVersion: 2,
|
|
6297
6419
|
ledgerSha256: sha256(contents),
|
|
6298
|
-
|
|
6299
|
-
|
|
6300
|
-
|
|
6301
|
-
|
|
6302
|
-
|
|
6303
|
-
blocked: ledger.
|
|
6420
|
+
budgetUnits: ledger.budgetUnits,
|
|
6421
|
+
reserveUnits: ledger.reserveUnits,
|
|
6422
|
+
plannedSpendCeilingUnits: nanounitsToBudgetUnits(budgetNanounits - reserveNanounits),
|
|
6423
|
+
totalSpentUnits: nanounitsToBudgetUnits(spentNanounits),
|
|
6424
|
+
remainingBudgetUnits: nanounitsToBudgetUnits(budgetNanounits - spentNanounits),
|
|
6425
|
+
blocked: ledger.blockedEvent !== void 0,
|
|
6304
6426
|
cumulative,
|
|
6305
6427
|
...normalizedRunId ? {
|
|
6306
6428
|
run: {
|
|
@@ -6320,52 +6442,86 @@ function resolveRate(model) {
|
|
|
6320
6442
|
return match[1];
|
|
6321
6443
|
}
|
|
6322
6444
|
function assertModelAllowed(model, config) {
|
|
6323
|
-
|
|
6445
|
+
try {
|
|
6446
|
+
resolveRate(model);
|
|
6447
|
+
} catch (error) {
|
|
6448
|
+
throw new BenchmarkRunBlockedError(
|
|
6449
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6450
|
+
"Configured Codex model is unsupported by the bounded budget.",
|
|
6451
|
+
{ cause: error }
|
|
6452
|
+
);
|
|
6453
|
+
}
|
|
6324
6454
|
if (SOL_MODEL.test(model) && !config.allowSol) {
|
|
6325
|
-
throw new
|
|
6326
|
-
|
|
6455
|
+
throw new BenchmarkRunBlockedError(
|
|
6456
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6457
|
+
"Configured Codex model is disallowed by bounded-budget policy.",
|
|
6458
|
+
{
|
|
6459
|
+
cause: new Error(
|
|
6460
|
+
"gpt-5.6-sol is disabled for bounded benchmark runs because it is the most expensive GPT-5.6 tier. Use gpt-5.6-terra or gpt-5.6-luna, or explicitly set REMNIC_BENCH_CODEX_ALLOW_SOL=1."
|
|
6461
|
+
)
|
|
6462
|
+
}
|
|
6327
6463
|
);
|
|
6328
6464
|
}
|
|
6329
6465
|
}
|
|
6330
6466
|
async function readLedger(config) {
|
|
6331
6467
|
try {
|
|
6332
|
-
const
|
|
6333
|
-
|
|
6334
|
-
)
|
|
6335
|
-
if (parsed.budgetCredits !== config.budgetCredits || parsed.reserveCredits !== config.reserveCredits) {
|
|
6468
|
+
const contents = await readFile3(config.ledgerPath);
|
|
6469
|
+
const parsed = parseLedger(JSON.parse(contents.toString("utf8")), contents);
|
|
6470
|
+
if (parsed.budgetUnits !== config.budgetCredits || parsed.reserveUnits !== config.reserveCredits) {
|
|
6336
6471
|
throw new Error("ledger schema or budget does not match this run");
|
|
6337
6472
|
}
|
|
6338
6473
|
return parsed;
|
|
6339
6474
|
} catch (error) {
|
|
6340
6475
|
if (error.code !== "ENOENT") {
|
|
6341
|
-
throw new
|
|
6476
|
+
throw new BenchmarkRunBlockedError(
|
|
6477
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6478
|
+
"Codex credit ledger is invalid or incompatible with this run.",
|
|
6479
|
+
{ cause: new Error(`Invalid Codex credit ledger at ${config.ledgerPath}: ${String(error)}`, { cause: error }) }
|
|
6480
|
+
);
|
|
6342
6481
|
}
|
|
6343
6482
|
return {
|
|
6344
|
-
schemaVersion:
|
|
6345
|
-
|
|
6346
|
-
|
|
6347
|
-
|
|
6483
|
+
schemaVersion: 2,
|
|
6484
|
+
budgetUnits: config.budgetCredits,
|
|
6485
|
+
reserveUnits: config.reserveCredits,
|
|
6486
|
+
spentUnits: 0,
|
|
6348
6487
|
entries: []
|
|
6349
6488
|
};
|
|
6350
6489
|
}
|
|
6351
6490
|
}
|
|
6352
|
-
function parseLedger(parsed) {
|
|
6353
|
-
|
|
6354
|
-
|
|
6355
|
-
|
|
6356
|
-
)
|
|
6357
|
-
const reconciliationCredits = parsed.reconciliations === void 0 ? 0 : Array.isArray(parsed.reconciliations) ? parsed.reconciliations.reduce(
|
|
6358
|
-
(sum, reconciliation) => sum + (typeof reconciliation?.credits === "number" ? reconciliation.credits ?? 0 : 0),
|
|
6359
|
-
0
|
|
6360
|
-
) : Number.NaN;
|
|
6361
|
-
if (parsed.schemaVersion !== 1 || typeof parsed.budgetCredits !== "number" || !Number.isFinite(parsed.budgetCredits) || parsed.budgetCredits <= 0 || typeof parsed.reserveCredits !== "number" || !Number.isFinite(parsed.reserveCredits) || parsed.reserveCredits < 0 || parsed.reserveCredits >= parsed.budgetCredits || typeof parsed.spentCredits !== "number" || !Number.isFinite(parsed.spentCredits) || parsed.spentCredits < 0 || !Array.isArray(parsed.entries) || !parsed.entries.every(isLedgerEntry) || parsed.reconciliations !== void 0 && (!Array.isArray(parsed.reconciliations) || !parsed.reconciliations.every(
|
|
6362
|
-
(reconciliation) => isLedgerReconciliationWithinBudget(reconciliation, parsed.budgetCredits)
|
|
6363
|
-
)) || Math.abs(entryCredits + reconciliationCredits - parsed.spentCredits) > 1e-9 || parsed.blockedReason !== void 0 && typeof parsed.blockedReason !== "string") {
|
|
6491
|
+
function parseLedger(parsed, sourceContents) {
|
|
6492
|
+
if (isLedgerV1(parsed)) return migrateLedgerV1(parsed, sourceContents);
|
|
6493
|
+
if (!parsed || typeof parsed !== "object") throw new Error("ledger schema is invalid");
|
|
6494
|
+
const candidate = parsed;
|
|
6495
|
+
if (candidate.schemaVersion !== 2 || !isPositiveFinite(candidate.budgetUnits) || !isSupportedBudgetUnits(candidate.budgetUnits) || !isNonNegativeFinite(candidate.reserveUnits) || !isSupportedBudgetUnits(candidate.reserveUnits) || candidate.reserveUnits >= candidate.budgetUnits || !isNonNegativeFinite(candidate.spentUnits) || !Array.isArray(candidate.entries) || !candidate.entries.every(isLedgerEntry) || candidate.resolutions !== void 0 && (!Array.isArray(candidate.resolutions) || !candidate.resolutions.every(isLedgerResolution)) || candidate.legacyReconciliations !== void 0 && (!Array.isArray(candidate.legacyReconciliations) || !candidate.legacyReconciliations.every(isLedgerReconciliationV1)) || !isLedgerSpentConsistent(candidate) || !isResolutionHistoryConsistent(candidate) || !isMigrationWitnessConsistent(candidate) || candidate.blockedEvent !== void 0 && !isBlockedEvent(candidate.blockedEvent) || candidate.migratedFromV1Sha256 !== void 0 && !isSha256(candidate.migratedFromV1Sha256)) {
|
|
6364
6496
|
throw new Error("ledger schema is invalid");
|
|
6365
6497
|
}
|
|
6366
|
-
return
|
|
6498
|
+
return candidate;
|
|
6367
6499
|
}
|
|
6368
|
-
function
|
|
6500
|
+
function isLedgerV1(value) {
|
|
6501
|
+
if (!value || typeof value !== "object") return false;
|
|
6502
|
+
const candidate = value;
|
|
6503
|
+
const entries = candidate.entries;
|
|
6504
|
+
const reconciliations = candidate.reconciliations;
|
|
6505
|
+
return candidate.schemaVersion === 1 && isPositiveFinite(candidate.budgetCredits) && isSupportedBudgetUnits(candidate.budgetCredits) && isNonNegativeFinite(candidate.reserveCredits) && isSupportedBudgetUnits(candidate.reserveCredits) && candidate.reserveCredits < candidate.budgetCredits && isNonNegativeFinite(candidate.spentCredits) && isSupportedBudgetUnits(candidate.spentCredits) && Array.isArray(entries) && entries.every(isLedgerEntryV1) && (reconciliations === void 0 || Array.isArray(reconciliations) && reconciliations.every((item) => isLedgerReconciliationWithinBudgetV1(item, candidate.budgetCredits))) && Math.abs(
|
|
6506
|
+
entries.reduce((sum, entry) => sum + entry.credits, 0) + (reconciliations ?? []).reduce((sum, item) => sum + item.credits, 0) - candidate.spentCredits
|
|
6507
|
+
) <= 1e-9 && (candidate.blockedReason === void 0 || typeof candidate.blockedReason === "string" && candidate.blockedReason.length > 0);
|
|
6508
|
+
}
|
|
6509
|
+
function migrateLedgerV1(ledger, sourceContents) {
|
|
6510
|
+
const source = sourceContents ? Buffer.from(sourceContents).toString("utf8") : `${JSON.stringify(ledger)}
|
|
6511
|
+
`;
|
|
6512
|
+
return {
|
|
6513
|
+
schemaVersion: 2,
|
|
6514
|
+
budgetUnits: ledger.budgetCredits,
|
|
6515
|
+
reserveUnits: ledger.reserveCredits,
|
|
6516
|
+
spentUnits: ledger.spentCredits,
|
|
6517
|
+
entries: ledger.entries.map(({ credits, ...entry }) => ({ ...entry, budgetUnits: credits })),
|
|
6518
|
+
...ledger.reconciliations?.length ? { legacyReconciliations: ledger.reconciliations } : {},
|
|
6519
|
+
migratedFromV1Sha256: sha256(source),
|
|
6520
|
+
migrationWitnessV1: { source },
|
|
6521
|
+
...ledger.blockedReason ? { blockedEvent: { reason: ledger.blockedReason } } : {}
|
|
6522
|
+
};
|
|
6523
|
+
}
|
|
6524
|
+
function isLedgerReconciliationV1(value) {
|
|
6369
6525
|
if (!value || typeof value !== "object") return false;
|
|
6370
6526
|
const candidate = value;
|
|
6371
6527
|
const forbiddenUsageFields = [
|
|
@@ -6377,26 +6533,118 @@ function isLedgerReconciliation(value) {
|
|
|
6377
6533
|
"outputTokens",
|
|
6378
6534
|
"reasoningOutputTokens"
|
|
6379
6535
|
];
|
|
6380
|
-
return forbiddenUsageFields.every((field) => !Object.prototype.hasOwnProperty.call(candidate, field)) && isIsoTimestamp(candidate.at) && candidate.basis === "operator-observed-original-budget-balance" && candidate.attribution === "account-wide-unattributed" && isSha256(candidate.priorLedgerSha256) && typeof candidate.originalBudgetCredits === "number" && Number.isFinite(candidate.originalBudgetCredits) && candidate.originalBudgetCredits > 0 && typeof candidate.priorRecordedSpentCredits === "number" && Number.isFinite(candidate.priorRecordedSpentCredits) && candidate.priorRecordedSpentCredits >= 0 && typeof candidate.observedRemainingCredits === "number" && Number.isFinite(candidate.observedRemainingCredits) && candidate.observedRemainingCredits >= 0 && typeof candidate.credits === "number" && Number.isFinite(candidate.credits) && candidate.credits >= 0 && candidate.confirmations?.observedBalanceBelongsToOriginalBudget === true && candidate.confirmations.noCreditsAddedOrRefunded === true && candidate.confirmations.accountWideUnattributedChargeAccepted === true && isValidStoredRunId(candidate.affectedBlockedEvent?.runId) && typeof candidate.affectedBlockedEvent?.blockedReason === "string" && candidate.affectedBlockedEvent.blockedReason.length > 0;
|
|
6536
|
+
return forbiddenUsageFields.every((field) => !Object.prototype.hasOwnProperty.call(candidate, field)) && isIsoTimestamp(candidate.at) && candidate.basis === "operator-observed-original-budget-balance" && candidate.attribution === "account-wide-unattributed" && isSha256(candidate.priorLedgerSha256) && typeof candidate.originalBudgetCredits === "number" && Number.isFinite(candidate.originalBudgetCredits) && candidate.originalBudgetCredits > 0 && isSupportedBudgetUnits(candidate.originalBudgetCredits) && typeof candidate.priorRecordedSpentCredits === "number" && Number.isFinite(candidate.priorRecordedSpentCredits) && candidate.priorRecordedSpentCredits >= 0 && isSupportedBudgetUnits(candidate.priorRecordedSpentCredits) && typeof candidate.observedRemainingCredits === "number" && Number.isFinite(candidate.observedRemainingCredits) && candidate.observedRemainingCredits >= 0 && isSupportedBudgetUnits(candidate.observedRemainingCredits) && typeof candidate.credits === "number" && Number.isFinite(candidate.credits) && candidate.credits >= 0 && isSupportedBudgetUnits(candidate.credits) && candidate.confirmations?.observedBalanceBelongsToOriginalBudget === true && candidate.confirmations.noCreditsAddedOrRefunded === true && candidate.confirmations.accountWideUnattributedChargeAccepted === true && isValidStoredRunId(candidate.affectedBlockedEvent?.runId) && typeof candidate.affectedBlockedEvent?.blockedReason === "string" && candidate.affectedBlockedEvent.blockedReason.length > 0;
|
|
6537
|
+
}
|
|
6538
|
+
function isLedgerReconciliationWithinBudgetV1(value, budget) {
|
|
6539
|
+
return typeof budget === "number" && isLedgerReconciliationV1(value) && value.originalBudgetCredits === budget && value.observedRemainingCredits <= budget && value.credits <= budget && Math.abs(value.priorRecordedSpentCredits + value.credits + value.observedRemainingCredits - budget) <= 1e-9;
|
|
6540
|
+
}
|
|
6541
|
+
function isLedgerResolution(value) {
|
|
6542
|
+
if (!value || typeof value !== "object") return false;
|
|
6543
|
+
const candidate = value;
|
|
6544
|
+
return isIsoTimestamp(candidate.at) && candidate.basis === "operator-observed-account-balance-delta" && candidate.attribution === "account-wide-observation-window" && isSha256(candidate.priorLedgerSha256) && isNonNegativeFinite(candidate.priorRecordedSpentUnits) && Number.isSafeInteger(candidate.priorEntryCount) && candidate.priorEntryCount >= 0 && typeof candidate.beforeAccountBalance === "string" && isExactDecimal(candidate.beforeAccountBalance) && typeof candidate.afterAccountBalance === "string" && isExactDecimal(candidate.afterAccountBalance) && typeof candidate.observedAccountDebit === "string" && isExactDecimal(candidate.observedAccountDebit) && subtractExactDecimals(candidate.beforeAccountBalance, candidate.afterAccountBalance) === candidate.observedAccountDebit && (candidate.localBudgetChargeUnits === 0 || candidate.localBudgetChargeUnits === MAX_BOUNDED_CALL_CREDITS) && (candidate.observedAccountDebit === "0" ? candidate.localBudgetChargeUnits === 0 : candidate.localBudgetChargeUnits === MAX_BOUNDED_CALL_CREDITS) && candidate.confirmations?.sameAccount === true && candidate.confirmations.snapshotsBracketBlockedEvent === true && candidate.confirmations.balanceSettled === true && candidate.confirmations.noCreditsAddedOrRefunded === true && (candidate.observedAccountDebit === "0" || candidate.confirmations.noInterveningCodexActivity === true) && isValidStoredRunId(candidate.affectedRunId) && isBlockedEvent(candidate.affectedBlockedEvent) && (candidate.affectedBlockedEvent.runId === void 0 || candidate.affectedBlockedEvent.runId === candidate.affectedRunId);
|
|
6545
|
+
}
|
|
6546
|
+
function isResolutionHistoryConsistent(ledger) {
|
|
6547
|
+
const resolutions = ledger.resolutions ?? [];
|
|
6548
|
+
if (resolutions.length === 0) return true;
|
|
6549
|
+
const legacyNanounits = sumBudgetUnitNanounits(
|
|
6550
|
+
(ledger.legacyReconciliations ?? []).map((reconciliation) => reconciliation.credits)
|
|
6551
|
+
);
|
|
6552
|
+
let priorResolutionNanounits = 0n;
|
|
6553
|
+
let priorEntryCount = 0;
|
|
6554
|
+
const seenHashes = /* @__PURE__ */ new Set();
|
|
6555
|
+
for (let index = 0; index < resolutions.length; index += 1) {
|
|
6556
|
+
const resolution = resolutions[index];
|
|
6557
|
+
if (!resolution) return false;
|
|
6558
|
+
if (seenHashes.has(resolution.priorLedgerSha256)) return false;
|
|
6559
|
+
seenHashes.add(resolution.priorLedgerSha256);
|
|
6560
|
+
if (resolution.priorEntryCount < priorEntryCount || resolution.priorEntryCount > ledger.entries.length)
|
|
6561
|
+
return false;
|
|
6562
|
+
const entriesBeforeResolution = ledger.entries.slice(0, resolution.priorEntryCount);
|
|
6563
|
+
const expectedPriorSpentNanounits = sumBudgetUnitNanounits(entriesBeforeResolution.map((entry) => entry.budgetUnits)) + legacyNanounits + priorResolutionNanounits;
|
|
6564
|
+
if (budgetUnitsToNanounits(resolution.priorRecordedSpentUnits) !== expectedPriorSpentNanounits) return false;
|
|
6565
|
+
if (!(index === 0 && isDirectV1PredecessorResolution(ledger, resolution))) {
|
|
6566
|
+
const priorLedger = {
|
|
6567
|
+
...ledger,
|
|
6568
|
+
spentUnits: resolution.priorRecordedSpentUnits,
|
|
6569
|
+
entries: entriesBeforeResolution,
|
|
6570
|
+
resolutions: index > 0 ? resolutions.slice(0, index) : void 0,
|
|
6571
|
+
blockedEvent: resolution.affectedBlockedEvent
|
|
6572
|
+
};
|
|
6573
|
+
if (sha256(serializeLedger(priorLedger)) !== resolution.priorLedgerSha256) return false;
|
|
6574
|
+
}
|
|
6575
|
+
priorResolutionNanounits += budgetUnitsToNanounits(resolution.localBudgetChargeUnits);
|
|
6576
|
+
priorEntryCount = resolution.priorEntryCount;
|
|
6577
|
+
}
|
|
6578
|
+
return true;
|
|
6381
6579
|
}
|
|
6382
|
-
function
|
|
6383
|
-
|
|
6384
|
-
|
|
6385
|
-
|
|
6580
|
+
function isMigrationWitnessConsistent(ledger) {
|
|
6581
|
+
if (!ledger.migratedFromV1Sha256 && !ledger.migrationWitnessV1) return true;
|
|
6582
|
+
if (!ledger.migratedFromV1Sha256 || !ledger.migrationWitnessV1) return false;
|
|
6583
|
+
const source = ledger.migrationWitnessV1.source;
|
|
6584
|
+
if (typeof source !== "string" || sha256(source) !== ledger.migratedFromV1Sha256) return false;
|
|
6585
|
+
let predecessor;
|
|
6586
|
+
try {
|
|
6587
|
+
predecessor = JSON.parse(source);
|
|
6588
|
+
} catch {
|
|
6589
|
+
return false;
|
|
6590
|
+
}
|
|
6591
|
+
if (!isLedgerV1(predecessor)) return false;
|
|
6592
|
+
if (predecessor.budgetCredits !== ledger.budgetUnits || predecessor.reserveCredits !== ledger.reserveUnits)
|
|
6593
|
+
return false;
|
|
6594
|
+
if (JSON.stringify(predecessor.reconciliations ?? []) !== JSON.stringify(ledger.legacyReconciliations ?? [])) {
|
|
6595
|
+
return false;
|
|
6596
|
+
}
|
|
6597
|
+
const migratedEntries = predecessor.entries.map(({ credits, ...entry }) => ({ ...entry, budgetUnits: credits }));
|
|
6598
|
+
const firstResolution = ledger.resolutions?.[0];
|
|
6599
|
+
const entriesAtMigration = firstResolution ? ledger.entries.slice(0, firstResolution.priorEntryCount) : ledger.entries.slice(0, migratedEntries.length);
|
|
6600
|
+
if (JSON.stringify(migratedEntries) !== JSON.stringify(entriesAtMigration.slice(0, migratedEntries.length))) {
|
|
6601
|
+
return false;
|
|
6602
|
+
}
|
|
6603
|
+
return true;
|
|
6604
|
+
}
|
|
6605
|
+
function isDirectV1PredecessorResolution(ledger, resolution) {
|
|
6606
|
+
if (!ledger.migratedFromV1Sha256 || !ledger.migrationWitnessV1) return false;
|
|
6607
|
+
let predecessor;
|
|
6608
|
+
try {
|
|
6609
|
+
predecessor = JSON.parse(ledger.migrationWitnessV1.source);
|
|
6610
|
+
} catch {
|
|
6611
|
+
return false;
|
|
6612
|
+
}
|
|
6613
|
+
return isLedgerV1(predecessor) && resolution.priorEntryCount === predecessor.entries.length && resolution.priorLedgerSha256 === ledger.migratedFromV1Sha256 && budgetUnitsToNanounits(resolution.priorRecordedSpentUnits) === budgetUnitsToNanounits(predecessor.spentCredits) && predecessor.blockedReason === resolution.affectedBlockedEvent.reason;
|
|
6614
|
+
}
|
|
6615
|
+
function isBlockedEvent(value) {
|
|
6616
|
+
if (!value || typeof value !== "object") return false;
|
|
6617
|
+
const candidate = value;
|
|
6618
|
+
return typeof candidate.reason === "string" && candidate.reason.length > 0 && (candidate.at === void 0 || isIsoTimestamp(candidate.at)) && (candidate.runId === void 0 || isValidStoredRunId(candidate.runId)) && (candidate.model === void 0 || typeof candidate.model === "string" && candidate.model.length > 0);
|
|
6619
|
+
}
|
|
6620
|
+
function isLedgerEntryV1(entry) {
|
|
6621
|
+
if (!entry || typeof entry !== "object") return false;
|
|
6622
|
+
const candidate = entry;
|
|
6623
|
+
return isLedgerEntryCommon(candidate) && isNonNegativeFinite(candidate.credits) && isSupportedBudgetUnits(candidate.credits) && isEntryCreditConsistentV1(candidate);
|
|
6386
6624
|
}
|
|
6387
6625
|
function isLedgerEntry(entry) {
|
|
6388
6626
|
if (!entry || typeof entry !== "object") return false;
|
|
6389
6627
|
const candidate = entry;
|
|
6390
|
-
return
|
|
6628
|
+
return isIsoTimestamp(candidate.at) && typeof candidate.model === "string" && candidate.model.length > 0 && isNonNegativeFinite(candidate.budgetUnits) && (candidate.runId === void 0 || isValidStoredRunId(candidate.runId)) && readCounter(candidate.inputTokens) !== void 0 && readCounter(candidate.cachedInputTokens) !== void 0 && readCounter(candidate.outputTokens) !== void 0 && readCounter(candidate.reasoningOutputTokens) !== void 0 && (candidate.cachedInputTokens ?? 0) <= (candidate.inputTokens ?? 0) && isEntryBudgetUnitConsistent(candidate);
|
|
6629
|
+
}
|
|
6630
|
+
function isLedgerEntryCommon(candidate) {
|
|
6631
|
+
return isIsoTimestamp(candidate.at) && typeof candidate.model === "string" && candidate.model.length > 0 && (candidate.runId === void 0 || isValidStoredRunId(candidate.runId)) && readCounter(candidate.inputTokens) !== void 0 && readCounter(candidate.cachedInputTokens) !== void 0 && readCounter(candidate.outputTokens) !== void 0 && readCounter(candidate.reasoningOutputTokens) !== void 0 && (candidate.cachedInputTokens ?? 0) <= (candidate.inputTokens ?? 0);
|
|
6632
|
+
}
|
|
6633
|
+
function isEntryCreditConsistentV1(entry) {
|
|
6634
|
+
try {
|
|
6635
|
+
return Math.abs(calculateCodexBudgetUnits(entry.model, entry) - entry.credits) <= 1e-9;
|
|
6636
|
+
} catch {
|
|
6637
|
+
return false;
|
|
6638
|
+
}
|
|
6391
6639
|
}
|
|
6392
|
-
function
|
|
6640
|
+
function isEntryBudgetUnitConsistent(entry) {
|
|
6393
6641
|
try {
|
|
6394
|
-
return Math.abs(
|
|
6642
|
+
return Math.abs(calculateCodexBudgetUnits(entry.model, entry) - entry.budgetUnits) <= 1e-9;
|
|
6395
6643
|
} catch {
|
|
6396
6644
|
return false;
|
|
6397
6645
|
}
|
|
6398
6646
|
}
|
|
6399
|
-
function summarizeLedgerEntries(entries,
|
|
6647
|
+
function summarizeLedgerEntries(entries, resolutions = [], legacyReconciliations = []) {
|
|
6400
6648
|
const byModel = /* @__PURE__ */ new Map();
|
|
6401
6649
|
for (const entry of entries) {
|
|
6402
6650
|
const modelEntries = byModel.get(entry.model) ?? [];
|
|
@@ -6406,17 +6654,18 @@ function summarizeLedgerEntries(entries, reconciliations = []) {
|
|
|
6406
6654
|
const totals = summarizeUsage(entries);
|
|
6407
6655
|
return {
|
|
6408
6656
|
calls: entries.length,
|
|
6409
|
-
|
|
6410
|
-
|
|
6411
|
-
|
|
6412
|
-
|
|
6413
|
-
|
|
6657
|
+
budgetUnits: nanounitsToBudgetUnits(
|
|
6658
|
+
sumBudgetUnitNanounits(entries.map((entry) => entry.budgetUnits)) + sumBudgetUnitNanounits(resolutions.map((resolution) => resolution.localBudgetChargeUnits)) + sumBudgetUnitNanounits(legacyReconciliations.map((reconciliation) => reconciliation.credits))
|
|
6659
|
+
),
|
|
6660
|
+
accountBalanceResolutionCount: resolutions.length + legacyReconciliations.length,
|
|
6661
|
+
conservativeResolutionChargeUnits: nanounitsToBudgetUnits(
|
|
6662
|
+
sumBudgetUnitNanounits(resolutions.map((resolution) => resolution.localBudgetChargeUnits)) + sumBudgetUnitNanounits(legacyReconciliations.map((reconciliation) => reconciliation.credits))
|
|
6414
6663
|
),
|
|
6415
6664
|
...totals,
|
|
6416
6665
|
models: [...byModel.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([model, modelEntries]) => ({
|
|
6417
6666
|
model,
|
|
6418
6667
|
calls: modelEntries.length,
|
|
6419
|
-
|
|
6668
|
+
budgetUnits: nanounitsToBudgetUnits(sumBudgetUnitNanounits(modelEntries.map((entry) => entry.budgetUnits))),
|
|
6420
6669
|
...summarizeUsage(modelEntries)
|
|
6421
6670
|
}))
|
|
6422
6671
|
};
|
|
@@ -6433,14 +6682,23 @@ function summarizeUsage(entries) {
|
|
|
6433
6682
|
);
|
|
6434
6683
|
}
|
|
6435
6684
|
async function writeLedger(filePath, ledger) {
|
|
6685
|
+
if (failNextLedgerWriteForTest) {
|
|
6686
|
+
failNextLedgerWriteForTest = false;
|
|
6687
|
+
throw new Error(`injected ledger write failure for ${filePath}`);
|
|
6688
|
+
}
|
|
6436
6689
|
await mkdir2(path3.dirname(filePath), { recursive: true, mode: 448 });
|
|
6437
6690
|
const tempPath = `${filePath}.${process.pid}.tmp`;
|
|
6438
|
-
|
|
6439
|
-
|
|
6691
|
+
const contents = serializeLedger(ledger);
|
|
6692
|
+
await writeFile2(tempPath, contents, {
|
|
6440
6693
|
encoding: "utf8",
|
|
6441
6694
|
mode: 384
|
|
6442
6695
|
});
|
|
6443
6696
|
await rename2(tempPath, filePath);
|
|
6697
|
+
return { contents, sha256: sha256(contents) };
|
|
6698
|
+
}
|
|
6699
|
+
function serializeLedger(ledger) {
|
|
6700
|
+
return `${JSON.stringify(ledger, null, 2)}
|
|
6701
|
+
`;
|
|
6444
6702
|
}
|
|
6445
6703
|
function readCounter(value) {
|
|
6446
6704
|
return Number.isSafeInteger(value) && value >= 0 ? value : void 0;
|
|
@@ -6451,6 +6709,55 @@ function readOptionalCounter(value) {
|
|
|
6451
6709
|
function safeErrorMessage(error) {
|
|
6452
6710
|
return error instanceof Error ? error.message : String(error);
|
|
6453
6711
|
}
|
|
6712
|
+
function resourceLockedError(cause) {
|
|
6713
|
+
const original = cause instanceof Error ? cause : new Error("Codex credit ledger lock acquisition failed.");
|
|
6714
|
+
return new BenchmarkRunBlockedError(
|
|
6715
|
+
BenchmarkRunBlockReason.ResourceLocked,
|
|
6716
|
+
"Codex credit ledger resource is locked.",
|
|
6717
|
+
{ cause: original }
|
|
6718
|
+
);
|
|
6719
|
+
}
|
|
6720
|
+
function infrastructureUnavailableError(cause) {
|
|
6721
|
+
const original = cause instanceof Error ? cause : new Error("Codex ledger infrastructure setup failed.");
|
|
6722
|
+
return new BenchmarkRunBlockedError(
|
|
6723
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6724
|
+
"Codex ledger infrastructure is unavailable.",
|
|
6725
|
+
{ cause: original }
|
|
6726
|
+
);
|
|
6727
|
+
}
|
|
6728
|
+
function accountingPersistenceBlockedError(persistenceError, underlyingRunError) {
|
|
6729
|
+
const persistenceCause = persistenceError instanceof Error ? persistenceError : new Error("Codex ledger persistence failed.");
|
|
6730
|
+
const cause = underlyingRunError === void 0 ? persistenceCause : new AggregateError(
|
|
6731
|
+
[persistenceCause, underlyingRunError],
|
|
6732
|
+
"Codex ledger persistence failed after an uncertain dispatch outcome."
|
|
6733
|
+
);
|
|
6734
|
+
return new BenchmarkRunBlockedError(
|
|
6735
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6736
|
+
"Codex usage accounting could not be persisted; manual reconciliation is required.",
|
|
6737
|
+
{ cause }
|
|
6738
|
+
);
|
|
6739
|
+
}
|
|
6740
|
+
async function prepareLedgerDirectory(lockPath) {
|
|
6741
|
+
if (failNextLedgerSetupForTest) {
|
|
6742
|
+
failNextLedgerSetupForTest = false;
|
|
6743
|
+
throw new Error(`injected setup failure for ${lockPath}`);
|
|
6744
|
+
}
|
|
6745
|
+
await mkdir2(path3.dirname(lockPath), { recursive: true, mode: 448 });
|
|
6746
|
+
}
|
|
6747
|
+
async function bestEffortSettleLock(lock) {
|
|
6748
|
+
await writeLockState(lock, "settled").catch(() => void 0);
|
|
6749
|
+
}
|
|
6750
|
+
async function bestEffortCloseLock(lock) {
|
|
6751
|
+
await lock.close().catch(() => void 0);
|
|
6752
|
+
}
|
|
6753
|
+
async function bestEffortRemoveOwnedLedgerLock(lockPath) {
|
|
6754
|
+
if (failNextOwnedLockRemovalForTest) {
|
|
6755
|
+
failNextOwnedLockRemovalForTest = false;
|
|
6756
|
+
failNextLedgerSetupForTest = false;
|
|
6757
|
+
return;
|
|
6758
|
+
}
|
|
6759
|
+
await removeOwnedLedgerLock(lockPath).catch(() => void 0);
|
|
6760
|
+
}
|
|
6454
6761
|
function expandHomeRelativePath(value) {
|
|
6455
6762
|
if (value === "~") return os.homedir();
|
|
6456
6763
|
if (value.startsWith("~/") || value.startsWith("~\\")) {
|
|
@@ -6472,6 +6779,81 @@ function parseNonNegativeNumber(value, name) {
|
|
|
6472
6779
|
}
|
|
6473
6780
|
return parsed;
|
|
6474
6781
|
}
|
|
6782
|
+
function isPositiveFinite(value) {
|
|
6783
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0;
|
|
6784
|
+
}
|
|
6785
|
+
function isNonNegativeFinite(value) {
|
|
6786
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
6787
|
+
}
|
|
6788
|
+
function isSupportedBudgetUnits(value) {
|
|
6789
|
+
if (typeof value !== "number") return false;
|
|
6790
|
+
try {
|
|
6791
|
+
budgetUnitsToNanounits(value);
|
|
6792
|
+
return true;
|
|
6793
|
+
} catch {
|
|
6794
|
+
return false;
|
|
6795
|
+
}
|
|
6796
|
+
}
|
|
6797
|
+
var BUDGET_UNIT_SCALE = 1e9;
|
|
6798
|
+
function rateNanounitsPerToken(rate) {
|
|
6799
|
+
const scaled = rate * 1e3;
|
|
6800
|
+
if (!Number.isSafeInteger(scaled)) throw new Error("Codex credit rate exceeds nanounit precision");
|
|
6801
|
+
return BigInt(scaled);
|
|
6802
|
+
}
|
|
6803
|
+
function budgetUnitsToNanounits(value) {
|
|
6804
|
+
if (!Number.isFinite(value) || value < 0) throw new Error("budget units must be finite and non-negative");
|
|
6805
|
+
const fixed = value.toFixed(9);
|
|
6806
|
+
if (Number(fixed) !== value) {
|
|
6807
|
+
throw new Error("budget units exceed supported nanounit precision");
|
|
6808
|
+
}
|
|
6809
|
+
const [integer = "0", fraction = ""] = fixed.split(".");
|
|
6810
|
+
const nanounits = BigInt(integer) * BigInt(BUDGET_UNIT_SCALE) + BigInt(fraction.padEnd(9, "0"));
|
|
6811
|
+
if (nanounits > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("budget units exceed safe integer range");
|
|
6812
|
+
return nanounits;
|
|
6813
|
+
}
|
|
6814
|
+
function nanounitsToBudgetUnits(value) {
|
|
6815
|
+
return Number(value) / BUDGET_UNIT_SCALE;
|
|
6816
|
+
}
|
|
6817
|
+
function sumBudgetUnitNanounits(values) {
|
|
6818
|
+
return values.reduce((sum, value) => sum + budgetUnitsToNanounits(value), 0n);
|
|
6819
|
+
}
|
|
6820
|
+
function ledgerSpentNanounits(ledger) {
|
|
6821
|
+
return sumBudgetUnitNanounits(ledger.entries.map((entry) => entry.budgetUnits)) + sumBudgetUnitNanounits((ledger.resolutions ?? []).map((resolution) => resolution.localBudgetChargeUnits)) + sumBudgetUnitNanounits((ledger.legacyReconciliations ?? []).map((reconciliation) => reconciliation.credits));
|
|
6822
|
+
}
|
|
6823
|
+
function isLedgerSpentConsistent(ledger) {
|
|
6824
|
+
try {
|
|
6825
|
+
return budgetUnitsToNanounits(ledger.spentUnits) === ledgerSpentNanounits(ledger);
|
|
6826
|
+
} catch {
|
|
6827
|
+
return false;
|
|
6828
|
+
}
|
|
6829
|
+
}
|
|
6830
|
+
function parseExactDecimal(value, name) {
|
|
6831
|
+
if (typeof value !== "string" || value !== value.trim() || !isExactDecimal(value)) {
|
|
6832
|
+
throw new Error(`${name} must be a non-negative plain decimal string`);
|
|
6833
|
+
}
|
|
6834
|
+
return value;
|
|
6835
|
+
}
|
|
6836
|
+
function isExactDecimal(value) {
|
|
6837
|
+
return /^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value) && value.length <= 128;
|
|
6838
|
+
}
|
|
6839
|
+
function subtractExactDecimals(before, after) {
|
|
6840
|
+
const beforeParts = splitExactDecimal(before);
|
|
6841
|
+
const afterParts = splitExactDecimal(after);
|
|
6842
|
+
const scale = Math.max(beforeParts.fraction.length, afterParts.fraction.length);
|
|
6843
|
+
const beforeScaled = BigInt(`${beforeParts.integer}${beforeParts.fraction.padEnd(scale, "0")}`);
|
|
6844
|
+
const afterScaled = BigInt(`${afterParts.integer}${afterParts.fraction.padEnd(scale, "0")}`);
|
|
6845
|
+
const difference = beforeScaled - afterScaled;
|
|
6846
|
+
const sign = difference < 0n ? "-" : "";
|
|
6847
|
+
const digits = (difference < 0n ? -difference : difference).toString().padStart(scale + 1, "0");
|
|
6848
|
+
if (scale === 0) return `${sign}${digits}`;
|
|
6849
|
+
const integer = digits.slice(0, -scale);
|
|
6850
|
+
const fraction = digits.slice(-scale).replace(/0+$/, "");
|
|
6851
|
+
return fraction ? `${sign}${integer}.${fraction}` : `${sign}${integer}`;
|
|
6852
|
+
}
|
|
6853
|
+
function splitExactDecimal(value) {
|
|
6854
|
+
const [integer, fraction = ""] = value.split(".");
|
|
6855
|
+
return { integer: integer ?? "", fraction };
|
|
6856
|
+
}
|
|
6475
6857
|
function parseOptionalRunId(value) {
|
|
6476
6858
|
const runId = value?.trim();
|
|
6477
6859
|
if (!runId) return void 0;
|
|
@@ -6503,9 +6885,6 @@ function isSha256(value) {
|
|
|
6503
6885
|
function sha256(value) {
|
|
6504
6886
|
return createHash4("sha256").update(value).digest("hex");
|
|
6505
6887
|
}
|
|
6506
|
-
function normalizeZero(value) {
|
|
6507
|
-
return Object.is(value, -0) ? 0 : value;
|
|
6508
|
-
}
|
|
6509
6888
|
function isIsoTimestamp(value) {
|
|
6510
6889
|
if (typeof value !== "string") return false;
|
|
6511
6890
|
try {
|
|
@@ -10303,7 +10682,15 @@ var CodexCliProvider = class {
|
|
|
10303
10682
|
resolveBenchmarkRunId()
|
|
10304
10683
|
);
|
|
10305
10684
|
if (creditBudget) {
|
|
10306
|
-
|
|
10685
|
+
try {
|
|
10686
|
+
await this.assertChatGptCreditAuth();
|
|
10687
|
+
} catch (error) {
|
|
10688
|
+
throw new BenchmarkRunBlockedError(
|
|
10689
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
10690
|
+
"Codex CLI ChatGPT authentication is unavailable for this bounded benchmark run.",
|
|
10691
|
+
{ cause: error }
|
|
10692
|
+
);
|
|
10693
|
+
}
|
|
10307
10694
|
}
|
|
10308
10695
|
const maxAttempts = normalizeCodexCliMaxAttempts(
|
|
10309
10696
|
creditBudget ? 1 : this.config.retryOptions?.maxAttempts
|
|
@@ -10455,6 +10842,10 @@ ${result.stderr}`
|
|
|
10455
10842
|
}
|
|
10456
10843
|
return { ok: true, verdict, telemetry };
|
|
10457
10844
|
} catch (error) {
|
|
10845
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
10846
|
+
if (blocked) {
|
|
10847
|
+
throw blocked;
|
|
10848
|
+
}
|
|
10458
10849
|
const aborted = isCodexStructuredJudgeAbort(error, request.signal);
|
|
10459
10850
|
const errorCode = aborted ? "aborted" : "transport_error";
|
|
10460
10851
|
return {
|
|
@@ -16282,7 +16673,11 @@ async function llmJudgeScoreDetailed(judge, question, predicted, expected) {
|
|
|
16282
16673
|
tokens: { input: 0, output: 0 },
|
|
16283
16674
|
latencyMs: durationMs
|
|
16284
16675
|
};
|
|
16285
|
-
} catch {
|
|
16676
|
+
} catch (error) {
|
|
16677
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
16678
|
+
if (blocked) {
|
|
16679
|
+
throw blocked;
|
|
16680
|
+
}
|
|
16286
16681
|
return {
|
|
16287
16682
|
score: deterministicJudgeFallback(predicted, expected),
|
|
16288
16683
|
tokens: { input: 0, output: 0 },
|
|
@@ -16302,7 +16697,11 @@ async function llmBinaryJudgeScoreDetailed(judge, prompt, fallback) {
|
|
|
16302
16697
|
const startedAt = performance.now();
|
|
16303
16698
|
try {
|
|
16304
16699
|
return await judge.scoreBinaryPrompt(prompt);
|
|
16305
|
-
} catch {
|
|
16700
|
+
} catch (error) {
|
|
16701
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
16702
|
+
if (blocked) {
|
|
16703
|
+
throw blocked;
|
|
16704
|
+
}
|
|
16306
16705
|
return {
|
|
16307
16706
|
score: deterministicJudgeFallback(fallback.predicted, fallback.expected),
|
|
16308
16707
|
tokens: { input: 0, output: 0 },
|
|
@@ -20247,37 +20646,51 @@ async function executePlanTrials(ctx, trials, options) {
|
|
|
20247
20646
|
}
|
|
20248
20647
|
return;
|
|
20249
20648
|
}
|
|
20250
|
-
|
|
20251
|
-
|
|
20252
|
-
|
|
20253
|
-
|
|
20254
|
-
|
|
20255
|
-
|
|
20256
|
-
|
|
20257
|
-
|
|
20649
|
+
for (let batchStart = 0; batchStart < trials.length; batchStart += options.trialConcurrency) {
|
|
20650
|
+
const batch = trials.slice(
|
|
20651
|
+
batchStart,
|
|
20652
|
+
batchStart + options.trialConcurrency
|
|
20653
|
+
);
|
|
20654
|
+
const settled = await Promise.allSettled(
|
|
20655
|
+
batch.map(
|
|
20656
|
+
(trial) => executeTrialWithFailure(
|
|
20657
|
+
ctx,
|
|
20658
|
+
trial,
|
|
20659
|
+
options.planIndex,
|
|
20660
|
+
options.answerSupportGate
|
|
20661
|
+
)
|
|
20662
|
+
)
|
|
20663
|
+
);
|
|
20664
|
+
const unexpectedRejection = settled.find(
|
|
20665
|
+
(result) => result.status === "rejected" && findBenchmarkRunBlockedError(result.reason) === void 0
|
|
20666
|
+
);
|
|
20667
|
+
if (unexpectedRejection) {
|
|
20668
|
+
throw unexpectedRejection.reason;
|
|
20258
20669
|
}
|
|
20259
|
-
|
|
20260
|
-
|
|
20261
|
-
|
|
20262
|
-
|
|
20263
|
-
|
|
20264
|
-
|
|
20265
|
-
|
|
20670
|
+
const terminalOffset = settled.findIndex(
|
|
20671
|
+
(result) => result.status === "rejected" && findBenchmarkRunBlockedError(result.reason) !== void 0
|
|
20672
|
+
);
|
|
20673
|
+
const emitLimit = terminalOffset < 0 ? settled.length : terminalOffset;
|
|
20674
|
+
for (let offset = 0; offset < emitLimit; offset += 1) {
|
|
20675
|
+
const result = settled[offset];
|
|
20676
|
+
if (result?.status !== "fulfilled") {
|
|
20677
|
+
throw new Error(
|
|
20678
|
+
`PublishedBenchmarkHarness: concurrent trial ${batchStart + offset} did not settle before canonical emission.`
|
|
20679
|
+
);
|
|
20266
20680
|
}
|
|
20267
|
-
|
|
20268
|
-
ctx,
|
|
20269
|
-
trials[trialIndex],
|
|
20270
|
-
options.planIndex,
|
|
20271
|
-
options.answerSupportGate
|
|
20272
|
-
);
|
|
20273
|
-
completed[trialIndex] = true;
|
|
20274
|
-
emitCompletedPrefix();
|
|
20681
|
+
appendCompletedTask(ctx, options.tasks, result.value);
|
|
20275
20682
|
}
|
|
20276
|
-
|
|
20277
|
-
|
|
20278
|
-
|
|
20279
|
-
|
|
20280
|
-
|
|
20683
|
+
if (terminalOffset >= 0) {
|
|
20684
|
+
const terminalResult = settled[terminalOffset];
|
|
20685
|
+
const terminalError = terminalResult?.status === "rejected" ? findBenchmarkRunBlockedError(terminalResult.reason) : void 0;
|
|
20686
|
+
if (!terminalError) {
|
|
20687
|
+
throw new Error(
|
|
20688
|
+
`PublishedBenchmarkHarness: concurrent trial ${batchStart + terminalOffset} lost its terminal error before canonical emission.`
|
|
20689
|
+
);
|
|
20690
|
+
}
|
|
20691
|
+
throw terminalError;
|
|
20692
|
+
}
|
|
20693
|
+
}
|
|
20281
20694
|
}
|
|
20282
20695
|
function appendCompletedTask(ctx, tasks, task) {
|
|
20283
20696
|
tasks.push(task);
|
|
@@ -20288,6 +20701,10 @@ async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate)
|
|
|
20288
20701
|
try {
|
|
20289
20702
|
return await executeTrial(ctx, trial, answerSupportGate);
|
|
20290
20703
|
} catch (err) {
|
|
20704
|
+
const blocked = findBenchmarkRunBlockedError(err);
|
|
20705
|
+
if (blocked) {
|
|
20706
|
+
throw blocked;
|
|
20707
|
+
}
|
|
20291
20708
|
const message = err instanceof Error ? err.message : String(err);
|
|
20292
20709
|
console.error(` [WARN] harness trial plan-${planIndex}/${trialId} failed: ${message}`);
|
|
20293
20710
|
return {
|
|
@@ -20532,6 +20949,10 @@ async function assessRecallSupport(ctx, trial, recalledText) {
|
|
|
20532
20949
|
validateRecallSupportAssessment(assessment);
|
|
20533
20950
|
return assessment;
|
|
20534
20951
|
} catch (error) {
|
|
20952
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
20953
|
+
if (blocked) {
|
|
20954
|
+
throw blocked;
|
|
20955
|
+
}
|
|
20535
20956
|
return {
|
|
20536
20957
|
status: "backend_failure",
|
|
20537
20958
|
reason: error instanceof Error ? error.message : String(error)
|
|
@@ -20599,6 +21020,9 @@ async function scoreTrialJudge(ctx, trial, answeredText) {
|
|
|
20599
21020
|
);
|
|
20600
21021
|
}
|
|
20601
21022
|
function answerWithTrialFallback(trial, recalledText, error) {
|
|
21023
|
+
if (isBenchmarkRunBlockedError(error)) {
|
|
21024
|
+
throw error;
|
|
21025
|
+
}
|
|
20602
21026
|
const fallback = trial.answerFallback?.({
|
|
20603
21027
|
question: trial.question,
|
|
20604
21028
|
recalledText,
|
|
@@ -44580,6 +45004,7 @@ export {
|
|
|
44580
45004
|
buildProviderFreeLoCoMoRetrievalConfig,
|
|
44581
45005
|
buildSchemaTierFixture,
|
|
44582
45006
|
buildSchemaTierSmokeFixture,
|
|
45007
|
+
calculateCodexBudgetUnits,
|
|
44583
45008
|
calendarFixture,
|
|
44584
45009
|
canonicalJsonStringify,
|
|
44585
45010
|
captureLoCoMoRetrievalTrace,
|