@remnic/bench 9.7.4 → 9.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +27 -22
- package/dist/index.js +644 -206
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5886,9 +5886,51 @@ import { createHash as createHash4 } from "crypto";
|
|
|
5886
5886
|
import { mkdir as mkdir2, open, readFile as readFile3, rename as rename2, rmdir, unlink as unlink2, writeFile as writeFile2 } from "fs/promises";
|
|
5887
5887
|
import os from "os";
|
|
5888
5888
|
import path3 from "path";
|
|
5889
|
-
|
|
5890
|
-
|
|
5889
|
+
|
|
5890
|
+
// src/benchmark-run-blocked-error.ts
|
|
5891
|
+
var BenchmarkRunBlockReason = {
|
|
5892
|
+
InfrastructureUnavailable: "infrastructure_unavailable",
|
|
5893
|
+
ManualReconciliationRequired: "manual_reconciliation_required",
|
|
5894
|
+
SpendHeadroomExhausted: "spend_headroom_exhausted",
|
|
5895
|
+
SpendCeilingExceeded: "spend_ceiling_exceeded",
|
|
5896
|
+
ResourceLocked: "resource_locked"
|
|
5897
|
+
};
|
|
5898
|
+
var BENCHMARK_RUN_BLOCKED_ERROR_CODE = "REMNIC_BENCHMARK_RUN_BLOCKED";
|
|
5899
|
+
var BLOCK_REASONS = new Set(
|
|
5900
|
+
Object.values(BenchmarkRunBlockReason)
|
|
5901
|
+
);
|
|
5902
|
+
var BenchmarkRunBlockedError = class extends Error {
|
|
5903
|
+
code = BENCHMARK_RUN_BLOCKED_ERROR_CODE;
|
|
5904
|
+
reason;
|
|
5905
|
+
constructor(reason, message, options) {
|
|
5906
|
+
super(message, options);
|
|
5907
|
+
this.name = "BenchmarkRunBlockedError";
|
|
5908
|
+
this.reason = reason;
|
|
5909
|
+
}
|
|
5910
|
+
};
|
|
5911
|
+
function findBenchmarkRunBlockedError(error) {
|
|
5912
|
+
const visited = /* @__PURE__ */ new Set();
|
|
5913
|
+
let current = error;
|
|
5914
|
+
while (typeof current === "object" && current !== null) {
|
|
5915
|
+
if (visited.has(current)) {
|
|
5916
|
+
return void 0;
|
|
5917
|
+
}
|
|
5918
|
+
visited.add(current);
|
|
5919
|
+
const candidate = current;
|
|
5920
|
+
if (candidate.code === BENCHMARK_RUN_BLOCKED_ERROR_CODE && typeof candidate.reason === "string" && BLOCK_REASONS.has(candidate.reason)) {
|
|
5921
|
+
return current;
|
|
5922
|
+
}
|
|
5923
|
+
current = candidate.cause;
|
|
5924
|
+
}
|
|
5925
|
+
return void 0;
|
|
5926
|
+
}
|
|
5927
|
+
function isBenchmarkRunBlockedError(error) {
|
|
5928
|
+
return findBenchmarkRunBlockedError(error) !== void 0;
|
|
5929
|
+
}
|
|
5930
|
+
|
|
5931
|
+
// src/providers/codex-credit-budget.ts
|
|
5891
5932
|
var MAX_BOUNDED_CALL_CREDITS = 300;
|
|
5933
|
+
var LEGACY_LEDGER_FLOAT_DRIFT_TOLERANCE = 1e-9;
|
|
5892
5934
|
var SOL_MODEL = /^gpt-5\.6-sol$/i;
|
|
5893
5935
|
var CREDIT_RATES = [
|
|
5894
5936
|
[/^gpt-5\.6-sol$/i, { input: 125, cachedInput: 12.5, output: 750 }],
|
|
@@ -5901,6 +5943,10 @@ var CREDIT_RATES = [
|
|
|
5901
5943
|
[/^gpt-5\.2$/i, { input: 43.75, cachedInput: 4.375, output: 350 }]
|
|
5902
5944
|
];
|
|
5903
5945
|
var completionQueue = Promise.resolve();
|
|
5946
|
+
var failNextSettledLockWriteForTest = false;
|
|
5947
|
+
var failNextOwnedLockRemovalForTest = false;
|
|
5948
|
+
var failNextLedgerSetupForTest = false;
|
|
5949
|
+
var failNextLedgerWriteForTest = false;
|
|
5904
5950
|
var CodexCreditAccountingError = class extends Error {
|
|
5905
5951
|
constructor(message) {
|
|
5906
5952
|
super(message);
|
|
@@ -5914,26 +5960,34 @@ var CodexCreditDispatchError = class extends Error {
|
|
|
5914
5960
|
}
|
|
5915
5961
|
};
|
|
5916
5962
|
async function reconcileCodexCreditLedger(args) {
|
|
5917
|
-
if (args.
|
|
5918
|
-
throw new Error(
|
|
5919
|
-
"Codex credit reconciliation requires confirmation that the observed balance belongs to the ledger's original budget"
|
|
5920
|
-
);
|
|
5963
|
+
if (args.sameAccountConfirmed !== true) {
|
|
5964
|
+
throw new Error("Codex credit reconciliation requires confirmation that both balances belong to the same account");
|
|
5921
5965
|
}
|
|
5922
|
-
if (args.
|
|
5923
|
-
throw new Error(
|
|
5924
|
-
|
|
5925
|
-
|
|
5966
|
+
if (args.snapshotsBracketBlockedEventConfirmed !== true) {
|
|
5967
|
+
throw new Error("Codex credit reconciliation requires confirmation that the snapshots bracket the blocked event");
|
|
5968
|
+
}
|
|
5969
|
+
if (args.balanceSettledConfirmed !== true) {
|
|
5970
|
+
throw new Error("Codex credit reconciliation requires confirmation that the displayed balances are settled");
|
|
5926
5971
|
}
|
|
5927
|
-
if (args.
|
|
5972
|
+
if (args.noCreditsAddedOrRefundedConfirmed !== true) {
|
|
5928
5973
|
throw new Error(
|
|
5929
|
-
"Codex credit reconciliation requires
|
|
5974
|
+
"Codex credit reconciliation requires confirmation that no credits were added or refunded between snapshots"
|
|
5930
5975
|
);
|
|
5931
5976
|
}
|
|
5932
5977
|
if (!isSha256(args.priorLedgerSha256)) {
|
|
5933
5978
|
throw new Error("priorLedgerSha256 must be a lowercase SHA-256 digest");
|
|
5934
5979
|
}
|
|
5935
|
-
|
|
5936
|
-
|
|
5980
|
+
const beforeAccountBalance = parseExactDecimal(args.beforeAccountBalance, "beforeAccountBalance");
|
|
5981
|
+
const afterAccountBalance = parseExactDecimal(args.afterAccountBalance, "afterAccountBalance");
|
|
5982
|
+
const observedAccountDebit = subtractExactDecimals(beforeAccountBalance, afterAccountBalance);
|
|
5983
|
+
if (observedAccountDebit.startsWith("-")) {
|
|
5984
|
+
throw new Error("afterAccountBalance exceeds beforeAccountBalance; reconciliation refused");
|
|
5985
|
+
}
|
|
5986
|
+
const localBudgetChargeUnits = observedAccountDebit === "0" ? 0 : MAX_BOUNDED_CALL_CREDITS;
|
|
5987
|
+
if (localBudgetChargeUnits > 0 && args.noInterveningCodexActivityConfirmed !== true) {
|
|
5988
|
+
throw new Error(
|
|
5989
|
+
"a positive account-wide debit requires confirmation that no other Codex activity occurred between snapshots"
|
|
5990
|
+
);
|
|
5937
5991
|
}
|
|
5938
5992
|
const affectedRunId = parseRequiredRunId(args.affectedRunId, "affectedRunId");
|
|
5939
5993
|
const ledgerPath = path3.resolve(expandHomeRelativePath(args.ledgerPath));
|
|
@@ -5946,7 +6000,11 @@ async function reconcileCodexCreditLedger(args) {
|
|
|
5946
6000
|
const lockPath = `${ledgerPath}.lock`;
|
|
5947
6001
|
let lock;
|
|
5948
6002
|
try {
|
|
5949
|
-
|
|
6003
|
+
try {
|
|
6004
|
+
await prepareLedgerDirectory(lockPath);
|
|
6005
|
+
} catch (error) {
|
|
6006
|
+
throw infrastructureUnavailableError(error);
|
|
6007
|
+
}
|
|
5950
6008
|
lock = await acquireLedgerLock(lockPath);
|
|
5951
6009
|
const contents = await readFile3(ledgerPath);
|
|
5952
6010
|
const currentSha256 = sha256(contents);
|
|
@@ -5955,74 +6013,81 @@ async function reconcileCodexCreditLedger(args) {
|
|
|
5955
6013
|
`Codex credit ledger changed since operator observation: expected ${args.priorLedgerSha256}, found ${currentSha256}; obtain a fresh ledger hash and remaining balance`
|
|
5956
6014
|
);
|
|
5957
6015
|
}
|
|
5958
|
-
const ledger = parseLedger(JSON.parse(contents.toString("utf8")));
|
|
5959
|
-
if (!ledger.
|
|
6016
|
+
const ledger = parseLedger(JSON.parse(contents.toString("utf8")), contents);
|
|
6017
|
+
if (!ledger.blockedEvent) {
|
|
5960
6018
|
throw new Error("Codex credit ledger is not blocked; reconciliation is not permitted");
|
|
5961
6019
|
}
|
|
5962
|
-
if (
|
|
5963
|
-
throw new Error(
|
|
6020
|
+
if (ledger.blockedEvent.runId && ledger.blockedEvent.runId !== affectedRunId) {
|
|
6021
|
+
throw new Error(
|
|
6022
|
+
`affectedRunId does not match the blocked event run ID ${JSON.stringify(ledger.blockedEvent.runId)}`
|
|
6023
|
+
);
|
|
5964
6024
|
}
|
|
5965
|
-
const
|
|
5966
|
-
|
|
6025
|
+
const plannedSpendCeilingNanounits = budgetUnitsToNanounits(ledger.budgetUnits) - budgetUnitsToNanounits(ledger.reserveUnits);
|
|
6026
|
+
const spentNanounits = ledgerSpentNanounits(ledger);
|
|
6027
|
+
const localBudgetChargeNanounits = budgetUnitsToNanounits(localBudgetChargeUnits);
|
|
6028
|
+
if (plannedSpendCeilingNanounits - spentNanounits < localBudgetChargeNanounits) {
|
|
5967
6029
|
throw new Error(
|
|
5968
|
-
|
|
6030
|
+
`reconciliation requires ${localBudgetChargeUnits} local budget units but only ${nanounitsToBudgetUnits(
|
|
6031
|
+
plannedSpendCeilingNanounits > spentNanounits ? plannedSpendCeilingNanounits - spentNanounits : 0n
|
|
6032
|
+
)} remain below the planned-spend ceiling`
|
|
5969
6033
|
);
|
|
5970
6034
|
}
|
|
5971
|
-
const
|
|
5972
|
-
const
|
|
6035
|
+
const totalSpentNanounits = spentNanounits + localBudgetChargeNanounits;
|
|
6036
|
+
const totalSpentUnits = nanounitsToBudgetUnits(totalSpentNanounits);
|
|
5973
6037
|
const at = (/* @__PURE__ */ new Date()).toISOString();
|
|
5974
|
-
const
|
|
6038
|
+
const affectedBlockedEvent = ledger.blockedEvent;
|
|
6039
|
+
const resolution = {
|
|
5975
6040
|
at,
|
|
5976
|
-
basis: "operator-observed-
|
|
5977
|
-
attribution: "account-wide-
|
|
6041
|
+
basis: "operator-observed-account-balance-delta",
|
|
6042
|
+
attribution: "account-wide-observation-window",
|
|
5978
6043
|
priorLedgerSha256: currentSha256,
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
6044
|
+
priorRecordedSpentUnits: nanounitsToBudgetUnits(spentNanounits),
|
|
6045
|
+
priorEntryCount: ledger.entries.length,
|
|
6046
|
+
beforeAccountBalance,
|
|
6047
|
+
afterAccountBalance,
|
|
6048
|
+
observedAccountDebit,
|
|
6049
|
+
localBudgetChargeUnits,
|
|
5983
6050
|
confirmations: {
|
|
5984
|
-
|
|
6051
|
+
sameAccount: true,
|
|
6052
|
+
snapshotsBracketBlockedEvent: true,
|
|
6053
|
+
balanceSettled: true,
|
|
5985
6054
|
noCreditsAddedOrRefunded: true,
|
|
5986
|
-
|
|
6055
|
+
...args.noInterveningCodexActivityConfirmed === true ? { noInterveningCodexActivity: true } : {}
|
|
5987
6056
|
},
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
blockedReason: ledger.blockedReason
|
|
5991
|
-
}
|
|
6057
|
+
affectedRunId,
|
|
6058
|
+
affectedBlockedEvent
|
|
5992
6059
|
};
|
|
5993
6060
|
const nextLedger = {
|
|
5994
6061
|
...ledger,
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
6062
|
+
spentUnits: totalSpentUnits,
|
|
6063
|
+
resolutions: [...ledger.resolutions ?? [], resolution],
|
|
6064
|
+
blockedEvent: void 0
|
|
5998
6065
|
};
|
|
5999
|
-
await writeLedger(ledgerPath, nextLedger);
|
|
6000
|
-
await
|
|
6001
|
-
const nextContents = await readFile3(ledgerPath);
|
|
6066
|
+
const committed = await writeLedger(ledgerPath, nextLedger);
|
|
6067
|
+
await bestEffortSettleLock(lock);
|
|
6002
6068
|
return {
|
|
6003
|
-
schemaVersion:
|
|
6069
|
+
schemaVersion: 2,
|
|
6004
6070
|
priorLedgerSha256: currentSha256,
|
|
6005
|
-
ledgerSha256: sha256
|
|
6071
|
+
ledgerSha256: committed.sha256,
|
|
6006
6072
|
at,
|
|
6007
|
-
attribution: "account-wide-
|
|
6073
|
+
attribution: "account-wide-observation-window",
|
|
6008
6074
|
affectedRunId,
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
)
|
|
6075
|
+
priorRecordedSpentUnits: nanounitsToBudgetUnits(spentNanounits),
|
|
6076
|
+
beforeAccountBalance,
|
|
6077
|
+
afterAccountBalance,
|
|
6078
|
+
observedAccountDebit,
|
|
6079
|
+
localBudgetChargeUnits,
|
|
6080
|
+
totalSpentUnits,
|
|
6081
|
+
remainingPlannedSpendUnits: nanounitsToBudgetUnits(plannedSpendCeilingNanounits - totalSpentNanounits),
|
|
6082
|
+
affectedBlockedEventSha256: sha256(JSON.stringify(affectedBlockedEvent))
|
|
6018
6083
|
};
|
|
6019
6084
|
} finally {
|
|
6020
6085
|
try {
|
|
6021
6086
|
if (lock) {
|
|
6022
6087
|
try {
|
|
6023
|
-
await lock
|
|
6088
|
+
await bestEffortCloseLock(lock);
|
|
6024
6089
|
} finally {
|
|
6025
|
-
await
|
|
6090
|
+
await bestEffortRemoveOwnedLedgerLock(lockPath);
|
|
6026
6091
|
}
|
|
6027
6092
|
}
|
|
6028
6093
|
} finally {
|
|
@@ -6033,18 +6098,13 @@ async function reconcileCodexCreditLedger(args) {
|
|
|
6033
6098
|
function resolveCodexCreditBudgetConfig(env = process.env, fallbackRunId) {
|
|
6034
6099
|
const rawBudget = env.REMNIC_BENCH_CODEX_CREDIT_BUDGET?.trim();
|
|
6035
6100
|
if (!rawBudget) return void 0;
|
|
6036
|
-
const budgetCredits = parsePositiveNumber(
|
|
6037
|
-
rawBudget,
|
|
6038
|
-
"REMNIC_BENCH_CODEX_CREDIT_BUDGET"
|
|
6039
|
-
);
|
|
6101
|
+
const budgetCredits = parsePositiveNumber(rawBudget, "REMNIC_BENCH_CODEX_CREDIT_BUDGET");
|
|
6040
6102
|
const reserveCredits = parseNonNegativeNumber(
|
|
6041
6103
|
env.REMNIC_BENCH_CODEX_CREDIT_RESERVE?.trim() ?? "473",
|
|
6042
6104
|
"REMNIC_BENCH_CODEX_CREDIT_RESERVE"
|
|
6043
6105
|
);
|
|
6044
6106
|
if (reserveCredits >= budgetCredits) {
|
|
6045
|
-
throw new Error(
|
|
6046
|
-
"REMNIC_BENCH_CODEX_CREDIT_RESERVE must be smaller than REMNIC_BENCH_CODEX_CREDIT_BUDGET"
|
|
6047
|
-
);
|
|
6107
|
+
throw new Error("REMNIC_BENCH_CODEX_CREDIT_RESERVE must be smaller than REMNIC_BENCH_CODEX_CREDIT_BUDGET");
|
|
6048
6108
|
}
|
|
6049
6109
|
if (reserveCredits < MAX_BOUNDED_CALL_CREDITS) {
|
|
6050
6110
|
throw new Error(
|
|
@@ -6052,18 +6112,14 @@ function resolveCodexCreditBudgetConfig(env = process.env, fallbackRunId) {
|
|
|
6052
6112
|
);
|
|
6053
6113
|
}
|
|
6054
6114
|
const ledgerPath = path3.resolve(
|
|
6055
|
-
expandHomeRelativePath(
|
|
6056
|
-
env.REMNIC_BENCH_CODEX_CREDIT_LEDGER?.trim() || ".remnic/bench/codex-credit-ledger.json"
|
|
6057
|
-
)
|
|
6115
|
+
expandHomeRelativePath(env.REMNIC_BENCH_CODEX_CREDIT_LEDGER?.trim() || ".remnic/bench/codex-credit-ledger.json")
|
|
6058
6116
|
);
|
|
6059
6117
|
const runId = parseOptionalRunId(env.REMNIC_BENCH_RUN_ID) ?? parseOptionalRunId(fallbackRunId);
|
|
6060
6118
|
return {
|
|
6061
6119
|
budgetCredits,
|
|
6062
6120
|
reserveCredits,
|
|
6063
6121
|
ledgerPath,
|
|
6064
|
-
allowSol: /^(?:1|true|yes|on)$/i.test(
|
|
6065
|
-
env.REMNIC_BENCH_CODEX_ALLOW_SOL?.trim() ?? ""
|
|
6066
|
-
),
|
|
6122
|
+
allowSol: /^(?:1|true|yes|on)$/i.test(env.REMNIC_BENCH_CODEX_ALLOW_SOL?.trim() ?? ""),
|
|
6067
6123
|
...runId ? { runId } : {}
|
|
6068
6124
|
};
|
|
6069
6125
|
}
|
|
@@ -6071,6 +6127,12 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6071
6127
|
if (!args.config) {
|
|
6072
6128
|
return (await args.run()).value;
|
|
6073
6129
|
}
|
|
6130
|
+
try {
|
|
6131
|
+
budgetUnitsToNanounits(args.config.budgetCredits);
|
|
6132
|
+
budgetUnitsToNanounits(args.config.reserveCredits);
|
|
6133
|
+
} catch (error) {
|
|
6134
|
+
throw infrastructureUnavailableError(error);
|
|
6135
|
+
}
|
|
6074
6136
|
const previous = completionQueue;
|
|
6075
6137
|
let release;
|
|
6076
6138
|
completionQueue = new Promise((resolve) => {
|
|
@@ -6081,21 +6143,36 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6081
6143
|
let lock;
|
|
6082
6144
|
let dispatchStarted = false;
|
|
6083
6145
|
let accountingSettled = false;
|
|
6146
|
+
let ledgerCommitted = false;
|
|
6084
6147
|
try {
|
|
6085
|
-
|
|
6148
|
+
try {
|
|
6149
|
+
await prepareLedgerDirectory(lockPath);
|
|
6150
|
+
} catch (error) {
|
|
6151
|
+
throw infrastructureUnavailableError(error);
|
|
6152
|
+
}
|
|
6086
6153
|
lock = await acquireLedgerLock(lockPath);
|
|
6087
6154
|
assertModelAllowed(args.model, args.config);
|
|
6088
6155
|
const ledger = await readLedger(args.config);
|
|
6089
|
-
if (ledger.
|
|
6090
|
-
throw new
|
|
6091
|
-
|
|
6156
|
+
if (ledger.blockedEvent) {
|
|
6157
|
+
throw new BenchmarkRunBlockedError(
|
|
6158
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6159
|
+
"Codex credit ledger requires manual reconciliation.",
|
|
6160
|
+
{ cause: new Error(`Private ledger block: ${ledger.blockedEvent.reason}`) }
|
|
6092
6161
|
);
|
|
6093
6162
|
}
|
|
6094
|
-
const
|
|
6095
|
-
const
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6163
|
+
const usableNanounits = budgetUnitsToNanounits(args.config.budgetCredits) - budgetUnitsToNanounits(args.config.reserveCredits);
|
|
6164
|
+
const usableCredits = nanounitsToBudgetUnits(usableNanounits);
|
|
6165
|
+
const spentNanounits = ledgerSpentNanounits(ledger);
|
|
6166
|
+
const dispatchHeadroomNanounits = usableNanounits - spentNanounits;
|
|
6167
|
+
if (dispatchHeadroomNanounits < budgetUnitsToNanounits(MAX_BOUNDED_CALL_CREDITS)) {
|
|
6168
|
+
throw new BenchmarkRunBlockedError(
|
|
6169
|
+
BenchmarkRunBlockReason.SpendHeadroomExhausted,
|
|
6170
|
+
"Codex credit budget lacks conservative dispatch headroom.",
|
|
6171
|
+
{
|
|
6172
|
+
cause: new Error(
|
|
6173
|
+
`${nanounitsToBudgetUnits(spentNanounits)} local units spent; ${nanounitsToBudgetUnits(dispatchHeadroomNanounits)} available; ${MAX_BOUNDED_CALL_CREDITS} required.`
|
|
6174
|
+
)
|
|
6175
|
+
}
|
|
6099
6176
|
);
|
|
6100
6177
|
}
|
|
6101
6178
|
await writeLockState(lock, "in-flight");
|
|
@@ -6105,42 +6182,74 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6105
6182
|
result = await args.run();
|
|
6106
6183
|
} catch (error) {
|
|
6107
6184
|
if (error instanceof CodexCreditDispatchError) {
|
|
6108
|
-
await writeLockState(lock, "settled");
|
|
6109
|
-
accountingSettled = true;
|
|
6110
|
-
} else {
|
|
6111
|
-
const blockedLedger = {
|
|
6112
|
-
...ledger,
|
|
6113
|
-
blockedReason: error instanceof CodexCreditAccountingError ? error.message : `Codex dispatch outcome is unknown after an unexpected error: ${safeErrorMessage(error)}`
|
|
6114
|
-
};
|
|
6115
|
-
await writeLedger(args.config.ledgerPath, blockedLedger);
|
|
6116
|
-
await writeLockState(lock, "settled");
|
|
6117
6185
|
accountingSettled = true;
|
|
6186
|
+
await bestEffortSettleLock(lock);
|
|
6187
|
+
throw new BenchmarkRunBlockedError(
|
|
6188
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6189
|
+
"Codex CLI infrastructure was unavailable before dispatch.",
|
|
6190
|
+
{ cause: error }
|
|
6191
|
+
);
|
|
6118
6192
|
}
|
|
6119
|
-
|
|
6193
|
+
const blockedLedger = {
|
|
6194
|
+
...ledger,
|
|
6195
|
+
blockedEvent: {
|
|
6196
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6197
|
+
...args.config.runId ? { runId: args.config.runId } : {},
|
|
6198
|
+
model: args.model,
|
|
6199
|
+
reason: error instanceof CodexCreditAccountingError ? error.message : `Codex dispatch outcome is unknown after an unexpected error: ${safeErrorMessage(error)}`
|
|
6200
|
+
}
|
|
6201
|
+
};
|
|
6202
|
+
try {
|
|
6203
|
+
await writeLedger(args.config.ledgerPath, blockedLedger);
|
|
6204
|
+
} catch (persistenceError) {
|
|
6205
|
+
throw accountingPersistenceBlockedError(persistenceError, error);
|
|
6206
|
+
}
|
|
6207
|
+
ledgerCommitted = true;
|
|
6208
|
+
accountingSettled = true;
|
|
6209
|
+
await bestEffortSettleLock(lock);
|
|
6210
|
+
throw new BenchmarkRunBlockedError(
|
|
6211
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6212
|
+
"Codex usage accounting is uncertain; manual reconciliation is required.",
|
|
6213
|
+
{
|
|
6214
|
+
cause: error
|
|
6215
|
+
}
|
|
6216
|
+
);
|
|
6120
6217
|
}
|
|
6121
|
-
const
|
|
6122
|
-
const
|
|
6218
|
+
const creditNanounits = calculateCodexBudgetNanounits(args.model, result.usage);
|
|
6219
|
+
const credits = nanounitsToBudgetUnits(creditNanounits);
|
|
6220
|
+
const nextSpentNanounits = spentNanounits + creditNanounits;
|
|
6221
|
+
const nextSpent = nanounitsToBudgetUnits(nextSpentNanounits);
|
|
6123
6222
|
const nextLedger = {
|
|
6124
6223
|
...ledger,
|
|
6125
|
-
|
|
6224
|
+
spentUnits: nextSpent,
|
|
6126
6225
|
entries: [
|
|
6127
6226
|
...ledger.entries,
|
|
6128
6227
|
{
|
|
6129
6228
|
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6130
6229
|
model: args.model,
|
|
6131
|
-
credits,
|
|
6230
|
+
budgetUnits: credits,
|
|
6132
6231
|
...args.config.runId ? { runId: args.config.runId } : {},
|
|
6133
6232
|
...result.usage
|
|
6134
6233
|
}
|
|
6135
6234
|
]
|
|
6136
6235
|
};
|
|
6137
|
-
|
|
6138
|
-
|
|
6236
|
+
try {
|
|
6237
|
+
await writeLedger(args.config.ledgerPath, nextLedger);
|
|
6238
|
+
} catch (error) {
|
|
6239
|
+
throw accountingPersistenceBlockedError(error);
|
|
6240
|
+
}
|
|
6241
|
+
ledgerCommitted = true;
|
|
6139
6242
|
accountingSettled = true;
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6243
|
+
await bestEffortSettleLock(lock);
|
|
6244
|
+
try {
|
|
6245
|
+
args.onUsagePersisted?.(result.usage);
|
|
6246
|
+
} catch {
|
|
6247
|
+
}
|
|
6248
|
+
if (nextSpentNanounits > usableNanounits) {
|
|
6249
|
+
throw new BenchmarkRunBlockedError(
|
|
6250
|
+
BenchmarkRunBlockReason.SpendCeilingExceeded,
|
|
6251
|
+
"Codex planned-spend ceiling was exceeded by committed usage.",
|
|
6252
|
+
{ cause: new Error(`${nextSpent} local units spent exceeds the ${usableCredits} planned ceiling.`) }
|
|
6144
6253
|
);
|
|
6145
6254
|
}
|
|
6146
6255
|
return result.value;
|
|
@@ -6148,10 +6257,10 @@ async function runWithinCodexCreditBudget(args) {
|
|
|
6148
6257
|
try {
|
|
6149
6258
|
if (lock) {
|
|
6150
6259
|
try {
|
|
6151
|
-
await lock
|
|
6260
|
+
await bestEffortCloseLock(lock);
|
|
6152
6261
|
} finally {
|
|
6153
|
-
if (!dispatchStarted || accountingSettled) {
|
|
6154
|
-
await
|
|
6262
|
+
if (!dispatchStarted || accountingSettled || ledgerCommitted) {
|
|
6263
|
+
await bestEffortRemoveOwnedLedgerLock(lockPath);
|
|
6155
6264
|
}
|
|
6156
6265
|
}
|
|
6157
6266
|
}
|
|
@@ -6165,11 +6274,13 @@ async function acquireLedgerLock(lockPath) {
|
|
|
6165
6274
|
try {
|
|
6166
6275
|
await mkdir2(lockPath, { mode: 448 });
|
|
6167
6276
|
} catch (error) {
|
|
6168
|
-
if (error.code !== "EEXIST") throw error;
|
|
6277
|
+
if (error.code !== "EEXIST") throw infrastructureUnavailableError(error);
|
|
6169
6278
|
const owner = await readLockOwner(lockPath);
|
|
6170
6279
|
if (!owner || isProcessAlive(owner.pid) || owner.phase === "in-flight") {
|
|
6171
|
-
throw
|
|
6172
|
-
|
|
6280
|
+
throw resourceLockedError(
|
|
6281
|
+
new Error(
|
|
6282
|
+
`Codex credit ledger is locked${owner?.phase === "in-flight" ? " with unreconciled in-flight usage" : " by another benchmark process"} (${lockPath}); refusing credit spend.`
|
|
6283
|
+
)
|
|
6173
6284
|
);
|
|
6174
6285
|
}
|
|
6175
6286
|
await reclaimStaleLedgerLock(lockPath, owner);
|
|
@@ -6186,10 +6297,10 @@ async function acquireLedgerLock(lockPath) {
|
|
|
6186
6297
|
await unlink2(lockOwnerPath(lockPath)).catch(() => void 0);
|
|
6187
6298
|
await rmdir(lockHeldPath(lockPath)).catch(() => void 0);
|
|
6188
6299
|
await rmdir(lockPath).catch(() => void 0);
|
|
6189
|
-
throw error;
|
|
6300
|
+
throw infrastructureUnavailableError(error);
|
|
6190
6301
|
}
|
|
6191
6302
|
}
|
|
6192
|
-
throw new Error(`Unable to acquire Codex credit ledger lock (${lockPath})`);
|
|
6303
|
+
throw resourceLockedError(new Error(`Unable to acquire Codex credit ledger lock (${lockPath})`));
|
|
6193
6304
|
}
|
|
6194
6305
|
async function readLockOwner(lockPath) {
|
|
6195
6306
|
try {
|
|
@@ -6206,14 +6317,17 @@ async function reclaimStaleLedgerLock(lockPath, expectedOwner) {
|
|
|
6206
6317
|
try {
|
|
6207
6318
|
await rmdir(lockHeldPath(lockPath));
|
|
6208
6319
|
} catch (error) {
|
|
6209
|
-
throw
|
|
6210
|
-
|
|
6320
|
+
throw resourceLockedError(
|
|
6321
|
+
new Error(
|
|
6322
|
+
`Codex credit ledger stale-lock reclamation is already claimed or incomplete (${lockPath}); refusing credit spend: ${safeErrorMessage(error)}`,
|
|
6323
|
+
{ cause: error }
|
|
6324
|
+
)
|
|
6211
6325
|
);
|
|
6212
6326
|
}
|
|
6213
6327
|
const currentOwner = await readLockOwner(lockPath);
|
|
6214
6328
|
if (!currentOwner || currentOwner.pid !== expectedOwner.pid || currentOwner.phase !== expectedOwner.phase || isProcessAlive(currentOwner.pid) || currentOwner.phase === "in-flight") {
|
|
6215
|
-
throw
|
|
6216
|
-
`Codex credit ledger owner changed during stale-lock reclamation (${lockPath}); refusing credit spend.`
|
|
6329
|
+
throw resourceLockedError(
|
|
6330
|
+
new Error(`Codex credit ledger owner changed during stale-lock reclamation (${lockPath}); refusing credit spend.`)
|
|
6217
6331
|
);
|
|
6218
6332
|
}
|
|
6219
6333
|
await unlink2(lockOwnerPath(lockPath));
|
|
@@ -6231,6 +6345,10 @@ function lockHeldPath(lockPath) {
|
|
|
6231
6345
|
return path3.join(lockPath, "held");
|
|
6232
6346
|
}
|
|
6233
6347
|
async function writeLockState(lock, phase) {
|
|
6348
|
+
if (phase === "settled" && failNextSettledLockWriteForTest) {
|
|
6349
|
+
failNextSettledLockWriteForTest = false;
|
|
6350
|
+
throw new Error("injected settled lock-state failure");
|
|
6351
|
+
}
|
|
6234
6352
|
const contents = `${JSON.stringify({
|
|
6235
6353
|
pid: process.pid,
|
|
6236
6354
|
phase,
|
|
@@ -6259,12 +6377,8 @@ function parseCodexJsonlUsage(output) {
|
|
|
6259
6377
|
if (event.type !== "turn.completed" || !event.usage) continue;
|
|
6260
6378
|
const inputTokens = readCounter(event.usage.input_tokens);
|
|
6261
6379
|
const outputTokens = readCounter(event.usage.output_tokens);
|
|
6262
|
-
const cachedInputTokens = readOptionalCounter(
|
|
6263
|
-
|
|
6264
|
-
);
|
|
6265
|
-
const reasoningOutputTokens = readOptionalCounter(
|
|
6266
|
-
event.usage.reasoning_output_tokens
|
|
6267
|
-
);
|
|
6380
|
+
const cachedInputTokens = readOptionalCounter(event.usage.cached_input_tokens);
|
|
6381
|
+
const reasoningOutputTokens = readOptionalCounter(event.usage.reasoning_output_tokens);
|
|
6268
6382
|
if (inputTokens !== void 0 && outputTokens !== void 0 && cachedInputTokens !== void 0 && reasoningOutputTokens !== void 0) {
|
|
6269
6383
|
usage = {
|
|
6270
6384
|
inputTokens,
|
|
@@ -6278,29 +6392,38 @@ function parseCodexJsonlUsage(output) {
|
|
|
6278
6392
|
}
|
|
6279
6393
|
return usage;
|
|
6280
6394
|
}
|
|
6281
|
-
function
|
|
6395
|
+
function calculateCodexBudgetUnits(model, usage) {
|
|
6396
|
+
return nanounitsToBudgetUnits(calculateCodexBudgetNanounits(model, usage));
|
|
6397
|
+
}
|
|
6398
|
+
function calculateCodexBudgetNanounits(model, usage) {
|
|
6282
6399
|
const rate = resolveRate(model);
|
|
6283
6400
|
const cached = Math.min(usage.inputTokens, usage.cachedInputTokens);
|
|
6284
6401
|
const uncached = usage.inputTokens - cached;
|
|
6285
|
-
return (uncached * rate.input + cached * rate.cachedInput + usage.outputTokens * rate.output)
|
|
6402
|
+
return BigInt(uncached) * rateNanounitsPerToken(rate.input) + BigInt(cached) * rateNanounitsPerToken(rate.cachedInput) + BigInt(usage.outputTokens) * rateNanounitsPerToken(rate.output);
|
|
6286
6403
|
}
|
|
6287
6404
|
async function buildCodexCreditReceipt(ledgerPath, runId) {
|
|
6288
6405
|
const resolvedPath = path3.resolve(expandHomeRelativePath(ledgerPath));
|
|
6289
6406
|
const contents = await readFile3(resolvedPath);
|
|
6290
|
-
const ledger = parseLedger(JSON.parse(contents.toString("utf8")));
|
|
6407
|
+
const ledger = parseLedger(JSON.parse(contents.toString("utf8")), contents);
|
|
6291
6408
|
const normalizedRunId = parseOptionalRunId(runId);
|
|
6292
|
-
const
|
|
6293
|
-
const
|
|
6409
|
+
const spentNanounits = ledgerSpentNanounits(ledger);
|
|
6410
|
+
const budgetNanounits = budgetUnitsToNanounits(ledger.budgetUnits);
|
|
6411
|
+
const reserveNanounits = budgetUnitsToNanounits(ledger.reserveUnits);
|
|
6412
|
+
const cumulative = summarizeLedgerEntries(
|
|
6413
|
+
ledger.entries,
|
|
6414
|
+
ledger.resolutions ?? [],
|
|
6415
|
+
ledger.legacyReconciliations ?? []
|
|
6416
|
+
);
|
|
6294
6417
|
const runEntries = normalizedRunId ? ledger.entries.filter((entry) => entry.runId === normalizedRunId) : [];
|
|
6295
6418
|
return {
|
|
6296
|
-
schemaVersion:
|
|
6419
|
+
schemaVersion: 2,
|
|
6297
6420
|
ledgerSha256: sha256(contents),
|
|
6298
|
-
|
|
6299
|
-
|
|
6300
|
-
|
|
6301
|
-
|
|
6302
|
-
|
|
6303
|
-
blocked: ledger.
|
|
6421
|
+
budgetUnits: ledger.budgetUnits,
|
|
6422
|
+
reserveUnits: ledger.reserveUnits,
|
|
6423
|
+
plannedSpendCeilingUnits: nanounitsToBudgetUnits(budgetNanounits - reserveNanounits),
|
|
6424
|
+
totalSpentUnits: nanounitsToBudgetUnits(spentNanounits),
|
|
6425
|
+
remainingBudgetUnits: nanounitsToBudgetUnits(budgetNanounits - spentNanounits),
|
|
6426
|
+
blocked: ledger.blockedEvent !== void 0,
|
|
6304
6427
|
cumulative,
|
|
6305
6428
|
...normalizedRunId ? {
|
|
6306
6429
|
run: {
|
|
@@ -6320,52 +6443,85 @@ function resolveRate(model) {
|
|
|
6320
6443
|
return match[1];
|
|
6321
6444
|
}
|
|
6322
6445
|
function assertModelAllowed(model, config) {
|
|
6323
|
-
|
|
6446
|
+
try {
|
|
6447
|
+
resolveRate(model);
|
|
6448
|
+
} catch (error) {
|
|
6449
|
+
throw new BenchmarkRunBlockedError(
|
|
6450
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6451
|
+
"Configured Codex model is unsupported by the bounded budget.",
|
|
6452
|
+
{ cause: error }
|
|
6453
|
+
);
|
|
6454
|
+
}
|
|
6324
6455
|
if (SOL_MODEL.test(model) && !config.allowSol) {
|
|
6325
|
-
throw new
|
|
6326
|
-
|
|
6456
|
+
throw new BenchmarkRunBlockedError(
|
|
6457
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6458
|
+
"Configured Codex model is disallowed by bounded-budget policy.",
|
|
6459
|
+
{
|
|
6460
|
+
cause: new Error(
|
|
6461
|
+
"gpt-5.6-sol is disabled for bounded benchmark runs because it is the most expensive GPT-5.6 tier. Use gpt-5.6-terra or gpt-5.6-luna, or explicitly set REMNIC_BENCH_CODEX_ALLOW_SOL=1."
|
|
6462
|
+
)
|
|
6463
|
+
}
|
|
6327
6464
|
);
|
|
6328
6465
|
}
|
|
6329
6466
|
}
|
|
6330
6467
|
async function readLedger(config) {
|
|
6331
6468
|
try {
|
|
6332
|
-
const
|
|
6333
|
-
|
|
6334
|
-
)
|
|
6335
|
-
if (parsed.budgetCredits !== config.budgetCredits || parsed.reserveCredits !== config.reserveCredits) {
|
|
6469
|
+
const contents = await readFile3(config.ledgerPath);
|
|
6470
|
+
const parsed = parseLedger(JSON.parse(contents.toString("utf8")), contents);
|
|
6471
|
+
if (parsed.budgetUnits !== config.budgetCredits || parsed.reserveUnits !== config.reserveCredits) {
|
|
6336
6472
|
throw new Error("ledger schema or budget does not match this run");
|
|
6337
6473
|
}
|
|
6338
6474
|
return parsed;
|
|
6339
6475
|
} catch (error) {
|
|
6340
6476
|
if (error.code !== "ENOENT") {
|
|
6341
|
-
throw new
|
|
6477
|
+
throw new BenchmarkRunBlockedError(
|
|
6478
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6479
|
+
"Codex credit ledger is invalid or incompatible with this run.",
|
|
6480
|
+
{ cause: new Error(`Invalid Codex credit ledger at ${config.ledgerPath}: ${String(error)}`, { cause: error }) }
|
|
6481
|
+
);
|
|
6342
6482
|
}
|
|
6343
6483
|
return {
|
|
6344
|
-
schemaVersion:
|
|
6345
|
-
|
|
6346
|
-
|
|
6347
|
-
|
|
6484
|
+
schemaVersion: 2,
|
|
6485
|
+
budgetUnits: config.budgetCredits,
|
|
6486
|
+
reserveUnits: config.reserveCredits,
|
|
6487
|
+
spentUnits: 0,
|
|
6348
6488
|
entries: []
|
|
6349
6489
|
};
|
|
6350
6490
|
}
|
|
6351
6491
|
}
|
|
6352
|
-
function parseLedger(parsed) {
|
|
6353
|
-
|
|
6354
|
-
|
|
6355
|
-
|
|
6356
|
-
)
|
|
6357
|
-
const reconciliationCredits = parsed.reconciliations === void 0 ? 0 : Array.isArray(parsed.reconciliations) ? parsed.reconciliations.reduce(
|
|
6358
|
-
(sum, reconciliation) => sum + (typeof reconciliation?.credits === "number" ? reconciliation.credits ?? 0 : 0),
|
|
6359
|
-
0
|
|
6360
|
-
) : Number.NaN;
|
|
6361
|
-
if (parsed.schemaVersion !== 1 || typeof parsed.budgetCredits !== "number" || !Number.isFinite(parsed.budgetCredits) || parsed.budgetCredits <= 0 || typeof parsed.reserveCredits !== "number" || !Number.isFinite(parsed.reserveCredits) || parsed.reserveCredits < 0 || parsed.reserveCredits >= parsed.budgetCredits || typeof parsed.spentCredits !== "number" || !Number.isFinite(parsed.spentCredits) || parsed.spentCredits < 0 || !Array.isArray(parsed.entries) || !parsed.entries.every(isLedgerEntry) || parsed.reconciliations !== void 0 && (!Array.isArray(parsed.reconciliations) || !parsed.reconciliations.every(
|
|
6362
|
-
(reconciliation) => isLedgerReconciliationWithinBudget(reconciliation, parsed.budgetCredits)
|
|
6363
|
-
)) || Math.abs(entryCredits + reconciliationCredits - parsed.spentCredits) > 1e-9 || parsed.blockedReason !== void 0 && typeof parsed.blockedReason !== "string") {
|
|
6492
|
+
function parseLedger(parsed, sourceContents) {
|
|
6493
|
+
if (isLedgerV1(parsed)) return migrateLedgerV1(parsed, sourceContents);
|
|
6494
|
+
if (!parsed || typeof parsed !== "object") throw new Error("ledger schema is invalid");
|
|
6495
|
+
const candidate = parsed;
|
|
6496
|
+
if (candidate.schemaVersion !== 2 || !isPositiveFinite(candidate.budgetUnits) || !isSupportedBudgetUnits(candidate.budgetUnits) || !isNonNegativeFinite(candidate.reserveUnits) || !isSupportedBudgetUnits(candidate.reserveUnits) || candidate.reserveUnits >= candidate.budgetUnits || !isNonNegativeFinite(candidate.spentUnits) || !Array.isArray(candidate.entries) || !candidate.entries.every(isLedgerEntry) || candidate.resolutions !== void 0 && (!Array.isArray(candidate.resolutions) || !candidate.resolutions.every(isLedgerResolution)) || candidate.legacyReconciliations !== void 0 && (!Array.isArray(candidate.legacyReconciliations) || !candidate.legacyReconciliations.every(isLedgerReconciliationV1)) || !isLedgerSpentConsistent(candidate) || !isResolutionHistoryConsistent(candidate) || !isMigrationWitnessConsistent(candidate) || candidate.blockedEvent !== void 0 && !isBlockedEvent(candidate.blockedEvent) || candidate.migratedFromV1Sha256 !== void 0 && !isSha256(candidate.migratedFromV1Sha256)) {
|
|
6364
6497
|
throw new Error("ledger schema is invalid");
|
|
6365
6498
|
}
|
|
6366
|
-
return
|
|
6499
|
+
return candidate;
|
|
6500
|
+
}
|
|
6501
|
+
function isLedgerV1(value) {
|
|
6502
|
+
if (!value || typeof value !== "object") return false;
|
|
6503
|
+
const candidate = value;
|
|
6504
|
+
const entries = candidate.entries;
|
|
6505
|
+
const reconciliations = candidate.reconciliations;
|
|
6506
|
+
return candidate.schemaVersion === 1 && isPositiveFinite(candidate.budgetCredits) && isSupportedBudgetUnits(candidate.budgetCredits) && isNonNegativeFinite(candidate.reserveCredits) && isSupportedBudgetUnits(candidate.reserveCredits) && candidate.reserveCredits < candidate.budgetCredits && isNonNegativeFinite(candidate.spentCredits) && Array.isArray(entries) && entries.every(isLedgerEntryV1) && (reconciliations === void 0 || Array.isArray(reconciliations) && reconciliations.every((item) => isLedgerReconciliationWithinBudgetV1(item, candidate.budgetCredits))) && isLedgerV1SpentConsistent(candidate) && (candidate.blockedReason === void 0 || typeof candidate.blockedReason === "string" && candidate.blockedReason.length > 0);
|
|
6367
6507
|
}
|
|
6368
|
-
function
|
|
6508
|
+
function migrateLedgerV1(ledger, sourceContents) {
|
|
6509
|
+
const source = sourceContents ? Buffer.from(sourceContents).toString("utf8") : `${JSON.stringify(ledger)}
|
|
6510
|
+
`;
|
|
6511
|
+
const spentUnits = nanounitsToBudgetUnits(ledgerV1SpentNanounits(ledger));
|
|
6512
|
+
return {
|
|
6513
|
+
schemaVersion: 2,
|
|
6514
|
+
budgetUnits: ledger.budgetCredits,
|
|
6515
|
+
reserveUnits: ledger.reserveCredits,
|
|
6516
|
+
spentUnits,
|
|
6517
|
+
entries: ledger.entries.map(({ credits, ...entry }) => ({ ...entry, budgetUnits: credits })),
|
|
6518
|
+
...ledger.reconciliations?.length ? { legacyReconciliations: ledger.reconciliations } : {},
|
|
6519
|
+
migratedFromV1Sha256: sha256(source),
|
|
6520
|
+
migrationWitnessV1: { source },
|
|
6521
|
+
...ledger.blockedReason ? { blockedEvent: { reason: ledger.blockedReason } } : {}
|
|
6522
|
+
};
|
|
6523
|
+
}
|
|
6524
|
+
function isLedgerReconciliationV1(value) {
|
|
6369
6525
|
if (!value || typeof value !== "object") return false;
|
|
6370
6526
|
const candidate = value;
|
|
6371
6527
|
const forbiddenUsageFields = [
|
|
@@ -6377,26 +6533,118 @@ function isLedgerReconciliation(value) {
|
|
|
6377
6533
|
"outputTokens",
|
|
6378
6534
|
"reasoningOutputTokens"
|
|
6379
6535
|
];
|
|
6380
|
-
return forbiddenUsageFields.every((field) => !Object.prototype.hasOwnProperty.call(candidate, field)) && isIsoTimestamp(candidate.at) && candidate.basis === "operator-observed-original-budget-balance" && candidate.attribution === "account-wide-unattributed" && isSha256(candidate.priorLedgerSha256) && typeof candidate.originalBudgetCredits === "number" && Number.isFinite(candidate.originalBudgetCredits) && candidate.originalBudgetCredits > 0 && typeof candidate.priorRecordedSpentCredits === "number" && Number.isFinite(candidate.priorRecordedSpentCredits) && candidate.priorRecordedSpentCredits >= 0 && typeof candidate.observedRemainingCredits === "number" && Number.isFinite(candidate.observedRemainingCredits) && candidate.observedRemainingCredits >= 0 && typeof candidate.credits === "number" && Number.isFinite(candidate.credits) && candidate.credits >= 0 && candidate.confirmations?.observedBalanceBelongsToOriginalBudget === true && candidate.confirmations.noCreditsAddedOrRefunded === true && candidate.confirmations.accountWideUnattributedChargeAccepted === true && isValidStoredRunId(candidate.affectedBlockedEvent?.runId) && typeof candidate.affectedBlockedEvent?.blockedReason === "string" && candidate.affectedBlockedEvent.blockedReason.length > 0;
|
|
6536
|
+
return forbiddenUsageFields.every((field) => !Object.prototype.hasOwnProperty.call(candidate, field)) && isIsoTimestamp(candidate.at) && candidate.basis === "operator-observed-original-budget-balance" && candidate.attribution === "account-wide-unattributed" && isSha256(candidate.priorLedgerSha256) && typeof candidate.originalBudgetCredits === "number" && Number.isFinite(candidate.originalBudgetCredits) && candidate.originalBudgetCredits > 0 && isSupportedBudgetUnits(candidate.originalBudgetCredits) && typeof candidate.priorRecordedSpentCredits === "number" && Number.isFinite(candidate.priorRecordedSpentCredits) && candidate.priorRecordedSpentCredits >= 0 && isSupportedBudgetUnits(candidate.priorRecordedSpentCredits) && typeof candidate.observedRemainingCredits === "number" && Number.isFinite(candidate.observedRemainingCredits) && candidate.observedRemainingCredits >= 0 && isSupportedBudgetUnits(candidate.observedRemainingCredits) && typeof candidate.credits === "number" && Number.isFinite(candidate.credits) && candidate.credits >= 0 && isSupportedBudgetUnits(candidate.credits) && candidate.confirmations?.observedBalanceBelongsToOriginalBudget === true && candidate.confirmations.noCreditsAddedOrRefunded === true && candidate.confirmations.accountWideUnattributedChargeAccepted === true && isValidStoredRunId(candidate.affectedBlockedEvent?.runId) && typeof candidate.affectedBlockedEvent?.blockedReason === "string" && candidate.affectedBlockedEvent.blockedReason.length > 0;
|
|
6381
6537
|
}
|
|
6382
|
-
function
|
|
6383
|
-
return typeof budget === "number" &&
|
|
6384
|
-
|
|
6385
|
-
|
|
6538
|
+
function isLedgerReconciliationWithinBudgetV1(value, budget) {
|
|
6539
|
+
return typeof budget === "number" && isLedgerReconciliationV1(value) && value.originalBudgetCredits === budget && value.observedRemainingCredits <= budget && value.credits <= budget && Math.abs(value.priorRecordedSpentCredits + value.credits + value.observedRemainingCredits - budget) <= 1e-9;
|
|
6540
|
+
}
|
|
6541
|
+
function isLedgerResolution(value) {
|
|
6542
|
+
if (!value || typeof value !== "object") return false;
|
|
6543
|
+
const candidate = value;
|
|
6544
|
+
return isIsoTimestamp(candidate.at) && candidate.basis === "operator-observed-account-balance-delta" && candidate.attribution === "account-wide-observation-window" && isSha256(candidate.priorLedgerSha256) && isNonNegativeFinite(candidate.priorRecordedSpentUnits) && Number.isSafeInteger(candidate.priorEntryCount) && candidate.priorEntryCount >= 0 && typeof candidate.beforeAccountBalance === "string" && isExactDecimal(candidate.beforeAccountBalance) && typeof candidate.afterAccountBalance === "string" && isExactDecimal(candidate.afterAccountBalance) && typeof candidate.observedAccountDebit === "string" && isExactDecimal(candidate.observedAccountDebit) && subtractExactDecimals(candidate.beforeAccountBalance, candidate.afterAccountBalance) === candidate.observedAccountDebit && (candidate.localBudgetChargeUnits === 0 || candidate.localBudgetChargeUnits === MAX_BOUNDED_CALL_CREDITS) && (candidate.observedAccountDebit === "0" ? candidate.localBudgetChargeUnits === 0 : candidate.localBudgetChargeUnits === MAX_BOUNDED_CALL_CREDITS) && candidate.confirmations?.sameAccount === true && candidate.confirmations.snapshotsBracketBlockedEvent === true && candidate.confirmations.balanceSettled === true && candidate.confirmations.noCreditsAddedOrRefunded === true && (candidate.observedAccountDebit === "0" || candidate.confirmations.noInterveningCodexActivity === true) && isValidStoredRunId(candidate.affectedRunId) && isBlockedEvent(candidate.affectedBlockedEvent) && (candidate.affectedBlockedEvent.runId === void 0 || candidate.affectedBlockedEvent.runId === candidate.affectedRunId);
|
|
6545
|
+
}
|
|
6546
|
+
function isResolutionHistoryConsistent(ledger) {
|
|
6547
|
+
const resolutions = ledger.resolutions ?? [];
|
|
6548
|
+
if (resolutions.length === 0) return true;
|
|
6549
|
+
const legacyNanounits = sumBudgetUnitNanounits(
|
|
6550
|
+
(ledger.legacyReconciliations ?? []).map((reconciliation) => reconciliation.credits)
|
|
6551
|
+
);
|
|
6552
|
+
let priorResolutionNanounits = 0n;
|
|
6553
|
+
let priorEntryCount = 0;
|
|
6554
|
+
const seenHashes = /* @__PURE__ */ new Set();
|
|
6555
|
+
for (let index = 0; index < resolutions.length; index += 1) {
|
|
6556
|
+
const resolution = resolutions[index];
|
|
6557
|
+
if (!resolution) return false;
|
|
6558
|
+
if (seenHashes.has(resolution.priorLedgerSha256)) return false;
|
|
6559
|
+
seenHashes.add(resolution.priorLedgerSha256);
|
|
6560
|
+
if (resolution.priorEntryCount < priorEntryCount || resolution.priorEntryCount > ledger.entries.length)
|
|
6561
|
+
return false;
|
|
6562
|
+
const entriesBeforeResolution = ledger.entries.slice(0, resolution.priorEntryCount);
|
|
6563
|
+
const expectedPriorSpentNanounits = sumBudgetUnitNanounits(entriesBeforeResolution.map((entry) => entry.budgetUnits)) + legacyNanounits + priorResolutionNanounits;
|
|
6564
|
+
if (budgetUnitsToNanounits(resolution.priorRecordedSpentUnits) !== expectedPriorSpentNanounits) return false;
|
|
6565
|
+
if (!(index === 0 && isDirectV1PredecessorResolution(ledger, resolution))) {
|
|
6566
|
+
const priorLedger = {
|
|
6567
|
+
...ledger,
|
|
6568
|
+
spentUnits: resolution.priorRecordedSpentUnits,
|
|
6569
|
+
entries: entriesBeforeResolution,
|
|
6570
|
+
resolutions: index > 0 ? resolutions.slice(0, index) : void 0,
|
|
6571
|
+
blockedEvent: resolution.affectedBlockedEvent
|
|
6572
|
+
};
|
|
6573
|
+
if (sha256(serializeLedger(priorLedger)) !== resolution.priorLedgerSha256) return false;
|
|
6574
|
+
}
|
|
6575
|
+
priorResolutionNanounits += budgetUnitsToNanounits(resolution.localBudgetChargeUnits);
|
|
6576
|
+
priorEntryCount = resolution.priorEntryCount;
|
|
6577
|
+
}
|
|
6578
|
+
return true;
|
|
6579
|
+
}
|
|
6580
|
+
function isMigrationWitnessConsistent(ledger) {
|
|
6581
|
+
if (!ledger.migratedFromV1Sha256 && !ledger.migrationWitnessV1) return true;
|
|
6582
|
+
if (!ledger.migratedFromV1Sha256 || !ledger.migrationWitnessV1) return false;
|
|
6583
|
+
const source = ledger.migrationWitnessV1.source;
|
|
6584
|
+
if (typeof source !== "string" || sha256(source) !== ledger.migratedFromV1Sha256) return false;
|
|
6585
|
+
let predecessor;
|
|
6586
|
+
try {
|
|
6587
|
+
predecessor = JSON.parse(source);
|
|
6588
|
+
} catch {
|
|
6589
|
+
return false;
|
|
6590
|
+
}
|
|
6591
|
+
if (!isLedgerV1(predecessor)) return false;
|
|
6592
|
+
if (predecessor.budgetCredits !== ledger.budgetUnits || predecessor.reserveCredits !== ledger.reserveUnits)
|
|
6593
|
+
return false;
|
|
6594
|
+
if (JSON.stringify(predecessor.reconciliations ?? []) !== JSON.stringify(ledger.legacyReconciliations ?? [])) {
|
|
6595
|
+
return false;
|
|
6596
|
+
}
|
|
6597
|
+
const migratedEntries = predecessor.entries.map(({ credits, ...entry }) => ({ ...entry, budgetUnits: credits }));
|
|
6598
|
+
const firstResolution = ledger.resolutions?.[0];
|
|
6599
|
+
const entriesAtMigration = firstResolution ? ledger.entries.slice(0, firstResolution.priorEntryCount) : ledger.entries.slice(0, migratedEntries.length);
|
|
6600
|
+
if (JSON.stringify(migratedEntries) !== JSON.stringify(entriesAtMigration.slice(0, migratedEntries.length))) {
|
|
6601
|
+
return false;
|
|
6602
|
+
}
|
|
6603
|
+
return true;
|
|
6604
|
+
}
|
|
6605
|
+
function isDirectV1PredecessorResolution(ledger, resolution) {
|
|
6606
|
+
if (!ledger.migratedFromV1Sha256 || !ledger.migrationWitnessV1) return false;
|
|
6607
|
+
let predecessor;
|
|
6608
|
+
try {
|
|
6609
|
+
predecessor = JSON.parse(ledger.migrationWitnessV1.source);
|
|
6610
|
+
} catch {
|
|
6611
|
+
return false;
|
|
6612
|
+
}
|
|
6613
|
+
return isLedgerV1(predecessor) && resolution.priorEntryCount === predecessor.entries.length && resolution.priorLedgerSha256 === ledger.migratedFromV1Sha256 && budgetUnitsToNanounits(resolution.priorRecordedSpentUnits) === ledgerV1SpentNanounits(predecessor) && predecessor.blockedReason === resolution.affectedBlockedEvent.reason;
|
|
6614
|
+
}
|
|
6615
|
+
function isBlockedEvent(value) {
|
|
6616
|
+
if (!value || typeof value !== "object") return false;
|
|
6617
|
+
const candidate = value;
|
|
6618
|
+
return typeof candidate.reason === "string" && candidate.reason.length > 0 && (candidate.at === void 0 || isIsoTimestamp(candidate.at)) && (candidate.runId === void 0 || isValidStoredRunId(candidate.runId)) && (candidate.model === void 0 || typeof candidate.model === "string" && candidate.model.length > 0);
|
|
6619
|
+
}
|
|
6620
|
+
function isLedgerEntryV1(entry) {
|
|
6621
|
+
if (!entry || typeof entry !== "object") return false;
|
|
6622
|
+
const candidate = entry;
|
|
6623
|
+
return isLedgerEntryCommon(candidate) && isNonNegativeFinite(candidate.credits) && isSupportedBudgetUnits(candidate.credits) && isEntryCreditConsistentV1(candidate);
|
|
6386
6624
|
}
|
|
6387
6625
|
function isLedgerEntry(entry) {
|
|
6388
6626
|
if (!entry || typeof entry !== "object") return false;
|
|
6389
6627
|
const candidate = entry;
|
|
6390
|
-
return
|
|
6628
|
+
return isIsoTimestamp(candidate.at) && typeof candidate.model === "string" && candidate.model.length > 0 && isNonNegativeFinite(candidate.budgetUnits) && (candidate.runId === void 0 || isValidStoredRunId(candidate.runId)) && readCounter(candidate.inputTokens) !== void 0 && readCounter(candidate.cachedInputTokens) !== void 0 && readCounter(candidate.outputTokens) !== void 0 && readCounter(candidate.reasoningOutputTokens) !== void 0 && (candidate.cachedInputTokens ?? 0) <= (candidate.inputTokens ?? 0) && isEntryBudgetUnitConsistent(candidate);
|
|
6391
6629
|
}
|
|
6392
|
-
function
|
|
6630
|
+
function isLedgerEntryCommon(candidate) {
|
|
6631
|
+
return isIsoTimestamp(candidate.at) && typeof candidate.model === "string" && candidate.model.length > 0 && (candidate.runId === void 0 || isValidStoredRunId(candidate.runId)) && readCounter(candidate.inputTokens) !== void 0 && readCounter(candidate.cachedInputTokens) !== void 0 && readCounter(candidate.outputTokens) !== void 0 && readCounter(candidate.reasoningOutputTokens) !== void 0 && (candidate.cachedInputTokens ?? 0) <= (candidate.inputTokens ?? 0);
|
|
6632
|
+
}
|
|
6633
|
+
function isEntryCreditConsistentV1(entry) {
|
|
6393
6634
|
try {
|
|
6394
|
-
return Math.abs(
|
|
6635
|
+
return Math.abs(calculateCodexBudgetUnits(entry.model, entry) - entry.credits) <= 1e-9;
|
|
6395
6636
|
} catch {
|
|
6396
6637
|
return false;
|
|
6397
6638
|
}
|
|
6398
6639
|
}
|
|
6399
|
-
function
|
|
6640
|
+
function isEntryBudgetUnitConsistent(entry) {
|
|
6641
|
+
try {
|
|
6642
|
+
return Math.abs(calculateCodexBudgetUnits(entry.model, entry) - entry.budgetUnits) <= 1e-9;
|
|
6643
|
+
} catch {
|
|
6644
|
+
return false;
|
|
6645
|
+
}
|
|
6646
|
+
}
|
|
6647
|
+
function summarizeLedgerEntries(entries, resolutions = [], legacyReconciliations = []) {
|
|
6400
6648
|
const byModel = /* @__PURE__ */ new Map();
|
|
6401
6649
|
for (const entry of entries) {
|
|
6402
6650
|
const modelEntries = byModel.get(entry.model) ?? [];
|
|
@@ -6406,17 +6654,18 @@ function summarizeLedgerEntries(entries, reconciliations = []) {
|
|
|
6406
6654
|
const totals = summarizeUsage(entries);
|
|
6407
6655
|
return {
|
|
6408
6656
|
calls: entries.length,
|
|
6409
|
-
|
|
6410
|
-
|
|
6411
|
-
|
|
6412
|
-
|
|
6413
|
-
|
|
6657
|
+
budgetUnits: nanounitsToBudgetUnits(
|
|
6658
|
+
sumBudgetUnitNanounits(entries.map((entry) => entry.budgetUnits)) + sumBudgetUnitNanounits(resolutions.map((resolution) => resolution.localBudgetChargeUnits)) + sumBudgetUnitNanounits(legacyReconciliations.map((reconciliation) => reconciliation.credits))
|
|
6659
|
+
),
|
|
6660
|
+
accountBalanceResolutionCount: resolutions.length + legacyReconciliations.length,
|
|
6661
|
+
conservativeResolutionChargeUnits: nanounitsToBudgetUnits(
|
|
6662
|
+
sumBudgetUnitNanounits(resolutions.map((resolution) => resolution.localBudgetChargeUnits)) + sumBudgetUnitNanounits(legacyReconciliations.map((reconciliation) => reconciliation.credits))
|
|
6414
6663
|
),
|
|
6415
6664
|
...totals,
|
|
6416
6665
|
models: [...byModel.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([model, modelEntries]) => ({
|
|
6417
6666
|
model,
|
|
6418
6667
|
calls: modelEntries.length,
|
|
6419
|
-
|
|
6668
|
+
budgetUnits: nanounitsToBudgetUnits(sumBudgetUnitNanounits(modelEntries.map((entry) => entry.budgetUnits))),
|
|
6420
6669
|
...summarizeUsage(modelEntries)
|
|
6421
6670
|
}))
|
|
6422
6671
|
};
|
|
@@ -6433,14 +6682,23 @@ function summarizeUsage(entries) {
|
|
|
6433
6682
|
);
|
|
6434
6683
|
}
|
|
6435
6684
|
async function writeLedger(filePath, ledger) {
|
|
6685
|
+
if (failNextLedgerWriteForTest) {
|
|
6686
|
+
failNextLedgerWriteForTest = false;
|
|
6687
|
+
throw new Error(`injected ledger write failure for ${filePath}`);
|
|
6688
|
+
}
|
|
6436
6689
|
await mkdir2(path3.dirname(filePath), { recursive: true, mode: 448 });
|
|
6437
6690
|
const tempPath = `${filePath}.${process.pid}.tmp`;
|
|
6438
|
-
|
|
6439
|
-
|
|
6691
|
+
const contents = serializeLedger(ledger);
|
|
6692
|
+
await writeFile2(tempPath, contents, {
|
|
6440
6693
|
encoding: "utf8",
|
|
6441
6694
|
mode: 384
|
|
6442
6695
|
});
|
|
6443
6696
|
await rename2(tempPath, filePath);
|
|
6697
|
+
return { contents, sha256: sha256(contents) };
|
|
6698
|
+
}
|
|
6699
|
+
function serializeLedger(ledger) {
|
|
6700
|
+
return `${JSON.stringify(ledger, null, 2)}
|
|
6701
|
+
`;
|
|
6444
6702
|
}
|
|
6445
6703
|
function readCounter(value) {
|
|
6446
6704
|
return Number.isSafeInteger(value) && value >= 0 ? value : void 0;
|
|
@@ -6451,6 +6709,55 @@ function readOptionalCounter(value) {
|
|
|
6451
6709
|
function safeErrorMessage(error) {
|
|
6452
6710
|
return error instanceof Error ? error.message : String(error);
|
|
6453
6711
|
}
|
|
6712
|
+
function resourceLockedError(cause) {
|
|
6713
|
+
const original = cause instanceof Error ? cause : new Error("Codex credit ledger lock acquisition failed.");
|
|
6714
|
+
return new BenchmarkRunBlockedError(
|
|
6715
|
+
BenchmarkRunBlockReason.ResourceLocked,
|
|
6716
|
+
"Codex credit ledger resource is locked.",
|
|
6717
|
+
{ cause: original }
|
|
6718
|
+
);
|
|
6719
|
+
}
|
|
6720
|
+
function infrastructureUnavailableError(cause) {
|
|
6721
|
+
const original = cause instanceof Error ? cause : new Error("Codex ledger infrastructure setup failed.");
|
|
6722
|
+
return new BenchmarkRunBlockedError(
|
|
6723
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
6724
|
+
"Codex ledger infrastructure is unavailable.",
|
|
6725
|
+
{ cause: original }
|
|
6726
|
+
);
|
|
6727
|
+
}
|
|
6728
|
+
function accountingPersistenceBlockedError(persistenceError, underlyingRunError) {
|
|
6729
|
+
const persistenceCause = persistenceError instanceof Error ? persistenceError : new Error("Codex ledger persistence failed.");
|
|
6730
|
+
const cause = underlyingRunError === void 0 ? persistenceCause : new AggregateError(
|
|
6731
|
+
[persistenceCause, underlyingRunError],
|
|
6732
|
+
"Codex ledger persistence failed after an uncertain dispatch outcome."
|
|
6733
|
+
);
|
|
6734
|
+
return new BenchmarkRunBlockedError(
|
|
6735
|
+
BenchmarkRunBlockReason.ManualReconciliationRequired,
|
|
6736
|
+
"Codex usage accounting could not be persisted; manual reconciliation is required.",
|
|
6737
|
+
{ cause }
|
|
6738
|
+
);
|
|
6739
|
+
}
|
|
6740
|
+
async function prepareLedgerDirectory(lockPath) {
|
|
6741
|
+
if (failNextLedgerSetupForTest) {
|
|
6742
|
+
failNextLedgerSetupForTest = false;
|
|
6743
|
+
throw new Error(`injected setup failure for ${lockPath}`);
|
|
6744
|
+
}
|
|
6745
|
+
await mkdir2(path3.dirname(lockPath), { recursive: true, mode: 448 });
|
|
6746
|
+
}
|
|
6747
|
+
async function bestEffortSettleLock(lock) {
|
|
6748
|
+
await writeLockState(lock, "settled").catch(() => void 0);
|
|
6749
|
+
}
|
|
6750
|
+
async function bestEffortCloseLock(lock) {
|
|
6751
|
+
await lock.close().catch(() => void 0);
|
|
6752
|
+
}
|
|
6753
|
+
async function bestEffortRemoveOwnedLedgerLock(lockPath) {
|
|
6754
|
+
if (failNextOwnedLockRemovalForTest) {
|
|
6755
|
+
failNextOwnedLockRemovalForTest = false;
|
|
6756
|
+
failNextLedgerSetupForTest = false;
|
|
6757
|
+
return;
|
|
6758
|
+
}
|
|
6759
|
+
await removeOwnedLedgerLock(lockPath).catch(() => void 0);
|
|
6760
|
+
}
|
|
6454
6761
|
function expandHomeRelativePath(value) {
|
|
6455
6762
|
if (value === "~") return os.homedir();
|
|
6456
6763
|
if (value.startsWith("~/") || value.startsWith("~\\")) {
|
|
@@ -6472,6 +6779,94 @@ function parseNonNegativeNumber(value, name) {
|
|
|
6472
6779
|
}
|
|
6473
6780
|
return parsed;
|
|
6474
6781
|
}
|
|
6782
|
+
function isPositiveFinite(value) {
|
|
6783
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0;
|
|
6784
|
+
}
|
|
6785
|
+
function isNonNegativeFinite(value) {
|
|
6786
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
6787
|
+
}
|
|
6788
|
+
function isSupportedBudgetUnits(value) {
|
|
6789
|
+
if (typeof value !== "number") return false;
|
|
6790
|
+
try {
|
|
6791
|
+
budgetUnitsToNanounits(value);
|
|
6792
|
+
return true;
|
|
6793
|
+
} catch {
|
|
6794
|
+
return false;
|
|
6795
|
+
}
|
|
6796
|
+
}
|
|
6797
|
+
var BUDGET_UNIT_SCALE = 1e9;
|
|
6798
|
+
function rateNanounitsPerToken(rate) {
|
|
6799
|
+
const scaled = rate * 1e3;
|
|
6800
|
+
if (!Number.isSafeInteger(scaled)) throw new Error("Codex credit rate exceeds nanounit precision");
|
|
6801
|
+
return BigInt(scaled);
|
|
6802
|
+
}
|
|
6803
|
+
function budgetUnitsToNanounits(value) {
|
|
6804
|
+
if (!Number.isFinite(value) || value < 0) throw new Error("budget units must be finite and non-negative");
|
|
6805
|
+
const fixed = value.toFixed(9);
|
|
6806
|
+
if (Number(fixed) !== value) {
|
|
6807
|
+
throw new Error("budget units exceed supported nanounit precision");
|
|
6808
|
+
}
|
|
6809
|
+
const [integer = "0", fraction = ""] = fixed.split(".");
|
|
6810
|
+
const nanounits = BigInt(integer) * BigInt(BUDGET_UNIT_SCALE) + BigInt(fraction.padEnd(9, "0"));
|
|
6811
|
+
if (nanounits > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("budget units exceed safe integer range");
|
|
6812
|
+
return nanounits;
|
|
6813
|
+
}
|
|
6814
|
+
function nanounitsToBudgetUnits(value) {
|
|
6815
|
+
return Number(value) / BUDGET_UNIT_SCALE;
|
|
6816
|
+
}
|
|
6817
|
+
function sumBudgetUnitNanounits(values) {
|
|
6818
|
+
return values.reduce((sum, value) => sum + budgetUnitsToNanounits(value), 0n);
|
|
6819
|
+
}
|
|
6820
|
+
function ledgerV1SpentNanounits(ledger) {
|
|
6821
|
+
return sumBudgetUnitNanounits(ledger.entries.map((entry) => entry.credits)) + sumBudgetUnitNanounits((ledger.reconciliations ?? []).map((reconciliation) => reconciliation.credits));
|
|
6822
|
+
}
|
|
6823
|
+
function isLedgerV1SpentConsistent(ledger) {
|
|
6824
|
+
try {
|
|
6825
|
+
const exactSpentNanounits = ledgerV1SpentNanounits(ledger);
|
|
6826
|
+
if (exactSpentNanounits > BigInt(Number.MAX_SAFE_INTEGER)) return false;
|
|
6827
|
+
const exactSpentUnits = nanounitsToBudgetUnits(exactSpentNanounits);
|
|
6828
|
+
return Math.abs(exactSpentUnits - ledger.spentCredits) <= LEGACY_LEDGER_FLOAT_DRIFT_TOLERANCE;
|
|
6829
|
+
} catch {
|
|
6830
|
+
return false;
|
|
6831
|
+
}
|
|
6832
|
+
}
|
|
6833
|
+
function ledgerSpentNanounits(ledger) {
|
|
6834
|
+
return sumBudgetUnitNanounits(ledger.entries.map((entry) => entry.budgetUnits)) + sumBudgetUnitNanounits((ledger.resolutions ?? []).map((resolution) => resolution.localBudgetChargeUnits)) + sumBudgetUnitNanounits((ledger.legacyReconciliations ?? []).map((reconciliation) => reconciliation.credits));
|
|
6835
|
+
}
|
|
6836
|
+
function isLedgerSpentConsistent(ledger) {
|
|
6837
|
+
try {
|
|
6838
|
+
return budgetUnitsToNanounits(ledger.spentUnits) === ledgerSpentNanounits(ledger);
|
|
6839
|
+
} catch {
|
|
6840
|
+
return false;
|
|
6841
|
+
}
|
|
6842
|
+
}
|
|
6843
|
+
function parseExactDecimal(value, name) {
|
|
6844
|
+
if (typeof value !== "string" || value !== value.trim() || !isExactDecimal(value)) {
|
|
6845
|
+
throw new Error(`${name} must be a non-negative plain decimal string`);
|
|
6846
|
+
}
|
|
6847
|
+
return value;
|
|
6848
|
+
}
|
|
6849
|
+
function isExactDecimal(value) {
|
|
6850
|
+
return /^(?:0|[1-9]\d*)(?:\.\d+)?$/.test(value) && value.length <= 128;
|
|
6851
|
+
}
|
|
6852
|
+
function subtractExactDecimals(before, after) {
|
|
6853
|
+
const beforeParts = splitExactDecimal(before);
|
|
6854
|
+
const afterParts = splitExactDecimal(after);
|
|
6855
|
+
const scale = Math.max(beforeParts.fraction.length, afterParts.fraction.length);
|
|
6856
|
+
const beforeScaled = BigInt(`${beforeParts.integer}${beforeParts.fraction.padEnd(scale, "0")}`);
|
|
6857
|
+
const afterScaled = BigInt(`${afterParts.integer}${afterParts.fraction.padEnd(scale, "0")}`);
|
|
6858
|
+
const difference = beforeScaled - afterScaled;
|
|
6859
|
+
const sign = difference < 0n ? "-" : "";
|
|
6860
|
+
const digits = (difference < 0n ? -difference : difference).toString().padStart(scale + 1, "0");
|
|
6861
|
+
if (scale === 0) return `${sign}${digits}`;
|
|
6862
|
+
const integer = digits.slice(0, -scale);
|
|
6863
|
+
const fraction = digits.slice(-scale).replace(/0+$/, "");
|
|
6864
|
+
return fraction ? `${sign}${integer}.${fraction}` : `${sign}${integer}`;
|
|
6865
|
+
}
|
|
6866
|
+
function splitExactDecimal(value) {
|
|
6867
|
+
const [integer, fraction = ""] = value.split(".");
|
|
6868
|
+
return { integer: integer ?? "", fraction };
|
|
6869
|
+
}
|
|
6475
6870
|
function parseOptionalRunId(value) {
|
|
6476
6871
|
const runId = value?.trim();
|
|
6477
6872
|
if (!runId) return void 0;
|
|
@@ -6503,9 +6898,6 @@ function isSha256(value) {
|
|
|
6503
6898
|
function sha256(value) {
|
|
6504
6899
|
return createHash4("sha256").update(value).digest("hex");
|
|
6505
6900
|
}
|
|
6506
|
-
function normalizeZero(value) {
|
|
6507
|
-
return Object.is(value, -0) ? 0 : value;
|
|
6508
|
-
}
|
|
6509
6901
|
function isIsoTimestamp(value) {
|
|
6510
6902
|
if (typeof value !== "string") return false;
|
|
6511
6903
|
try {
|
|
@@ -10303,7 +10695,15 @@ var CodexCliProvider = class {
|
|
|
10303
10695
|
resolveBenchmarkRunId()
|
|
10304
10696
|
);
|
|
10305
10697
|
if (creditBudget) {
|
|
10306
|
-
|
|
10698
|
+
try {
|
|
10699
|
+
await this.assertChatGptCreditAuth();
|
|
10700
|
+
} catch (error) {
|
|
10701
|
+
throw new BenchmarkRunBlockedError(
|
|
10702
|
+
BenchmarkRunBlockReason.InfrastructureUnavailable,
|
|
10703
|
+
"Codex CLI ChatGPT authentication is unavailable for this bounded benchmark run.",
|
|
10704
|
+
{ cause: error }
|
|
10705
|
+
);
|
|
10706
|
+
}
|
|
10307
10707
|
}
|
|
10308
10708
|
const maxAttempts = normalizeCodexCliMaxAttempts(
|
|
10309
10709
|
creditBudget ? 1 : this.config.retryOptions?.maxAttempts
|
|
@@ -10455,6 +10855,10 @@ ${result.stderr}`
|
|
|
10455
10855
|
}
|
|
10456
10856
|
return { ok: true, verdict, telemetry };
|
|
10457
10857
|
} catch (error) {
|
|
10858
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
10859
|
+
if (blocked) {
|
|
10860
|
+
throw blocked;
|
|
10861
|
+
}
|
|
10458
10862
|
const aborted = isCodexStructuredJudgeAbort(error, request.signal);
|
|
10459
10863
|
const errorCode = aborted ? "aborted" : "transport_error";
|
|
10460
10864
|
return {
|
|
@@ -16282,7 +16686,11 @@ async function llmJudgeScoreDetailed(judge, question, predicted, expected) {
|
|
|
16282
16686
|
tokens: { input: 0, output: 0 },
|
|
16283
16687
|
latencyMs: durationMs
|
|
16284
16688
|
};
|
|
16285
|
-
} catch {
|
|
16689
|
+
} catch (error) {
|
|
16690
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
16691
|
+
if (blocked) {
|
|
16692
|
+
throw blocked;
|
|
16693
|
+
}
|
|
16286
16694
|
return {
|
|
16287
16695
|
score: deterministicJudgeFallback(predicted, expected),
|
|
16288
16696
|
tokens: { input: 0, output: 0 },
|
|
@@ -16302,7 +16710,11 @@ async function llmBinaryJudgeScoreDetailed(judge, prompt, fallback) {
|
|
|
16302
16710
|
const startedAt = performance.now();
|
|
16303
16711
|
try {
|
|
16304
16712
|
return await judge.scoreBinaryPrompt(prompt);
|
|
16305
|
-
} catch {
|
|
16713
|
+
} catch (error) {
|
|
16714
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
16715
|
+
if (blocked) {
|
|
16716
|
+
throw blocked;
|
|
16717
|
+
}
|
|
16306
16718
|
return {
|
|
16307
16719
|
score: deterministicJudgeFallback(fallback.predicted, fallback.expected),
|
|
16308
16720
|
tokens: { input: 0, output: 0 },
|
|
@@ -20247,37 +20659,51 @@ async function executePlanTrials(ctx, trials, options) {
|
|
|
20247
20659
|
}
|
|
20248
20660
|
return;
|
|
20249
20661
|
}
|
|
20250
|
-
|
|
20251
|
-
|
|
20252
|
-
|
|
20253
|
-
|
|
20254
|
-
|
|
20255
|
-
|
|
20256
|
-
|
|
20257
|
-
|
|
20662
|
+
for (let batchStart = 0; batchStart < trials.length; batchStart += options.trialConcurrency) {
|
|
20663
|
+
const batch = trials.slice(
|
|
20664
|
+
batchStart,
|
|
20665
|
+
batchStart + options.trialConcurrency
|
|
20666
|
+
);
|
|
20667
|
+
const settled = await Promise.allSettled(
|
|
20668
|
+
batch.map(
|
|
20669
|
+
(trial) => executeTrialWithFailure(
|
|
20670
|
+
ctx,
|
|
20671
|
+
trial,
|
|
20672
|
+
options.planIndex,
|
|
20673
|
+
options.answerSupportGate
|
|
20674
|
+
)
|
|
20675
|
+
)
|
|
20676
|
+
);
|
|
20677
|
+
const unexpectedRejection = settled.find(
|
|
20678
|
+
(result) => result.status === "rejected" && findBenchmarkRunBlockedError(result.reason) === void 0
|
|
20679
|
+
);
|
|
20680
|
+
if (unexpectedRejection) {
|
|
20681
|
+
throw unexpectedRejection.reason;
|
|
20258
20682
|
}
|
|
20259
|
-
|
|
20260
|
-
|
|
20261
|
-
|
|
20262
|
-
|
|
20263
|
-
|
|
20264
|
-
|
|
20265
|
-
|
|
20683
|
+
const terminalOffset = settled.findIndex(
|
|
20684
|
+
(result) => result.status === "rejected" && findBenchmarkRunBlockedError(result.reason) !== void 0
|
|
20685
|
+
);
|
|
20686
|
+
const emitLimit = terminalOffset < 0 ? settled.length : terminalOffset;
|
|
20687
|
+
for (let offset = 0; offset < emitLimit; offset += 1) {
|
|
20688
|
+
const result = settled[offset];
|
|
20689
|
+
if (result?.status !== "fulfilled") {
|
|
20690
|
+
throw new Error(
|
|
20691
|
+
`PublishedBenchmarkHarness: concurrent trial ${batchStart + offset} did not settle before canonical emission.`
|
|
20692
|
+
);
|
|
20266
20693
|
}
|
|
20267
|
-
|
|
20268
|
-
ctx,
|
|
20269
|
-
trials[trialIndex],
|
|
20270
|
-
options.planIndex,
|
|
20271
|
-
options.answerSupportGate
|
|
20272
|
-
);
|
|
20273
|
-
completed[trialIndex] = true;
|
|
20274
|
-
emitCompletedPrefix();
|
|
20694
|
+
appendCompletedTask(ctx, options.tasks, result.value);
|
|
20275
20695
|
}
|
|
20276
|
-
|
|
20277
|
-
|
|
20278
|
-
|
|
20279
|
-
|
|
20280
|
-
|
|
20696
|
+
if (terminalOffset >= 0) {
|
|
20697
|
+
const terminalResult = settled[terminalOffset];
|
|
20698
|
+
const terminalError = terminalResult?.status === "rejected" ? findBenchmarkRunBlockedError(terminalResult.reason) : void 0;
|
|
20699
|
+
if (!terminalError) {
|
|
20700
|
+
throw new Error(
|
|
20701
|
+
`PublishedBenchmarkHarness: concurrent trial ${batchStart + terminalOffset} lost its terminal error before canonical emission.`
|
|
20702
|
+
);
|
|
20703
|
+
}
|
|
20704
|
+
throw terminalError;
|
|
20705
|
+
}
|
|
20706
|
+
}
|
|
20281
20707
|
}
|
|
20282
20708
|
function appendCompletedTask(ctx, tasks, task) {
|
|
20283
20709
|
tasks.push(task);
|
|
@@ -20288,6 +20714,10 @@ async function executeTrialWithFailure(ctx, trial, planIndex, answerSupportGate)
|
|
|
20288
20714
|
try {
|
|
20289
20715
|
return await executeTrial(ctx, trial, answerSupportGate);
|
|
20290
20716
|
} catch (err) {
|
|
20717
|
+
const blocked = findBenchmarkRunBlockedError(err);
|
|
20718
|
+
if (blocked) {
|
|
20719
|
+
throw blocked;
|
|
20720
|
+
}
|
|
20291
20721
|
const message = err instanceof Error ? err.message : String(err);
|
|
20292
20722
|
console.error(` [WARN] harness trial plan-${planIndex}/${trialId} failed: ${message}`);
|
|
20293
20723
|
return {
|
|
@@ -20532,6 +20962,10 @@ async function assessRecallSupport(ctx, trial, recalledText) {
|
|
|
20532
20962
|
validateRecallSupportAssessment(assessment);
|
|
20533
20963
|
return assessment;
|
|
20534
20964
|
} catch (error) {
|
|
20965
|
+
const blocked = findBenchmarkRunBlockedError(error);
|
|
20966
|
+
if (blocked) {
|
|
20967
|
+
throw blocked;
|
|
20968
|
+
}
|
|
20535
20969
|
return {
|
|
20536
20970
|
status: "backend_failure",
|
|
20537
20971
|
reason: error instanceof Error ? error.message : String(error)
|
|
@@ -20599,6 +21033,9 @@ async function scoreTrialJudge(ctx, trial, answeredText) {
|
|
|
20599
21033
|
);
|
|
20600
21034
|
}
|
|
20601
21035
|
function answerWithTrialFallback(trial, recalledText, error) {
|
|
21036
|
+
if (isBenchmarkRunBlockedError(error)) {
|
|
21037
|
+
throw error;
|
|
21038
|
+
}
|
|
20602
21039
|
const fallback = trial.answerFallback?.({
|
|
20603
21040
|
question: trial.question,
|
|
20604
21041
|
recalledText,
|
|
@@ -44580,6 +45017,7 @@ export {
|
|
|
44580
45017
|
buildProviderFreeLoCoMoRetrievalConfig,
|
|
44581
45018
|
buildSchemaTierFixture,
|
|
44582
45019
|
buildSchemaTierSmokeFixture,
|
|
45020
|
+
calculateCodexBudgetUnits,
|
|
44583
45021
|
calendarFixture,
|
|
44584
45022
|
canonicalJsonStringify,
|
|
44585
45023
|
captureLoCoMoRetrievalTrace,
|