@tangle-network/agent-runtime 0.94.6 → 0.94.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +1 -1
- package/dist/agent.js +4 -4
- package/dist/{agentic-generator-CL1q0H91.d.ts → agentic-generator-1g7GDba7.d.ts} +73 -6
- package/dist/analyst-loop.d.ts +1 -1
- package/dist/analyst-loop.js +2 -2
- package/dist/{chunk-GKZ6DFDN.js → chunk-7HH22XN4.js} +105 -27
- package/dist/chunk-7HH22XN4.js.map +1 -0
- package/dist/{chunk-JMT5VYY3.js → chunk-7NCP74SL.js} +174 -48
- package/dist/chunk-7NCP74SL.js.map +1 -0
- package/dist/{chunk-WQA5Y6PD.js → chunk-EP6RVHMX.js} +3 -3
- package/dist/chunk-EP6RVHMX.js.map +1 -0
- package/dist/{chunk-LWNMCOEA.js → chunk-FRXDOBFP.js} +3 -3
- package/dist/chunk-GHXW56PO.js +526 -0
- package/dist/chunk-GHXW56PO.js.map +1 -0
- package/dist/{chunk-63TIZHMI.js → chunk-KYPVUEJ4.js} +4 -4
- package/dist/{chunk-FVJ7M3DA.js → chunk-M22PL4GH.js} +1 -1
- package/dist/chunk-M22PL4GH.js.map +1 -0
- package/dist/{chunk-4VMRCEI6.js → chunk-MSUWXCHD.js} +2 -2
- package/dist/{chunk-ZQZX77MM.js → chunk-QDSOD7RC.js} +2 -2
- package/dist/{chunk-RNWDH5Z3.js → chunk-TNUBPPZA.js} +2 -2
- package/dist/{completion-gate-CfX2thvL.d.ts → completion-gate-tzwyyD-E.d.ts} +2 -2
- package/dist/{coordination-pOGZuYS7.d.ts → coordination-o0TzS7Ms.d.ts} +5 -5
- package/dist/environment-provider.d.ts +2 -2
- package/dist/{improve-Ciuvu2jh.d.ts → improve-DZeatcs7.d.ts} +19 -8
- package/dist/index.d.ts +20 -17
- package/dist/index.js +37 -20
- package/dist/index.js.map +1 -1
- package/dist/intelligence.d.ts +6 -6
- package/dist/intelligence.js +41 -17
- package/dist/intelligence.js.map +1 -1
- package/dist/knowledge.d.ts +7 -7
- package/dist/knowledge.js +4 -4
- package/dist/lifecycle.d.ts +3 -3
- package/dist/lifecycle.js +2 -2
- package/dist/{local-harness-CePtFrwu.d.ts → local-harness-ZLMkKdZc.d.ts} +1 -1
- package/dist/{loop-runner-bin-B4PwjVaF.d.ts → loop-runner-bin-iTSxSmpv.d.ts} +2 -2
- package/dist/loop-runner-bin.d.ts +6 -6
- package/dist/loop-runner-bin.js +6 -6
- package/dist/loops.d.ts +36 -23
- package/dist/loops.js +4 -4
- package/dist/mcp/bin.js +3 -3
- package/dist/mcp/index.d.ts +10 -10
- package/dist/mcp/index.js +5 -5
- package/dist/{mcp-serve-verifier-D7hCZIkT.d.ts → mcp-serve-verifier-BO4MEsR3.d.ts} +1 -1
- package/dist/{openai-tools-BMOu1t3A.d.ts → openai-tools-0wACz8f8.d.ts} +1 -1
- package/dist/profiles.d.ts +1 -1
- package/dist/{sanitize-Dr3SliRC.d.ts → sanitize-Dcbjp0tU.d.ts} +1 -1
- package/dist/{supervise-Y3w878fN.d.ts → supervise-CRwaJIkd.d.ts} +4 -4
- package/dist/{types-Kjr5W6OF.d.ts → types-CKmyZ9TP.d.ts} +4 -7
- package/dist/{types-DHLXdLVN.d.ts → types-CogNCaw7.d.ts} +1 -1
- package/dist/{worktree-fanout-DmBvR-_O.d.ts → worktree-fanout-BJy24LYe.d.ts} +4 -4
- package/package.json +3 -3
- package/dist/chunk-4WWSICFY.js +0 -210
- package/dist/chunk-4WWSICFY.js.map +0 -1
- package/dist/chunk-FVJ7M3DA.js.map +0 -1
- package/dist/chunk-GKZ6DFDN.js.map +0 -1
- package/dist/chunk-JMT5VYY3.js.map +0 -1
- package/dist/chunk-WQA5Y6PD.js.map +0 -1
- /package/dist/{chunk-LWNMCOEA.js.map → chunk-FRXDOBFP.js.map} +0 -0
- /package/dist/{chunk-63TIZHMI.js.map → chunk-KYPVUEJ4.js.map} +0 -0
- /package/dist/{chunk-4VMRCEI6.js.map → chunk-MSUWXCHD.js.map} +0 -0
- /package/dist/{chunk-ZQZX77MM.js.map → chunk-QDSOD7RC.js.map} +0 -0
- /package/dist/{chunk-RNWDH5Z3.js.map → chunk-TNUBPPZA.js.map} +0 -0
package/dist/agent.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as _tangle_network_agent_eval from '@tangle-network/agent-eval';
|
|
2
2
|
import { TraceAnalystKindSpec, AnalystFinding } from '@tangle-network/agent-eval';
|
|
3
3
|
import { A as ArtifactKind, C as CandidateGenerator, P as PromotionGate } from './generator-YkAQrOoD.js';
|
|
4
|
-
import { a as RuntimeStreamEvent, S as SandboxClient, c as OutputAdapter, d as AgentRunSpec } from './types-
|
|
4
|
+
import { a as RuntimeStreamEvent, S as SandboxClient, c as OutputAdapter, d as AgentRunSpec } from './types-CKmyZ9TP.js';
|
|
5
5
|
import { A as AgentSurfaces } from './improvement-adapter-BieWeK5J.js';
|
|
6
6
|
export { C as CreateSurfaceImprovementAdapterOpts, D as DraftPatchInput, a as DraftPatchOutput, R as ResolvedSurface, S as SurfaceImprovementEdit, b as SurfaceValidationIssue, c as createSurfaceImprovementAdapter, r as renderSurfaceIssues, d as resolveSubjectPath, v as validateSurfaces } from './improvement-adapter-BieWeK5J.js';
|
|
7
7
|
import { K as KnowledgeAdapter, a as RunAnalystLoopResult } from './types-BC3bZpH0.js';
|
package/dist/agent.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import "./chunk-
|
|
1
|
+
import "./chunk-7HH22XN4.js";
|
|
2
2
|
import {
|
|
3
3
|
createSandboxForSpec
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-FRXDOBFP.js";
|
|
5
5
|
import "./chunk-DPEUKJRO.js";
|
|
6
6
|
import "./chunk-ISTDY47H.js";
|
|
7
7
|
import "./chunk-WIPGQ4GT.js";
|
|
@@ -9,8 +9,8 @@ import "./chunk-VSWBYWFK.js";
|
|
|
9
9
|
import "./chunk-3MDZX7YU.js";
|
|
10
10
|
import {
|
|
11
11
|
mapSandboxEvent
|
|
12
|
-
} from "./chunk-
|
|
13
|
-
import "./chunk-
|
|
12
|
+
} from "./chunk-M22PL4GH.js";
|
|
13
|
+
import "./chunk-EP6RVHMX.js";
|
|
14
14
|
import {
|
|
15
15
|
ValidationError
|
|
16
16
|
} from "./chunk-YEJR7IXO.js";
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { AnalystFinding } from '@tangle-network/agent-eval';
|
|
2
|
-
import {
|
|
1
|
+
import { AnalystFinding, CostLedger, MaximumCharge } from '@tangle-network/agent-eval';
|
|
2
|
+
import { AgentProfile, ReasoningEffort } from '@tangle-network/agent-interface';
|
|
3
|
+
import { L as LocalHarness, C as CodexTokenUsage, a as CodexExecutionEvidence, r as runLocalHarness } from './local-harness-ZLMkKdZc.js';
|
|
3
4
|
import { LabeledScenarioStore, WorktreeAdapter, SurfaceProposer } from '@tangle-network/agent-eval/campaign';
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -42,7 +43,7 @@ interface CandidateGenerator {
|
|
|
42
43
|
* for a guaranteed no-op. Default `false`. */
|
|
43
44
|
proposesWithoutFindings?: boolean;
|
|
44
45
|
generate(args: {
|
|
45
|
-
/** The candidate worktree — a
|
|
46
|
+
/** The candidate worktree — a clean checkout of the current incumbent. */
|
|
46
47
|
worktreePath: string;
|
|
47
48
|
/** Phase-2 research report (analyst findings + diff), opaque. */
|
|
48
49
|
report: unknown;
|
|
@@ -54,6 +55,13 @@ interface CandidateGenerator {
|
|
|
54
55
|
* reflective generator ignores it). */
|
|
55
56
|
maxShots: number;
|
|
56
57
|
signal: AbortSignal;
|
|
58
|
+
/** Improvement-loop coordinates. Present when called through improvementDriver. */
|
|
59
|
+
generation?: number;
|
|
60
|
+
candidateIndex?: number;
|
|
61
|
+
/** Shared run-wide paid-call account supplied by agent-eval 0.117+. */
|
|
62
|
+
costLedger?: CostLedger;
|
|
63
|
+
/** Receipt attribution phase supplied alongside `costLedger`. */
|
|
64
|
+
costPhase?: string;
|
|
57
65
|
}): Promise<{
|
|
58
66
|
applied: boolean;
|
|
59
67
|
summary: string;
|
|
@@ -62,11 +70,12 @@ interface CandidateGenerator {
|
|
|
62
70
|
interface ImprovementDriverOptions {
|
|
63
71
|
worktree: WorktreeAdapter;
|
|
64
72
|
generator: CandidateGenerator;
|
|
65
|
-
/**
|
|
73
|
+
/** Root ref for first-generation/direct callers. Default `main`.
|
|
74
|
+
* Later code generations retain the incumbent's original root. */
|
|
66
75
|
baseRef?: string;
|
|
67
76
|
}
|
|
68
77
|
interface ManagedImprovementDriver extends SurfaceProposer<AnalystFinding> {
|
|
69
|
-
/** Remove every
|
|
78
|
+
/** Remove every owned candidate except explicitly retained finalized winners. */
|
|
70
79
|
cleanup(retainWorktreeRefs?: readonly string[]): Promise<void>;
|
|
71
80
|
}
|
|
72
81
|
/** The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. */
|
|
@@ -106,6 +115,9 @@ declare function improvementDriver(opts: ImprovementDriverOptions): ManagedImpro
|
|
|
106
115
|
* @experimental
|
|
107
116
|
*/
|
|
108
117
|
|
|
118
|
+
/** Dedicated ephemeral root for generic author-profile files. Every declared
|
|
119
|
+
* file must live below this root so cleanup cannot alter candidate-owned files. */
|
|
120
|
+
declare const AGENTIC_PROFILE_RESOURCE_ROOT = ".agent-runtime-profile-resources";
|
|
109
121
|
/** Outcome of verifying a candidate worktree. `feedback` (compiler errors,
|
|
110
122
|
* failing test output) is fed into the next shot when `ok` is false. */
|
|
111
123
|
interface VerifyResult {
|
|
@@ -116,9 +128,64 @@ interface VerifyResult {
|
|
|
116
128
|
* (a candidate that fails verification returns `{ok:false}`, it does not
|
|
117
129
|
* throw). */
|
|
118
130
|
type Verifier = (worktreePath: string) => Promise<VerifyResult> | VerifyResult;
|
|
131
|
+
interface AgenticGeneratorShotReceipt {
|
|
132
|
+
readonly schemaVersion: 1;
|
|
133
|
+
readonly generation: number | null;
|
|
134
|
+
readonly candidateIndex: number | null;
|
|
135
|
+
/** One-based shot number within this candidate. */
|
|
136
|
+
readonly shot: number;
|
|
137
|
+
readonly maxShots: number;
|
|
138
|
+
readonly harness: LocalHarness;
|
|
139
|
+
readonly model: string | null;
|
|
140
|
+
readonly reasoningEffort: ReasoningEffort | null;
|
|
141
|
+
readonly promptSha256: `sha256:${string}`;
|
|
142
|
+
readonly startedAt: string;
|
|
143
|
+
readonly completedAt: string;
|
|
144
|
+
readonly durationMs: number;
|
|
145
|
+
readonly exitCode: number | null;
|
|
146
|
+
readonly timedOut: boolean;
|
|
147
|
+
readonly killedBySignal: NodeJS.Signals | null;
|
|
148
|
+
readonly stdoutBytes: number | null;
|
|
149
|
+
readonly stdoutSha256: `sha256:${string}` | null;
|
|
150
|
+
readonly stderrBytes: number | null;
|
|
151
|
+
readonly stderrSha256: `sha256:${string}` | null;
|
|
152
|
+
readonly usage: CodexTokenUsage | null;
|
|
153
|
+
/** Digest of the exact profile-file workspace plan applied for this shot. */
|
|
154
|
+
readonly profileWorkspacePlanDigest: string | null;
|
|
155
|
+
readonly profileWorkspaceFileCount: number;
|
|
156
|
+
/** Shared run-ledger call id for this exact shot. */
|
|
157
|
+
readonly costCallId: string | null;
|
|
158
|
+
/** Whether dollars came from the provider, the pricing table, or are unknown. */
|
|
159
|
+
readonly costBasis: 'provider-reported' | 'estimated-pricing' | 'unknown';
|
|
160
|
+
readonly costUsd: number | null;
|
|
161
|
+
/** True only for a provider-reported amount, never for a pricing estimate. */
|
|
162
|
+
readonly costUsdKnown: boolean;
|
|
163
|
+
readonly evidence: CodexExecutionEvidence | null;
|
|
164
|
+
readonly error: {
|
|
165
|
+
readonly name: string;
|
|
166
|
+
readonly message: string;
|
|
167
|
+
} | null;
|
|
168
|
+
}
|
|
119
169
|
interface AgenticGeneratorOptions {
|
|
120
170
|
/** Local coding harness to run in the worktree. Default `claude`. */
|
|
121
171
|
harness?: LocalHarness;
|
|
172
|
+
/** Author profile rendered through the canonical harness mapper. Required
|
|
173
|
+
* for reproducible Codex so model and reasoning settings are explicit. */
|
|
174
|
+
profile?: AgentProfile;
|
|
175
|
+
/** Run Codex with isolated configuration, exact prompt evidence, and required
|
|
176
|
+
* terminal token usage. Requires `harness: 'codex'` and `profile`. */
|
|
177
|
+
codexReproducible?: boolean;
|
|
178
|
+
/** Absolute paths reproducible Codex must not read. A function can derive
|
|
179
|
+
* candidate-specific paths after the driver creates its worktree. */
|
|
180
|
+
codexReadDeniedPaths?: ReadonlyArray<string> | ((worktreePath: string) => ReadonlyArray<string>);
|
|
181
|
+
/** Awaited once for every attempted author shot, including process failures.
|
|
182
|
+
* Throwing aborts the candidate so receipt persistence can fail closed. */
|
|
183
|
+
onShotCompleted?: (receipt: AgenticGeneratorShotReceipt) => void | Promise<void>;
|
|
184
|
+
/** Optional hard upper bound passed to the run-wide CostLedger before each
|
|
185
|
+
* author shot. This MUST be enforced by the provider or executor; a planning
|
|
186
|
+
* estimate is not an admissible bound. Omit for an uncapped ledger. A capped
|
|
187
|
+
* ledger rejects before model dispatch when this is absent. */
|
|
188
|
+
maximumCharge?: MaximumCharge;
|
|
122
189
|
/** Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). */
|
|
123
190
|
timeoutMs?: number;
|
|
124
191
|
/** Build the harness task prompt from the report + findings. Override for
|
|
@@ -148,4 +215,4 @@ declare function agenticGenerator(opts?: AgenticGeneratorOptions): CandidateGene
|
|
|
148
215
|
* silent fallback). */
|
|
149
216
|
declare function commandVerifier(command: string, args?: string[], timeoutMs?: number): Verifier;
|
|
150
217
|
|
|
151
|
-
export {
|
|
218
|
+
export { AGENTIC_PROFILE_RESOURCE_ROOT as A, type CandidateGenerator as C, type ImprovementDriverOptions as I, type ManagedImprovementDriver as M, type Verifier as V, type AgenticGeneratorOptions as a, type AgenticGeneratorShotReceipt as b, type VerifyResult as c, agenticGenerator as d, commandVerifier as e, improvementDriver as i };
|
package/dist/analyst-loop.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { TraceAnalystByteBudgets, TraceAnalysisStore } from '@tangle-network/agent-eval';
|
|
2
|
-
import { I as Iteration } from './types-
|
|
2
|
+
import { I as Iteration } from './types-CKmyZ9TP.js';
|
|
3
3
|
import { R as RunAnalystLoopOpts, a as RunAnalystLoopResult } from './types-BC3bZpH0.js';
|
|
4
4
|
export { A as AnalystLoopEvent, b as AnalystRegistryLike, c as AnalystRegistryStreamingLike, d as AutoApplyPolicy, F as FindingsStoreLike, I as ImprovementAdapter, e as ImprovementEditBatch, f as ImprovementReport, K as KnowledgeAdapter, g as KnowledgeProposalBatch, h as KnowledgeReport } from './types-BC3bZpH0.js';
|
|
5
5
|
import '@tangle-network/sandbox';
|
package/dist/analyst-loop.js
CHANGED
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
settledToIteration,
|
|
14
14
|
supervise,
|
|
15
15
|
withDriverExecutor
|
|
16
|
-
} from "./chunk-
|
|
16
|
+
} from "./chunk-FRXDOBFP.js";
|
|
17
17
|
import {
|
|
18
18
|
InMemoryResultBlobStore,
|
|
19
19
|
InMemorySpawnJournal
|
|
@@ -30,7 +30,7 @@ import {
|
|
|
30
30
|
createSandboxToolPartState,
|
|
31
31
|
mapSandboxEvent,
|
|
32
32
|
mapSandboxToolEvent
|
|
33
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-M22PL4GH.js";
|
|
34
34
|
import {
|
|
35
35
|
AnalystError,
|
|
36
36
|
BackendTransportError,
|
|
@@ -627,12 +627,6 @@ import {
|
|
|
627
627
|
} from "@tangle-network/agent-eval/campaign";
|
|
628
628
|
import { collectAgentResponseText } from "@tangle-network/sandbox";
|
|
629
629
|
|
|
630
|
-
// src/runtime/report-usage.ts
|
|
631
|
-
function reportLoopUsage(cost, result, source = "loop") {
|
|
632
|
-
cost.observe(result.costUsd, source);
|
|
633
|
-
cost.observeTokens(result.tokenUsage);
|
|
634
|
-
}
|
|
635
|
-
|
|
636
630
|
// src/runtime/loop-dispatch.ts
|
|
637
631
|
function campaignTraceToLoopEmitter(trace) {
|
|
638
632
|
return {
|
|
@@ -643,23 +637,61 @@ function campaignTraceToLoopEmitter(trace) {
|
|
|
643
637
|
}
|
|
644
638
|
async function runLoopForCell(opts, scenario, profile, ctx) {
|
|
645
639
|
const loopOptions = opts.toLoopOptions(scenario, profile);
|
|
646
|
-
return runLoopWithCampaignContext(opts, loopOptions, ctx
|
|
640
|
+
return runLoopWithCampaignContext(opts, loopOptions, ctx, {
|
|
641
|
+
model: profile.model?.default ?? modelFromLoopOptions(loopOptions),
|
|
642
|
+
maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario, profile) : opts.maximumCharge,
|
|
643
|
+
resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario, profile) : void 0
|
|
644
|
+
});
|
|
647
645
|
}
|
|
648
|
-
async function runLoopWithCampaignContext(opts, loopOptions, ctx) {
|
|
649
|
-
const
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
646
|
+
async function runLoopWithCampaignContext(opts, loopOptions, ctx, cost) {
|
|
647
|
+
const paid = await ctx.cost.runPaidCall({
|
|
648
|
+
channel: "agent",
|
|
649
|
+
actor: opts.costSource ?? "loop",
|
|
650
|
+
model: cost.model,
|
|
651
|
+
signal: ctx.signal,
|
|
652
|
+
...cost.maximumCharge ? { maximumCharge: cost.maximumCharge } : {},
|
|
653
|
+
execute: (executionSignal) => runLoop({
|
|
654
|
+
...loopOptions,
|
|
655
|
+
ctx: {
|
|
656
|
+
sandboxClient: opts.sandboxClient,
|
|
657
|
+
signal: executionSignal,
|
|
658
|
+
traceEmitter: opts.forwardTrace === false ? void 0 : campaignTraceToLoopEmitter(ctx.trace)
|
|
659
|
+
}
|
|
660
|
+
}),
|
|
661
|
+
receipt: (result2) => loopCostReceipt(result2, cost.resolveModel?.(result2) ?? cost.model)
|
|
656
662
|
});
|
|
657
|
-
|
|
663
|
+
if (!paid.succeeded) {
|
|
664
|
+
throw paid.error;
|
|
665
|
+
}
|
|
666
|
+
const result = paid.value;
|
|
658
667
|
const toArtifact = opts.toArtifact ?? ((r) => r.winner?.output);
|
|
659
668
|
return toArtifact(result);
|
|
660
669
|
}
|
|
670
|
+
function loopCostReceipt(result, model) {
|
|
671
|
+
return {
|
|
672
|
+
model,
|
|
673
|
+
inputTokens: result.tokenUsage.input,
|
|
674
|
+
outputTokens: result.tokenUsage.output,
|
|
675
|
+
...result.costUsd > 0 ? { actualCostUsd: result.costUsd } : {}
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
function modelFromLoopOptions(options) {
|
|
679
|
+
const profiles = options.agentRun ? [options.agentRun.profile] : options.agentRuns?.map((run) => run.profile) ?? [];
|
|
680
|
+
const models = new Set(
|
|
681
|
+
profiles.map((profile) => profile.model?.default).filter((model) => !!model)
|
|
682
|
+
);
|
|
683
|
+
if (models.size === 1) return [...models][0];
|
|
684
|
+
return models.size > 1 ? "mixed" : "unknown";
|
|
685
|
+
}
|
|
661
686
|
function loopCampaignDispatch(opts) {
|
|
662
|
-
return (scenario, ctx) =>
|
|
687
|
+
return (scenario, ctx) => {
|
|
688
|
+
const loopOptions = opts.toLoopOptions(scenario);
|
|
689
|
+
return runLoopWithCampaignContext(opts, loopOptions, ctx, {
|
|
690
|
+
model: modelFromLoopOptions(loopOptions),
|
|
691
|
+
maximumCharge: typeof opts.maximumCharge === "function" ? opts.maximumCharge(scenario) : opts.maximumCharge,
|
|
692
|
+
resolveModel: opts.resolveCostModel ? (result) => opts.resolveCostModel?.(result, scenario) : void 0
|
|
693
|
+
});
|
|
694
|
+
};
|
|
663
695
|
}
|
|
664
696
|
function loopDispatch(opts) {
|
|
665
697
|
return (profile, scenario, ctx) => runLoopForCell(opts, scenario, profile, ctx);
|
|
@@ -981,9 +1013,23 @@ function defineLeaderboard(spec) {
|
|
|
981
1013
|
return p;
|
|
982
1014
|
};
|
|
983
1015
|
let shotNonce = 0;
|
|
1016
|
+
const maximumCharge = spec.maximumCharge;
|
|
984
1017
|
const dispatch = spec.dispatch ?? ((profile, scenario, dispatchCtx) => {
|
|
985
1018
|
const cellDispatch = loopDispatch({
|
|
986
1019
|
sandboxClient,
|
|
1020
|
+
maximumCharge: typeof maximumCharge === "function" ? (cellScenario, cellProfile) => maximumCharge(cellProfile, cellScenario) : maximumCharge,
|
|
1021
|
+
resolveCostModel: (result, _cellScenario, cellProfile) => {
|
|
1022
|
+
if (!spec.resolveModel) return cellProfile.model?.default;
|
|
1023
|
+
const served = new Set(
|
|
1024
|
+
result.iterations.map((iteration) => spec.resolveModel?.(iteration.events)).filter((model) => model !== void 0)
|
|
1025
|
+
);
|
|
1026
|
+
if (served.size > 1) {
|
|
1027
|
+
throw new Error(
|
|
1028
|
+
`defineLeaderboard(${spec.name}): one cell reported multiple served models: ${[...served].join(", ")}`
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
1031
|
+
return [...served][0] ?? cellProfile.model?.default;
|
|
1032
|
+
},
|
|
987
1033
|
toLoopOptions: (cellScenario, cellProfile) => {
|
|
988
1034
|
const axis = harnessAxisOf(cellProfile);
|
|
989
1035
|
const modelId = bareModel(axis?.model ?? models[0] ?? "");
|
|
@@ -1034,10 +1080,6 @@ function defineLeaderboard(spec) {
|
|
|
1034
1080
|
...iter.error ? { error: iter.error.message } : {},
|
|
1035
1081
|
...iter.verdict ? { verdict: { score: iter.verdict.score } } : {}
|
|
1036
1082
|
});
|
|
1037
|
-
if (spec.resolveModel) {
|
|
1038
|
-
const served = spec.resolveModel(iter.events);
|
|
1039
|
-
if (served !== void 0) dispatchCtx.cost.observeModel?.(served);
|
|
1040
|
-
}
|
|
1041
1083
|
}
|
|
1042
1084
|
return result.winner?.output;
|
|
1043
1085
|
}
|
|
@@ -2648,6 +2690,12 @@ function promotionGate(opts) {
|
|
|
2648
2690
|
};
|
|
2649
2691
|
}
|
|
2650
2692
|
|
|
2693
|
+
// src/runtime/report-usage.ts
|
|
2694
|
+
function reportLoopUsage(cost, result, source = "loop") {
|
|
2695
|
+
cost.observe(result.costUsd, source);
|
|
2696
|
+
cost.observeTokens(result.tokenUsage);
|
|
2697
|
+
}
|
|
2698
|
+
|
|
2651
2699
|
// src/runtime/run-benchmark.ts
|
|
2652
2700
|
import { pairedBootstrap as pairedBootstrap2, paretoFrontier } from "@tangle-network/agent-eval";
|
|
2653
2701
|
|
|
@@ -5740,13 +5788,43 @@ import { stuckLoopView, toolWasteView } from "@tangle-network/agent-eval/pipelin
|
|
|
5740
5788
|
async function analyzeTrace(source, runId = "worker") {
|
|
5741
5789
|
const spans = await source.collect();
|
|
5742
5790
|
const store = new InMemoryTraceStore();
|
|
5791
|
+
const laneSpanId = `${runId}-session-lane`;
|
|
5792
|
+
if (spans.length > 0) {
|
|
5793
|
+
const startedAt = Math.min(...spans.map((span) => span.startedAt));
|
|
5794
|
+
const endedAt = Math.max(...spans.map((span) => span.endedAt ?? span.startedAt));
|
|
5795
|
+
const agentSpanId = `${runId}-agent`;
|
|
5796
|
+
await store.appendSpan({
|
|
5797
|
+
spanId: agentSpanId,
|
|
5798
|
+
runId,
|
|
5799
|
+
kind: "agent",
|
|
5800
|
+
name: "trace-source-agent",
|
|
5801
|
+
startedAt,
|
|
5802
|
+
endedAt
|
|
5803
|
+
});
|
|
5804
|
+
await store.appendSpan({
|
|
5805
|
+
spanId: laneSpanId,
|
|
5806
|
+
parentSpanId: agentSpanId,
|
|
5807
|
+
runId,
|
|
5808
|
+
kind: "custom",
|
|
5809
|
+
name: "trace-source-session",
|
|
5810
|
+
startedAt,
|
|
5811
|
+
endedAt
|
|
5812
|
+
});
|
|
5813
|
+
}
|
|
5743
5814
|
for (let i = 0; i < spans.length; i += 1) {
|
|
5744
5815
|
const s = spans[i];
|
|
5745
|
-
if (s)
|
|
5816
|
+
if (s) {
|
|
5817
|
+
await store.appendSpan({
|
|
5818
|
+
...s,
|
|
5819
|
+
runId,
|
|
5820
|
+
spanId: `${runId}-t${i}`,
|
|
5821
|
+
parentSpanId: laneSpanId
|
|
5822
|
+
});
|
|
5823
|
+
}
|
|
5746
5824
|
}
|
|
5747
5825
|
const [trajectory, stuckLoop, toolWaste] = await Promise.all([
|
|
5748
5826
|
buildTrajectory(store, runId),
|
|
5749
|
-
stuckLoopView(store, { runId }),
|
|
5827
|
+
stuckLoopView(store, { runId, maxInterveningToolCalls: 1 }),
|
|
5750
5828
|
toolWasteView(store, { runId })
|
|
5751
5829
|
]);
|
|
5752
5830
|
return { trajectory, stuckLoop, toolWaste };
|
|
@@ -6248,7 +6326,6 @@ export {
|
|
|
6248
6326
|
stopSentinel,
|
|
6249
6327
|
sentinelCompletion,
|
|
6250
6328
|
deterministicCompletion,
|
|
6251
|
-
reportLoopUsage,
|
|
6252
6329
|
loopCampaignDispatch,
|
|
6253
6330
|
loopDispatch,
|
|
6254
6331
|
inlineSandboxClient,
|
|
@@ -6285,6 +6362,7 @@ export {
|
|
|
6285
6362
|
trajectoryReport,
|
|
6286
6363
|
equalKOnCost,
|
|
6287
6364
|
promotionGate,
|
|
6365
|
+
reportLoopUsage,
|
|
6288
6366
|
depthStrategy,
|
|
6289
6367
|
breadthStrategy,
|
|
6290
6368
|
sample,
|
|
@@ -6339,4 +6417,4 @@ export {
|
|
|
6339
6417
|
computeFindingId,
|
|
6340
6418
|
makeFinding2 as makeFinding
|
|
6341
6419
|
};
|
|
6342
|
-
//# sourceMappingURL=chunk-
|
|
6420
|
+
//# sourceMappingURL=chunk-7HH22XN4.js.map
|