@tangle-network/agent-eval 0.125.0 → 0.126.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -35
- package/README.md +270 -189
- package/dist/analyst/index.d.ts +15 -145
- package/dist/analyst/index.js +33 -47
- package/dist/analyst/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +45 -162
- package/dist/benchmarks/index.js +8 -9
- package/dist/campaign/index.d.ts +3674 -5393
- package/dist/campaign/index.js +21 -95
- package/dist/{chunk-R226UZOI.js → chunk-474LBSOX.js} +2 -2
- package/dist/{chunk-HM6V7F3M.js → chunk-FO7HEH76.js} +3 -3
- package/dist/chunk-IILEIWGW.js +635 -0
- package/dist/chunk-IILEIWGW.js.map +1 -0
- package/dist/{chunk-EQUK3RFS.js → chunk-J5SQWP6Y.js} +8 -5
- package/dist/chunk-J5SQWP6Y.js.map +1 -0
- package/dist/{chunk-W5B3ZGP3.js → chunk-KE2VWPZX.js} +8 -6
- package/dist/{chunk-W5B3ZGP3.js.map → chunk-KE2VWPZX.js.map} +1 -1
- package/dist/{chunk-DT7OXY3C.js → chunk-LUNF2SEL.js} +538 -851
- package/dist/chunk-LUNF2SEL.js.map +1 -0
- package/dist/chunk-NGUYT5CI.js +4637 -0
- package/dist/chunk-NGUYT5CI.js.map +1 -0
- package/dist/{chunk-QFQZ3U3X.js → chunk-OCFJACJU.js} +2 -2
- package/dist/{chunk-GID26AN4.js → chunk-P22LJ3Y2.js} +4 -6
- package/dist/{chunk-GID26AN4.js.map → chunk-P22LJ3Y2.js.map} +1 -1
- package/dist/{chunk-SJT4OBVL.js → chunk-SDPM6554.js} +3 -3
- package/dist/{chunk-D5JZ7UDZ.js → chunk-UCLVDLCH.js} +136 -50
- package/dist/chunk-UCLVDLCH.js.map +1 -0
- package/dist/chunk-VMUENW6F.js +7274 -0
- package/dist/chunk-VMUENW6F.js.map +1 -0
- package/dist/{chunk-JKDNAOF5.js → chunk-W4L6C2XT.js} +2 -2
- package/dist/chunk-WGXIEX7P.js +116 -0
- package/dist/chunk-WGXIEX7P.js.map +1 -0
- package/dist/{chunk-GRCDRKII.js → chunk-WS3NZZQQ.js} +58 -20
- package/dist/chunk-WS3NZZQQ.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/contract/index.d.ts +3220 -3094
- package/dist/contract/index.js +173 -42
- package/dist/contract/index.js.map +1 -1
- package/dist/control.js +2 -3
- package/dist/fuzz.d.ts +14 -1
- package/dist/fuzz.js +1 -1
- package/dist/hosted/index.d.ts +8 -1
- package/dist/index.d.ts +71 -687
- package/dist/index.js +178 -497
- package/dist/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +5 -100
- package/dist/rl.js +4 -5
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-I3JXKVAK.js → run-campaign-LVFKZCEU.js} +3 -3
- package/dist/traces.js +2 -3
- package/dist/wire/index.d.ts +14 -1
- package/dist/wire/index.js +3 -3
- package/docs/campaign-proposers.md +363 -168
- package/docs/design/loop-taxonomy.md +142 -190
- package/docs/design.md +1 -1
- package/docs/distributed-driver.md +8 -11
- package/docs/feature-guide.md +20 -19
- package/docs/knowledge-readiness.md +2 -5
- package/docs/multi-shot-optimization.md +35 -27
- package/docs/rollout.md +5 -5
- package/package.json +4 -4
- package/dist/chunk-A62YMFWA.js +0 -9269
- package/dist/chunk-A62YMFWA.js.map +0 -1
- package/dist/chunk-A6GT67HT.js +0 -550
- package/dist/chunk-A6GT67HT.js.map +0 -1
- package/dist/chunk-D5JZ7UDZ.js.map +0 -1
- package/dist/chunk-DT7OXY3C.js.map +0 -1
- package/dist/chunk-EQUK3RFS.js.map +0 -1
- package/dist/chunk-GC4ATIKK.js +0 -317
- package/dist/chunk-GC4ATIKK.js.map +0 -1
- package/dist/chunk-GRCDRKII.js.map +0 -1
- package/dist/chunk-LOW3U7JZ.js +0 -328
- package/dist/chunk-LOW3U7JZ.js.map +0 -1
- package/dist/chunk-PMITBABE.js +0 -3841
- package/dist/chunk-PMITBABE.js.map +0 -1
- /package/dist/{chunk-R226UZOI.js.map → chunk-474LBSOX.js.map} +0 -0
- /package/dist/{chunk-HM6V7F3M.js.map → chunk-FO7HEH76.js.map} +0 -0
- /package/dist/{chunk-QFQZ3U3X.js.map → chunk-OCFJACJU.js.map} +0 -0
- /package/dist/{chunk-SJT4OBVL.js.map → chunk-SDPM6554.js.map} +0 -0
- /package/dist/{chunk-JKDNAOF5.js.map → chunk-W4L6C2XT.js.map} +0 -0
- /package/dist/{run-campaign-I3JXKVAK.js.map → run-campaign-LVFKZCEU.js.map} +0 -0
package/dist/control.js
CHANGED
|
@@ -10,11 +10,10 @@ import {
|
|
|
10
10
|
stopOnNoProgress,
|
|
11
11
|
stopOnRepeatedAction,
|
|
12
12
|
subjectiveEval
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-474LBSOX.js";
|
|
14
14
|
import "./chunk-VQMK5FMP.js";
|
|
15
|
-
import "./chunk-
|
|
15
|
+
import "./chunk-IILEIWGW.js";
|
|
16
16
|
import "./chunk-MA6HLL3S.js";
|
|
17
|
-
import "./chunk-GC4ATIKK.js";
|
|
18
17
|
import "./chunk-VSMTAMNK.js";
|
|
19
18
|
import "./chunk-ONWEPEDO.js";
|
|
20
19
|
import "./chunk-PZ5AY32C.js";
|
package/dist/fuzz.d.ts
CHANGED
|
@@ -31,29 +31,40 @@ interface CostReceipt extends CostCallBase, CostUsage {
|
|
|
31
31
|
costUsd: number;
|
|
32
32
|
costUnknown: boolean;
|
|
33
33
|
usageUnknown?: boolean;
|
|
34
|
+
/** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */
|
|
34
35
|
pricing?: {
|
|
35
36
|
inputUsdPerThousand: number;
|
|
37
|
+
cachedInputUsdPerThousand?: number;
|
|
38
|
+
cacheWriteUsdPerThousand?: number;
|
|
36
39
|
outputUsdPerThousand: number;
|
|
37
40
|
};
|
|
41
|
+
/** Cost reported by the provider, not a local token-price calculation. */
|
|
38
42
|
actualCostUsd?: number;
|
|
39
43
|
error?: string;
|
|
40
44
|
}
|
|
41
45
|
interface CostReceiptInput extends CostUsage {
|
|
42
46
|
model: string;
|
|
47
|
+
/** Caller-supplied rates for a local estimate when the provider does not report billed cost. */
|
|
48
|
+
customTokenPricing?: CustomTokenPricing;
|
|
43
49
|
actualCostUsd?: number;
|
|
44
50
|
costUnknown?: boolean;
|
|
45
51
|
usageUnknown?: boolean;
|
|
46
52
|
}
|
|
47
53
|
/** Per-million token rates for a model or endpoint not covered by package pricing. */
|
|
48
54
|
interface CustomTokenPricing {
|
|
55
|
+
/** Non-cached input tokens. */
|
|
49
56
|
inputUsdPerMillion: number;
|
|
57
|
+
/** Cache-read tokens. Falls back to the normal input rate when omitted. */
|
|
58
|
+
cachedInputUsdPerMillion?: number;
|
|
59
|
+
/** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */
|
|
60
|
+
cacheWriteUsdPerMillion?: number;
|
|
50
61
|
outputUsdPerMillion: number;
|
|
51
62
|
}
|
|
52
63
|
type MaximumCharge = {
|
|
53
64
|
externallyEnforcedMaximumUsd: number;
|
|
54
65
|
} | ({
|
|
55
66
|
customTokenPricing: CustomTokenPricing;
|
|
56
|
-
} & Pick<CostUsage, 'inputTokens' | 'outputTokens'>) | ({
|
|
67
|
+
} & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>) | ({
|
|
57
68
|
model: string;
|
|
58
69
|
} & CostUsage);
|
|
59
70
|
interface RunPaidCallInput<T> {
|
|
@@ -121,6 +132,8 @@ interface CostLedgerFilter {
|
|
|
121
132
|
interface CostLedgerWaitOptions {
|
|
122
133
|
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
123
134
|
timeoutMs?: number;
|
|
135
|
+
/** Wait only for calls matching this attribution filter. */
|
|
136
|
+
filter?: CostLedgerFilter;
|
|
124
137
|
}
|
|
125
138
|
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
126
139
|
interface CostLedgerPersistence {
|
package/dist/fuzz.js
CHANGED
package/dist/hosted/index.d.ts
CHANGED
|
@@ -46,14 +46,21 @@ interface CodeSurface {
|
|
|
46
46
|
/** Human summary of what changed — rendered into the auto-PR body. */
|
|
47
47
|
readonly summary?: string;
|
|
48
48
|
}
|
|
49
|
+
/** Named text components optimized together as one candidate. */
|
|
50
|
+
interface ComponentSurface {
|
|
51
|
+
readonly kind: 'components';
|
|
52
|
+
readonly components: Readonly<Record<string, string>>;
|
|
53
|
+
}
|
|
49
54
|
/** The mutable surface a proposer changes. Tiers (see
|
|
50
55
|
* `docs/design/loop-taxonomy.md`):
|
|
51
56
|
* - `string` — tiers 1-2: system-prompt addendum / serialized tool
|
|
52
57
|
* config. Cheap, reversible, text-diffable.
|
|
58
|
+
* - `ComponentSurface` — named prompts, policies, and tool descriptions
|
|
59
|
+
* optimized together.
|
|
53
60
|
* - `CodeSurface` — tier 4: an implementation change behind a worktree ref.
|
|
54
61
|
* Tier 3 (knowledge) is owned by agent-knowledge and rides its own adapter,
|
|
55
62
|
* not this type. */
|
|
56
|
-
type MutableSurface = string | CodeSurface;
|
|
63
|
+
type MutableSurface = string | ComponentSurface | CodeSurface;
|
|
57
64
|
/** Five-valued verdict taxonomy (MOSS-paper alignment). */
|
|
58
65
|
type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
|
|
59
66
|
|