@tangle-network/agent-eval 0.125.0 → 0.126.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +62 -35
  2. package/README.md +270 -189
  3. package/dist/analyst/index.d.ts +15 -145
  4. package/dist/analyst/index.js +33 -47
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/benchmarks/index.d.ts +45 -162
  7. package/dist/benchmarks/index.js +8 -9
  8. package/dist/campaign/index.d.ts +3674 -5393
  9. package/dist/campaign/index.js +21 -95
  10. package/dist/{chunk-R226UZOI.js → chunk-474LBSOX.js} +2 -2
  11. package/dist/{chunk-HM6V7F3M.js → chunk-FO7HEH76.js} +3 -3
  12. package/dist/chunk-IILEIWGW.js +635 -0
  13. package/dist/chunk-IILEIWGW.js.map +1 -0
  14. package/dist/{chunk-EQUK3RFS.js → chunk-J5SQWP6Y.js} +8 -5
  15. package/dist/chunk-J5SQWP6Y.js.map +1 -0
  16. package/dist/{chunk-W5B3ZGP3.js → chunk-KE2VWPZX.js} +8 -6
  17. package/dist/{chunk-W5B3ZGP3.js.map → chunk-KE2VWPZX.js.map} +1 -1
  18. package/dist/{chunk-DT7OXY3C.js → chunk-LUNF2SEL.js} +538 -851
  19. package/dist/chunk-LUNF2SEL.js.map +1 -0
  20. package/dist/chunk-NGUYT5CI.js +4637 -0
  21. package/dist/chunk-NGUYT5CI.js.map +1 -0
  22. package/dist/{chunk-QFQZ3U3X.js → chunk-OCFJACJU.js} +2 -2
  23. package/dist/{chunk-GID26AN4.js → chunk-P22LJ3Y2.js} +4 -6
  24. package/dist/{chunk-GID26AN4.js.map → chunk-P22LJ3Y2.js.map} +1 -1
  25. package/dist/{chunk-SJT4OBVL.js → chunk-SDPM6554.js} +3 -3
  26. package/dist/{chunk-D5JZ7UDZ.js → chunk-UCLVDLCH.js} +136 -50
  27. package/dist/chunk-UCLVDLCH.js.map +1 -0
  28. package/dist/chunk-VMUENW6F.js +7274 -0
  29. package/dist/chunk-VMUENW6F.js.map +1 -0
  30. package/dist/{chunk-JKDNAOF5.js → chunk-W4L6C2XT.js} +2 -2
  31. package/dist/chunk-WGXIEX7P.js +116 -0
  32. package/dist/chunk-WGXIEX7P.js.map +1 -0
  33. package/dist/{chunk-GRCDRKII.js → chunk-WS3NZZQQ.js} +58 -20
  34. package/dist/chunk-WS3NZZQQ.js.map +1 -0
  35. package/dist/cli.js +3 -3
  36. package/dist/contract/index.d.ts +3220 -3094
  37. package/dist/contract/index.js +173 -42
  38. package/dist/contract/index.js.map +1 -1
  39. package/dist/control.js +2 -3
  40. package/dist/fuzz.d.ts +14 -1
  41. package/dist/fuzz.js +1 -1
  42. package/dist/hosted/index.d.ts +8 -1
  43. package/dist/index.d.ts +71 -687
  44. package/dist/index.js +178 -497
  45. package/dist/index.js.map +1 -1
  46. package/dist/openapi.json +1 -1
  47. package/dist/rl.d.ts +5 -100
  48. package/dist/rl.js +4 -5
  49. package/dist/rl.js.map +1 -1
  50. package/dist/{run-campaign-I3JXKVAK.js → run-campaign-LVFKZCEU.js} +3 -3
  51. package/dist/traces.js +2 -3
  52. package/dist/wire/index.d.ts +14 -1
  53. package/dist/wire/index.js +3 -3
  54. package/docs/campaign-proposers.md +363 -168
  55. package/docs/design/loop-taxonomy.md +142 -190
  56. package/docs/design.md +1 -1
  57. package/docs/distributed-driver.md +8 -11
  58. package/docs/feature-guide.md +20 -19
  59. package/docs/knowledge-readiness.md +2 -5
  60. package/docs/multi-shot-optimization.md +35 -27
  61. package/docs/rollout.md +5 -5
  62. package/package.json +4 -4
  63. package/dist/chunk-A62YMFWA.js +0 -9269
  64. package/dist/chunk-A62YMFWA.js.map +0 -1
  65. package/dist/chunk-A6GT67HT.js +0 -550
  66. package/dist/chunk-A6GT67HT.js.map +0 -1
  67. package/dist/chunk-D5JZ7UDZ.js.map +0 -1
  68. package/dist/chunk-DT7OXY3C.js.map +0 -1
  69. package/dist/chunk-EQUK3RFS.js.map +0 -1
  70. package/dist/chunk-GC4ATIKK.js +0 -317
  71. package/dist/chunk-GC4ATIKK.js.map +0 -1
  72. package/dist/chunk-GRCDRKII.js.map +0 -1
  73. package/dist/chunk-LOW3U7JZ.js +0 -328
  74. package/dist/chunk-LOW3U7JZ.js.map +0 -1
  75. package/dist/chunk-PMITBABE.js +0 -3841
  76. package/dist/chunk-PMITBABE.js.map +0 -1
  77. /package/dist/{chunk-R226UZOI.js.map → chunk-474LBSOX.js.map} +0 -0
  78. /package/dist/{chunk-HM6V7F3M.js.map → chunk-FO7HEH76.js.map} +0 -0
  79. /package/dist/{chunk-QFQZ3U3X.js.map → chunk-OCFJACJU.js.map} +0 -0
  80. /package/dist/{chunk-SJT4OBVL.js.map → chunk-SDPM6554.js.map} +0 -0
  81. /package/dist/{chunk-JKDNAOF5.js.map → chunk-W4L6C2XT.js.map} +0 -0
  82. /package/dist/{run-campaign-I3JXKVAK.js.map → run-campaign-LVFKZCEU.js.map} +0 -0
package/dist/control.js CHANGED
@@ -10,11 +10,10 @@ import {
10
10
  stopOnNoProgress,
11
11
  stopOnRepeatedAction,
12
12
  subjectiveEval
13
- } from "./chunk-R226UZOI.js";
13
+ } from "./chunk-474LBSOX.js";
14
14
  import "./chunk-VQMK5FMP.js";
15
- import "./chunk-LOW3U7JZ.js";
15
+ import "./chunk-IILEIWGW.js";
16
16
  import "./chunk-MA6HLL3S.js";
17
- import "./chunk-GC4ATIKK.js";
18
17
  import "./chunk-VSMTAMNK.js";
19
18
  import "./chunk-ONWEPEDO.js";
20
19
  import "./chunk-PZ5AY32C.js";
package/dist/fuzz.d.ts CHANGED
@@ -31,29 +31,40 @@ interface CostReceipt extends CostCallBase, CostUsage {
31
31
  costUsd: number;
32
32
  costUnknown: boolean;
33
33
  usageUnknown?: boolean;
34
+ /** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */
34
35
  pricing?: {
35
36
  inputUsdPerThousand: number;
37
+ cachedInputUsdPerThousand?: number;
38
+ cacheWriteUsdPerThousand?: number;
36
39
  outputUsdPerThousand: number;
37
40
  };
41
+ /** Cost reported by the provider, not a local token-price calculation. */
38
42
  actualCostUsd?: number;
39
43
  error?: string;
40
44
  }
41
45
  interface CostReceiptInput extends CostUsage {
42
46
  model: string;
47
+ /** Caller-supplied rates for a local estimate when the provider does not report billed cost. */
48
+ customTokenPricing?: CustomTokenPricing;
43
49
  actualCostUsd?: number;
44
50
  costUnknown?: boolean;
45
51
  usageUnknown?: boolean;
46
52
  }
47
53
  /** Per-million token rates for a model or endpoint not covered by package pricing. */
48
54
  interface CustomTokenPricing {
55
+ /** Non-cached input tokens. */
49
56
  inputUsdPerMillion: number;
57
+ /** Cache-read tokens. Falls back to the normal input rate when omitted. */
58
+ cachedInputUsdPerMillion?: number;
59
+ /** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */
60
+ cacheWriteUsdPerMillion?: number;
50
61
  outputUsdPerMillion: number;
51
62
  }
52
63
  type MaximumCharge = {
53
64
  externallyEnforcedMaximumUsd: number;
54
65
  } | ({
55
66
  customTokenPricing: CustomTokenPricing;
56
- } & Pick<CostUsage, 'inputTokens' | 'outputTokens'>) | ({
67
+ } & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>) | ({
57
68
  model: string;
58
69
  } & CostUsage);
59
70
  interface RunPaidCallInput<T> {
@@ -121,6 +132,8 @@ interface CostLedgerFilter {
121
132
  interface CostLedgerWaitOptions {
122
133
  /** Maximum time to wait for active provider calls. Default 5 seconds. */
123
134
  timeoutMs?: number;
135
+ /** Wait only for calls matching this attribution filter. */
136
+ filter?: CostLedgerFilter;
124
137
  }
125
138
  /** Append-only storage. `append` must atomically reject stale revisions. */
126
139
  interface CostLedgerPersistence {
package/dist/fuzz.js CHANGED
@@ -4,7 +4,7 @@ import {
4
4
  import {
5
5
  CostLedger,
6
6
  CostReceiptCaptureError
7
- } from "./chunk-GRCDRKII.js";
7
+ } from "./chunk-WS3NZZQQ.js";
8
8
  import "./chunk-VI2UW6B6.js";
9
9
  import {
10
10
  ValidationError
@@ -46,14 +46,21 @@ interface CodeSurface {
46
46
  /** Human summary of what changed — rendered into the auto-PR body. */
47
47
  readonly summary?: string;
48
48
  }
49
+ /** Named text components optimized together as one candidate. */
50
+ interface ComponentSurface {
51
+ readonly kind: 'components';
52
+ readonly components: Readonly<Record<string, string>>;
53
+ }
49
54
  /** The mutable surface a proposer changes. Tiers (see
50
55
  * `docs/design/loop-taxonomy.md`):
51
56
  * - `string` — tiers 1-2: system-prompt addendum / serialized tool
52
57
  * config. Cheap, reversible, text-diffable.
58
+ * - `ComponentSurface` — named prompts, policies, and tool descriptions
59
+ * optimized together.
53
60
  * - `CodeSurface` — tier 4: an implementation change behind a worktree ref.
54
61
  * Tier 3 (knowledge) is owned by agent-knowledge and rides its own adapter,
55
62
  * not this type. */
56
- type MutableSurface = string | CodeSurface;
63
+ type MutableSurface = string | ComponentSurface | CodeSurface;
57
64
  /** Five-valued verdict taxonomy (MOSS-paper alignment). */
58
65
  type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
59
66