@peakinfer/cli 1.0.133

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (367) hide show
  1. package/.claude/settings.local.json +8 -0
  2. package/.env.example +6 -0
  3. package/.github/workflows/peakinfer.yml +64 -0
  4. package/CHANGELOG.md +31 -0
  5. package/LICENSE +190 -0
  6. package/README.md +335 -0
  7. package/data/inferencemax.json +274 -0
  8. package/dist/agent-analyzer.d.ts +45 -0
  9. package/dist/agent-analyzer.d.ts.map +1 -0
  10. package/dist/agent-analyzer.js +374 -0
  11. package/dist/agent-analyzer.js.map +1 -0
  12. package/dist/agent.d.ts +76 -0
  13. package/dist/agent.d.ts.map +1 -0
  14. package/dist/agent.js +965 -0
  15. package/dist/agent.js.map +1 -0
  16. package/dist/agents/correlation-analyzer.d.ts +34 -0
  17. package/dist/agents/correlation-analyzer.d.ts.map +1 -0
  18. package/dist/agents/correlation-analyzer.js +261 -0
  19. package/dist/agents/correlation-analyzer.js.map +1 -0
  20. package/dist/agents/index.d.ts +91 -0
  21. package/dist/agents/index.d.ts.map +1 -0
  22. package/dist/agents/index.js +111 -0
  23. package/dist/agents/index.js.map +1 -0
  24. package/dist/agents/runtime-analyzer.d.ts +38 -0
  25. package/dist/agents/runtime-analyzer.d.ts.map +1 -0
  26. package/dist/agents/runtime-analyzer.js +244 -0
  27. package/dist/agents/runtime-analyzer.js.map +1 -0
  28. package/dist/analysis-types.d.ts +500 -0
  29. package/dist/analysis-types.d.ts.map +1 -0
  30. package/dist/analysis-types.js +11 -0
  31. package/dist/analysis-types.js.map +1 -0
  32. package/dist/analytics.d.ts +25 -0
  33. package/dist/analytics.d.ts.map +1 -0
  34. package/dist/analytics.js +94 -0
  35. package/dist/analytics.js.map +1 -0
  36. package/dist/analyzer.d.ts +48 -0
  37. package/dist/analyzer.d.ts.map +1 -0
  38. package/dist/analyzer.js +547 -0
  39. package/dist/analyzer.js.map +1 -0
  40. package/dist/artifacts.d.ts +44 -0
  41. package/dist/artifacts.d.ts.map +1 -0
  42. package/dist/artifacts.js +165 -0
  43. package/dist/artifacts.js.map +1 -0
  44. package/dist/benchmarks/index.d.ts +88 -0
  45. package/dist/benchmarks/index.d.ts.map +1 -0
  46. package/dist/benchmarks/index.js +205 -0
  47. package/dist/benchmarks/index.js.map +1 -0
  48. package/dist/cli.d.ts +3 -0
  49. package/dist/cli.d.ts.map +1 -0
  50. package/dist/cli.js +427 -0
  51. package/dist/cli.js.map +1 -0
  52. package/dist/commands/ci.d.ts +19 -0
  53. package/dist/commands/ci.d.ts.map +1 -0
  54. package/dist/commands/ci.js +253 -0
  55. package/dist/commands/ci.js.map +1 -0
  56. package/dist/commands/config.d.ts +16 -0
  57. package/dist/commands/config.d.ts.map +1 -0
  58. package/dist/commands/config.js +249 -0
  59. package/dist/commands/config.js.map +1 -0
  60. package/dist/commands/demo.d.ts +15 -0
  61. package/dist/commands/demo.d.ts.map +1 -0
  62. package/dist/commands/demo.js +106 -0
  63. package/dist/commands/demo.js.map +1 -0
  64. package/dist/commands/export.d.ts +14 -0
  65. package/dist/commands/export.d.ts.map +1 -0
  66. package/dist/commands/export.js +209 -0
  67. package/dist/commands/export.js.map +1 -0
  68. package/dist/commands/history.d.ts +15 -0
  69. package/dist/commands/history.d.ts.map +1 -0
  70. package/dist/commands/history.js +389 -0
  71. package/dist/commands/history.js.map +1 -0
  72. package/dist/commands/template.d.ts +14 -0
  73. package/dist/commands/template.d.ts.map +1 -0
  74. package/dist/commands/template.js +341 -0
  75. package/dist/commands/template.js.map +1 -0
  76. package/dist/commands/validate-map.d.ts +12 -0
  77. package/dist/commands/validate-map.d.ts.map +1 -0
  78. package/dist/commands/validate-map.js +274 -0
  79. package/dist/commands/validate-map.js.map +1 -0
  80. package/dist/commands/whatif.d.ts +17 -0
  81. package/dist/commands/whatif.d.ts.map +1 -0
  82. package/dist/commands/whatif.js +206 -0
  83. package/dist/commands/whatif.js.map +1 -0
  84. package/dist/comparison.d.ts +38 -0
  85. package/dist/comparison.d.ts.map +1 -0
  86. package/dist/comparison.js +223 -0
  87. package/dist/comparison.js.map +1 -0
  88. package/dist/config.d.ts +42 -0
  89. package/dist/config.d.ts.map +1 -0
  90. package/dist/config.js +158 -0
  91. package/dist/config.js.map +1 -0
  92. package/dist/connectors/helicone.d.ts +9 -0
  93. package/dist/connectors/helicone.d.ts.map +1 -0
  94. package/dist/connectors/helicone.js +106 -0
  95. package/dist/connectors/helicone.js.map +1 -0
  96. package/dist/connectors/index.d.ts +37 -0
  97. package/dist/connectors/index.d.ts.map +1 -0
  98. package/dist/connectors/index.js +65 -0
  99. package/dist/connectors/index.js.map +1 -0
  100. package/dist/connectors/langsmith.d.ts +9 -0
  101. package/dist/connectors/langsmith.d.ts.map +1 -0
  102. package/dist/connectors/langsmith.js +122 -0
  103. package/dist/connectors/langsmith.js.map +1 -0
  104. package/dist/connectors/types.d.ts +83 -0
  105. package/dist/connectors/types.d.ts.map +1 -0
  106. package/dist/connectors/types.js +98 -0
  107. package/dist/connectors/types.js.map +1 -0
  108. package/dist/cost-estimator.d.ts +46 -0
  109. package/dist/cost-estimator.d.ts.map +1 -0
  110. package/dist/cost-estimator.js +104 -0
  111. package/dist/cost-estimator.js.map +1 -0
  112. package/dist/costs.d.ts +57 -0
  113. package/dist/costs.d.ts.map +1 -0
  114. package/dist/costs.js +251 -0
  115. package/dist/costs.js.map +1 -0
  116. package/dist/counterfactuals.d.ts +29 -0
  117. package/dist/counterfactuals.d.ts.map +1 -0
  118. package/dist/counterfactuals.js +448 -0
  119. package/dist/counterfactuals.js.map +1 -0
  120. package/dist/enhancement-prompts.d.ts +41 -0
  121. package/dist/enhancement-prompts.d.ts.map +1 -0
  122. package/dist/enhancement-prompts.js +88 -0
  123. package/dist/enhancement-prompts.js.map +1 -0
  124. package/dist/envelopes.d.ts +20 -0
  125. package/dist/envelopes.d.ts.map +1 -0
  126. package/dist/envelopes.js +790 -0
  127. package/dist/envelopes.js.map +1 -0
  128. package/dist/format-normalizer.d.ts +71 -0
  129. package/dist/format-normalizer.d.ts.map +1 -0
  130. package/dist/format-normalizer.js +1331 -0
  131. package/dist/format-normalizer.js.map +1 -0
  132. package/dist/history.d.ts +79 -0
  133. package/dist/history.d.ts.map +1 -0
  134. package/dist/history.js +313 -0
  135. package/dist/history.js.map +1 -0
  136. package/dist/html.d.ts +11 -0
  137. package/dist/html.d.ts.map +1 -0
  138. package/dist/html.js +463 -0
  139. package/dist/html.js.map +1 -0
  140. package/dist/impact.d.ts +42 -0
  141. package/dist/impact.d.ts.map +1 -0
  142. package/dist/impact.js +443 -0
  143. package/dist/impact.js.map +1 -0
  144. package/dist/index.d.ts +26 -0
  145. package/dist/index.d.ts.map +1 -0
  146. package/dist/index.js +34 -0
  147. package/dist/index.js.map +1 -0
  148. package/dist/insights.d.ts +5 -0
  149. package/dist/insights.d.ts.map +1 -0
  150. package/dist/insights.js +271 -0
  151. package/dist/insights.js.map +1 -0
  152. package/dist/joiner.d.ts +9 -0
  153. package/dist/joiner.d.ts.map +1 -0
  154. package/dist/joiner.js +247 -0
  155. package/dist/joiner.js.map +1 -0
  156. package/dist/orchestrator.d.ts +34 -0
  157. package/dist/orchestrator.d.ts.map +1 -0
  158. package/dist/orchestrator.js +827 -0
  159. package/dist/orchestrator.js.map +1 -0
  160. package/dist/pdf.d.ts +26 -0
  161. package/dist/pdf.d.ts.map +1 -0
  162. package/dist/pdf.js +84 -0
  163. package/dist/pdf.js.map +1 -0
  164. package/dist/prediction.d.ts +33 -0
  165. package/dist/prediction.d.ts.map +1 -0
  166. package/dist/prediction.js +316 -0
  167. package/dist/prediction.js.map +1 -0
  168. package/dist/prompts/loader.d.ts +38 -0
  169. package/dist/prompts/loader.d.ts.map +1 -0
  170. package/dist/prompts/loader.js +60 -0
  171. package/dist/prompts/loader.js.map +1 -0
  172. package/dist/renderer.d.ts +64 -0
  173. package/dist/renderer.d.ts.map +1 -0
  174. package/dist/renderer.js +923 -0
  175. package/dist/renderer.js.map +1 -0
  176. package/dist/runid.d.ts +57 -0
  177. package/dist/runid.d.ts.map +1 -0
  178. package/dist/runid.js +199 -0
  179. package/dist/runid.js.map +1 -0
  180. package/dist/runtime.d.ts +29 -0
  181. package/dist/runtime.d.ts.map +1 -0
  182. package/dist/runtime.js +366 -0
  183. package/dist/runtime.js.map +1 -0
  184. package/dist/scanner.d.ts +11 -0
  185. package/dist/scanner.d.ts.map +1 -0
  186. package/dist/scanner.js +426 -0
  187. package/dist/scanner.js.map +1 -0
  188. package/dist/templates.d.ts +120 -0
  189. package/dist/templates.d.ts.map +1 -0
  190. package/dist/templates.js +429 -0
  191. package/dist/templates.js.map +1 -0
  192. package/dist/tools/index.d.ts +153 -0
  193. package/dist/tools/index.d.ts.map +1 -0
  194. package/dist/tools/index.js +177 -0
  195. package/dist/tools/index.js.map +1 -0
  196. package/dist/types.d.ts +3647 -0
  197. package/dist/types.d.ts.map +1 -0
  198. package/dist/types.js +703 -0
  199. package/dist/types.js.map +1 -0
  200. package/dist/version.d.ts +7 -0
  201. package/dist/version.d.ts.map +1 -0
  202. package/dist/version.js +23 -0
  203. package/dist/version.js.map +1 -0
  204. package/docs/demo-guide.md +423 -0
  205. package/docs/events-format.md +295 -0
  206. package/docs/inferencemap-spec.md +344 -0
  207. package/docs/migration-v2.md +293 -0
  208. package/fixtures/demo/precomputed.json +142 -0
  209. package/fixtures/demo-project/README.md +52 -0
  210. package/fixtures/demo-project/ai-service.ts +65 -0
  211. package/fixtures/demo-project/sample-events.jsonl +15 -0
  212. package/fixtures/demo-project/src/ai-service.ts +128 -0
  213. package/fixtures/demo-project/src/llm-client.ts +155 -0
  214. package/package.json +65 -0
  215. package/prompts/agent-analyzer.yaml +47 -0
  216. package/prompts/ci-gate.yaml +98 -0
  217. package/prompts/correlation-analyzer.yaml +178 -0
  218. package/prompts/format-normalizer.yaml +46 -0
  219. package/prompts/peak-performance.yaml +180 -0
  220. package/prompts/pr-comment.yaml +111 -0
  221. package/prompts/runtime-analyzer.yaml +189 -0
  222. package/prompts/unified-analyzer.yaml +241 -0
  223. package/schemas/inference-map.v0.1.json +215 -0
  224. package/scripts/benchmark.ts +394 -0
  225. package/scripts/demo-v1.5.sh +158 -0
  226. package/scripts/sync-from-site.sh +197 -0
  227. package/scripts/validate-sync.sh +178 -0
  228. package/src/agent-analyzer.ts +481 -0
  229. package/src/agent.ts +1232 -0
  230. package/src/agents/correlation-analyzer.ts +353 -0
  231. package/src/agents/index.ts +235 -0
  232. package/src/agents/runtime-analyzer.ts +343 -0
  233. package/src/analysis-types.ts +558 -0
  234. package/src/analytics.ts +100 -0
  235. package/src/analyzer.ts +692 -0
  236. package/src/artifacts.ts +218 -0
  237. package/src/benchmarks/index.ts +309 -0
  238. package/src/cli.ts +503 -0
  239. package/src/commands/ci.ts +336 -0
  240. package/src/commands/config.ts +288 -0
  241. package/src/commands/demo.ts +175 -0
  242. package/src/commands/export.ts +297 -0
  243. package/src/commands/history.ts +425 -0
  244. package/src/commands/template.ts +385 -0
  245. package/src/commands/validate-map.ts +324 -0
  246. package/src/commands/whatif.ts +272 -0
  247. package/src/comparison.ts +283 -0
  248. package/src/config.ts +188 -0
  249. package/src/connectors/helicone.ts +164 -0
  250. package/src/connectors/index.ts +93 -0
  251. package/src/connectors/langsmith.ts +179 -0
  252. package/src/connectors/types.ts +180 -0
  253. package/src/cost-estimator.ts +146 -0
  254. package/src/costs.ts +347 -0
  255. package/src/counterfactuals.ts +516 -0
  256. package/src/enhancement-prompts.ts +118 -0
  257. package/src/envelopes.ts +814 -0
  258. package/src/format-normalizer.ts +1486 -0
  259. package/src/history.ts +400 -0
  260. package/src/html.ts +512 -0
  261. package/src/impact.ts +522 -0
  262. package/src/index.ts +83 -0
  263. package/src/insights.ts +341 -0
  264. package/src/joiner.ts +289 -0
  265. package/src/orchestrator.ts +1015 -0
  266. package/src/pdf.ts +110 -0
  267. package/src/prediction.ts +392 -0
  268. package/src/prompts/loader.ts +88 -0
  269. package/src/renderer.ts +1045 -0
  270. package/src/runid.ts +261 -0
  271. package/src/runtime.ts +450 -0
  272. package/src/scanner.ts +508 -0
  273. package/src/templates.ts +561 -0
  274. package/src/tools/index.ts +214 -0
  275. package/src/types.ts +873 -0
  276. package/src/version.ts +24 -0
  277. package/templates/context-accumulation.yaml +23 -0
  278. package/templates/cost-concentration.yaml +20 -0
  279. package/templates/dead-code.yaml +20 -0
  280. package/templates/latency-explainer.yaml +23 -0
  281. package/templates/optimizations/ab-testing-framework.yaml +74 -0
  282. package/templates/optimizations/api-gateway-optimization.yaml +81 -0
  283. package/templates/optimizations/api-model-routing-strategy.yaml +126 -0
  284. package/templates/optimizations/auto-scaling-optimization.yaml +85 -0
  285. package/templates/optimizations/batch-utilization-diagnostic.yaml +142 -0
  286. package/templates/optimizations/comprehensive-apm.yaml +76 -0
  287. package/templates/optimizations/context-window-optimization.yaml +91 -0
  288. package/templates/optimizations/cost-sensitive-batch-processing.yaml +77 -0
  289. package/templates/optimizations/distributed-training-optimization.yaml +77 -0
  290. package/templates/optimizations/document-analysis-edge.yaml +77 -0
  291. package/templates/optimizations/document-pipeline-optimization.yaml +78 -0
  292. package/templates/optimizations/domain-specific-distillation.yaml +78 -0
  293. package/templates/optimizations/error-handling-optimization.yaml +76 -0
  294. package/templates/optimizations/gptq-4bit-quantization.yaml +96 -0
  295. package/templates/optimizations/long-context-memory-management.yaml +78 -0
  296. package/templates/optimizations/max-tokens-optimization.yaml +76 -0
  297. package/templates/optimizations/memory-bandwidth-optimization.yaml +73 -0
  298. package/templates/optimizations/multi-framework-resilience.yaml +75 -0
  299. package/templates/optimizations/multi-tenant-optimization.yaml +75 -0
  300. package/templates/optimizations/prompt-caching-optimization.yaml +143 -0
  301. package/templates/optimizations/pytorch-to-onnx-migration.yaml +109 -0
  302. package/templates/optimizations/quality-monitoring.yaml +74 -0
  303. package/templates/optimizations/realtime-budget-controls.yaml +74 -0
  304. package/templates/optimizations/realtime-latency-optimization.yaml +74 -0
  305. package/templates/optimizations/sglang-concurrency-optimization.yaml +78 -0
  306. package/templates/optimizations/smart-model-routing.yaml +96 -0
  307. package/templates/optimizations/streaming-batch-selection.yaml +167 -0
  308. package/templates/optimizations/system-prompt-optimization.yaml +75 -0
  309. package/templates/optimizations/tensorrt-llm-performance.yaml +77 -0
  310. package/templates/optimizations/vllm-high-throughput-optimization.yaml +93 -0
  311. package/templates/optimizations/vllm-migration-memory-bound.yaml +78 -0
  312. package/templates/overpowered-extraction.yaml +32 -0
  313. package/templates/overpowered-model.yaml +31 -0
  314. package/templates/prompt-bloat.yaml +24 -0
  315. package/templates/retry-explosion.yaml +28 -0
  316. package/templates/schema/insight.schema.json +113 -0
  317. package/templates/schema/optimization.schema.json +180 -0
  318. package/templates/streaming-drift.yaml +30 -0
  319. package/templates/throughput-gap.yaml +21 -0
  320. package/templates/token-underutilization.yaml +28 -0
  321. package/templates/untested-fallback.yaml +21 -0
  322. package/tests/accuracy/drift-detection.test.ts +184 -0
  323. package/tests/accuracy/false-positives.test.ts +166 -0
  324. package/tests/accuracy/templates.test.ts +205 -0
  325. package/tests/action/commands.test.ts +125 -0
  326. package/tests/action/comments.test.ts +347 -0
  327. package/tests/cli.test.ts +203 -0
  328. package/tests/comparison.test.ts +309 -0
  329. package/tests/correlation-analyzer.test.ts +534 -0
  330. package/tests/counterfactuals.test.ts +347 -0
  331. package/tests/fixtures/events/missing-id.jsonl +1 -0
  332. package/tests/fixtures/events/missing-input.jsonl +1 -0
  333. package/tests/fixtures/events/missing-latency.jsonl +1 -0
  334. package/tests/fixtures/events/missing-model.jsonl +1 -0
  335. package/tests/fixtures/events/missing-output.jsonl +1 -0
  336. package/tests/fixtures/events/missing-provider.jsonl +1 -0
  337. package/tests/fixtures/events/missing-ts.jsonl +1 -0
  338. package/tests/fixtures/events/valid.csv +3 -0
  339. package/tests/fixtures/events/valid.json +1 -0
  340. package/tests/fixtures/events/valid.jsonl +2 -0
  341. package/tests/fixtures/events/with-callsite.jsonl +1 -0
  342. package/tests/fixtures/events/with-intent.jsonl +1 -0
  343. package/tests/fixtures/events/wrong-type.jsonl +1 -0
  344. package/tests/fixtures/repos/empty/.gitkeep +0 -0
  345. package/tests/fixtures/repos/hybrid-router/router.py +35 -0
  346. package/tests/fixtures/repos/saas-anthropic/agent.ts +27 -0
  347. package/tests/fixtures/repos/saas-openai/assistant.js +33 -0
  348. package/tests/fixtures/repos/saas-openai/client.py +26 -0
  349. package/tests/fixtures/repos/self-hosted-vllm/inference.py +22 -0
  350. package/tests/github-action.test.ts +292 -0
  351. package/tests/insights.test.ts +878 -0
  352. package/tests/joiner.test.ts +168 -0
  353. package/tests/performance/action-latency.test.ts +132 -0
  354. package/tests/performance/benchmark.test.ts +189 -0
  355. package/tests/performance/cli-latency.test.ts +102 -0
  356. package/tests/pr-comment.test.ts +313 -0
  357. package/tests/prediction.test.ts +296 -0
  358. package/tests/runtime-analyzer.test.ts +375 -0
  359. package/tests/runtime.test.ts +205 -0
  360. package/tests/scanner.test.ts +122 -0
  361. package/tests/template-conformance.test.ts +526 -0
  362. package/tests/unit/cost-calculator.test.ts +303 -0
  363. package/tests/unit/credits.test.ts +180 -0
  364. package/tests/unit/inference-map.test.ts +276 -0
  365. package/tests/unit/schema.test.ts +300 -0
  366. package/tsconfig.json +20 -0
  367. package/vitest.config.ts +14 -0
@@ -0,0 +1,878 @@
1
+ import { describe, it, expect, beforeAll } from 'vitest';
2
+ import { evaluate } from '../src/insights.js';
3
+ import { setTestPricing } from '../src/costs.js';
4
+ import type { InsightTemplate, EnrichedCallsite, JoinedOutput } from '../src/types.js';
5
+
6
+ // Set up mock pricing data before tests
7
+ beforeAll(() => {
8
+ setTestPricing({
9
+ 'gpt-4o': { input: 5.0, output: 15.0 }, // $5/$15 per 1M tokens
10
+ 'gpt-4o-mini': { input: 0.15, output: 0.6 }, // $0.15/$0.60 per 1M tokens
11
+ 'gpt-4': { input: 30.0, output: 60.0 }, // $30/$60 per 1M tokens
12
+ 'claude-3-opus': { input: 15.0, output: 75.0 },
13
+ 'claude-3-5-sonnet-20241022': { input: 3.0, output: 15.0 },
14
+ });
15
+ });
16
+
17
+ // =============================================================================
18
+ // TEST TEMPLATES (matching the 12 templates in peakinfer_templates repo)
19
+ // =============================================================================
20
+
21
+ const templates: InsightTemplate[] = [
22
+ // COST TEMPLATES (4)
23
+ {
24
+ id: 'prompt-bloat',
25
+ version: '1.0',
26
+ name: 'Prompt Bloat Detection',
27
+ description: 'Detects high input/output token ratio',
28
+ category: 'cost',
29
+ severity: 'warning',
30
+ tags: ['tokens', 'prompt'],
31
+ match: {
32
+ scope: 'callsite',
33
+ conditions: [
34
+ { field: 'usage.tokens_in', op: 'ratio_gt', compare_to: 'usage.tokens_out', value: 20 },
35
+ ],
36
+ },
37
+ output: {
38
+ headline: '{{input_output_ratio}}x more input than output tokens',
39
+ evidence: '{{location}}: Sending {{tokens_in}} tokens, receiving {{tokens_out}}.',
40
+ },
41
+ recommends: [],
42
+ },
43
+ {
44
+ id: 'retry-explosion',
45
+ version: '1.0',
46
+ name: 'Retry Storm Detection',
47
+ description: 'Detects retry storms via latency ratio',
48
+ category: 'cost',
49
+ severity: 'critical',
50
+ tags: ['retry', 'error-handling'],
51
+ match: {
52
+ scope: 'callsite',
53
+ conditions: [
54
+ { field: 'usage.calls', op: 'gt', value: 10 },
55
+ { field: 'usage.latency_p99', op: 'ratio_gt', compare_to: 'usage.latency_p50', value: 5 },
56
+ ],
57
+ },
58
+ output: {
59
+ headline: 'Possible retry storm at {{location}}',
60
+ evidence: '{{calls}} calls with p99/p50 latency ratio of {{ratio}}x.',
61
+ },
62
+ recommends: [],
63
+ },
64
+ {
65
+ id: 'cost-concentration',
66
+ version: '1.0',
67
+ name: 'Cost Concentration Detection',
68
+ description: 'Single callsite dominates cost',
69
+ category: 'cost',
70
+ severity: 'warning',
71
+ tags: ['cost', 'concentration'],
72
+ match: {
73
+ scope: 'global',
74
+ conditions: [
75
+ { field: 'top_callsite_cost_percent', op: 'gt', value: 50 },
76
+ ],
77
+ },
78
+ output: {
79
+ headline: '{{percent}}% of cost from one callsite',
80
+ evidence: '{{model}} at {{location}} dominates spend.',
81
+ },
82
+ recommends: [],
83
+ },
84
+ {
85
+ id: 'overpowered-extraction',
86
+ version: '1.0',
87
+ name: 'Overpowered Model for Simple Tasks',
88
+ description: 'Premium model for small outputs',
89
+ category: 'cost',
90
+ severity: 'warning',
91
+ tags: ['model-selection'],
92
+ match: {
93
+ scope: 'callsite',
94
+ conditions: [
95
+ { field: 'model', op: 'in', value: ['gpt-4o', 'gpt-4', 'claude-3-opus', 'claude-3-5-sonnet-20241022'] },
96
+ { field: 'avg_tokens', op: 'lt', value: 100 },
97
+ ],
98
+ },
99
+ output: {
100
+ headline: 'Using {{model}} for {{avg_tokens}}-token outputs',
101
+ evidence: '{{location}}: Consider gpt-4o-mini for simple tasks.',
102
+ },
103
+ recommends: [],
104
+ },
105
+
106
+ // DRIFT TEMPLATES (3)
107
+ {
108
+ id: 'dead-code',
109
+ version: '1.0',
110
+ name: 'Dead Code Detection',
111
+ description: 'Callsites with no runtime events',
112
+ category: 'drift',
113
+ severity: 'warning',
114
+ tags: ['dead-code', 'drift'],
115
+ match: {
116
+ scope: 'joined',
117
+ conditions: [
118
+ { field: 'codeOnly.length', op: 'gt', value: 0 },
119
+ ],
120
+ },
121
+ output: {
122
+ headline: '{{count}} callsites in code with no runtime events',
123
+ evidence: '{{locations}}',
124
+ },
125
+ recommends: [],
126
+ },
127
+ {
128
+ id: 'streaming-drift',
129
+ version: '1.0',
130
+ name: 'Streaming Drift Detection',
131
+ description: 'Streaming declared but high latency',
132
+ category: 'drift',
133
+ severity: 'warning',
134
+ tags: ['streaming', 'drift'],
135
+ match: {
136
+ scope: 'callsite',
137
+ conditions: [
138
+ { field: 'patterns.streaming', op: 'eq', value: true },
139
+ { field: 'usage.latency_p50', op: 'gt', value: 2000 },
140
+ ],
141
+ },
142
+ output: {
143
+ headline: 'Streaming declared but p50 latency is {{p50}}ms',
144
+ evidence: '{{location}}: Code says stream=True but response times suggest buffering.',
145
+ },
146
+ recommends: [],
147
+ },
148
+ {
149
+ id: 'untested-fallback',
150
+ version: '1.0',
151
+ name: 'Untested Fallback Detection',
152
+ description: 'Fallback pattern rarely exercised',
153
+ category: 'drift',
154
+ severity: 'info',
155
+ tags: ['fallback', 'reliability'],
156
+ match: {
157
+ scope: 'callsite',
158
+ conditions: [
159
+ { field: 'patterns.fallback', op: 'eq', value: true },
160
+ { field: 'usage.calls', op: 'lt', value: 5 },
161
+ ],
162
+ },
163
+ output: {
164
+ headline: 'Fallback at {{location}} has rarely fired',
165
+ evidence: 'Only {{calls}} calls recorded.',
166
+ },
167
+ recommends: [],
168
+ },
169
+
170
+ // PERFORMANCE TEMPLATES (3)
171
+ {
172
+ id: 'throughput-gap',
173
+ version: '1.0',
174
+ name: 'Throughput Gap Detection',
175
+ description: 'Running below achievable throughput',
176
+ category: 'performance',
177
+ severity: 'warning',
178
+ tags: ['throughput', 'performance'],
179
+ match: {
180
+ scope: 'envelope',
181
+ conditions: [
182
+ { field: 'actual_tps', op: 'ratio_lt', compare_to: 'envelope.tps_median', value: 0.5 },
183
+ ],
184
+ },
185
+ output: {
186
+ headline: 'Running at {{percent}}% of achievable throughput',
187
+ evidence: 'Your {{model}}: {{actual}} tok/s, reference: {{reference}} tok/s.',
188
+ },
189
+ recommends: [],
190
+ },
191
+ {
192
+ id: 'latency-explainer',
193
+ version: '1.0',
194
+ name: 'High Latency Without Streaming',
195
+ description: 'High p95 without streaming enabled',
196
+ category: 'performance',
197
+ severity: 'warning',
198
+ tags: ['latency', 'streaming'],
199
+ match: {
200
+ scope: 'callsite',
201
+ conditions: [
202
+ { field: 'patterns.streaming', op: 'neq', value: true },
203
+ { field: 'usage.latency_p95', op: 'gt', value: 3000 },
204
+ ],
205
+ },
206
+ output: {
207
+ headline: 'p95 latency {{p95}}ms without streaming',
208
+ evidence: '{{location}}: Enable streaming to improve perceived latency.',
209
+ },
210
+ recommends: [],
211
+ },
212
+ {
213
+ id: 'context-accumulation',
214
+ version: '1.0',
215
+ name: 'Context Window Bloat Detection',
216
+ description: 'Very high input token counts',
217
+ category: 'performance',
218
+ severity: 'warning',
219
+ tags: ['context', 'tokens'],
220
+ match: {
221
+ scope: 'callsite',
222
+ conditions: [
223
+ { field: 'usage.tokens_in', op: 'gt', value: 50000 },
224
+ ],
225
+ },
226
+ output: {
227
+ headline: 'High context usage at {{location}}',
228
+ evidence: 'Averaging {{avg_tokens_in}} input tokens per call.',
229
+ },
230
+ recommends: [],
231
+ },
232
+
233
+ // WASTE TEMPLATES (2)
234
+ {
235
+ id: 'overpowered-model',
236
+ version: '1.0',
237
+ name: 'Overpowered Model Detection',
238
+ description: 'Premium model with tiny outputs',
239
+ category: 'waste',
240
+ severity: 'info',
241
+ tags: ['model-selection', 'cost'],
242
+ match: {
243
+ scope: 'callsite',
244
+ conditions: [
245
+ { field: 'model', op: 'in', value: ['gpt-4o', 'gpt-4', 'claude-3-opus'] },
246
+ { field: 'avg_tokens', op: 'lt', value: 50 },
247
+ ],
248
+ },
249
+ output: {
250
+ headline: '{{model}} generating only {{avg_tokens}} tokens average',
251
+ evidence: '{{location}}: Short outputs suggest cheaper models may work.',
252
+ },
253
+ recommends: [],
254
+ },
255
+ {
256
+ id: 'token-underutilization',
257
+ version: '1.0',
258
+ name: 'Token Budget Underutilization',
259
+ description: 'Low output token counts',
260
+ category: 'waste',
261
+ severity: 'info',
262
+ tags: ['tokens', 'max-tokens'],
263
+ match: {
264
+ scope: 'callsite',
265
+ conditions: [
266
+ { field: 'usage.tokens_out', op: 'exists' },
267
+ { field: 'avg_tokens', op: 'lt', value: 200 },
268
+ ],
269
+ },
270
+ output: {
271
+ headline: 'Low output utilization at {{location}}',
272
+ evidence: 'Averaging {{avg_tokens}} output tokens.',
273
+ },
274
+ recommends: [],
275
+ },
276
+ ];
277
+
278
+ // =============================================================================
279
+ // HELPER: Create enriched callsite
280
+ // =============================================================================
281
+
282
+ function createCallsite(overrides: Partial<EnrichedCallsite> & { file: string; line: number }): EnrichedCallsite {
283
+ return {
284
+ id: `${overrides.file}:${overrides.line}`,
285
+ file: overrides.file,
286
+ line: overrides.line,
287
+ snippet: overrides.snippet || 'openai.chat.completions.create(...)',
288
+ provider: overrides.provider || 'openai',
289
+ model: overrides.model || 'gpt-4o-mini',
290
+ patterns: overrides.patterns || {},
291
+ usage: overrides.usage,
292
+ };
293
+ }
294
+
295
+ // =============================================================================
296
+ // COST TEMPLATE TESTS
297
+ // =============================================================================
298
+
299
+ describe('Cost Templates', () => {
300
+ describe('prompt-bloat', () => {
301
+ it('triggers when input/output ratio > 20', () => {
302
+ const callsites: EnrichedCallsite[] = [
303
+ createCallsite({
304
+ file: 'src/api/chat.ts',
305
+ line: 45,
306
+ usage: {
307
+ calls: 100,
308
+ tokens_in: 50000, // 50000 / 500 = 100x ratio
309
+ tokens_out: 500,
310
+ latency_p50: 1000,
311
+ latency_p95: 2000,
312
+ latency_p99: 3000,
313
+ },
314
+ }),
315
+ ];
316
+
317
+ const insights = evaluate({ callsites }, [templates[0]]);
318
+
319
+ expect(insights).toHaveLength(1);
320
+ expect(insights[0].templateId).toBe('prompt-bloat');
321
+ expect(insights[0].headline).toContain('100x more input than output');
322
+ });
323
+
324
+ it('does not trigger when ratio <= 20', () => {
325
+ const callsites: EnrichedCallsite[] = [
326
+ createCallsite({
327
+ file: 'src/api/chat.ts',
328
+ line: 45,
329
+ usage: {
330
+ calls: 100,
331
+ tokens_in: 1000,
332
+ tokens_out: 100, // 10x ratio
333
+ latency_p50: 1000,
334
+ latency_p95: 2000,
335
+ latency_p99: 3000,
336
+ },
337
+ }),
338
+ ];
339
+
340
+ const insights = evaluate({ callsites }, [templates[0]]);
341
+ expect(insights).toHaveLength(0);
342
+ });
343
+ });
344
+
345
+ describe('retry-explosion', () => {
346
+ it('triggers when calls > 10 AND latency ratio > 5', () => {
347
+ const callsites: EnrichedCallsite[] = [
348
+ createCallsite({
349
+ file: 'src/services/llm.ts',
350
+ line: 120,
351
+ usage: {
352
+ calls: 50,
353
+ tokens_in: 1000,
354
+ tokens_out: 100,
355
+ latency_p50: 500,
356
+ latency_p95: 2000,
357
+ latency_p99: 3000, // 3000/500 = 6x ratio
358
+ },
359
+ }),
360
+ ];
361
+
362
+ const insights = evaluate({ callsites }, [templates[1]]);
363
+
364
+ expect(insights).toHaveLength(1);
365
+ expect(insights[0].templateId).toBe('retry-explosion');
366
+ expect(insights[0].severity).toBe('critical');
367
+ });
368
+
369
+ it('does not trigger when calls <= 10', () => {
370
+ const callsites: EnrichedCallsite[] = [
371
+ createCallsite({
372
+ file: 'src/services/llm.ts',
373
+ line: 120,
374
+ usage: {
375
+ calls: 5, // Too few calls
376
+ tokens_in: 1000,
377
+ tokens_out: 100,
378
+ latency_p50: 500,
379
+ latency_p95: 2000,
380
+ latency_p99: 3000,
381
+ },
382
+ }),
383
+ ];
384
+
385
+ const insights = evaluate({ callsites }, [templates[1]]);
386
+ expect(insights).toHaveLength(0);
387
+ });
388
+ });
389
+
390
+ describe('cost-concentration', () => {
391
+ it('triggers when one callsite > 50% of total cost', () => {
392
+ const callsites: EnrichedCallsite[] = [
393
+ createCallsite({
394
+ file: 'src/api/expensive.ts',
395
+ line: 10,
396
+ model: 'gpt-4o',
397
+ usage: {
398
+ calls: 1000,
399
+ tokens_in: 100000,
400
+ tokens_out: 50000,
401
+ latency_p50: 1000,
402
+ latency_p95: 2000,
403
+ latency_p99: 3000,
404
+ },
405
+ }),
406
+ createCallsite({
407
+ file: 'src/api/cheap.ts',
408
+ line: 20,
409
+ model: 'gpt-4o-mini',
410
+ usage: {
411
+ calls: 100,
412
+ tokens_in: 1000,
413
+ tokens_out: 500,
414
+ latency_p50: 500,
415
+ latency_p95: 1000,
416
+ latency_p99: 1500,
417
+ },
418
+ }),
419
+ ];
420
+
421
+ const insights = evaluate({ callsites }, [templates[2]]);
422
+
423
+ expect(insights).toHaveLength(1);
424
+ expect(insights[0].templateId).toBe('cost-concentration');
425
+ });
426
+ });
427
+
428
+ describe('overpowered-extraction', () => {
429
+ it('triggers for premium model with small outputs', () => {
430
+ const callsites: EnrichedCallsite[] = [
431
+ createCallsite({
432
+ file: 'src/extractors/ner.ts',
433
+ line: 55,
434
+ model: 'gpt-4o',
435
+ usage: {
436
+ calls: 500,
437
+ tokens_in: 5000,
438
+ tokens_out: 2500, // avg = 5 tokens
439
+ latency_p50: 800,
440
+ latency_p95: 1200,
441
+ latency_p99: 1500,
442
+ },
443
+ }),
444
+ ];
445
+
446
+ const insights = evaluate({ callsites }, [templates[3]]);
447
+
448
+ expect(insights).toHaveLength(1);
449
+ expect(insights[0].templateId).toBe('overpowered-extraction');
450
+ expect(insights[0].headline).toContain('gpt-4o');
451
+ });
452
+
453
+ it('does not trigger for cheap models', () => {
454
+ const callsites: EnrichedCallsite[] = [
455
+ createCallsite({
456
+ file: 'src/extractors/ner.ts',
457
+ line: 55,
458
+ model: 'gpt-4o-mini', // Cheap model
459
+ usage: {
460
+ calls: 500,
461
+ tokens_in: 5000,
462
+ tokens_out: 2500,
463
+ latency_p50: 800,
464
+ latency_p95: 1200,
465
+ latency_p99: 1500,
466
+ },
467
+ }),
468
+ ];
469
+
470
+ const insights = evaluate({ callsites }, [templates[3]]);
471
+ expect(insights).toHaveLength(0);
472
+ });
473
+ });
474
+ });
475
+
476
+ // =============================================================================
477
+ // DRIFT TEMPLATE TESTS
478
+ // =============================================================================
479
+
480
+ describe('Drift Templates', () => {
481
+ describe('dead-code', () => {
482
+ it('triggers when callsites exist with no runtime events', () => {
483
+ const joined: JoinedOutput = {
484
+ matched: [],
485
+ codeOnly: [
486
+ createCallsite({ file: 'src/unused/old.ts', line: 10 }),
487
+ createCallsite({ file: 'src/unused/deprecated.ts', line: 20 }),
488
+ ],
489
+ runtimeOnly: [],
490
+ drift: { codeOnly: 2, runtimeOnly: 0 },
491
+ };
492
+
493
+ const insights = evaluate(joined, [templates[4]]);
494
+
495
+ expect(insights).toHaveLength(1);
496
+ expect(insights[0].templateId).toBe('dead-code');
497
+ expect(insights[0].headline).toContain('2 callsites');
498
+ });
499
+
500
+ it('does not trigger when all callsites have runtime data', () => {
501
+ const joined: JoinedOutput = {
502
+ matched: [createCallsite({ file: 'src/api/active.ts', line: 10 })],
503
+ codeOnly: [],
504
+ runtimeOnly: [],
505
+ drift: { codeOnly: 0, runtimeOnly: 0 },
506
+ };
507
+
508
+ const insights = evaluate(joined, [templates[4]]);
509
+ expect(insights).toHaveLength(0);
510
+ });
511
+ });
512
+
513
+ describe('streaming-drift', () => {
514
+ it('triggers when streaming=true but high latency', () => {
515
+ const callsites: EnrichedCallsite[] = [
516
+ createCallsite({
517
+ file: 'src/api/stream.ts',
518
+ line: 30,
519
+ patterns: { streaming: true },
520
+ usage: {
521
+ calls: 100,
522
+ tokens_in: 1000,
523
+ tokens_out: 500,
524
+ latency_p50: 3500, // > 2000ms
525
+ latency_p95: 5000,
526
+ latency_p99: 7000,
527
+ },
528
+ }),
529
+ ];
530
+
531
+ const insights = evaluate({ callsites }, [templates[5]]);
532
+
533
+ expect(insights).toHaveLength(1);
534
+ expect(insights[0].templateId).toBe('streaming-drift');
535
+ expect(insights[0].headline).toContain('3500ms');
536
+ });
537
+
538
+ it('does not trigger for non-streaming callsites', () => {
539
+ const callsites: EnrichedCallsite[] = [
540
+ createCallsite({
541
+ file: 'src/api/batch.ts',
542
+ line: 30,
543
+ patterns: { streaming: false },
544
+ usage: {
545
+ calls: 100,
546
+ tokens_in: 1000,
547
+ tokens_out: 500,
548
+ latency_p50: 3500,
549
+ latency_p95: 5000,
550
+ latency_p99: 7000,
551
+ },
552
+ }),
553
+ ];
554
+
555
+ const insights = evaluate({ callsites }, [templates[5]]);
556
+ expect(insights).toHaveLength(0);
557
+ });
558
+ });
559
+
560
+ describe('untested-fallback', () => {
561
+ it('triggers when fallback=true but calls < 5', () => {
562
+ const callsites: EnrichedCallsite[] = [
563
+ createCallsite({
564
+ file: 'src/resilience/fallback.ts',
565
+ line: 80,
566
+ patterns: { fallback: true },
567
+ usage: {
568
+ calls: 2, // Rarely exercised
569
+ tokens_in: 200,
570
+ tokens_out: 100,
571
+ latency_p50: 1000,
572
+ latency_p95: 2000,
573
+ latency_p99: 3000,
574
+ },
575
+ }),
576
+ ];
577
+
578
+ const insights = evaluate({ callsites }, [templates[6]]);
579
+
580
+ expect(insights).toHaveLength(1);
581
+ expect(insights[0].templateId).toBe('untested-fallback');
582
+ expect(insights[0].severity).toBe('info');
583
+ });
584
+
585
+ it('does not trigger when fallback is well-exercised', () => {
586
+ const callsites: EnrichedCallsite[] = [
587
+ createCallsite({
588
+ file: 'src/resilience/fallback.ts',
589
+ line: 80,
590
+ patterns: { fallback: true },
591
+ usage: {
592
+ calls: 100, // Well exercised
593
+ tokens_in: 200,
594
+ tokens_out: 100,
595
+ latency_p50: 1000,
596
+ latency_p95: 2000,
597
+ latency_p99: 3000,
598
+ },
599
+ }),
600
+ ];
601
+
602
+ const insights = evaluate({ callsites }, [templates[6]]);
603
+ expect(insights).toHaveLength(0);
604
+ });
605
+ });
606
+ });
607
+
608
+ // =============================================================================
609
+ // PERFORMANCE TEMPLATE TESTS
610
+ // =============================================================================
611
+
612
+ describe('Performance Templates', () => {
613
+ describe('throughput-gap', () => {
614
+ it('triggers when actual TPS < 50% of envelope median', () => {
615
+ // This test requires envelope data - skipping for now as it needs
616
+ // actual InferenceMAX envelope integration
617
+ // The envelope scope is tested in the actual integration
618
+ expect(true).toBe(true);
619
+ });
620
+ });
621
+
622
+ describe('latency-explainer', () => {
623
+ it('triggers when no streaming and p95 > 3000ms', () => {
624
+ const callsites: EnrichedCallsite[] = [
625
+ createCallsite({
626
+ file: 'src/api/slow.ts',
627
+ line: 100,
628
+ patterns: { streaming: false },
629
+ usage: {
630
+ calls: 50,
631
+ tokens_in: 2000,
632
+ tokens_out: 1000,
633
+ latency_p50: 2000,
634
+ latency_p95: 4500, // > 3000ms
635
+ latency_p99: 6000,
636
+ },
637
+ }),
638
+ ];
639
+
640
+ const insights = evaluate({ callsites }, [templates[8]]);
641
+
642
+ expect(insights).toHaveLength(1);
643
+ expect(insights[0].templateId).toBe('latency-explainer');
644
+ expect(insights[0].headline).toContain('4500ms');
645
+ });
646
+
647
+ it('does not trigger when streaming is enabled', () => {
648
+ const callsites: EnrichedCallsite[] = [
649
+ createCallsite({
650
+ file: 'src/api/slow.ts',
651
+ line: 100,
652
+ patterns: { streaming: true }, // Streaming enabled
653
+ usage: {
654
+ calls: 50,
655
+ tokens_in: 2000,
656
+ tokens_out: 1000,
657
+ latency_p50: 2000,
658
+ latency_p95: 4500,
659
+ latency_p99: 6000,
660
+ },
661
+ }),
662
+ ];
663
+
664
+ const insights = evaluate({ callsites }, [templates[8]]);
665
+ expect(insights).toHaveLength(0);
666
+ });
667
+ });
668
+
669
+ describe('context-accumulation', () => {
670
+ it('triggers when tokens_in > 50000', () => {
671
+ const callsites: EnrichedCallsite[] = [
672
+ createCallsite({
673
+ file: 'src/chat/conversation.ts',
674
+ line: 200,
675
+ usage: {
676
+ calls: 10,
677
+ tokens_in: 75000, // > 50000
678
+ tokens_out: 5000,
679
+ latency_p50: 5000,
680
+ latency_p95: 8000,
681
+ latency_p99: 10000,
682
+ },
683
+ }),
684
+ ];
685
+
686
+ const insights = evaluate({ callsites }, [templates[9]]);
687
+
688
+ expect(insights).toHaveLength(1);
689
+ expect(insights[0].templateId).toBe('context-accumulation');
690
+ });
691
+
692
+ it('does not trigger for normal context sizes', () => {
693
+ const callsites: EnrichedCallsite[] = [
694
+ createCallsite({
695
+ file: 'src/chat/conversation.ts',
696
+ line: 200,
697
+ usage: {
698
+ calls: 10,
699
+ tokens_in: 5000, // Normal
700
+ tokens_out: 1000,
701
+ latency_p50: 1000,
702
+ latency_p95: 2000,
703
+ latency_p99: 3000,
704
+ },
705
+ }),
706
+ ];
707
+
708
+ const insights = evaluate({ callsites }, [templates[9]]);
709
+ expect(insights).toHaveLength(0);
710
+ });
711
+ });
712
+ });
713
+
714
+ // =============================================================================
715
+ // WASTE TEMPLATE TESTS
716
+ // =============================================================================
717
+
718
+ describe('Waste Templates', () => {
719
+ describe('overpowered-model', () => {
720
+ it('triggers for gpt-4o with avg_tokens < 50', () => {
721
+ const callsites: EnrichedCallsite[] = [
722
+ createCallsite({
723
+ file: 'src/classify/sentiment.ts',
724
+ line: 15,
725
+ model: 'gpt-4o',
726
+ usage: {
727
+ calls: 1000,
728
+ tokens_in: 10000,
729
+ tokens_out: 10000, // avg = 10 tokens
730
+ latency_p50: 500,
731
+ latency_p95: 800,
732
+ latency_p99: 1000,
733
+ },
734
+ }),
735
+ ];
736
+
737
+ const insights = evaluate({ callsites }, [templates[10]]);
738
+
739
+ expect(insights).toHaveLength(1);
740
+ expect(insights[0].templateId).toBe('overpowered-model');
741
+ expect(insights[0].headline).toContain('gpt-4o');
742
+ expect(insights[0].headline).toContain('10 tokens');
743
+ });
744
+
745
+ it('does not trigger for larger outputs', () => {
746
+ const callsites: EnrichedCallsite[] = [
747
+ createCallsite({
748
+ file: 'src/generate/essay.ts',
749
+ line: 15,
750
+ model: 'gpt-4o',
751
+ usage: {
752
+ calls: 100,
753
+ tokens_in: 10000,
754
+ tokens_out: 50000, // avg = 500 tokens
755
+ latency_p50: 3000,
756
+ latency_p95: 5000,
757
+ latency_p99: 7000,
758
+ },
759
+ }),
760
+ ];
761
+
762
+ const insights = evaluate({ callsites }, [templates[10]]);
763
+ expect(insights).toHaveLength(0);
764
+ });
765
+ });
766
+
767
+ describe('token-underutilization', () => {
768
+ it('triggers when avg_tokens < 200', () => {
769
+ const callsites: EnrichedCallsite[] = [
770
+ createCallsite({
771
+ file: 'src/api/short.ts',
772
+ line: 50,
773
+ usage: {
774
+ calls: 100,
775
+ tokens_in: 5000,
776
+ tokens_out: 5000, // avg = 50 tokens
777
+ latency_p50: 500,
778
+ latency_p95: 800,
779
+ latency_p99: 1000,
780
+ },
781
+ }),
782
+ ];
783
+
784
+ const insights = evaluate({ callsites }, [templates[11]]);
785
+
786
+ expect(insights).toHaveLength(1);
787
+ expect(insights[0].templateId).toBe('token-underutilization');
788
+ });
789
+
790
+ it('does not trigger for higher output counts', () => {
791
+ const callsites: EnrichedCallsite[] = [
792
+ createCallsite({
793
+ file: 'src/api/verbose.ts',
794
+ line: 50,
795
+ usage: {
796
+ calls: 100,
797
+ tokens_in: 5000,
798
+ tokens_out: 50000, // avg = 500 tokens
799
+ latency_p50: 2000,
800
+ latency_p95: 3000,
801
+ latency_p99: 4000,
802
+ },
803
+ }),
804
+ ];
805
+
806
+ const insights = evaluate({ callsites }, [templates[11]]);
807
+ expect(insights).toHaveLength(0);
808
+ });
809
+ });
810
+ });
811
+
812
+ // =============================================================================
813
+ // COMBINED SCENARIOS
814
+ // =============================================================================
815
+
816
+ describe('Combined Scenarios', () => {
817
+ it('multiple templates can trigger on the same data', () => {
818
+ const callsites: EnrichedCallsite[] = [
819
+ createCallsite({
820
+ file: 'src/api/problematic.ts',
821
+ line: 100,
822
+ model: 'gpt-4o',
823
+ patterns: { streaming: false },
824
+ usage: {
825
+ calls: 500,
826
+ tokens_in: 100000, // High input (prompt bloat)
827
+ tokens_out: 2500, // Low output (avg = 5 tokens → overpowered-model + overpowered-extraction)
828
+ latency_p50: 2000,
829
+ latency_p95: 4000, // High latency (latency-explainer)
830
+ latency_p99: 6000,
831
+ },
832
+ }),
833
+ ];
834
+
835
+ const insights = evaluate({ callsites }, templates);
836
+
837
+ // Should trigger multiple templates
838
+ const templateIds = insights.map(i => i.templateId);
839
+ expect(templateIds).toContain('prompt-bloat');
840
+ expect(templateIds).toContain('overpowered-extraction');
841
+ expect(templateIds).toContain('overpowered-model');
842
+ expect(templateIds).toContain('latency-explainer');
843
+ expect(templateIds).toContain('token-underutilization');
844
+ });
845
+
846
+ it('well-optimized callsite triggers no insights', () => {
847
+ const callsites: EnrichedCallsite[] = [
848
+ createCallsite({
849
+ file: 'src/api/optimized.ts',
850
+ line: 50,
851
+ model: 'gpt-4o-mini', // Cheap model
852
+ patterns: { streaming: true }, // Streaming enabled
853
+ usage: {
854
+ calls: 100,
855
+ tokens_in: 1000,
856
+ tokens_out: 50000, // Good output
857
+ latency_p50: 500,
858
+ latency_p95: 800, // Low latency
859
+ latency_p99: 1000,
860
+ },
861
+ }),
862
+ ];
863
+
864
+ const joined: JoinedOutput = {
865
+ matched: callsites,
866
+ codeOnly: [], // No dead code
867
+ runtimeOnly: [],
868
+ drift: { codeOnly: 0, runtimeOnly: 0 },
869
+ };
870
+
871
+ const insights = evaluate(joined, templates);
872
+
873
+ // Only cost-concentration might trigger (depends on global stats)
874
+ // Filter it out for this test
875
+ const filtered = insights.filter(i => i.templateId !== 'cost-concentration');
876
+ expect(filtered).toHaveLength(0);
877
+ });
878
+ });