@outputai/cli 0.1.2 → 0.1.3-dev.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (278) hide show
  1. package/dist/api/generated/api.d.ts +820 -0
  2. package/dist/api/generated/api.js +226 -0
  3. package/dist/api/http_client.d.ts +27 -0
  4. package/dist/api/http_client.js +71 -0
  5. package/dist/api/orval_post_process.d.ts +11 -0
  6. package/dist/api/orval_post_process.js +46 -0
  7. package/dist/api/parser.d.ts +17 -0
  8. package/dist/api/parser.js +68 -0
  9. package/dist/assets/config/costs.yml +309 -0
  10. package/dist/assets/docker/docker-compose-dev.yml +146 -0
  11. package/dist/commands/credentials/edit.d.ts +10 -0
  12. package/dist/commands/credentials/edit.js +67 -0
  13. package/dist/commands/credentials/edit.spec.d.ts +1 -0
  14. package/dist/commands/credentials/edit.spec.js +73 -0
  15. package/dist/commands/credentials/get.d.ts +13 -0
  16. package/dist/commands/credentials/get.js +46 -0
  17. package/dist/commands/credentials/get.spec.d.ts +1 -0
  18. package/dist/commands/credentials/get.spec.js +74 -0
  19. package/dist/commands/credentials/init.d.ts +11 -0
  20. package/dist/commands/credentials/init.js +45 -0
  21. package/dist/commands/credentials/init.spec.d.ts +1 -0
  22. package/dist/commands/credentials/init.spec.js +68 -0
  23. package/dist/commands/credentials/show.d.ts +10 -0
  24. package/dist/commands/credentials/show.js +33 -0
  25. package/dist/commands/credentials/show.spec.d.ts +1 -0
  26. package/dist/commands/credentials/show.spec.js +57 -0
  27. package/dist/commands/dev/eject.d.ts +11 -0
  28. package/dist/commands/dev/eject.js +58 -0
  29. package/dist/commands/dev/eject.spec.d.ts +1 -0
  30. package/dist/commands/dev/eject.spec.js +109 -0
  31. package/dist/commands/dev/index.d.ts +14 -0
  32. package/dist/commands/dev/index.js +173 -0
  33. package/dist/commands/dev/index.spec.d.ts +1 -0
  34. package/dist/commands/dev/index.spec.js +239 -0
  35. package/dist/commands/init.d.ts +12 -0
  36. package/dist/commands/init.js +37 -0
  37. package/dist/commands/init.spec.d.ts +1 -0
  38. package/dist/commands/init.spec.js +100 -0
  39. package/dist/commands/update.d.ts +14 -0
  40. package/dist/commands/update.js +120 -0
  41. package/dist/commands/update.spec.d.ts +1 -0
  42. package/dist/commands/update.spec.js +178 -0
  43. package/dist/commands/workflow/cost.d.ts +16 -0
  44. package/dist/commands/workflow/cost.js +71 -0
  45. package/dist/commands/workflow/cost.spec.d.ts +1 -0
  46. package/dist/commands/workflow/cost.spec.js +47 -0
  47. package/dist/commands/workflow/dataset/generate.d.ts +22 -0
  48. package/dist/commands/workflow/dataset/generate.js +143 -0
  49. package/dist/commands/workflow/dataset/list.d.ts +12 -0
  50. package/dist/commands/workflow/dataset/list.js +87 -0
  51. package/dist/commands/workflow/debug.d.ts +16 -0
  52. package/dist/commands/workflow/debug.js +60 -0
  53. package/dist/commands/workflow/debug.spec.d.ts +1 -0
  54. package/dist/commands/workflow/debug.spec.js +34 -0
  55. package/dist/commands/workflow/generate.d.ts +17 -0
  56. package/dist/commands/workflow/generate.js +85 -0
  57. package/dist/commands/workflow/generate.spec.d.ts +1 -0
  58. package/dist/commands/workflow/generate.spec.js +115 -0
  59. package/dist/commands/workflow/list.d.ts +22 -0
  60. package/dist/commands/workflow/list.js +152 -0
  61. package/dist/commands/workflow/list.spec.d.ts +1 -0
  62. package/dist/commands/workflow/list.spec.js +99 -0
  63. package/dist/commands/workflow/plan.d.ts +12 -0
  64. package/dist/commands/workflow/plan.js +66 -0
  65. package/dist/commands/workflow/plan.spec.d.ts +1 -0
  66. package/dist/commands/workflow/plan.spec.js +341 -0
  67. package/dist/commands/workflow/reset.d.ts +14 -0
  68. package/dist/commands/workflow/reset.js +51 -0
  69. package/dist/commands/workflow/result.d.ts +13 -0
  70. package/dist/commands/workflow/result.js +46 -0
  71. package/dist/commands/workflow/result.spec.d.ts +1 -0
  72. package/dist/commands/workflow/result.spec.js +23 -0
  73. package/dist/commands/workflow/run.d.ts +16 -0
  74. package/dist/commands/workflow/run.js +97 -0
  75. package/dist/commands/workflow/run.spec.d.ts +1 -0
  76. package/dist/commands/workflow/run.spec.js +110 -0
  77. package/dist/commands/workflow/runs/list.d.ts +14 -0
  78. package/dist/commands/workflow/runs/list.js +104 -0
  79. package/dist/commands/workflow/start.d.ts +15 -0
  80. package/dist/commands/workflow/start.js +62 -0
  81. package/dist/commands/workflow/start.spec.d.ts +1 -0
  82. package/dist/commands/workflow/start.spec.js +28 -0
  83. package/dist/commands/workflow/status.d.ts +13 -0
  84. package/dist/commands/workflow/status.js +57 -0
  85. package/dist/commands/workflow/status.spec.d.ts +1 -0
  86. package/dist/commands/workflow/status.spec.js +33 -0
  87. package/dist/commands/workflow/stop.d.ts +10 -0
  88. package/dist/commands/workflow/stop.js +31 -0
  89. package/dist/commands/workflow/stop.spec.d.ts +1 -0
  90. package/dist/commands/workflow/stop.spec.js +17 -0
  91. package/dist/commands/workflow/terminate.d.ts +13 -0
  92. package/dist/commands/workflow/terminate.js +39 -0
  93. package/dist/commands/workflow/test_eval.d.ts +20 -0
  94. package/dist/commands/workflow/test_eval.js +151 -0
  95. package/dist/config.d.ts +47 -0
  96. package/dist/config.js +47 -0
  97. package/dist/generated/framework_version.json +3 -0
  98. package/dist/hooks/init.d.ts +3 -0
  99. package/dist/hooks/init.js +30 -0
  100. package/dist/hooks/init.spec.d.ts +1 -0
  101. package/dist/hooks/init.spec.js +54 -0
  102. package/dist/index.d.ts +1 -0
  103. package/dist/index.js +1 -0
  104. package/dist/index.spec.d.ts +1 -0
  105. package/dist/index.spec.js +6 -0
  106. package/dist/services/claude_client.d.ts +30 -0
  107. package/dist/services/claude_client.integration.test.d.ts +1 -0
  108. package/dist/services/claude_client.integration.test.js +43 -0
  109. package/dist/services/claude_client.js +215 -0
  110. package/dist/services/claude_client.spec.d.ts +1 -0
  111. package/dist/services/claude_client.spec.js +145 -0
  112. package/dist/services/coding_agents.d.ts +36 -0
  113. package/dist/services/coding_agents.js +236 -0
  114. package/dist/services/coding_agents.spec.d.ts +1 -0
  115. package/dist/services/coding_agents.spec.js +256 -0
  116. package/dist/services/copy_assets.spec.d.ts +1 -0
  117. package/dist/services/copy_assets.spec.js +22 -0
  118. package/dist/services/cost_calculator.d.ts +18 -0
  119. package/dist/services/cost_calculator.js +359 -0
  120. package/dist/services/cost_calculator.spec.d.ts +1 -0
  121. package/dist/services/cost_calculator.spec.js +540 -0
  122. package/dist/services/credentials_service.d.ts +12 -0
  123. package/dist/services/credentials_service.integration.test.d.ts +1 -0
  124. package/dist/services/credentials_service.integration.test.js +66 -0
  125. package/dist/services/credentials_service.js +64 -0
  126. package/dist/services/credentials_service.spec.d.ts +1 -0
  127. package/dist/services/credentials_service.spec.js +106 -0
  128. package/dist/services/datasets.d.ts +20 -0
  129. package/dist/services/datasets.js +132 -0
  130. package/dist/services/docker.d.ts +39 -0
  131. package/dist/services/docker.js +160 -0
  132. package/dist/services/docker.spec.d.ts +1 -0
  133. package/dist/services/docker.spec.js +124 -0
  134. package/dist/services/env_configurator.d.ts +15 -0
  135. package/dist/services/env_configurator.js +163 -0
  136. package/dist/services/env_configurator.spec.d.ts +1 -0
  137. package/dist/services/env_configurator.spec.js +192 -0
  138. package/dist/services/generate_plan_name@v1.prompt +24 -0
  139. package/dist/services/messages.d.ts +9 -0
  140. package/dist/services/messages.js +338 -0
  141. package/dist/services/messages.spec.d.ts +1 -0
  142. package/dist/services/messages.spec.js +55 -0
  143. package/dist/services/npm_update_service.d.ts +6 -0
  144. package/dist/services/npm_update_service.js +87 -0
  145. package/dist/services/npm_update_service.spec.d.ts +1 -0
  146. package/dist/services/npm_update_service.spec.js +104 -0
  147. package/dist/services/project_scaffold.d.ts +31 -0
  148. package/dist/services/project_scaffold.js +212 -0
  149. package/dist/services/project_scaffold.spec.d.ts +1 -0
  150. package/dist/services/project_scaffold.spec.js +122 -0
  151. package/dist/services/s3_trace_downloader.d.ts +12 -0
  152. package/dist/services/s3_trace_downloader.js +57 -0
  153. package/dist/services/template_processor.d.ts +14 -0
  154. package/dist/services/template_processor.js +57 -0
  155. package/dist/services/trace_reader.d.ts +16 -0
  156. package/dist/services/trace_reader.js +57 -0
  157. package/dist/services/trace_reader.spec.d.ts +1 -0
  158. package/dist/services/trace_reader.spec.js +78 -0
  159. package/dist/services/version_check.d.ts +6 -0
  160. package/dist/services/version_check.js +52 -0
  161. package/dist/services/version_check.spec.d.ts +1 -0
  162. package/dist/services/version_check.spec.js +106 -0
  163. package/dist/services/workflow_builder.d.ts +16 -0
  164. package/dist/services/workflow_builder.js +86 -0
  165. package/dist/services/workflow_builder.spec.d.ts +1 -0
  166. package/dist/services/workflow_builder.spec.js +165 -0
  167. package/dist/services/workflow_generator.d.ts +5 -0
  168. package/dist/services/workflow_generator.js +40 -0
  169. package/dist/services/workflow_generator.spec.d.ts +1 -0
  170. package/dist/services/workflow_generator.spec.js +77 -0
  171. package/dist/services/workflow_planner.d.ts +15 -0
  172. package/dist/services/workflow_planner.js +48 -0
  173. package/dist/services/workflow_planner.spec.d.ts +1 -0
  174. package/dist/services/workflow_planner.spec.js +122 -0
  175. package/dist/services/workflow_runs.d.ts +14 -0
  176. package/dist/services/workflow_runs.js +25 -0
  177. package/dist/templates/agent_instructions/CLAUDE.md.template +19 -0
  178. package/dist/templates/agent_instructions/dotclaude/settings.json.template +29 -0
  179. package/dist/templates/project/.env.example.template +9 -0
  180. package/dist/templates/project/.gitignore.template +35 -0
  181. package/dist/templates/project/README.md.template +100 -0
  182. package/dist/templates/project/config/costs.yml.template +29 -0
  183. package/dist/templates/project/package.json.template +25 -0
  184. package/dist/templates/project/src/clients/jina.ts.template +30 -0
  185. package/dist/templates/project/src/shared/utils/string.ts.template +3 -0
  186. package/dist/templates/project/src/shared/utils/url.ts.template +15 -0
  187. package/dist/templates/project/src/workflows/blog_evaluator/evaluators.ts.template +23 -0
  188. package/dist/templates/project/src/workflows/blog_evaluator/prompts/signal_noise@v1.prompt.template +26 -0
  189. package/dist/templates/project/src/workflows/blog_evaluator/scenarios/paulgraham_hwh.json.template +3 -0
  190. package/dist/templates/project/src/workflows/blog_evaluator/steps.ts.template +27 -0
  191. package/dist/templates/project/src/workflows/blog_evaluator/types.ts.template +30 -0
  192. package/dist/templates/project/src/workflows/blog_evaluator/utils.ts.template +15 -0
  193. package/dist/templates/project/src/workflows/blog_evaluator/workflow.ts.template +27 -0
  194. package/dist/templates/project/tsconfig.json.template +20 -0
  195. package/dist/templates/workflow/README.md.template +216 -0
  196. package/dist/templates/workflow/evaluators.ts.template +21 -0
  197. package/dist/templates/workflow/prompts/example@v1.prompt.template +15 -0
  198. package/dist/templates/workflow/scenarios/test_input.json.template +3 -0
  199. package/dist/templates/workflow/steps.ts.template +20 -0
  200. package/dist/templates/workflow/types.ts.template +13 -0
  201. package/dist/templates/workflow/workflow.ts.template +23 -0
  202. package/dist/test_helpers/mocks.d.ts +38 -0
  203. package/dist/test_helpers/mocks.js +77 -0
  204. package/dist/types/cost.d.ts +149 -0
  205. package/dist/types/cost.js +6 -0
  206. package/dist/types/domain.d.ts +20 -0
  207. package/dist/types/domain.js +4 -0
  208. package/dist/types/errors.d.ts +68 -0
  209. package/dist/types/errors.js +100 -0
  210. package/dist/types/errors.spec.d.ts +1 -0
  211. package/dist/types/errors.spec.js +18 -0
  212. package/dist/types/generator.d.ts +26 -0
  213. package/dist/types/generator.js +1 -0
  214. package/dist/types/trace.d.ts +161 -0
  215. package/dist/types/trace.js +18 -0
  216. package/dist/utils/claude.d.ts +5 -0
  217. package/dist/utils/claude.js +19 -0
  218. package/dist/utils/claude.spec.d.ts +1 -0
  219. package/dist/utils/claude.spec.js +119 -0
  220. package/dist/utils/constants.d.ts +5 -0
  221. package/dist/utils/constants.js +4 -0
  222. package/dist/utils/cost_formatter.d.ts +5 -0
  223. package/dist/utils/cost_formatter.js +218 -0
  224. package/dist/utils/date_formatter.d.ts +23 -0
  225. package/dist/utils/date_formatter.js +49 -0
  226. package/dist/utils/env_loader.d.ts +1 -0
  227. package/dist/utils/env_loader.js +22 -0
  228. package/dist/utils/env_loader.spec.d.ts +1 -0
  229. package/dist/utils/env_loader.spec.js +43 -0
  230. package/dist/utils/error_handler.d.ts +8 -0
  231. package/dist/utils/error_handler.js +71 -0
  232. package/dist/utils/error_utils.d.ts +24 -0
  233. package/dist/utils/error_utils.js +87 -0
  234. package/dist/utils/file_system.d.ts +3 -0
  235. package/dist/utils/file_system.js +33 -0
  236. package/dist/utils/format_workflow_result.d.ts +5 -0
  237. package/dist/utils/format_workflow_result.js +18 -0
  238. package/dist/utils/format_workflow_result.spec.d.ts +1 -0
  239. package/dist/utils/format_workflow_result.spec.js +81 -0
  240. package/dist/utils/framework_version.d.ts +4 -0
  241. package/dist/utils/framework_version.js +4 -0
  242. package/dist/utils/framework_version.spec.d.ts +1 -0
  243. package/dist/utils/framework_version.spec.js +13 -0
  244. package/dist/utils/header_utils.d.ts +12 -0
  245. package/dist/utils/header_utils.js +29 -0
  246. package/dist/utils/header_utils.spec.d.ts +1 -0
  247. package/dist/utils/header_utils.spec.js +52 -0
  248. package/dist/utils/input_parser.d.ts +1 -0
  249. package/dist/utils/input_parser.js +19 -0
  250. package/dist/utils/output_formatter.d.ts +2 -0
  251. package/dist/utils/output_formatter.js +11 -0
  252. package/dist/utils/paths.d.ts +25 -0
  253. package/dist/utils/paths.js +36 -0
  254. package/dist/utils/process.d.ts +4 -0
  255. package/dist/utils/process.js +50 -0
  256. package/dist/utils/resolve_input.d.ts +1 -0
  257. package/dist/utils/resolve_input.js +22 -0
  258. package/dist/utils/scenario_resolver.d.ts +9 -0
  259. package/dist/utils/scenario_resolver.js +93 -0
  260. package/dist/utils/scenario_resolver.spec.d.ts +1 -0
  261. package/dist/utils/scenario_resolver.spec.js +214 -0
  262. package/dist/utils/secret_sanitizer.d.ts +1 -0
  263. package/dist/utils/secret_sanitizer.js +29 -0
  264. package/dist/utils/sleep.d.ts +5 -0
  265. package/dist/utils/sleep.js +5 -0
  266. package/dist/utils/template.d.ts +9 -0
  267. package/dist/utils/template.js +30 -0
  268. package/dist/utils/template.spec.d.ts +1 -0
  269. package/dist/utils/template.spec.js +77 -0
  270. package/dist/utils/trace_extractor.d.ts +27 -0
  271. package/dist/utils/trace_extractor.js +53 -0
  272. package/dist/utils/trace_formatter.d.ts +11 -0
  273. package/dist/utils/trace_formatter.js +402 -0
  274. package/dist/utils/validation.d.ts +13 -0
  275. package/dist/utils/validation.js +25 -0
  276. package/dist/utils/validation.spec.d.ts +1 -0
  277. package/dist/utils/validation.spec.js +140 -0
  278. package/package.json +4 -4
@@ -0,0 +1,540 @@
1
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
2
+ const mockReadFileSync = vi.fn();
3
+ const mockExistsSync = vi.fn();
4
+ vi.mock('node:fs', () => ({
5
+ readFileSync: (...args) => mockReadFileSync(...args),
6
+ existsSync: (...args) => mockExistsSync(...args)
7
+ }));
8
+ const mockLoad = vi.fn();
9
+ vi.mock('js-yaml', () => ({
10
+ default: { load: (...args) => mockLoad(...args) }
11
+ }));
12
+ import { extractValue, findLLMCalls, findHTTPCalls, calculateLLMCallCost, identifyService, calculateServiceCost, calculateCost, loadPricingConfig } from '#services/cost_calculator.js';
13
+ const llmTrace = {
14
+ id: 'test-trace-1',
15
+ kind: 'workflow',
16
+ name: 'test_workflow',
17
+ startedAt: 1700000000000,
18
+ endedAt: 1700000100000,
19
+ children: [
20
+ {
21
+ id: 'step-1',
22
+ kind: 'step',
23
+ name: 'test_workflow#generate_summary',
24
+ children: [
25
+ {
26
+ id: 'llm-1',
27
+ kind: 'llm',
28
+ name: 'generate_summary',
29
+ input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
30
+ output: { usage: { inputTokens: 1000, outputTokens: 500 } }
31
+ }
32
+ ]
33
+ },
34
+ {
35
+ id: 'step-2',
36
+ kind: 'step',
37
+ name: 'test_workflow#analyze_data',
38
+ children: [
39
+ {
40
+ id: 'llm-2',
41
+ kind: 'llm',
42
+ name: 'analyze_data',
43
+ input: { loadedPrompt: { config: { model: 'claude-haiku-4-5' } } },
44
+ output: { usage: { inputTokens: 2000, outputTokens: 1000, cachedInputTokens: 500 } }
45
+ }
46
+ ]
47
+ }
48
+ ]
49
+ };
50
+ const httpTrace = {
51
+ id: 'test-trace-2',
52
+ kind: 'workflow',
53
+ name: 'test_workflow',
54
+ startedAt: 1700000000000,
55
+ endedAt: 1700000100000,
56
+ children: [
57
+ {
58
+ id: 'step-1',
59
+ kind: 'step',
60
+ name: 'test_workflow#fetch_content',
61
+ children: [
62
+ {
63
+ id: 'http-1',
64
+ kind: 'http',
65
+ name: 'jina_request',
66
+ input: { url: 'https://r.jina.ai/https://example.com', method: 'GET' },
67
+ output: { status: 200, body: { data: { usage: { tokens: 5000 } } } }
68
+ }
69
+ ]
70
+ },
71
+ {
72
+ id: 'step-2',
73
+ kind: 'step',
74
+ name: 'test_workflow#search',
75
+ children: [
76
+ {
77
+ id: 'http-2',
78
+ kind: 'http',
79
+ name: 'exa_request',
80
+ input: { url: 'https://api.exa.ai/research', method: 'GET' },
81
+ output: {
82
+ status: 200,
83
+ body: {
84
+ model: 'exa-research',
85
+ costDollars: { total: 0.15, numSearches: 1, numPages: 5 }
86
+ }
87
+ }
88
+ }
89
+ ]
90
+ }
91
+ ]
92
+ };
93
+ const duplicateTrace = {
94
+ id: 'test-trace-3',
95
+ kind: 'workflow',
96
+ name: 'test_workflow',
97
+ startedAt: 1700000000000,
98
+ endedAt: 1700000100000,
99
+ children: [
100
+ {
101
+ id: 'step-1',
102
+ kind: 'step',
103
+ name: 'test_workflow#step_one',
104
+ children: [
105
+ {
106
+ id: 'llm-same-id',
107
+ kind: 'llm',
108
+ name: 'generate',
109
+ input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
110
+ output: { usage: { inputTokens: 1000, outputTokens: 500 } }
111
+ }
112
+ ]
113
+ },
114
+ {
115
+ id: 'child-workflow',
116
+ kind: 'workflow',
117
+ name: 'child_workflow',
118
+ children: [
119
+ {
120
+ id: 'llm-same-id',
121
+ kind: 'llm',
122
+ name: 'generate',
123
+ input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
124
+ output: { usage: { inputTokens: 1000, outputTokens: 500 } }
125
+ }
126
+ ]
127
+ }
128
+ ]
129
+ };
130
+ const testConfig = {
131
+ models: {
132
+ 'claude-sonnet-4-5': { provider: 'anthropic', input: 3.0, output: 15.0, cached_input: 0.30 },
133
+ 'claude-haiku-4-5': { provider: 'anthropic', input: 1.0, output: 5.0, cached_input: 0.10 }
134
+ },
135
+ services: {
136
+ jina: {
137
+ type: 'token',
138
+ url_pattern: 'r.jina.ai',
139
+ usage_path: 'body.data.usage.tokens',
140
+ per_million: 0.045
141
+ },
142
+ exa: {
143
+ type: 'response_cost',
144
+ url_pattern: 'api.exa.ai',
145
+ cost_path: 'output.body.costDollars.total',
146
+ billable_method: 'POST'
147
+ }
148
+ }
149
+ };
150
+ describe('extractValue', () => {
151
+ it('extracts nested values with dot notation', () => {
152
+ const obj = { a: { b: { c: 42 } } };
153
+ expect(extractValue(obj, 'a.b.c')).toBe(42);
154
+ });
155
+ it('extracts array values with bracket notation', () => {
156
+ const obj = { items: [{ name: 'first' }, { name: 'second' }] };
157
+ expect(extractValue(obj, 'items[0].name')).toBe('first');
158
+ expect(extractValue(obj, 'items[1].name')).toBe('second');
159
+ });
160
+ it('returns undefined for missing paths', () => {
161
+ const obj = { a: 1 };
162
+ expect(extractValue(obj, 'b.c.d')).toBeUndefined();
163
+ });
164
+ it('handles null/undefined input', () => {
165
+ expect(extractValue(null, 'a.b')).toBeNull();
166
+ expect(extractValue(undefined, 'a.b')).toBeUndefined();
167
+ });
168
+ });
169
+ describe('findLLMCalls', () => {
170
+ it('finds nested LLM calls', () => {
171
+ const calls = findLLMCalls(llmTrace);
172
+ expect(calls).toHaveLength(2);
173
+ });
174
+ it('extracts step names correctly', () => {
175
+ const calls = findLLMCalls(llmTrace);
176
+ expect(calls[0].stepName).toBe('generate_summary');
177
+ expect(calls[1].stepName).toBe('analyze_data');
178
+ });
179
+ it('extracts model names from loadedPrompt config', () => {
180
+ const calls = findLLMCalls(llmTrace);
181
+ expect(calls[0].model).toBe('claude-sonnet-4-5');
182
+ expect(calls[1].model).toBe('claude-haiku-4-5');
183
+ });
184
+ it('extracts token usage', () => {
185
+ const calls = findLLMCalls(llmTrace);
186
+ expect(calls[0].usage.inputTokens).toBe(1000);
187
+ expect(calls[0].usage.outputTokens).toBe(500);
188
+ expect(calls[1].usage.cachedInputTokens).toBe(500);
189
+ });
190
+ it('deduplicates by ID', () => {
191
+ const calls = findLLMCalls(duplicateTrace);
192
+ expect(calls).toHaveLength(1);
193
+ });
194
+ });
195
+ describe('findHTTPCalls', () => {
196
+ it('finds nested HTTP calls', () => {
197
+ const calls = findHTTPCalls(httpTrace);
198
+ expect(calls).toHaveLength(2);
199
+ });
200
+ it('extracts URLs and methods', () => {
201
+ const calls = findHTTPCalls(httpTrace);
202
+ expect(calls[0].url).toContain('r.jina.ai');
203
+ expect(calls[0].method).toBe('GET');
204
+ });
205
+ it('extracts step names', () => {
206
+ const calls = findHTTPCalls(httpTrace);
207
+ expect(calls[0].stepName).toBe('fetch_content');
208
+ expect(calls[1].stepName).toBe('search');
209
+ });
210
+ });
211
+ describe('calculateLLMCallCost', () => {
212
+ it('calculates cost for known model', () => {
213
+ const usage = { inputTokens: 1000000, outputTokens: 500000 };
214
+ const modelPricing = { provider: 'anthropic', input: 3.0, output: 15.0 };
215
+ const { cost } = calculateLLMCallCost(usage, modelPricing);
216
+ // 1M input * $3/M + 0.5M output * $15/M = $3 + $7.5 = $10.5
217
+ expect(cost).toBeCloseTo(10.5, 2);
218
+ });
219
+ it('includes cached input tokens at reduced rate', () => {
220
+ const usage = { inputTokens: 1000000, outputTokens: 0, cachedInputTokens: 1000000 };
221
+ const modelPricing = { provider: 'anthropic', input: 3.0, output: 15.0, cached_input: 0.3 };
222
+ const { cost } = calculateLLMCallCost(usage, modelPricing);
223
+ // 1M input * $3/M + 1M cached * $0.3/M = $3 + $0.3 = $3.3
224
+ expect(cost).toBeCloseTo(3.3, 2);
225
+ });
226
+ it('returns zero with warning for unknown model', () => {
227
+ const usage = { inputTokens: 1000, outputTokens: 500 };
228
+ const { cost, warning } = calculateLLMCallCost(usage, undefined);
229
+ expect(cost).toBe(0);
230
+ expect(warning).toBe('unknown model');
231
+ });
232
+ });
233
+ describe('identifyService', () => {
234
+ it('identifies Jina by URL pattern', () => {
235
+ const call = {
236
+ stepName: 'test',
237
+ url: 'https://r.jina.ai/https://example.com',
238
+ method: 'GET',
239
+ input: {},
240
+ output: {}
241
+ };
242
+ const result = identifyService(call, testConfig.services);
243
+ expect(result?.serviceName).toBe('jina');
244
+ });
245
+ it('identifies Exa by URL pattern', () => {
246
+ const call = {
247
+ stepName: 'test',
248
+ url: 'https://api.exa.ai/research',
249
+ method: 'GET',
250
+ input: {},
251
+ output: {}
252
+ };
253
+ const result = identifyService(call, testConfig.services);
254
+ expect(result?.serviceName).toBe('exa');
255
+ });
256
+ it('returns null for unknown URLs', () => {
257
+ const call = {
258
+ stepName: 'test',
259
+ url: 'https://unknown-api.com/endpoint',
260
+ method: 'GET',
261
+ input: {},
262
+ output: {}
263
+ };
264
+ const result = identifyService(call, testConfig.services);
265
+ expect(result).toBeNull();
266
+ });
267
+ });
268
+ describe('calculateServiceCost', () => {
269
+ it('calculates Jina token-based cost', () => {
270
+ const call = {
271
+ stepName: 'test',
272
+ url: 'https://r.jina.ai/https://example.com',
273
+ method: 'GET',
274
+ input: {},
275
+ output: {
276
+ body: { data: { usage: { tokens: 1000000 } } }
277
+ }
278
+ };
279
+ const serviceInfo = identifyService(call, testConfig.services);
280
+ const result = calculateServiceCost(call, serviceInfo);
281
+ // 1M tokens * $0.045/M = $0.045
282
+ expect(result.cost).toBeCloseTo(0.045, 4);
283
+ });
284
+ it('extracts Exa cost from response body', () => {
285
+ const call = {
286
+ stepName: 'test',
287
+ url: 'https://api.exa.ai/research',
288
+ method: 'GET',
289
+ input: {},
290
+ output: {
291
+ body: {
292
+ model: 'exa-research',
293
+ costDollars: { total: 0.15, numSearches: 1, numPages: 5 }
294
+ }
295
+ }
296
+ };
297
+ const serviceInfo = identifyService(call, testConfig.services);
298
+ const result = calculateServiceCost(call, serviceInfo);
299
+ expect(result.cost).toBe(0.15);
300
+ expect(result.usage).toContain('1 searches');
301
+ expect(result.usage).toContain('5 pages');
302
+ });
303
+ it('returns zero cost for Exa response without costDollars', () => {
304
+ const call = {
305
+ stepName: 'test',
306
+ url: 'https://api.exa.ai/research',
307
+ method: 'GET',
308
+ input: {},
309
+ output: {
310
+ body: { status: 'pending' }
311
+ }
312
+ };
313
+ const serviceInfo = identifyService(call, testConfig.services);
314
+ const result = calculateServiceCost(call, serviceInfo);
315
+ expect(result.cost).toBe(0);
316
+ expect(result.warning).toBe('no cost data');
317
+ });
318
+ });
319
+ describe('response_cost filtering in calculateCost', () => {
320
+ it('skips Exa polling requests without cost data', () => {
321
+ const trace = {
322
+ kind: 'workflow',
323
+ name: 'test_workflow',
324
+ startedAt: 1700000000000,
325
+ endedAt: 1700000100000,
326
+ children: [
327
+ {
328
+ id: 'step-exa',
329
+ kind: 'step',
330
+ name: 'test_workflow#search',
331
+ children: [
332
+ {
333
+ id: 'http-exa-poll',
334
+ kind: 'http',
335
+ name: 'exa_poll',
336
+ input: { url: 'https://api.exa.ai/research/task-123', method: 'GET' },
337
+ output: { status: 200, body: { status: 'in_progress' } }
338
+ }
339
+ ]
340
+ }
341
+ ]
342
+ };
343
+ const report = calculateCost(trace, testConfig, 'test.json');
344
+ expect(report.serviceTotalCost).toBe(0);
345
+ expect(report.services).toHaveLength(0);
346
+ });
347
+ it('counts Exa responses that have costDollars', () => {
348
+ const trace = {
349
+ kind: 'workflow',
350
+ name: 'test_workflow',
351
+ startedAt: 1700000000000,
352
+ endedAt: 1700000100000,
353
+ children: [
354
+ {
355
+ id: 'step-exa',
356
+ kind: 'step',
357
+ name: 'test_workflow#search',
358
+ children: [
359
+ {
360
+ id: 'http-exa-poll',
361
+ kind: 'http',
362
+ name: 'exa_poll',
363
+ input: { url: 'https://api.exa.ai/research/task-123', method: 'GET' },
364
+ output: { status: 200, body: { status: 'in_progress' } }
365
+ },
366
+ {
367
+ id: 'http-exa-result',
368
+ kind: 'http',
369
+ name: 'exa_result',
370
+ input: { url: 'https://api.exa.ai/research/task-123', method: 'GET' },
371
+ output: {
372
+ status: 200,
373
+ body: {
374
+ model: 'exa-research',
375
+ costDollars: { total: 0.08, numSearches: 1, numPages: 3 }
376
+ }
377
+ }
378
+ }
379
+ ]
380
+ }
381
+ ]
382
+ };
383
+ const report = calculateCost(trace, testConfig, 'test.json');
384
+ expect(report.services).toHaveLength(1);
385
+ expect(report.services[0].calls).toHaveLength(1);
386
+ expect(report.services[0].totalCost).toBeCloseTo(0.08, 4);
387
+ });
388
+ });
389
+ describe('calculateCost', () => {
390
+ it('calculates total cost for LLM trace', () => {
391
+ const report = calculateCost(llmTrace, testConfig, 'test.json');
392
+ expect(report.llmCalls).toHaveLength(2);
393
+ expect(report.workflowName).toBe('test_workflow');
394
+ expect(report.llmTotalCost).toBeGreaterThan(0);
395
+ expect(report.totalCost).toBe(report.llmTotalCost + report.serviceTotalCost);
396
+ });
397
+ it('calculates total cost for HTTP trace', () => {
398
+ const report = calculateCost(httpTrace, testConfig, 'test.json');
399
+ expect(report.services.length).toBeGreaterThan(0);
400
+ expect(report.serviceTotalCost).toBeGreaterThan(0);
401
+ });
402
+ it('calculates duration from timestamps', () => {
403
+ const report = calculateCost(llmTrace, testConfig, 'test.json');
404
+ // 1700000100000 - 1700000000000 = 100000ms
405
+ expect(report.durationMs).toBe(100000);
406
+ });
407
+ it('handles deduplication in cost calculation', () => {
408
+ const report = calculateCost(duplicateTrace, testConfig, 'test.json');
409
+ expect(report.llmCalls).toHaveLength(1);
410
+ });
411
+ it('matches versioned model names by prefix', () => {
412
+ const trace = {
413
+ kind: 'workflow',
414
+ name: 'test',
415
+ children: [{
416
+ id: 'step-1',
417
+ kind: 'step',
418
+ name: 'test#gen',
419
+ children: [{
420
+ id: 'llm-1',
421
+ kind: 'llm',
422
+ name: 'gen',
423
+ input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5-20250514' } } },
424
+ output: { usage: { inputTokens: 1000, outputTokens: 500 } }
425
+ }]
426
+ }]
427
+ };
428
+ const report = calculateCost(trace, testConfig, 'test.json');
429
+ expect(report.llmTotalCost).toBeGreaterThan(0);
430
+ expect(report.unknownModels).toHaveLength(0);
431
+ expect(report.llmCalls[0].warning).toBe('priced as claude-sonnet-4-5');
432
+ });
433
+ it('reports unknown model when no prefix match exists', () => {
434
+ const trace = {
435
+ kind: 'workflow',
436
+ name: 'test',
437
+ children: [{
438
+ id: 'step-1',
439
+ kind: 'step',
440
+ name: 'test#gen',
441
+ children: [{
442
+ id: 'llm-1',
443
+ kind: 'llm',
444
+ name: 'gen',
445
+ input: { loadedPrompt: { config: { model: 'totally-unknown-model' } } },
446
+ output: { usage: { inputTokens: 1000, outputTokens: 500 } }
447
+ }]
448
+ }]
449
+ };
450
+ const report = calculateCost(trace, testConfig, 'test.json');
451
+ expect(report.llmTotalCost).toBe(0);
452
+ expect(report.unknownModels).toContain('totally-unknown-model');
453
+ expect(report.llmCalls[0].warning).toBe('unknown model');
454
+ });
455
+ it('prefers exact model match over prefix', () => {
456
+ const trace = {
457
+ kind: 'workflow',
458
+ name: 'test',
459
+ children: [{
460
+ id: 'step-1',
461
+ kind: 'step',
462
+ name: 'test#gen',
463
+ children: [{
464
+ id: 'llm-1',
465
+ kind: 'llm',
466
+ name: 'gen',
467
+ input: { loadedPrompt: { config: { model: 'claude-sonnet-4-5' } } },
468
+ output: { usage: { inputTokens: 1000, outputTokens: 500 } }
469
+ }]
470
+ }]
471
+ };
472
+ const report = calculateCost(trace, testConfig, 'test.json');
473
+ expect(report.llmCalls[0].warning).toBeUndefined();
474
+ });
475
+ });
476
+ describe('loadPricingConfig', () => {
477
+ beforeEach(() => {
478
+ mockReadFileSync.mockReset();
479
+ mockLoad.mockReset();
480
+ mockExistsSync.mockReset();
481
+ mockExistsSync.mockReturnValue(false);
482
+ });
483
+ it('loads bundled config when no project config exists', () => {
484
+ const yamlContent = 'models: {}';
485
+ const parsed = { models: {}, services: {} };
486
+ mockReadFileSync.mockReturnValue(yamlContent);
487
+ mockLoad.mockReturnValue(parsed);
488
+ const result = loadPricingConfig();
489
+ expect(mockReadFileSync).toHaveBeenCalledTimes(1);
490
+ expect(mockLoad).toHaveBeenCalledWith(yamlContent);
491
+ expect(result).toEqual(parsed);
492
+ });
493
+ it('loads config from custom path without merging', () => {
494
+ const yamlContent = 'models: {}';
495
+ const parsed = { models: {}, services: {} };
496
+ mockReadFileSync.mockReturnValue(yamlContent);
497
+ mockLoad.mockReturnValue(parsed);
498
+ loadPricingConfig('/custom/path/costs.yml');
499
+ expect(mockReadFileSync).toHaveBeenCalledWith('/custom/path/costs.yml', 'utf-8');
500
+ expect(mockExistsSync).not.toHaveBeenCalled();
501
+ });
502
+ it('merges project config over bundled config', () => {
503
+ const bundled = {
504
+ models: { 'claude-sonnet-4-5': { provider: 'anthropic', input: 3.0, output: 15.0 } },
505
+ services: { jina: { type: 'token', url_pattern: 'r.jina.ai', per_million: 0.045 } }
506
+ };
507
+ const project = {
508
+ models: { 'custom-model': { provider: 'custom', input: 1.0, output: 2.0 } },
509
+ services: {}
510
+ };
511
+ mockExistsSync.mockReturnValue(true);
512
+ mockReadFileSync.mockReturnValue('yaml');
513
+ mockLoad
514
+ .mockReturnValueOnce(bundled)
515
+ .mockReturnValueOnce(project);
516
+ const result = loadPricingConfig();
517
+ expect(mockReadFileSync).toHaveBeenCalledTimes(2);
518
+ expect(result.models).toHaveProperty('claude-sonnet-4-5');
519
+ expect(result.models).toHaveProperty('custom-model');
520
+ expect(result.services).toHaveProperty('jina');
521
+ });
522
+ it('project config overrides bundled model prices', () => {
523
+ const bundled = {
524
+ models: { 'claude-sonnet-4-5': { provider: 'anthropic', input: 3.0, output: 15.0 } },
525
+ services: {}
526
+ };
527
+ const project = {
528
+ models: { 'claude-sonnet-4-5': { provider: 'anthropic', input: 2.0, output: 10.0 } },
529
+ services: {}
530
+ };
531
+ mockExistsSync.mockReturnValue(true);
532
+ mockReadFileSync.mockReturnValue('yaml');
533
+ mockLoad
534
+ .mockReturnValueOnce(bundled)
535
+ .mockReturnValueOnce(project);
536
+ const result = loadPricingConfig();
537
+ expect(result.models['claude-sonnet-4-5'].input).toBe(2.0);
538
+ expect(result.models['claude-sonnet-4-5'].output).toBe(10.0);
539
+ });
540
+ });
@@ -0,0 +1,12 @@
1
+ export type CredentialsEnvironment = string | undefined;
2
+ export type WorkflowTarget = string | undefined;
3
+ export declare const resolveCredentialsPath: (environment: CredentialsEnvironment, workflow?: WorkflowTarget) => string;
4
+ export declare const resolveKeyPath: (environment: CredentialsEnvironment, workflow?: WorkflowTarget) => string;
5
+ export declare const resolveKey: (environment: CredentialsEnvironment, workflow?: WorkflowTarget) => string;
6
+ export declare const credentialsExist: (environment: CredentialsEnvironment, workflow?: WorkflowTarget) => boolean;
7
+ export declare const decryptCredentials: (environment: CredentialsEnvironment, workflow?: WorkflowTarget) => string;
8
+ export declare const writeEncrypted: (environment: CredentialsEnvironment, plaintext: string, workflow?: WorkflowTarget) => void;
9
+ export declare const initCredentials: (environment: CredentialsEnvironment, workflow?: WorkflowTarget) => {
10
+ keyPath: string;
11
+ credPath: string;
12
+ };
@@ -0,0 +1,66 @@
1
+ import { describe, it, expect, afterAll } from 'vitest';
2
+ import fs from 'node:fs';
3
+ import path from 'node:path';
4
+ import os from 'node:os';
5
+ import { load as parseYaml } from 'js-yaml';
6
+ import { encrypt, decrypt, generateKey } from '@outputai/credentials';
7
+ describe('credentials service integration', () => {
8
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'output-creds-'));
9
+ afterAll(() => {
10
+ fs.rmSync(tmpDir, { recursive: true, force: true });
11
+ });
12
+ describe('encryption round-trip', () => {
13
+ it('should generate a valid hex key', () => {
14
+ const key = generateKey();
15
+ expect(key).toMatch(/^[0-9a-f]{64}$/);
16
+ });
17
+ it('should encrypt and decrypt plaintext', () => {
18
+ const key = generateKey();
19
+ const plaintext = 'anthropic:\n api_key: sk-test-123\n';
20
+ const ciphertext = encrypt(plaintext, key);
21
+ const decrypted = decrypt(ciphertext, key);
22
+ expect(decrypted).toBe(plaintext);
23
+ });
24
+ it('should produce different ciphertext for the same plaintext (nonce)', () => {
25
+ const key = generateKey();
26
+ const plaintext = 'test: value\n';
27
+ const a = encrypt(plaintext, key);
28
+ const b = encrypt(plaintext, key);
29
+ expect(a).not.toBe(b);
30
+ expect(decrypt(a, key)).toBe(plaintext);
31
+ expect(decrypt(b, key)).toBe(plaintext);
32
+ });
33
+ });
34
+ describe('init, decrypt, and re-encrypt workflow', () => {
35
+ const key = generateKey();
36
+ const configDir = path.join(tmpDir, 'config');
37
+ const keyPath = path.join(configDir, 'credentials.key');
38
+ const credPath = path.join(configDir, 'credentials.yml.enc');
39
+ const template = 'anthropic:\n api_key: ""\nopenai:\n api_key: ""\n';
40
+ // Set up files once for all tests in this block
41
+ fs.mkdirSync(configDir, { recursive: true });
42
+ fs.writeFileSync(keyPath, key, { mode: 0o600 });
43
+ fs.writeFileSync(credPath, encrypt(template, key), 'utf8');
44
+ it('should create key and encrypted credentials on disk', () => {
45
+ expect(fs.existsSync(keyPath)).toBe(true);
46
+ expect(fs.existsSync(credPath)).toBe(true);
47
+ });
48
+ it('should decrypt to valid YAML', () => {
49
+ const ciphertext = fs.readFileSync(credPath, 'utf8').trim();
50
+ const plaintext = decrypt(ciphertext, key);
51
+ const parsed = parseYaml(plaintext);
52
+ expect(parsed).toHaveProperty('anthropic');
53
+ expect(parsed).toHaveProperty('openai');
54
+ });
55
+ it('should re-encrypt updated credentials and round-trip', () => {
56
+ const updated = 'anthropic:\n api_key: sk-new-key-456\nopenai:\n api_key: sk-openai-789\n';
57
+ fs.writeFileSync(credPath, encrypt(updated, key), 'utf8');
58
+ const ciphertext = fs.readFileSync(credPath, 'utf8').trim();
59
+ const decrypted = decrypt(ciphertext, key);
60
+ expect(decrypted).toBe(updated);
61
+ const parsed = parseYaml(decrypted);
62
+ expect(parsed.anthropic.api_key).toBe('sk-new-key-456');
63
+ expect(parsed.openai.api_key).toBe('sk-openai-789');
64
+ });
65
+ });
66
+ });
@@ -0,0 +1,64 @@
1
+ import fs from 'node:fs';
2
+ import path from 'node:path';
3
+ import { dump as stringifyYaml } from 'js-yaml';
4
+ import { encrypt, decrypt, generateKey, resolveCredentialsPath as resolveCredPath, resolveKeyPath as resolveKPath, resolveKeyEnvVar, resolveWorkflowCredentialsPath, resolveWorkflowKeyPath, resolveWorkflowKeyEnvVar } from '@outputai/credentials';
5
+ const WORKFLOWS_DIR = path.join('src', 'workflows');
6
+ const resolveWorkflowDir = (workflow) => path.resolve(process.cwd(), WORKFLOWS_DIR, workflow);
7
+ export const resolveCredentialsPath = (environment, workflow) => workflow ?
8
+ resolveWorkflowCredentialsPath(resolveWorkflowDir(workflow)) :
9
+ resolveCredPath(process.cwd(), environment);
10
+ export const resolveKeyPath = (environment, workflow) => workflow ?
11
+ resolveWorkflowKeyPath(resolveWorkflowDir(workflow)) :
12
+ resolveKPath(process.cwd(), environment);
13
+ export const resolveKey = (environment, workflow) => {
14
+ if (workflow) {
15
+ const wfEnvVar = resolveWorkflowKeyEnvVar(workflow);
16
+ if (process.env[wfEnvVar]) {
17
+ return process.env[wfEnvVar];
18
+ }
19
+ const wfKeyPath = resolveKeyPath(undefined, workflow);
20
+ if (fs.existsSync(wfKeyPath)) {
21
+ return fs.readFileSync(wfKeyPath, 'utf8').trim();
22
+ }
23
+ return resolveKey(undefined);
24
+ }
25
+ const envVar = resolveKeyEnvVar(environment);
26
+ if (process.env[envVar]) {
27
+ return process.env[envVar];
28
+ }
29
+ const keyPath = resolveKeyPath(environment);
30
+ if (fs.existsSync(keyPath)) {
31
+ return fs.readFileSync(keyPath, 'utf8').trim();
32
+ }
33
+ throw new Error(`No key found. Set ${envVar} env var or create ${keyPath}.`);
34
+ };
35
+ export const credentialsExist = (environment, workflow) => fs.existsSync(resolveCredentialsPath(environment, workflow));
36
+ export const decryptCredentials = (environment, workflow) => {
37
+ const key = resolveKey(environment, workflow);
38
+ const credPath = resolveCredentialsPath(environment, workflow);
39
+ if (!fs.existsSync(credPath)) {
40
+ throw new Error(`Credentials file not found: ${credPath}`);
41
+ }
42
+ const ciphertext = fs.readFileSync(credPath, 'utf8').trim();
43
+ return decrypt(ciphertext, key);
44
+ };
45
+ export const writeEncrypted = (environment, plaintext, workflow) => {
46
+ const key = resolveKey(environment, workflow);
47
+ const credPath = resolveCredentialsPath(environment, workflow);
48
+ fs.mkdirSync(path.dirname(credPath), { recursive: true });
49
+ fs.writeFileSync(credPath, encrypt(plaintext, key), 'utf8');
50
+ };
51
+ export const initCredentials = (environment, workflow) => {
52
+ const credPath = resolveCredentialsPath(environment, workflow);
53
+ const keyPath = resolveKeyPath(environment, workflow);
54
+ fs.mkdirSync(path.dirname(keyPath), { recursive: true });
55
+ fs.mkdirSync(path.dirname(credPath), { recursive: true });
56
+ const key = generateKey();
57
+ fs.writeFileSync(keyPath, key, { mode: 0o600 });
58
+ const template = stringifyYaml({
59
+ anthropic: { api_key: '' },
60
+ openai: { api_key: '' }
61
+ });
62
+ fs.writeFileSync(credPath, encrypt(template, key), 'utf8');
63
+ return { keyPath, credPath };
64
+ };
@@ -0,0 +1 @@
1
+ export {};