@outputai/cli 0.1.2-dev.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/dist/api/generated/api.d.ts +0 -820
- package/dist/api/generated/api.js +0 -226
- package/dist/api/http_client.d.ts +0 -27
- package/dist/api/http_client.js +0 -71
- package/dist/api/orval_post_process.d.ts +0 -11
- package/dist/api/orval_post_process.js +0 -46
- package/dist/api/parser.d.ts +0 -17
- package/dist/api/parser.js +0 -68
- package/dist/assets/config/costs.yml +0 -309
- package/dist/assets/docker/docker-compose-dev.yml +0 -146
- package/dist/commands/credentials/edit.d.ts +0 -10
- package/dist/commands/credentials/edit.js +0 -67
- package/dist/commands/credentials/edit.spec.d.ts +0 -1
- package/dist/commands/credentials/edit.spec.js +0 -73
- package/dist/commands/credentials/get.d.ts +0 -13
- package/dist/commands/credentials/get.js +0 -46
- package/dist/commands/credentials/get.spec.d.ts +0 -1
- package/dist/commands/credentials/get.spec.js +0 -74
- package/dist/commands/credentials/init.d.ts +0 -11
- package/dist/commands/credentials/init.js +0 -45
- package/dist/commands/credentials/init.spec.d.ts +0 -1
- package/dist/commands/credentials/init.spec.js +0 -68
- package/dist/commands/credentials/show.d.ts +0 -10
- package/dist/commands/credentials/show.js +0 -33
- package/dist/commands/credentials/show.spec.d.ts +0 -1
- package/dist/commands/credentials/show.spec.js +0 -57
- package/dist/commands/dev/eject.d.ts +0 -11
- package/dist/commands/dev/eject.js +0 -58
- package/dist/commands/dev/eject.spec.d.ts +0 -1
- package/dist/commands/dev/eject.spec.js +0 -109
- package/dist/commands/dev/index.d.ts +0 -13
- package/dist/commands/dev/index.js +0 -162
- package/dist/commands/dev/index.spec.d.ts +0 -1
- package/dist/commands/dev/index.spec.js +0 -239
- package/dist/commands/init.d.ts +0 -12
- package/dist/commands/init.js +0 -37
- package/dist/commands/init.spec.d.ts +0 -1
- package/dist/commands/init.spec.js +0 -100
- package/dist/commands/update.d.ts +0 -14
- package/dist/commands/update.js +0 -120
- package/dist/commands/update.spec.d.ts +0 -1
- package/dist/commands/update.spec.js +0 -178
- package/dist/commands/workflow/cost.d.ts +0 -16
- package/dist/commands/workflow/cost.js +0 -71
- package/dist/commands/workflow/cost.spec.d.ts +0 -1
- package/dist/commands/workflow/cost.spec.js +0 -47
- package/dist/commands/workflow/dataset/generate.d.ts +0 -22
- package/dist/commands/workflow/dataset/generate.js +0 -143
- package/dist/commands/workflow/dataset/list.d.ts +0 -12
- package/dist/commands/workflow/dataset/list.js +0 -87
- package/dist/commands/workflow/debug.d.ts +0 -16
- package/dist/commands/workflow/debug.js +0 -60
- package/dist/commands/workflow/debug.spec.d.ts +0 -1
- package/dist/commands/workflow/debug.spec.js +0 -34
- package/dist/commands/workflow/generate.d.ts +0 -17
- package/dist/commands/workflow/generate.js +0 -85
- package/dist/commands/workflow/generate.spec.d.ts +0 -1
- package/dist/commands/workflow/generate.spec.js +0 -115
- package/dist/commands/workflow/list.d.ts +0 -22
- package/dist/commands/workflow/list.js +0 -152
- package/dist/commands/workflow/list.spec.d.ts +0 -1
- package/dist/commands/workflow/list.spec.js +0 -99
- package/dist/commands/workflow/plan.d.ts +0 -12
- package/dist/commands/workflow/plan.js +0 -66
- package/dist/commands/workflow/plan.spec.d.ts +0 -1
- package/dist/commands/workflow/plan.spec.js +0 -341
- package/dist/commands/workflow/reset.d.ts +0 -14
- package/dist/commands/workflow/reset.js +0 -51
- package/dist/commands/workflow/result.d.ts +0 -13
- package/dist/commands/workflow/result.js +0 -46
- package/dist/commands/workflow/result.spec.d.ts +0 -1
- package/dist/commands/workflow/result.spec.js +0 -23
- package/dist/commands/workflow/run.d.ts +0 -16
- package/dist/commands/workflow/run.js +0 -97
- package/dist/commands/workflow/run.spec.d.ts +0 -1
- package/dist/commands/workflow/run.spec.js +0 -110
- package/dist/commands/workflow/runs/list.d.ts +0 -14
- package/dist/commands/workflow/runs/list.js +0 -104
- package/dist/commands/workflow/start.d.ts +0 -15
- package/dist/commands/workflow/start.js +0 -62
- package/dist/commands/workflow/start.spec.d.ts +0 -1
- package/dist/commands/workflow/start.spec.js +0 -28
- package/dist/commands/workflow/status.d.ts +0 -13
- package/dist/commands/workflow/status.js +0 -57
- package/dist/commands/workflow/status.spec.d.ts +0 -1
- package/dist/commands/workflow/status.spec.js +0 -33
- package/dist/commands/workflow/stop.d.ts +0 -10
- package/dist/commands/workflow/stop.js +0 -31
- package/dist/commands/workflow/stop.spec.d.ts +0 -1
- package/dist/commands/workflow/stop.spec.js +0 -17
- package/dist/commands/workflow/terminate.d.ts +0 -13
- package/dist/commands/workflow/terminate.js +0 -39
- package/dist/commands/workflow/test_eval.d.ts +0 -20
- package/dist/commands/workflow/test_eval.js +0 -151
- package/dist/config.d.ts +0 -47
- package/dist/config.js +0 -47
- package/dist/generated/framework_version.json +0 -3
- package/dist/hooks/init.d.ts +0 -3
- package/dist/hooks/init.js +0 -30
- package/dist/hooks/init.spec.d.ts +0 -1
- package/dist/hooks/init.spec.js +0 -54
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/index.spec.d.ts +0 -1
- package/dist/index.spec.js +0 -6
- package/dist/services/claude_client.d.ts +0 -30
- package/dist/services/claude_client.integration.test.d.ts +0 -1
- package/dist/services/claude_client.integration.test.js +0 -43
- package/dist/services/claude_client.js +0 -215
- package/dist/services/claude_client.spec.d.ts +0 -1
- package/dist/services/claude_client.spec.js +0 -145
- package/dist/services/coding_agents.d.ts +0 -36
- package/dist/services/coding_agents.js +0 -236
- package/dist/services/coding_agents.spec.d.ts +0 -1
- package/dist/services/coding_agents.spec.js +0 -256
- package/dist/services/copy_assets.spec.d.ts +0 -1
- package/dist/services/copy_assets.spec.js +0 -22
- package/dist/services/cost_calculator.d.ts +0 -18
- package/dist/services/cost_calculator.js +0 -359
- package/dist/services/cost_calculator.spec.d.ts +0 -1
- package/dist/services/cost_calculator.spec.js +0 -540
- package/dist/services/credentials_service.d.ts +0 -12
- package/dist/services/credentials_service.integration.test.d.ts +0 -1
- package/dist/services/credentials_service.integration.test.js +0 -66
- package/dist/services/credentials_service.js +0 -64
- package/dist/services/credentials_service.spec.d.ts +0 -1
- package/dist/services/credentials_service.spec.js +0 -106
- package/dist/services/datasets.d.ts +0 -20
- package/dist/services/datasets.js +0 -132
- package/dist/services/docker.d.ts +0 -38
- package/dist/services/docker.js +0 -148
- package/dist/services/docker.spec.d.ts +0 -1
- package/dist/services/docker.spec.js +0 -124
- package/dist/services/env_configurator.d.ts +0 -15
- package/dist/services/env_configurator.js +0 -163
- package/dist/services/env_configurator.spec.d.ts +0 -1
- package/dist/services/env_configurator.spec.js +0 -192
- package/dist/services/generate_plan_name@v1.prompt +0 -24
- package/dist/services/messages.d.ts +0 -9
- package/dist/services/messages.js +0 -338
- package/dist/services/messages.spec.d.ts +0 -1
- package/dist/services/messages.spec.js +0 -55
- package/dist/services/npm_update_service.d.ts +0 -6
- package/dist/services/npm_update_service.js +0 -87
- package/dist/services/npm_update_service.spec.d.ts +0 -1
- package/dist/services/npm_update_service.spec.js +0 -104
- package/dist/services/project_scaffold.d.ts +0 -31
- package/dist/services/project_scaffold.js +0 -212
- package/dist/services/project_scaffold.spec.d.ts +0 -1
- package/dist/services/project_scaffold.spec.js +0 -122
- package/dist/services/s3_trace_downloader.d.ts +0 -12
- package/dist/services/s3_trace_downloader.js +0 -57
- package/dist/services/template_processor.d.ts +0 -14
- package/dist/services/template_processor.js +0 -57
- package/dist/services/trace_reader.d.ts +0 -16
- package/dist/services/trace_reader.js +0 -57
- package/dist/services/trace_reader.spec.d.ts +0 -1
- package/dist/services/trace_reader.spec.js +0 -78
- package/dist/services/version_check.d.ts +0 -6
- package/dist/services/version_check.js +0 -52
- package/dist/services/version_check.spec.d.ts +0 -1
- package/dist/services/version_check.spec.js +0 -106
- package/dist/services/workflow_builder.d.ts +0 -16
- package/dist/services/workflow_builder.js +0 -86
- package/dist/services/workflow_builder.spec.d.ts +0 -1
- package/dist/services/workflow_builder.spec.js +0 -165
- package/dist/services/workflow_generator.d.ts +0 -5
- package/dist/services/workflow_generator.js +0 -40
- package/dist/services/workflow_generator.spec.d.ts +0 -1
- package/dist/services/workflow_generator.spec.js +0 -77
- package/dist/services/workflow_planner.d.ts +0 -15
- package/dist/services/workflow_planner.js +0 -48
- package/dist/services/workflow_planner.spec.d.ts +0 -1
- package/dist/services/workflow_planner.spec.js +0 -122
- package/dist/services/workflow_runs.d.ts +0 -14
- package/dist/services/workflow_runs.js +0 -25
- package/dist/templates/agent_instructions/CLAUDE.md.template +0 -19
- package/dist/templates/agent_instructions/dotclaude/settings.json.template +0 -29
- package/dist/templates/project/.env.example.template +0 -9
- package/dist/templates/project/.gitignore.template +0 -35
- package/dist/templates/project/README.md.template +0 -100
- package/dist/templates/project/config/costs.yml.template +0 -29
- package/dist/templates/project/package.json.template +0 -25
- package/dist/templates/project/src/clients/jina.ts.template +0 -30
- package/dist/templates/project/src/shared/utils/string.ts.template +0 -3
- package/dist/templates/project/src/shared/utils/url.ts.template +0 -15
- package/dist/templates/project/src/workflows/blog_evaluator/evaluators.ts.template +0 -23
- package/dist/templates/project/src/workflows/blog_evaluator/prompts/signal_noise@v1.prompt.template +0 -26
- package/dist/templates/project/src/workflows/blog_evaluator/scenarios/paulgraham_hwh.json.template +0 -3
- package/dist/templates/project/src/workflows/blog_evaluator/steps.ts.template +0 -27
- package/dist/templates/project/src/workflows/blog_evaluator/types.ts.template +0 -30
- package/dist/templates/project/src/workflows/blog_evaluator/utils.ts.template +0 -15
- package/dist/templates/project/src/workflows/blog_evaluator/workflow.ts.template +0 -27
- package/dist/templates/project/tsconfig.json.template +0 -20
- package/dist/templates/workflow/README.md.template +0 -216
- package/dist/templates/workflow/evaluators.ts.template +0 -21
- package/dist/templates/workflow/prompts/example@v1.prompt.template +0 -15
- package/dist/templates/workflow/scenarios/test_input.json.template +0 -3
- package/dist/templates/workflow/steps.ts.template +0 -20
- package/dist/templates/workflow/types.ts.template +0 -13
- package/dist/templates/workflow/workflow.ts.template +0 -23
- package/dist/test_helpers/mocks.d.ts +0 -38
- package/dist/test_helpers/mocks.js +0 -77
- package/dist/types/cost.d.ts +0 -149
- package/dist/types/cost.js +0 -6
- package/dist/types/domain.d.ts +0 -20
- package/dist/types/domain.js +0 -4
- package/dist/types/errors.d.ts +0 -68
- package/dist/types/errors.js +0 -100
- package/dist/types/errors.spec.d.ts +0 -1
- package/dist/types/errors.spec.js +0 -18
- package/dist/types/generator.d.ts +0 -26
- package/dist/types/generator.js +0 -1
- package/dist/types/trace.d.ts +0 -161
- package/dist/types/trace.js +0 -18
- package/dist/utils/claude.d.ts +0 -5
- package/dist/utils/claude.js +0 -19
- package/dist/utils/claude.spec.d.ts +0 -1
- package/dist/utils/claude.spec.js +0 -119
- package/dist/utils/constants.d.ts +0 -5
- package/dist/utils/constants.js +0 -4
- package/dist/utils/cost_formatter.d.ts +0 -5
- package/dist/utils/cost_formatter.js +0 -218
- package/dist/utils/date_formatter.d.ts +0 -23
- package/dist/utils/date_formatter.js +0 -49
- package/dist/utils/env_loader.d.ts +0 -1
- package/dist/utils/env_loader.js +0 -22
- package/dist/utils/env_loader.spec.d.ts +0 -1
- package/dist/utils/env_loader.spec.js +0 -43
- package/dist/utils/error_handler.d.ts +0 -8
- package/dist/utils/error_handler.js +0 -71
- package/dist/utils/error_utils.d.ts +0 -24
- package/dist/utils/error_utils.js +0 -87
- package/dist/utils/file_system.d.ts +0 -3
- package/dist/utils/file_system.js +0 -33
- package/dist/utils/format_workflow_result.d.ts +0 -5
- package/dist/utils/format_workflow_result.js +0 -18
- package/dist/utils/format_workflow_result.spec.d.ts +0 -1
- package/dist/utils/format_workflow_result.spec.js +0 -81
- package/dist/utils/framework_version.d.ts +0 -4
- package/dist/utils/framework_version.js +0 -4
- package/dist/utils/framework_version.spec.d.ts +0 -1
- package/dist/utils/framework_version.spec.js +0 -13
- package/dist/utils/header_utils.d.ts +0 -12
- package/dist/utils/header_utils.js +0 -29
- package/dist/utils/header_utils.spec.d.ts +0 -1
- package/dist/utils/header_utils.spec.js +0 -52
- package/dist/utils/input_parser.d.ts +0 -1
- package/dist/utils/input_parser.js +0 -19
- package/dist/utils/output_formatter.d.ts +0 -2
- package/dist/utils/output_formatter.js +0 -11
- package/dist/utils/paths.d.ts +0 -25
- package/dist/utils/paths.js +0 -36
- package/dist/utils/process.d.ts +0 -4
- package/dist/utils/process.js +0 -50
- package/dist/utils/resolve_input.d.ts +0 -1
- package/dist/utils/resolve_input.js +0 -22
- package/dist/utils/scenario_resolver.d.ts +0 -9
- package/dist/utils/scenario_resolver.js +0 -93
- package/dist/utils/scenario_resolver.spec.d.ts +0 -1
- package/dist/utils/scenario_resolver.spec.js +0 -214
- package/dist/utils/secret_sanitizer.d.ts +0 -1
- package/dist/utils/secret_sanitizer.js +0 -29
- package/dist/utils/sleep.d.ts +0 -5
- package/dist/utils/sleep.js +0 -5
- package/dist/utils/template.d.ts +0 -9
- package/dist/utils/template.js +0 -30
- package/dist/utils/template.spec.d.ts +0 -1
- package/dist/utils/template.spec.js +0 -77
- package/dist/utils/trace_extractor.d.ts +0 -27
- package/dist/utils/trace_extractor.js +0 -53
- package/dist/utils/trace_formatter.d.ts +0 -11
- package/dist/utils/trace_formatter.js +0 -402
- package/dist/utils/validation.d.ts +0 -13
- package/dist/utils/validation.js +0 -25
- package/dist/utils/validation.spec.d.ts +0 -1
- package/dist/utils/validation.spec.js +0 -140
|
@@ -1,256 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
2
|
-
import { checkAgentStructure, prepareTemplateVariables, initializeAgentConfig, ensureOutputAISystem, ensureClaudePlugin } from './coding_agents.js';
|
|
3
|
-
import { access } from 'node:fs/promises';
|
|
4
|
-
import fs from 'node:fs/promises';
|
|
5
|
-
vi.mock('node:fs/promises');
|
|
6
|
-
vi.mock('../utils/paths.js', () => ({
|
|
7
|
-
getTemplateDir: vi.fn().mockReturnValue('/templates')
|
|
8
|
-
}));
|
|
9
|
-
vi.mock('../utils/template.js', () => ({
|
|
10
|
-
processTemplate: vi.fn().mockImplementation((content) => content)
|
|
11
|
-
}));
|
|
12
|
-
vi.mock('../utils/claude.js', () => ({
|
|
13
|
-
executeClaudeCommand: vi.fn().mockResolvedValue(undefined)
|
|
14
|
-
}));
|
|
15
|
-
vi.mock('@oclif/core', () => ({
|
|
16
|
-
ux: {
|
|
17
|
-
warn: vi.fn(),
|
|
18
|
-
stdout: vi.fn(),
|
|
19
|
-
colorize: vi.fn().mockImplementation((_color, text) => text)
|
|
20
|
-
}
|
|
21
|
-
}));
|
|
22
|
-
vi.mock('@inquirer/prompts', () => ({
|
|
23
|
-
confirm: vi.fn()
|
|
24
|
-
}));
|
|
25
|
-
describe('coding_agents service', () => {
|
|
26
|
-
beforeEach(() => {
|
|
27
|
-
vi.clearAllMocks();
|
|
28
|
-
});
|
|
29
|
-
describe('checkAgentStructure', () => {
|
|
30
|
-
it('should return needsInit true when settings.json does not exist', async () => {
|
|
31
|
-
vi.mocked(access).mockRejectedValue({ code: 'ENOENT' });
|
|
32
|
-
vi.mocked(fs.readFile).mockRejectedValue({ code: 'ENOENT' });
|
|
33
|
-
const result = await checkAgentStructure('/test/project');
|
|
34
|
-
expect(result).toEqual({
|
|
35
|
-
isComplete: false,
|
|
36
|
-
needsInit: true
|
|
37
|
-
});
|
|
38
|
-
});
|
|
39
|
-
it('should return complete when settings and CLAUDE.md exist with valid configuration', async () => {
|
|
40
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
41
|
-
vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({
|
|
42
|
-
extraKnownMarketplaces: {
|
|
43
|
-
'team-tools': {
|
|
44
|
-
source: {
|
|
45
|
-
source: 'github',
|
|
46
|
-
repo: 'growthxai/output-claude-plugins'
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
},
|
|
50
|
-
enabledPlugins: {
|
|
51
|
-
'outputai@outputai': true
|
|
52
|
-
}
|
|
53
|
-
}));
|
|
54
|
-
const result = await checkAgentStructure('/test/project');
|
|
55
|
-
expect(result).toEqual({
|
|
56
|
-
isComplete: true,
|
|
57
|
-
needsInit: false
|
|
58
|
-
});
|
|
59
|
-
});
|
|
60
|
-
it('should return needsInit true when settings.json has wrong marketplace repo', async () => {
|
|
61
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
62
|
-
vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({
|
|
63
|
-
extraKnownMarketplaces: {
|
|
64
|
-
'team-tools': {
|
|
65
|
-
source: {
|
|
66
|
-
source: 'github',
|
|
67
|
-
repo: 'wrong/repo'
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
},
|
|
71
|
-
enabledPlugins: {
|
|
72
|
-
'outputai@outputai': true
|
|
73
|
-
}
|
|
74
|
-
}));
|
|
75
|
-
const result = await checkAgentStructure('/test/project');
|
|
76
|
-
expect(result.isComplete).toBe(false);
|
|
77
|
-
expect(result.needsInit).toBe(true);
|
|
78
|
-
});
|
|
79
|
-
it('should return needsInit true when plugin is not enabled', async () => {
|
|
80
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
81
|
-
vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({
|
|
82
|
-
extraKnownMarketplaces: {
|
|
83
|
-
'team-tools': {
|
|
84
|
-
source: {
|
|
85
|
-
source: 'github',
|
|
86
|
-
repo: 'growthxai/output-claude-plugins'
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
},
|
|
90
|
-
enabledPlugins: {
|
|
91
|
-
'outputai@outputai': false
|
|
92
|
-
}
|
|
93
|
-
}));
|
|
94
|
-
const result = await checkAgentStructure('/test/project');
|
|
95
|
-
expect(result.isComplete).toBe(false);
|
|
96
|
-
expect(result.needsInit).toBe(true);
|
|
97
|
-
});
|
|
98
|
-
});
|
|
99
|
-
describe('prepareTemplateVariables', () => {
|
|
100
|
-
it('should return template variables with formatted date', () => {
|
|
101
|
-
const variables = prepareTemplateVariables();
|
|
102
|
-
expect(variables).toHaveProperty('date');
|
|
103
|
-
expect(typeof variables.date).toBe('string');
|
|
104
|
-
expect(variables.date).toMatch(/^[A-Z][a-z]+ \d{1,2}, \d{4}$/);
|
|
105
|
-
});
|
|
106
|
-
});
|
|
107
|
-
describe('initializeAgentConfig', () => {
|
|
108
|
-
beforeEach(() => {
|
|
109
|
-
vi.mocked(fs.mkdir).mockResolvedValue(undefined);
|
|
110
|
-
vi.mocked(access).mockRejectedValue({ code: 'ENOENT' });
|
|
111
|
-
vi.mocked(fs.readFile).mockResolvedValue('template content');
|
|
112
|
-
vi.mocked(fs.writeFile).mockResolvedValue(undefined);
|
|
113
|
-
});
|
|
114
|
-
it('should create exactly 2 outputs: settings.json and CLAUDE.md file', async () => {
|
|
115
|
-
await initializeAgentConfig({
|
|
116
|
-
projectRoot: '/test/project',
|
|
117
|
-
force: false
|
|
118
|
-
});
|
|
119
|
-
expect(fs.mkdir).toHaveBeenCalledTimes(1);
|
|
120
|
-
expect(fs.mkdir).toHaveBeenCalledWith('/test/project/.claude', expect.objectContaining({ recursive: true }));
|
|
121
|
-
expect(fs.writeFile).toHaveBeenCalledWith('/test/project/.claude/settings.json', expect.any(String), 'utf-8');
|
|
122
|
-
expect(fs.writeFile).toHaveBeenCalledWith('/test/project/CLAUDE.md', expect.any(String), 'utf-8');
|
|
123
|
-
// No symlink should be created - CLAUDE.md is now a real file
|
|
124
|
-
expect(fs.symlink).not.toHaveBeenCalled();
|
|
125
|
-
});
|
|
126
|
-
it('should skip existing files when force is false', async () => {
|
|
127
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
128
|
-
await initializeAgentConfig({
|
|
129
|
-
projectRoot: '/test/project',
|
|
130
|
-
force: false
|
|
131
|
-
});
|
|
132
|
-
expect(fs.writeFile).not.toHaveBeenCalled();
|
|
133
|
-
});
|
|
134
|
-
it('should overwrite existing files when force is true', async () => {
|
|
135
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
136
|
-
vi.mocked(fs.unlink).mockResolvedValue(undefined);
|
|
137
|
-
await initializeAgentConfig({
|
|
138
|
-
projectRoot: '/test/project',
|
|
139
|
-
force: true
|
|
140
|
-
});
|
|
141
|
-
expect(fs.writeFile).toHaveBeenCalled();
|
|
142
|
-
});
|
|
143
|
-
});
|
|
144
|
-
describe('ensureClaudePlugin', () => {
|
|
145
|
-
beforeEach(() => {
|
|
146
|
-
vi.mocked(fs.mkdir).mockResolvedValue(undefined);
|
|
147
|
-
vi.mocked(access).mockRejectedValue({ code: 'ENOENT' });
|
|
148
|
-
vi.mocked(fs.readFile).mockResolvedValue('template content');
|
|
149
|
-
vi.mocked(fs.writeFile).mockResolvedValue(undefined);
|
|
150
|
-
});
|
|
151
|
-
it('should call registerPluginMarketplace and installOutputAIPlugin', async () => {
|
|
152
|
-
const { executeClaudeCommand } = await import('../utils/claude.js');
|
|
153
|
-
await ensureClaudePlugin('/test/project');
|
|
154
|
-
expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'marketplace', 'add', 'growthxai/output-claude-plugins'], '/test/project', { ignoreFailure: true });
|
|
155
|
-
expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'marketplace', 'update', 'outputai'], '/test/project');
|
|
156
|
-
expect(executeClaudeCommand).toHaveBeenCalledWith(['plugin', 'install', 'outputai@outputai', '--scope', 'project'], '/test/project');
|
|
157
|
-
});
|
|
158
|
-
it('should show error and prompt user when plugin commands fail', async () => {
|
|
159
|
-
const { executeClaudeCommand } = await import('../utils/claude.js');
|
|
160
|
-
const { confirm } = await import('@inquirer/prompts');
|
|
161
|
-
vi.mocked(executeClaudeCommand)
|
|
162
|
-
.mockResolvedValueOnce(undefined) // marketplace add
|
|
163
|
-
.mockRejectedValueOnce(new Error('Plugin update failed')); // marketplace update
|
|
164
|
-
vi.mocked(confirm).mockResolvedValue(true);
|
|
165
|
-
await expect(ensureClaudePlugin('/test/project')).resolves.not.toThrow();
|
|
166
|
-
expect(confirm).toHaveBeenCalledWith(expect.objectContaining({
|
|
167
|
-
message: expect.stringContaining('proceed')
|
|
168
|
-
}));
|
|
169
|
-
});
|
|
170
|
-
it('should allow user to proceed without plugin setup if they confirm', async () => {
|
|
171
|
-
const { executeClaudeCommand } = await import('../utils/claude.js');
|
|
172
|
-
const { confirm } = await import('@inquirer/prompts');
|
|
173
|
-
vi.mocked(executeClaudeCommand)
|
|
174
|
-
.mockRejectedValue(new Error('All plugin commands fail'));
|
|
175
|
-
vi.mocked(confirm).mockResolvedValue(true);
|
|
176
|
-
await expect(ensureClaudePlugin('/test/project')).resolves.not.toThrow();
|
|
177
|
-
});
|
|
178
|
-
});
|
|
179
|
-
describe('ensureOutputAISystem', () => {
|
|
180
|
-
beforeEach(() => {
|
|
181
|
-
vi.mocked(fs.mkdir).mockResolvedValue(undefined);
|
|
182
|
-
vi.mocked(access).mockRejectedValue({ code: 'ENOENT' });
|
|
183
|
-
vi.mocked(fs.readFile).mockResolvedValue('template content');
|
|
184
|
-
vi.mocked(fs.writeFile).mockResolvedValue(undefined);
|
|
185
|
-
});
|
|
186
|
-
it('should return immediately when agent structure is complete', async () => {
|
|
187
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
188
|
-
vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({
|
|
189
|
-
extraKnownMarketplaces: {
|
|
190
|
-
'team-tools': {
|
|
191
|
-
source: { source: 'github', repo: 'growthxai/output-claude-plugins' }
|
|
192
|
-
}
|
|
193
|
-
},
|
|
194
|
-
enabledPlugins: { 'outputai@outputai': true }
|
|
195
|
-
}));
|
|
196
|
-
await ensureOutputAISystem('/test/project');
|
|
197
|
-
expect(fs.mkdir).not.toHaveBeenCalled();
|
|
198
|
-
});
|
|
199
|
-
it('should auto-initialize when settings.json is invalid', async () => {
|
|
200
|
-
vi.mocked(access).mockResolvedValue(undefined);
|
|
201
|
-
vi.mocked(fs.readFile).mockResolvedValue(JSON.stringify({
|
|
202
|
-
extraKnownMarketplaces: {
|
|
203
|
-
'team-tools': {
|
|
204
|
-
source: { source: 'github', repo: 'wrong/repo' }
|
|
205
|
-
}
|
|
206
|
-
},
|
|
207
|
-
enabledPlugins: { 'outputai@outputai': true }
|
|
208
|
-
}));
|
|
209
|
-
await ensureOutputAISystem('/test/project');
|
|
210
|
-
expect(fs.mkdir).toHaveBeenCalled();
|
|
211
|
-
});
|
|
212
|
-
});
|
|
213
|
-
describe('Claude plugin error handling', () => {
|
|
214
|
-
beforeEach(() => {
|
|
215
|
-
vi.mocked(fs.mkdir).mockResolvedValue(undefined);
|
|
216
|
-
vi.mocked(access).mockRejectedValue({ code: 'ENOENT' });
|
|
217
|
-
vi.mocked(fs.readFile).mockResolvedValue('template content');
|
|
218
|
-
vi.mocked(fs.writeFile).mockResolvedValue(undefined);
|
|
219
|
-
});
|
|
220
|
-
it('should show error and prompt user when registerPluginMarketplace fails', async () => {
|
|
221
|
-
const { executeClaudeCommand } = await import('../utils/claude.js');
|
|
222
|
-
const { confirm } = await import('@inquirer/prompts');
|
|
223
|
-
vi.mocked(executeClaudeCommand)
|
|
224
|
-
.mockResolvedValueOnce(undefined) // marketplace add
|
|
225
|
-
.mockRejectedValueOnce(new Error('Plugin update failed')); // marketplace update
|
|
226
|
-
vi.mocked(confirm).mockResolvedValue(true);
|
|
227
|
-
await expect(initializeAgentConfig({ projectRoot: '/test/project', force: true })).resolves.not.toThrow();
|
|
228
|
-
expect(confirm).toHaveBeenCalledWith(expect.objectContaining({
|
|
229
|
-
message: expect.stringContaining('proceed')
|
|
230
|
-
}));
|
|
231
|
-
});
|
|
232
|
-
it('should show error and prompt user when installOutputAIPlugin fails', async () => {
|
|
233
|
-
const { executeClaudeCommand } = await import('../utils/claude.js');
|
|
234
|
-
const { confirm } = await import('@inquirer/prompts');
|
|
235
|
-
vi.mocked(executeClaudeCommand)
|
|
236
|
-
.mockResolvedValueOnce(undefined) // marketplace add
|
|
237
|
-
.mockResolvedValueOnce(undefined) // marketplace update
|
|
238
|
-
.mockRejectedValueOnce(new Error('Plugin install failed')); // plugin install
|
|
239
|
-
vi.mocked(confirm).mockResolvedValue(true);
|
|
240
|
-
await expect(initializeAgentConfig({ projectRoot: '/test/project', force: true })).resolves.not.toThrow();
|
|
241
|
-
expect(confirm).toHaveBeenCalledWith(expect.objectContaining({
|
|
242
|
-
message: expect.stringContaining('proceed')
|
|
243
|
-
}));
|
|
244
|
-
});
|
|
245
|
-
it('should allow user to proceed without plugin setup if they confirm', async () => {
|
|
246
|
-
const { executeClaudeCommand } = await import('../utils/claude.js');
|
|
247
|
-
const { confirm } = await import('@inquirer/prompts');
|
|
248
|
-
vi.mocked(executeClaudeCommand)
|
|
249
|
-
.mockRejectedValue(new Error('All plugin commands fail'));
|
|
250
|
-
vi.mocked(confirm).mockResolvedValue(true);
|
|
251
|
-
await expect(initializeAgentConfig({ projectRoot: '/test/project', force: true })).resolves.not.toThrow();
|
|
252
|
-
// File operations should still complete
|
|
253
|
-
expect(fs.mkdir).toHaveBeenCalled();
|
|
254
|
-
});
|
|
255
|
-
});
|
|
256
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import * as fs from 'node:fs';
|
|
3
|
-
import * as path from 'node:path';
|
|
4
|
-
import { fileURLToPath } from 'node:url';
|
|
5
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
6
|
-
const __dirname = path.dirname(__filename);
|
|
7
|
-
// Resolve to the cli package root, then into dist/templates
|
|
8
|
-
const cliRoot = path.resolve(__dirname, '..', '..');
|
|
9
|
-
const distTemplatesDir = path.join(cliRoot, 'dist', 'templates', 'project');
|
|
10
|
-
describe('copy-assets build output', () => {
|
|
11
|
-
it('should include dotfile templates in dist/templates/project/', () => {
|
|
12
|
-
const dotfiles = ['.env.example.template', '.gitignore.template'];
|
|
13
|
-
for (const dotfile of dotfiles) {
|
|
14
|
-
const filePath = path.join(distTemplatesDir, dotfile);
|
|
15
|
-
expect(fs.existsSync(filePath), `Missing dotfile template in dist: ${dotfile}`).toBe(true);
|
|
16
|
-
}
|
|
17
|
-
});
|
|
18
|
-
it('should include config templates in dist/templates/project/', () => {
|
|
19
|
-
const configFile = path.join(distTemplatesDir, 'config', 'costs.yml.template');
|
|
20
|
-
expect(fs.existsSync(configFile), 'Missing config/costs.yml.template in dist').toBe(true);
|
|
21
|
-
});
|
|
22
|
-
});
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import type { TraceNode, LLMCall, HTTPCall, TokenUsage, PricingConfig, ModelPricing, ServiceConfig, ServiceCostResult, CostReport } from '#types/cost.js';
|
|
2
|
-
export declare function extractValue(obj: unknown, path: string): unknown;
|
|
3
|
-
export declare function loadPricingConfig(configPath?: string): PricingConfig;
|
|
4
|
-
export declare function findLLMCalls(node: TraceNode, parentStepName?: string | null, seenIds?: Set<string>): LLMCall[];
|
|
5
|
-
export declare function findHTTPCalls(node: TraceNode, parentStepName?: string | null, seenIds?: Set<string>): HTTPCall[];
|
|
6
|
-
export declare function calculateLLMCallCost(usage: TokenUsage, modelPricing: ModelPricing | undefined): {
|
|
7
|
-
cost: number;
|
|
8
|
-
warning?: string;
|
|
9
|
-
};
|
|
10
|
-
export declare function identifyService(httpCall: HTTPCall, services: Record<string, ServiceConfig>): {
|
|
11
|
-
serviceName: string;
|
|
12
|
-
config: ServiceConfig;
|
|
13
|
-
} | null;
|
|
14
|
-
export declare function calculateServiceCost(httpCall: HTTPCall, serviceInfo: {
|
|
15
|
-
serviceName: string;
|
|
16
|
-
config: ServiceConfig;
|
|
17
|
-
}): ServiceCostResult;
|
|
18
|
-
export declare function calculateCost(trace: TraceNode, config: PricingConfig, traceFile?: string): CostReport;
|
|
@@ -1,359 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import yaml from 'js-yaml';
|
|
4
|
-
const ARRAY_ACCESS_PATTERN = /^(\w+)\[(\d+)\]$/;
|
|
5
|
-
function tokenCost(tokens, pricePerMillion) {
|
|
6
|
-
return (tokens / 1_000_000) * pricePerMillion;
|
|
7
|
-
}
|
|
8
|
-
export function extractValue(obj, path) {
|
|
9
|
-
if (!path || !obj) {
|
|
10
|
-
return obj;
|
|
11
|
-
}
|
|
12
|
-
return path.split('.').reduce((current, part) => {
|
|
13
|
-
if (current === null || current === undefined) {
|
|
14
|
-
return current;
|
|
15
|
-
}
|
|
16
|
-
const arrayMatch = part.match(ARRAY_ACCESS_PATTERN);
|
|
17
|
-
if (arrayMatch) {
|
|
18
|
-
const [, key, index] = arrayMatch;
|
|
19
|
-
return current[key]?.[parseInt(index, 10)];
|
|
20
|
-
}
|
|
21
|
-
return current[part];
|
|
22
|
-
}, obj);
|
|
23
|
-
}
|
|
24
|
-
function loadYaml(filePath) {
|
|
25
|
-
return yaml.load(readFileSync(filePath, 'utf-8'));
|
|
26
|
-
}
|
|
27
|
-
export function loadPricingConfig(configPath) {
|
|
28
|
-
const bundledPath = new URL('../assets/config/costs.yml', import.meta.url).pathname;
|
|
29
|
-
const bundled = loadYaml(configPath ?? bundledPath);
|
|
30
|
-
const projectPath = join(process.cwd(), 'config', 'costs.yml');
|
|
31
|
-
if (!configPath && existsSync(projectPath)) {
|
|
32
|
-
const project = loadYaml(projectPath);
|
|
33
|
-
return {
|
|
34
|
-
models: { ...bundled.models, ...project.models },
|
|
35
|
-
services: { ...bundled.services, ...project.services }
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
return bundled;
|
|
39
|
-
}
|
|
40
|
-
function resolveStepName(node, parentStepName) {
|
|
41
|
-
if (node.kind === 'step' && node.name) {
|
|
42
|
-
return node.name.includes('#') ?
|
|
43
|
-
node.name.split('#').pop() :
|
|
44
|
-
node.name;
|
|
45
|
-
}
|
|
46
|
-
return parentStepName;
|
|
47
|
-
}
|
|
48
|
-
function findCalls(node, match, extract, parentStepName = null, seenIds = new Set()) {
|
|
49
|
-
const calls = [];
|
|
50
|
-
if (match(node)) {
|
|
51
|
-
const id = node.id;
|
|
52
|
-
if (id && seenIds.has(id)) {
|
|
53
|
-
return calls;
|
|
54
|
-
}
|
|
55
|
-
if (id) {
|
|
56
|
-
seenIds.add(id);
|
|
57
|
-
}
|
|
58
|
-
calls.push(extract(node, parentStepName));
|
|
59
|
-
}
|
|
60
|
-
const currentStepName = resolveStepName(node, parentStepName);
|
|
61
|
-
if (node.children) {
|
|
62
|
-
for (const child of node.children) {
|
|
63
|
-
calls.push(...findCalls(child, match, extract, currentStepName, seenIds));
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
return calls;
|
|
67
|
-
}
|
|
68
|
-
export function findLLMCalls(node, parentStepName = null, seenIds = new Set()) {
|
|
69
|
-
return findCalls(node, n => n.kind === 'llm' && !!n.output?.usage, (n, stepName) => {
|
|
70
|
-
const loadedPrompt = n.input?.loadedPrompt;
|
|
71
|
-
const outputRecord = n.output;
|
|
72
|
-
const inputRecord = n.input;
|
|
73
|
-
const model = loadedPrompt?.config?.model ||
|
|
74
|
-
outputRecord?.model ||
|
|
75
|
-
inputRecord?.model ||
|
|
76
|
-
'unknown';
|
|
77
|
-
return {
|
|
78
|
-
stepName: stepName || n.name || 'unknown',
|
|
79
|
-
llmName: n.name || 'llm',
|
|
80
|
-
model,
|
|
81
|
-
usage: n.output.usage
|
|
82
|
-
};
|
|
83
|
-
}, parentStepName, seenIds);
|
|
84
|
-
}
|
|
85
|
-
export function findHTTPCalls(node, parentStepName = null, seenIds = new Set()) {
|
|
86
|
-
return findCalls(node, n => n.kind === 'http', (n, stepName) => ({
|
|
87
|
-
stepName: stepName || 'unknown',
|
|
88
|
-
url: n.input?.url || '',
|
|
89
|
-
method: n.input?.method || 'GET',
|
|
90
|
-
input: n.input || {},
|
|
91
|
-
output: n.output || {},
|
|
92
|
-
status: n.output?.status
|
|
93
|
-
}), parentStepName, seenIds);
|
|
94
|
-
}
|
|
95
|
-
export function calculateLLMCallCost(usage, modelPricing) {
|
|
96
|
-
if (!modelPricing) {
|
|
97
|
-
return { cost: 0, warning: 'unknown model' };
|
|
98
|
-
}
|
|
99
|
-
const inputCost = tokenCost(usage.inputTokens ?? 0, modelPricing.input ?? 0);
|
|
100
|
-
const outputCost = tokenCost(usage.outputTokens ?? 0, modelPricing.output ?? 0);
|
|
101
|
-
const cachedCost = tokenCost(usage.cachedInputTokens ?? 0, modelPricing.cached_input ?? 0);
|
|
102
|
-
const reasoningCost = tokenCost(usage.reasoningTokens ?? 0, modelPricing.reasoning || modelPricing.output || 0);
|
|
103
|
-
return { cost: inputCost + outputCost + cachedCost + reasoningCost };
|
|
104
|
-
}
|
|
105
|
-
export function identifyService(httpCall, services) {
|
|
106
|
-
if (!services) {
|
|
107
|
-
return null;
|
|
108
|
-
}
|
|
109
|
-
for (const [serviceName, config] of Object.entries(services)) {
|
|
110
|
-
if (httpCall.url.includes(config.url_pattern)) {
|
|
111
|
-
return { serviceName, config };
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
return null;
|
|
115
|
-
}
|
|
116
|
-
function calculateTokenServiceCost(httpCall, config) {
|
|
117
|
-
if (!config.usage_path) {
|
|
118
|
-
return { step: httpCall.stepName, cost: 0, usage: 'no usage data', warning: 'no usage data' };
|
|
119
|
-
}
|
|
120
|
-
const usage = extractValue(httpCall.output, config.usage_path);
|
|
121
|
-
if (config.input_field && config.output_field) {
|
|
122
|
-
const usageObj = usage;
|
|
123
|
-
const inputTokens = usageObj?.[config.input_field] ?? 0;
|
|
124
|
-
const outputTokens = usageObj?.[config.output_field] ?? 0;
|
|
125
|
-
const inputCost = tokenCost(inputTokens, config.input_per_million ?? 0);
|
|
126
|
-
const outputCost = tokenCost(outputTokens, config.output_per_million ?? 0);
|
|
127
|
-
return {
|
|
128
|
-
step: httpCall.stepName,
|
|
129
|
-
cost: inputCost + outputCost,
|
|
130
|
-
usage: `${(inputTokens + outputTokens).toLocaleString('en-US')} tokens`
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
const tokens = typeof usage === 'number' ? usage : 0;
|
|
134
|
-
if (tokens === 0) {
|
|
135
|
-
return { step: httpCall.stepName, cost: 0, usage: 'no usage data', warning: 'no usage data' };
|
|
136
|
-
}
|
|
137
|
-
const cost = tokenCost(tokens, config.per_million ?? 0);
|
|
138
|
-
return { step: httpCall.stepName, cost, usage: `${tokens.toLocaleString('en-US')} tokens` };
|
|
139
|
-
}
|
|
140
|
-
function resolveUnitEndpoint(url, httpCall, config) {
|
|
141
|
-
if (!config.endpoints) {
|
|
142
|
-
return { units: 0, endpoint: 'unknown' };
|
|
143
|
-
}
|
|
144
|
-
for (const [endpointName, endpointConfig] of Object.entries(config.endpoints)) {
|
|
145
|
-
if (!url.includes(endpointConfig.pattern)) {
|
|
146
|
-
continue;
|
|
147
|
-
}
|
|
148
|
-
if (endpointConfig.units_per_request) {
|
|
149
|
-
return { units: endpointConfig.units_per_request, endpoint: endpointName };
|
|
150
|
-
}
|
|
151
|
-
if (endpointConfig.units_per_line) {
|
|
152
|
-
const body = httpCall.output?.body;
|
|
153
|
-
if (typeof body === 'string') {
|
|
154
|
-
const lines = body.split('\n').filter((l) => l.trim() && !l.startsWith('ERROR'));
|
|
155
|
-
const units = Math.max(0, lines.length - 1) * endpointConfig.units_per_line;
|
|
156
|
-
return { units, endpoint: endpointName };
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
return { units: 0, endpoint: endpointName };
|
|
160
|
-
}
|
|
161
|
-
return { units: 0, endpoint: 'unknown' };
|
|
162
|
-
}
|
|
163
|
-
function calculateUnitServiceCost(httpCall, config) {
|
|
164
|
-
const { units, endpoint } = resolveUnitEndpoint(httpCall.url, httpCall, config);
|
|
165
|
-
const cost = units * (config.price_per_unit || 0);
|
|
166
|
-
return {
|
|
167
|
-
step: httpCall.stepName,
|
|
168
|
-
cost,
|
|
169
|
-
usage: `${units.toLocaleString('en-US')} units`,
|
|
170
|
-
endpoint
|
|
171
|
-
};
|
|
172
|
-
}
|
|
173
|
-
function calculateRequestServiceCost(httpCall, config) {
|
|
174
|
-
if (config.models && config.model_path) {
|
|
175
|
-
const model = extractValue(httpCall.input, config.model_path);
|
|
176
|
-
const price = (model && config.models[model]) || config.default_price || 0;
|
|
177
|
-
return { step: httpCall.stepName, cost: price, usage: '1 request', model };
|
|
178
|
-
}
|
|
179
|
-
if (config.endpoints) {
|
|
180
|
-
for (const [endpointName, endpointConfig] of Object.entries(config.endpoints)) {
|
|
181
|
-
if (httpCall.url.includes(endpointConfig.pattern)) {
|
|
182
|
-
if (endpointConfig.price !== undefined) {
|
|
183
|
-
return {
|
|
184
|
-
step: httpCall.stepName,
|
|
185
|
-
cost: endpointConfig.price,
|
|
186
|
-
usage: '1 request',
|
|
187
|
-
endpoint: endpointName
|
|
188
|
-
};
|
|
189
|
-
}
|
|
190
|
-
if (endpointConfig.price_per_item && endpointConfig.items_path) {
|
|
191
|
-
const items = extractValue(httpCall.input, endpointConfig.items_path);
|
|
192
|
-
const count = Array.isArray(items) ? items.length : 0;
|
|
193
|
-
return {
|
|
194
|
-
step: httpCall.stepName,
|
|
195
|
-
cost: count * endpointConfig.price_per_item,
|
|
196
|
-
usage: `${count} items`,
|
|
197
|
-
endpoint: endpointName
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return { step: httpCall.stepName, cost: 0, usage: 'unknown endpoint', warning: 'unknown endpoint' };
|
|
204
|
-
}
|
|
205
|
-
function calculateResponseCostService(httpCall, config) {
|
|
206
|
-
const cost = extractValue(httpCall, config.cost_path);
|
|
207
|
-
if (typeof cost === 'number' && cost > 0) {
|
|
208
|
-
const costDollars = extractValue(httpCall, 'output.body.costDollars');
|
|
209
|
-
const model = extractValue(httpCall, 'output.body.model');
|
|
210
|
-
const numSearches = costDollars?.numSearches ?? 0;
|
|
211
|
-
const numPages = costDollars?.numPages ?? 0;
|
|
212
|
-
return {
|
|
213
|
-
step: httpCall.stepName,
|
|
214
|
-
cost,
|
|
215
|
-
usage: `${numSearches} searches, ${Math.round(numPages)} pages`,
|
|
216
|
-
model: model || 'unknown',
|
|
217
|
-
details: costDollars
|
|
218
|
-
};
|
|
219
|
-
}
|
|
220
|
-
if (config.fallback_models) {
|
|
221
|
-
const model = extractValue(httpCall, 'input.body.model') ||
|
|
222
|
-
extractValue(httpCall, 'output.body.model') ||
|
|
223
|
-
'unknown';
|
|
224
|
-
const fallbackPrice = config.fallback_models[model];
|
|
225
|
-
if (fallbackPrice) {
|
|
226
|
-
return {
|
|
227
|
-
step: httpCall.stepName,
|
|
228
|
-
cost: fallbackPrice,
|
|
229
|
-
usage: '1 request (estimated)',
|
|
230
|
-
model,
|
|
231
|
-
warning: 'using fallback estimate'
|
|
232
|
-
};
|
|
233
|
-
}
|
|
234
|
-
if (config.default_fallback) {
|
|
235
|
-
return {
|
|
236
|
-
step: httpCall.stepName,
|
|
237
|
-
cost: config.default_fallback,
|
|
238
|
-
usage: '1 request (estimated)',
|
|
239
|
-
model: 'unknown',
|
|
240
|
-
warning: 'using default estimate'
|
|
241
|
-
};
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
return { step: httpCall.stepName, cost: 0, usage: 'no cost data', warning: 'no cost data' };
|
|
245
|
-
}
|
|
246
|
-
export function calculateServiceCost(httpCall, serviceInfo) {
|
|
247
|
-
const { config } = serviceInfo;
|
|
248
|
-
switch (config.type) {
|
|
249
|
-
case 'token':
|
|
250
|
-
return calculateTokenServiceCost(httpCall, config);
|
|
251
|
-
case 'unit':
|
|
252
|
-
return calculateUnitServiceCost(httpCall, config);
|
|
253
|
-
case 'request':
|
|
254
|
-
return calculateRequestServiceCost(httpCall, config);
|
|
255
|
-
case 'response_cost':
|
|
256
|
-
return calculateResponseCostService(httpCall, config);
|
|
257
|
-
default:
|
|
258
|
-
return { step: httpCall.stepName, cost: 0, usage: 'unknown type', warning: 'unknown type' };
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
function findModelPricing(model, models) {
|
|
262
|
-
if (models[model]) {
|
|
263
|
-
return { pricing: models[model], matchedKey: model };
|
|
264
|
-
}
|
|
265
|
-
const prefixMatch = Object.entries(models).find(([key]) => model.startsWith(key));
|
|
266
|
-
return prefixMatch ?
|
|
267
|
-
{ pricing: prefixMatch[1], matchedKey: prefixMatch[0] } :
|
|
268
|
-
{ pricing: undefined, matchedKey: undefined };
|
|
269
|
-
}
|
|
270
|
-
function aggregateLLMCosts(llmCalls, config) {
|
|
271
|
-
const unknownModels = new Set();
|
|
272
|
-
const results = [];
|
|
273
|
-
const totals = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, reasoningTokens: 0, cost: 0 };
|
|
274
|
-
for (const call of llmCalls) {
|
|
275
|
-
const { pricing, matchedKey } = findModelPricing(call.model, config.models ?? {});
|
|
276
|
-
const { cost, warning } = calculateLLMCallCost(call.usage, pricing);
|
|
277
|
-
const prefixWarning = (pricing && matchedKey !== call.model) ?
|
|
278
|
-
`priced as ${matchedKey}` :
|
|
279
|
-
undefined;
|
|
280
|
-
if (!pricing) {
|
|
281
|
-
unknownModels.add(call.model);
|
|
282
|
-
}
|
|
283
|
-
results.push({
|
|
284
|
-
step: call.stepName,
|
|
285
|
-
model: call.model,
|
|
286
|
-
input: call.usage.inputTokens ?? 0,
|
|
287
|
-
output: call.usage.outputTokens ?? 0,
|
|
288
|
-
cached: call.usage.cachedInputTokens ?? 0,
|
|
289
|
-
reasoning: call.usage.reasoningTokens ?? 0,
|
|
290
|
-
cost,
|
|
291
|
-
warning: warning ?? prefixWarning
|
|
292
|
-
});
|
|
293
|
-
totals.inputTokens += call.usage.inputTokens ?? 0;
|
|
294
|
-
totals.outputTokens += call.usage.outputTokens ?? 0;
|
|
295
|
-
totals.cachedTokens += call.usage.cachedInputTokens ?? 0;
|
|
296
|
-
totals.reasoningTokens += call.usage.reasoningTokens ?? 0;
|
|
297
|
-
totals.cost += cost;
|
|
298
|
-
}
|
|
299
|
-
return {
|
|
300
|
-
results,
|
|
301
|
-
totalInputTokens: totals.inputTokens,
|
|
302
|
-
totalOutputTokens: totals.outputTokens,
|
|
303
|
-
totalCachedTokens: totals.cachedTokens,
|
|
304
|
-
totalReasoningTokens: totals.reasoningTokens,
|
|
305
|
-
llmTotalCost: totals.cost,
|
|
306
|
-
unknownModels: [...unknownModels]
|
|
307
|
-
};
|
|
308
|
-
}
|
|
309
|
-
export function calculateCost(trace, config, traceFile = '') {
|
|
310
|
-
const llmCalls = findLLMCalls(trace);
|
|
311
|
-
const httpCalls = findHTTPCalls(trace);
|
|
312
|
-
const serviceResults = {};
|
|
313
|
-
for (const call of httpCalls) {
|
|
314
|
-
if (call.status && call.status >= 400) {
|
|
315
|
-
continue;
|
|
316
|
-
}
|
|
317
|
-
const serviceInfo = identifyService(call, config.services);
|
|
318
|
-
if (!serviceInfo) {
|
|
319
|
-
continue;
|
|
320
|
-
}
|
|
321
|
-
if (serviceInfo.config.type === 'response_cost') {
|
|
322
|
-
const hasCostData = extractValue(call, serviceInfo.config.cost_path);
|
|
323
|
-
const isBillableMethod = serviceInfo.config.billable_method &&
|
|
324
|
-
call.method === serviceInfo.config.billable_method;
|
|
325
|
-
if (!hasCostData && !isBillableMethod) {
|
|
326
|
-
continue;
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
const result = calculateServiceCost(call, serviceInfo);
|
|
330
|
-
if (!serviceResults[serviceInfo.serviceName]) {
|
|
331
|
-
serviceResults[serviceInfo.serviceName] = {
|
|
332
|
-
serviceName: serviceInfo.serviceName,
|
|
333
|
-
calls: [],
|
|
334
|
-
totalCost: 0
|
|
335
|
-
};
|
|
336
|
-
}
|
|
337
|
-
serviceResults[serviceInfo.serviceName].calls.push(result);
|
|
338
|
-
serviceResults[serviceInfo.serviceName].totalCost += result.cost;
|
|
339
|
-
}
|
|
340
|
-
const { results: llmResults, totalInputTokens, totalOutputTokens, totalCachedTokens, totalReasoningTokens, llmTotalCost, unknownModels } = aggregateLLMCosts(llmCalls, config);
|
|
341
|
-
const serviceTotalCost = Object.values(serviceResults).reduce((sum, s) => sum + s.totalCost, 0);
|
|
342
|
-
const totalCost = llmTotalCost + serviceTotalCost;
|
|
343
|
-
const durationMs = trace.endedAt && trace.startedAt ? trace.endedAt - trace.startedAt : null;
|
|
344
|
-
return {
|
|
345
|
-
traceFile,
|
|
346
|
-
workflowName: trace.name || 'unknown',
|
|
347
|
-
durationMs,
|
|
348
|
-
llmCalls: llmResults,
|
|
349
|
-
llmTotalCost,
|
|
350
|
-
totalInputTokens,
|
|
351
|
-
totalOutputTokens,
|
|
352
|
-
totalCachedTokens,
|
|
353
|
-
totalReasoningTokens,
|
|
354
|
-
unknownModels,
|
|
355
|
-
services: Object.values(serviceResults),
|
|
356
|
-
serviceTotalCost,
|
|
357
|
-
totalCost
|
|
358
|
-
};
|
|
359
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|