openmerit 0.1.12 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +11 -27
- package/dist/pi/scout-extension.mjs +23 -10
- package/dist/terminal/scout-update.json +1 -1
- package/dist/terminal/src/browser.mjs +2 -4
- package/dist/terminal/src/cli.mjs +6 -56
- package/dist/terminal/src/install.mjs +3 -2
- package/dist/terminal/src/scout-instructions.js +23 -10
- package/dist/terminal/src/scout-pi.js +1 -1
- package/dist/terminal/src/scout.js +181 -0
- package/dist/terminal/src/telemetry.mjs +8 -60
- package/docs/agent-evaluations.md +8 -148
- package/docs/commands.md +4 -86
- package/docs/getting-started.md +2 -16
- package/docs/index.md +4 -27
- package/docs/inference-terminal.md +13 -655
- package/docs/pi-extension.md +1 -1
- package/docs/roadmap.md +4 -17
- package/docs/scout.md +34 -251
- package/docs/security.md +8 -33
- package/docs/testing.md +7 -47
- package/docs/troubleshooting.md +3 -3
- package/package.json +13 -46
- package/dist/core/src/index.d.ts +0 -101
- package/dist/core/src/index.js +0 -1649
- package/dist/core/src/store.d.ts +0 -35
- package/dist/core/src/store.js +0 -102
- package/dist/eval/src/adapters.mjs +0 -112
- package/dist/eval/src/cli.mjs +0 -151
- package/dist/eval/src/config.mjs +0 -108
- package/dist/eval/src/process.mjs +0 -44
- package/dist/eval/src/report.mjs +0 -59
- package/dist/eval/src/runner.mjs +0 -123
- package/dist/eval/src/scout.mjs +0 -48
- package/dist/protocol/src/index.d.ts +0 -484
- package/dist/protocol/src/index.js +0 -47
- package/dist/protocol/src/schemas.d.ts +0 -576
- package/dist/protocol/src/schemas.js +0 -280
- package/dist/terminal/public/app.js +0 -298
- package/dist/terminal/public/brands/anthropic.png +0 -0
- package/dist/terminal/public/brands/baai.png +0 -0
- package/dist/terminal/public/brands/baseten.png +0 -0
- package/dist/terminal/public/brands/cerebras.png +0 -0
- package/dist/terminal/public/brands/cohere.png +0 -0
- package/dist/terminal/public/brands/deepseek.ico +0 -0
- package/dist/terminal/public/brands/google.png +0 -0
- package/dist/terminal/public/brands/groq.ico +0 -0
- package/dist/terminal/public/brands/huggingface.svg +0 -8
- package/dist/terminal/public/brands/lm-studio.png +0 -0
- package/dist/terminal/public/brands/meta.ico +0 -0
- package/dist/terminal/public/brands/mistral.png +0 -0
- package/dist/terminal/public/brands/nomic.png +0 -0
- package/dist/terminal/public/brands/ollama.png +0 -0
- package/dist/terminal/public/brands/openai.png +0 -0
- package/dist/terminal/public/brands/openrouter.png +0 -0
- package/dist/terminal/public/brands/qwen.png +0 -0
- package/dist/terminal/public/brands/vllm.ico +0 -0
- package/dist/terminal/public/brands/vllm.png +0 -0
- package/dist/terminal/public/favicon.svg +0 -1
- package/dist/terminal/public/flow.css +0 -1
- package/dist/terminal/public/flow.js +0 -770
- package/dist/terminal/public/index.html +0 -22
- package/dist/terminal/public/styles.css +0 -1046
- package/dist/terminal/src/activity-merge.mjs +0 -66
- package/dist/terminal/src/collect.mjs +0 -370
- package/dist/terminal/src/coverage.mjs +0 -33
- package/dist/terminal/src/demo-policy.mjs +0 -205
- package/dist/terminal/src/discovery.mjs +0 -99
- package/dist/terminal/src/enterprise-sample.mjs +0 -151
- package/dist/terminal/src/hardware.mjs +0 -57
- package/dist/terminal/src/history.mjs +0 -32
- package/dist/terminal/src/log-continuity.mjs +0 -26
- package/dist/terminal/src/project-activity.mjs +0 -157
- package/dist/terminal/src/repository-background.mjs +0 -37
- package/dist/terminal/src/repository-index.mjs +0 -267
- package/dist/terminal/src/repository-policy.mjs +0 -3
- package/dist/terminal/src/repository-syntax.mjs +0 -236
- package/dist/terminal/src/repository-worker.mjs +0 -15
- package/dist/terminal/src/sample.mjs +0 -171
- package/dist/terminal/src/scout-credentials.d.ts +0 -12
- package/dist/terminal/src/scout-credentials.ts +0 -44
- package/dist/terminal/src/scout-instructions.d.ts +0 -5
- package/dist/terminal/src/scout-instructions.ts +0 -106
- package/dist/terminal/src/scout-pi.d.ts +0 -44
- package/dist/terminal/src/scout-pi.ts +0 -76
- package/dist/terminal/src/server.mjs +0 -95
- package/dist/terminal/src/service-registry.mjs +0 -78
- package/dist/terminal/src/services.mjs +0 -44
- package/dist/terminal/src/topology.mjs +0 -30
- package/dist/terminal/src/updates.d.ts +0 -62
- package/dist/terminal/src/updates.ts +0 -313
- package/docs/adapter-guide.md +0 -191
- package/docs/architecture.md +0 -61
- package/docs/automation.md +0 -76
- package/docs/budgets.md +0 -39
- package/docs/demo-backfill.md +0 -31
- package/docs/demo-fieldkit.md +0 -49
- package/docs/demo-placement.md +0 -32
- package/docs/demo-spam.md +0 -17
- package/docs/demo-support.md +0 -44
- package/docs/demo.md +0 -74
- package/docs/first-trial.md +0 -62
- package/docs/interactive-demo.md +0 -118
- package/docs/lifecycle.md +0 -32
- package/docs/metrics-and-evidence.md +0 -48
- package/docs/operations.md +0 -40
- package/docs/pareto-spec.md +0 -76
- package/docs/ux-reference.md +0 -32
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
// A reviewed model/service-offer catalog for the hosted fictional comparison only.
|
|
2
|
-
// Scores are publisher-reported point estimates, not insurance-workload evaluations.
|
|
3
|
-
export const EVIDENCE_DATE = '2026-10-10';
|
|
4
|
-
export const PREFERENCES = [
|
|
5
|
-
{ value: 'default', label: 'Default · within your stack', note: 'Search all configured services; choose the lowest modeled cost among qualified replacements.' },
|
|
6
|
-
{ value: 'zdr', label: 'Bias ZDR models', note: 'Prefer confirmed ZDR endpoints. Eligibility alone does not establish account approval or enabled settings.' },
|
|
7
|
-
{ value: 'open', label: 'Bias open-weights models', note: 'Prefer models with open weights.' },
|
|
8
|
-
{ value: 'closed', label: 'No open-weights models', note: 'Require a closed-weight assignment for every workload.' },
|
|
9
|
-
{ value: 'usa', label: 'Bias USA models', note: 'Prefer models from US developers. This does not specify hosting location or data residency.' },
|
|
10
|
-
{ value: 'china', label: 'Bias Chinese models', note: 'Prefer models from Chinese developers. This does not specify hosting location or data residency.' },
|
|
11
|
-
];
|
|
12
|
-
export const preferenceFor = value => PREFERENCES.find(item => item.value === value) ?? PREFERENCES[0];
|
|
13
|
-
const oa = 'https://openai.com/index/gpt-4-1/';
|
|
14
|
-
const claude = 'https://www-cdn.anthropic.com/bbd8ef16d70b7a1665f14f306ee88b53f686aa75/Claude%20Sonnet%204.6%20System%20Card.pdf';
|
|
15
|
-
const gemini = 'https://storage.googleapis.com/deepmind-media/gemini/gemini_3_flash_model_evaluation.pdf';
|
|
16
|
-
const metric = (axis, name, value, suite, source, settings) => ({ axis, name, value, suite, source, settings, unit: '%', direction: 'higher' });
|
|
17
|
-
const oaScores = (tool, capability, reasoning) => [
|
|
18
|
-
metric('tools', 'τ-bench retail', tool, 'openai-4.1-tau-retail-gpt4o-user', oa, '5-run average; original GPT-4o user simulator; no custom tools or prompting.'),
|
|
19
|
-
metric('capabilities', 'MMLU', capability, 'openai-4.1-mmlu', oa, 'OpenAI GPT-4.1 release comparison; GPT-4o snapshot 2024-11-20.'),
|
|
20
|
-
metric('reasoning', 'AIME 2024', reasoning, 'openai-4.1-aime24', oa, 'OpenAI GPT-4.1 release comparison, AIME 2024.'),
|
|
21
|
-
];
|
|
22
|
-
const claudeScores = (tool, capability, reasoning) => [
|
|
23
|
-
metric('tools', 'τ²-bench retail', tool, 'anthropic-sonnet46-tau2-retail', claude, 'Sonnet 4.6 system card §2.5; max effort, 10 trials for Sonnet; published Opus configuration.'),
|
|
24
|
-
metric('capabilities', 'MMMU-Pro · no tools', capability, 'anthropic-sonnet46-mmmupro-notools', claude, 'Sonnet 4.6 system card Table 2.1.A; publisher model configurations.'),
|
|
25
|
-
metric('reasoning', 'GPQA Diamond', reasoning, 'anthropic-sonnet46-gpqa', claude, 'Sonnet 4.6 system card §2.8; adaptive thinking, max effort for Sonnet; published Opus configuration.'),
|
|
26
|
-
];
|
|
27
|
-
const googleScores = (tool, capability, reasoning) => [
|
|
28
|
-
metric('tools', 'τ²-bench', tool, 'google-3flash-tau2-fixed-airline', gemini, 'Gemini 3 Pro user simulator; fixed Airline domain; mean of Retail, Airline and Telecom.'),
|
|
29
|
-
metric('capabilities', 'MMMU-Pro', capability, 'google-3flash-mmmupro', gemini, 'Thinking; mean of Standard and Vision settings; no tools.'),
|
|
30
|
-
metric('reasoning', 'GPQA Diamond', reasoning, 'google-3flash-gpqa', gemini, 'Thinking; no tools; Google December 2025 comparison.'),
|
|
31
|
-
];
|
|
32
|
-
const ossCard = 'https://arxiv.org/html/2508.10925v1';
|
|
33
|
-
const ossScores = (tool, capability, reasoning) => [
|
|
34
|
-
metric('tools', 'τ-bench retail', tool, 'openai-oss-high-retail', ossCard, 'Table 3; high reasoning effort; developer function calling.'),
|
|
35
|
-
metric('capabilities', 'MMLU', capability, 'openai-oss-high-mmlu', ossCard, 'Table 3; high reasoning effort, default system prompt, pass@1.'),
|
|
36
|
-
metric('reasoning', 'GPQA Diamond · no tools', reasoning, 'openai-oss-high-gpqa', ossCard, 'Table 3; high reasoning effort, no tools, pass@1.'),
|
|
37
|
-
];
|
|
38
|
-
const dsCard = 'https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash';
|
|
39
|
-
const dsScores = (tool, capability, reasoning) => [
|
|
40
|
-
metric('tools', 'AutomationBench', tool, 'deepseek41-automation', dsCard, 'Frontier instruct-model comparison; official scaffold, maximum reasoning effort, temperature 1, top-p .95.'),
|
|
41
|
-
metric('capabilities', "Agent’s Last Exam", capability, 'deepseek41-ale', dsCard, 'Frontier instruct-model comparison; broad agent tasks, official scaffold, maximum reasoning effort.'),
|
|
42
|
-
metric('reasoning', 'GPQA Diamond', reasoning, 'deepseek41-gpqa', dsCard, 'Frontier instruct-model comparison; pass@1, maximum reasoning effort. Published model scores, not endpoint measurements.'),
|
|
43
|
-
];
|
|
44
|
-
const qwenCard = 'https://huggingface.co/Qwen/Qwen3.6-27B';
|
|
45
|
-
const qwenScores = (tool, pass3, capability, reasoning) => [
|
|
46
|
-
metric('tools', 'ClawEval · average', tool, 'qwen36-claw-avg', qwenCard, 'Qwen 3.6 27B release comparison; published model configurations.'),
|
|
47
|
-
metric('tools', 'ClawEval · pass³', pass3, 'qwen36-claw-pass3', qwenCard, 'Same comparison, stricter repeated-task success. Both tool metrics must pass.'),
|
|
48
|
-
metric('capabilities', 'MMMU-Pro', capability, 'qwen36-mmmupro', qwenCard, 'Vision benchmark; published Qwen 3.6 comparison configurations.'),
|
|
49
|
-
metric('reasoning', 'GPQA Diamond', reasoning, 'qwen36-gpqa', qwenCard, 'Published Qwen 3.6 comparison configurations.'),
|
|
50
|
-
];
|
|
51
|
-
const cohortScores = (source, suite, names, values, settings) => ['tools', 'capabilities', 'reasoning'].map((axis, index) => metric(axis, names[index], values[index], `${suite}-${axis}`, source, settings));
|
|
52
|
-
const flashCard = 'https://huggingface.co/zai-org/GLM-5.3-Flash';
|
|
53
|
-
const flashScores = values => cohortScores(flashCard, 'glm53flash-release', ['AutomationBench v1.0.6', 'GDPval-AA v2', 'HLE · with tools'], values, 'Published release chart and footnotes; maximum effort; GDPval-AA rating, not percent.').map(item => item.axis === 'capabilities' ? { ...item, unit: ' rating' } : item);
|
|
54
|
-
const qwen38Card = 'https://huggingface.co/Qwen/Qwen3.8-27B';
|
|
55
|
-
const qwen38Scores = values => cohortScores(qwen38Card, 'qwen38-release', ['ClawEval-MM · average', 'IFBench', 'GPQA Diamond'], values, 'Qwen 3.8 model card comparison; published thinking configurations. ClawEval-MM is distinct from ClawEval.');
|
|
56
|
-
// The application-code profile screens generated SQL/Python used by the
|
|
57
|
-
// insurance product. These are application workers, not developer-harness usage.
|
|
58
|
-
// Keep publisher cohorts separate: LiveCodeBench v6 % is not LCB Pro Elo.
|
|
59
|
-
const glm47Card = 'https://huggingface.co/zai-org/GLM-4.7';
|
|
60
|
-
const glmCodeScores = values => cohortScores(glm47Card, 'glm47-application-code', ['τ²-bench', 'LiveCodeBench v6', 'GPQA Diamond'], values, 'GLM-4.7 release table; thinking enabled, preserved thinking for multi-turn tools; adjusted user prompts and fixed Airline domain. Published comparator configurations; not endpoint parity.');
|
|
61
|
-
const googleCodeScores = values => cohortScores(gemini, 'google-3flash-application-code', ['τ²-bench', 'LiveCodeBench Pro', 'GPQA Diamond'], values, 'December 2025 comparison: Gemini thinking, Sonnet 4.5 high/best available thinking. LCB Pro official leaderboard Elo; τ² fixed Airline, Gemini user simulator for Gemini and publisher-reported Sonnet score. Public comparison, not a controlled endpoint evaluation.').map(item => item.axis === 'capabilities' ? { ...item, unit: ' Elo' } : item);
|
|
62
|
-
const basetenPolicy = { confirmedZdr: true, policySource: 'https://www.baseten.co/security-practices/', endpointMode: 'Synchronous Model API; request/response logging off' };
|
|
63
|
-
const model = (id, name, provider, input, output, source, benchmarks = [], extra = {}) => ({ id, modelId: id, name, provider, input, output, source, benchmarks, creator: provider, origin: 'US', openWeights: false, outputType: 'text', confirmedZdr: false, ...extra });
|
|
64
|
-
const openai = name => `https://developers.openai.com/api/docs/models/${name}`;
|
|
65
|
-
const anthropic = 'https://platform.claude.com/docs/en/about-claude/pricing';
|
|
66
|
-
const google = 'https://ai.google.dev/gemini-api/docs/pricing';
|
|
67
|
-
const groq = 'https://console.groq.com/docs/models';
|
|
68
|
-
const baseten = 'https://www.baseten.co/pricing/';
|
|
69
|
-
const mistral = 'https://docs.mistral.ai/inference/pricing';
|
|
70
|
-
export const CATALOG = [
|
|
71
|
-
model('gpt4o', 'GPT-4o', 'OpenAI', 2.5, 10, openai('gpt-4o'), oaScores(60.3, 85.7, 13.1)),
|
|
72
|
-
model('gpt41', 'GPT-4.1', 'OpenAI', 2, 8, openai('gpt-4.1'), oaScores(68, 90.2, 48.1)),
|
|
73
|
-
model('gpt41mini', 'GPT-4.1 mini', 'OpenAI', .4, 1.6, openai('gpt-4.1-mini'), oaScores(55.8, 87.5, 49.6)),
|
|
74
|
-
model('gpt54', 'GPT-5.4', 'OpenAI', 2.5, 15, openai('gpt-5.4')),
|
|
75
|
-
model('gpt54mini', 'GPT-5.4 mini', 'OpenAI', .75, 4.5, openai('gpt-5.4-mini')),
|
|
76
|
-
model('opus45', 'Claude Opus 4.5', 'Anthropic', 5, 25, anthropic, [...claudeScores(88.9, 70.6, 87), ...qwenScores(76.6, 59.6, 70.6, 87)]),
|
|
77
|
-
model('sonnet46', 'Claude Sonnet 4.6', 'Anthropic', 3, 15, anthropic, claudeScores(91.7, 74.5, 89.9)),
|
|
78
|
-
model('sonnet45', 'Claude Sonnet 4.5', 'Anthropic', 3, 15, anthropic, [...glmCodeScores([87.2, 64.0, 83.4]), ...googleCodeScores([87.2, 1418, 83.4])], { configuration: 'Thinking enabled; published comparison configurations' }),
|
|
79
|
-
model('haiku45', 'Claude Haiku 4.5', 'Anthropic', 1, 5, anthropic),
|
|
80
|
-
model('gemini25pro', 'Gemini 2.5 Pro', 'Google', 1.25, 10, google, googleScores(77.8, 68, 86.4)),
|
|
81
|
-
model('gemini25flash', 'Gemini 2.5 Flash', 'Google', .3, 2.5, google, googleScores(79.5, 66.7, 82.8)),
|
|
82
|
-
model('gemini3flash', 'Gemini 3 Flash Preview', 'Google', .5, 3, google, [...googleScores(90.2, 81.2, 90.4), ...googleCodeScores([90.2, 2316, 90.4])]),
|
|
83
|
-
model('oss120', 'gpt-oss-120b', 'Groq', .15, .6, groq, ossScores(67.8, 90, 80.1), { creator: 'OpenAI', openWeights: true, configuration: 'High reasoning effort' }),
|
|
84
|
-
model('oss20', 'gpt-oss-20b', 'Groq', .075, .3, groq, ossScores(54.8, 85.3, 71.5), { creator: 'OpenAI', openWeights: true, configuration: 'High reasoning effort' }),
|
|
85
|
-
model('glm53', 'GLM-5.3', 'Baseten', 1.4, 4.4, baseten, dsScores(48.8, 28.5, 88.1), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Maximum reasoning effort' }),
|
|
86
|
-
model('glm53flash', 'GLM-5.3-Flash', 'Baseten', .15, .5, baseten, flashScores([48.8, 1773, 55.3]), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy }),
|
|
87
|
-
model('mistral35', 'Mistral Medium 3.5', 'Mistral', 1.5, 7.5, mistral, [], { origin: 'FR', openWeights: true, metadataSource: 'https://docs.mistral.ai/models/mistral-medium-3-5-26-04' }),
|
|
88
|
-
model('mistral4', 'Mistral Small 4', 'Mistral', .15, .6, mistral, [], { origin: 'FR', openWeights: true }),
|
|
89
|
-
model('oss120baseten', 'gpt-oss-120b', 'Baseten', .1, .5, baseten, ossScores(67.8, 90, 80.1), { modelId: 'oss120', creator: 'OpenAI', openWeights: true, ...basetenPolicy, configuration: 'High reasoning effort; same published checkpoint' }),
|
|
90
|
-
model('deepseek41flash', 'DeepSeek V4.1 Flash', 'Baseten', .3, 1.2, baseten, dsScores(54.8, 31.8, 90.9), { creator: 'DeepSeek', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Maximum reasoning effort (100)' }),
|
|
91
|
-
model('deepseek4pro0813', 'DeepSeek V4 Pro 0813', 'Baseten', 1.32, 3.96, baseten, [], { creator: 'DeepSeek', origin: 'CN', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813' }),
|
|
92
|
-
model('nemotron3ultra', 'Nemotron 3 Ultra', 'Baseten', .6, 2.4, baseten, cohortScores('https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16', 'nvidia-ultra-nemo', ['TauBench V3 · average', 'MMLU-Pro', 'GPQA · no tools'], [70.9, 86.8, 87.0], 'NVIDIA NeMo evaluation SDK; model-card configurations. TauBench V3 is not τ-bench or τ²-bench.'), { creator: 'NVIDIA', origin: 'US', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16' }),
|
|
93
|
-
model('glm52', 'GLM-5.2', 'Baseten', 1.4, 4.4, baseten, flashScores([26.2, 1504, 54.7]), { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, metadataSource: 'https://huggingface.co/zai-org/GLM-5.2' }),
|
|
94
|
-
model('glm47', 'GLM-4.7', 'Baseten', .6, 2.2, 'https://www.baseten.co/library/glm-4-7/', [...cohortScores('https://huggingface.co/zai-org/GLM-4.7', 'glm47-release', ['τ²-bench', 'MMLU-Pro', 'GPQA Diamond'], [87.4, 84.3, 85.7], 'GLM-4.7 release configurations; preserved thinking, adjusted user prompts and fixed Airline domain for τ²-bench.'), ...glmCodeScores([87.4, 84.9, 85.7])], { creator: 'Z.ai', origin: 'CN', openWeights: true, ...basetenPolicy, configuration: 'Thinking enabled', metadataSource: 'https://huggingface.co/zai-org/GLM-4.7' }),
|
|
95
|
-
model('qwen36', 'Qwen 3.6 · 27B', 'Groq', .6, 3, groq, [...qwenScores(72.4, 60.6, 75.8, 87.8), ...qwen38Scores([50.4, 69.1, 87.8])], { creator: 'Alibaba', origin: 'CN', openWeights: true, availability: 'Preview', metadataSource: qwenCard }),
|
|
96
|
-
model('qwen38', 'Qwen 3.8 · 27B', 'Groq', .8, 4, groq, qwen38Scores([56.9, 79.5, 89.2]), { creator: 'Alibaba', origin: 'CN', openWeights: true, availability: 'Preview', metadataSource: 'https://huggingface.co/Qwen/Qwen3.8-27B' }),
|
|
97
|
-
model('jev', 'Jev 1.13', 'TypeSafe', .042, 0, 'https://docs.typesafe.ai/models', [], { outputType: 'decision' }),
|
|
98
|
-
];
|
|
99
|
-
export const CONFIGURED_SERVICES = ['OpenAI', 'Anthropic', 'Google', 'Groq', 'Baseten', 'Mistral'];
|
|
100
|
-
const byId = id => CATALOG.find(item => item.id === id);
|
|
101
|
-
// Every workload is fictional application work. Names are plausible declarations,
|
|
102
|
-
// never represented as discoveries from a visitor's repository.
|
|
103
|
-
const APPLICATION_CODE_PROFILES = {
|
|
104
|
-
'actuarial.queries': { input: 14000, output: 700 },
|
|
105
|
-
'portfolio.reporting': { input: 18000, output: 900 },
|
|
106
|
-
'claims.reconciliation': { input: 5000, output: 2000 },
|
|
107
|
-
'reserves.analysis': { input: 6000, output: 2400 },
|
|
108
|
-
};
|
|
109
|
-
export const WORKLOADS = [
|
|
110
|
-
['claims.intake', 'Claim intake', 'gpt4o', ['parse_notice', 'extract_loss_date', 'normalize_address', 'link_policy', 'check_attachments', 'summarize_notice', 'prepare_case']],
|
|
111
|
-
['policy.extraction', 'Policy document extraction', 'gpt54', ['read_schedule', 'extract_limits', 'extract_deductibles', 'identify_endorsements', 'normalize_terms', 'link_sections', 'check_completeness']],
|
|
112
|
-
['coverage.review', 'Coverage review support', 'opus45', ['retrieve_wording', 'compare_endorsements', 'locate_exclusions', 'cite_conditions', 'summarize_scope', 'flag_ambiguity', 'prepare_review']],
|
|
113
|
-
['claims.correspondence', 'Claim correspondence', 'sonnet46', ['draft_acknowledgment', 'request_documents', 'summarize_progress', 'draft_follow_up', 'check_tone', 'cite_case_notes', 'prepare_handoff']],
|
|
114
|
-
['documents.verification', 'Document verification', 'gemini25pro', ['read_estimate', 'extract_line_items', 'match_invoices', 'check_totals', 'compare_versions', 'flag_missing_pages', 'prepare_evidence']],
|
|
115
|
-
['renewals.summary', 'Renewal preparation', 'gemini25flash', ['summarize_history', 'compare_schedules', 'list_changes', 'check_dates', 'extract_terms', 'prepare_brief', 'draft_reminder']],
|
|
116
|
-
['claims.tagging', 'Claim tagging', 'oss120', ['tag_document', 'label_channel', 'extract_reference', 'normalize_category', 'link_attachment', 'label_language', 'index_summary']],
|
|
117
|
-
['fraud.review', 'Fraud review support', 'glm53', ['summarize_signals', 'link_evidence', 'compare_statements', 'flag_inconsistency', 'build_timeline', 'prepare_case_note', 'route_to_reviewer']],
|
|
118
|
-
['broker.translation', 'Broker correspondence', 'mistral35', ['detect_language', 'translate_email', 'preserve_policy_terms', 'check_names', 'summarize_request', 'draft_reply', 'prepare_review']],
|
|
119
|
-
['claims.routing', 'Decision routing', 'gpt4o', ['route_notice', 'rank_queue', 'classify_document', 'score_completeness', 'flag_duplicates', 'choose_workflow', 'route_exception']],
|
|
120
|
-
['actuarial.queries', 'Actuarial query generation', 'sonnet45', ['read_schema', 'select_tables', 'generate_query', 'check_join_keys', 'validate_query', 'explain_result', 'prepare_review']],
|
|
121
|
-
['claims.reconciliation', 'Claim reconciliation scripts', 'sonnet45', ['read_line_items', 'normalize_columns', 'generate_transform', 'compare_totals', 'build_checks', 'explain_variance', 'prepare_review']],
|
|
122
|
-
['portfolio.reporting', 'Portfolio reporting queries', 'sonnet45', ['read_report_spec', 'map_dimensions', 'generate_query', 'check_aggregates', 'validate_periods', 'explain_result', 'prepare_report']],
|
|
123
|
-
['reserves.analysis', 'Reserve analysis scripts', 'sonnet45', ['read_triangle', 'normalize_periods', 'generate_calculation', 'check_assumptions', 'build_sensitivity', 'explain_projection', 'prepare_review']],
|
|
124
|
-
].map(([id, purpose, before, tasks]) => ({
|
|
125
|
-
id, purpose, before: byId(before), configuredServices: CONFIGURED_SERVICES, tasks,
|
|
126
|
-
benchmarkProfile: APPLICATION_CODE_PROFILES[id] ? 'Application code · tools, code generation, reasoning' : 'General application · tools, capabilities, reasoning',
|
|
127
|
-
tokenProfile: APPLICATION_CODE_PROFILES[id] ?? null,
|
|
128
|
-
requiredAxes: ['tools', 'capabilities', 'reasoning', ...(id === 'claims.routing' ? ['decisions'] : [])],
|
|
129
|
-
}));
|
|
130
|
-
|
|
131
|
-
export function modeledCost(model, usage) {
|
|
132
|
-
return model && [model.input, model.output, usage.inputTokens, usage.outputTokens].every(value => Number.isFinite(value) && value >= 0)
|
|
133
|
-
? (model.input * usage.inputTokens + model.output * usage.outputTokens) / 1e6 : null;
|
|
134
|
-
}
|
|
135
|
-
export function assessReplacement(workload, candidate, usage) {
|
|
136
|
-
const before = workload.before;
|
|
137
|
-
const checks = workload.requiredAxes.flatMap(axis => axis === 'decisions' ? ['decisionAccuracy', 'calibration'] : [axis]).flatMap(axis => {
|
|
138
|
-
const baselines = before.benchmarks.filter(item => item.axis === axis);
|
|
139
|
-
const pairs = baselines.flatMap(a => candidate.benchmarks.filter(b => b.axis === axis && a.suite === b.suite && a.name === b.name && a.unit === b.unit && a.direction === b.direction).map(b => ({ a, b })));
|
|
140
|
-
if (!pairs.length) return [{ axis, before: baselines[0] ?? null, after: candidate.benchmarks.find(item => item.axis === axis) ?? null, status: 'missing' }];
|
|
141
|
-
// Check every matched metric, not the first passing score in a cohort.
|
|
142
|
-
return pairs.map(({ a, b }) => {
|
|
143
|
-
const comparable = Boolean(a.source && b.source && Number.isFinite(a.value) && Number.isFinite(b.value));
|
|
144
|
-
const pass = comparable && (a.direction === 'lower' ? b.value <= a.value : b.value >= a.value);
|
|
145
|
-
return { axis, before: a, after: b, status: !comparable ? 'missing' : pass ? 'pass' : 'regression' };
|
|
146
|
-
});
|
|
147
|
-
});
|
|
148
|
-
const beforeCost = modeledCost(before, usage), afterCost = modeledCost(candidate, usage);
|
|
149
|
-
const cheaper = beforeCost !== null && afterCost !== null && afterCost < beforeCost;
|
|
150
|
-
// Only paired latency measurements with the same workload/configuration qualify.
|
|
151
|
-
// No latency data is fabricated or inferred from token generation speed in this demo.
|
|
152
|
-
const a = before.latency, b = candidate.latency;
|
|
153
|
-
const faster = Boolean(a && b && a.suite && a.metric && a.suite === b.suite && a.metric === b.metric && a.source && b.source && Number.isFinite(a.value) && Number.isFinite(b.value) && b.value > 0 && b.value < a.value);
|
|
154
|
-
const compatible = (workload.configuredServices ?? [before.provider]).includes(candidate.provider) && candidate.outputType === before.outputType && afterCost !== null;
|
|
155
|
-
const issues = [];
|
|
156
|
-
if (!(workload.configuredServices ?? [before.provider]).includes(candidate.provider)) issues.push('Service is not configured in this environment');
|
|
157
|
-
if (candidate.outputType !== before.outputType) issues.push('Different output contract; application adaptation required');
|
|
158
|
-
if (checks.some(check => check.status === 'missing')) issues.push('Comparable benchmark evidence is incomplete');
|
|
159
|
-
if (checks.some(check => check.status === 'regression')) issues.push('A required benchmark regresses');
|
|
160
|
-
if (afterCost === null) issues.push('Token price is unavailable');
|
|
161
|
-
if (!cheaper && !faster) issues.push('No verified cost or latency improvement');
|
|
162
|
-
return { candidate, checks, cheaper, faster, beforeCost, afterCost, compatible, issues, eligible: issues.length === 0 };
|
|
163
|
-
}
|
|
164
|
-
export function matchesPreference(model, preference) {
|
|
165
|
-
return preference === 'zdr' ? model.confirmedZdr === true : preference === 'open' ? model.openWeights === true : preference === 'closed' ? model.openWeights === false : preference === 'usa' ? model.origin === 'US' : preference === 'china' ? model.origin === 'CN' : true;
|
|
166
|
-
}
|
|
167
|
-
export function selectReplacement(workload, usage, preference = 'default') {
|
|
168
|
-
preference = preferenceFor(preference).value;
|
|
169
|
-
const assessments = (workload.candidates ?? CATALOG).filter(candidate => candidate.id !== workload.before.id).map(candidate => assessReplacement(workload, candidate, usage));
|
|
170
|
-
const allowed = item => item.eligible && (preference !== 'closed' || item.candidate.openWeights === false);
|
|
171
|
-
const byCost = (a, b) => (a.afterCost ?? Infinity) - (b.afterCost ?? Infinity) || a.candidate.id.localeCompare(b.candidate.id);
|
|
172
|
-
const qualified = assessments.filter(allowed).sort(byCost);
|
|
173
|
-
const defaultChoice = assessments.filter(item => item.eligible).sort(byCost)[0]?.candidate ?? workload.before;
|
|
174
|
-
// A preference first seeks a fully qualified match. Retaining an already
|
|
175
|
-
// matching baseline needs no exception. Only then relax benchmark/efficiency
|
|
176
|
-
// criteria, never the configured service, output contract or known pricing.
|
|
177
|
-
let selected = preference === 'default' ? qualified[0] : qualified.find(item => matchesPreference(item.candidate, preference));
|
|
178
|
-
if (!selected && preference !== 'default' && !matchesPreference(workload.before, preference)) {
|
|
179
|
-
const axes = (item, status) => new Set(item.checks.filter(check => check.status === status).map(check => check.axis)).size;
|
|
180
|
-
selected = assessments.filter(item => item.compatible && matchesPreference(item.candidate, preference)).sort((a, b) =>
|
|
181
|
-
axes(a, 'regression') - axes(b, 'regression') || axes(a, 'missing') - axes(b, 'missing') ||
|
|
182
|
-
Number(!a.cheaper && !a.faster) - Number(!b.cheaper && !b.faster) || byCost(a, b))[0];
|
|
183
|
-
if (!selected && preference !== 'closed') selected = qualified[0];
|
|
184
|
-
}
|
|
185
|
-
const proposed = selected?.candidate ?? workload.before;
|
|
186
|
-
const unresolved = preference === 'closed' && proposed.openWeights !== false;
|
|
187
|
-
const after = unresolved ? null : proposed;
|
|
188
|
-
const override = Boolean(after && selected && !selected.eligible);
|
|
189
|
-
const overrideReasons = override ? selected.issues : [];
|
|
190
|
-
const serviceChanged = Boolean(after && after.provider !== workload.before.provider);
|
|
191
|
-
const modelChanged = Boolean(after && after.modelId !== workload.before.modelId);
|
|
192
|
-
return {
|
|
193
|
-
id: workload.id, purpose: workload.purpose, benchmarkProfile: workload.benchmarkProfile, tokenProfile: workload.tokenProfile, before: workload.before, after,
|
|
194
|
-
changed: Boolean(after && selected), serviceChanged, modelChanged,
|
|
195
|
-
checks: selected?.checks ?? [], assessments, qualifiedCount: qualified.length,
|
|
196
|
-
preferenceMatched: Boolean(after && matchesPreference(after, preference)),
|
|
197
|
-
preferenceChanged: Boolean(after && after.id !== defaultChoice.id),
|
|
198
|
-
defaultChoice: defaultChoice.id, unresolved, override, overrideReasons,
|
|
199
|
-
reason: unresolved ? 'No compatible closed-weight assignment in the reviewed catalog. Service, output contract and known pricing remain required.'
|
|
200
|
-
: override ? `Selected for ${preferenceFor(preference).label}. No preferred replacement passed every primary check.`
|
|
201
|
-
: selected ? `${modelChanged ? 'Selected benchmark scores match or improve' : 'Same model checkpoint and published benchmark scores; service changes'}; ${selected.cheaper ? 'modeled token cost is lower' : 'paired latency is lower'}.`
|
|
202
|
-
: preference !== 'default' && matchesPreference(workload.before, preference) ? 'Baseline retained: already matches this preference; no qualifying preferred replacement.'
|
|
203
|
-
: 'Baseline retained. No reviewed candidate passes every required check.',
|
|
204
|
-
};
|
|
205
|
-
}
|
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
import { open, realpath, stat } from 'node:fs/promises';
|
|
2
|
-
import { constants } from 'node:fs';
|
|
3
|
-
import { relative, resolve, sep, isAbsolute } from 'node:path';
|
|
4
|
-
import { configuredServices } from './services.mjs';
|
|
5
|
-
import { SERVICE_CREDENTIAL_FIELDS, SERVICE_ENDPOINT_FIELDS } from './service-registry.mjs';
|
|
6
|
-
|
|
7
|
-
const LIMIT = 128 * 1024;
|
|
8
|
-
const inside = (root, path) => { const part = relative(root, path); return part !== '..' && !part.startsWith(`..${sep}`) && !isAbsolute(part); };
|
|
9
|
-
const credentials = new Set(SERVICE_CREDENTIAL_FIELDS);
|
|
10
|
-
const endpoints = new Set([...SERVICE_ENDPOINT_FIELDS, 'OLLAMA_HOST', 'OLLAMA_BASE_URL']);
|
|
11
|
-
const sdks = new Set(['openai', '@anthropic-ai/sdk', '@google/genai', '@google/generative-ai', 'groq-sdk', '@cerebras/cerebras_cloud_sdk', '@lmstudio/sdk', 'ollama', 'ollama-js', 'ai', 'litellm', 'anthropic', 'groq', 'cerebras-cloud-sdk', 'baseten', '@huggingface/inference', 'huggingface_hub', 'huggingface-hub', '@mistralai/mistralai', 'mistralai', 'cohere-ai', 'cohere', 'together-ai', 'together', 'fireworks-ai', 'replicate', 'voyageai', 'portkey-ai', 'xai-sdk', '@aws-sdk/client-bedrock-runtime', '@aws-sdk/client-sagemaker-runtime', '@google-cloud/vertexai', 'google-cloud-aiplatform', '@azure/ai-inference', 'azure-ai-inference']);
|
|
12
|
-
|
|
13
|
-
// Parse literal assignments only. Never source a shell file, expand variables,
|
|
14
|
-
// return credential contents, or guess how the application loads its config.
|
|
15
|
-
export function envDeclarations(body) {
|
|
16
|
-
const result = {};
|
|
17
|
-
for (const line of body.split(/\r?\n/)) {
|
|
18
|
-
const match = line.match(/^\s*(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=\s*(.*?)\s*$/);
|
|
19
|
-
if (!match || !credentials.has(match[1]) && !endpoints.has(match[1])) continue;
|
|
20
|
-
let value = match[2];
|
|
21
|
-
if (/^["']/.test(value)) { const quote = value[0], end = value.indexOf(quote, 1); if (end < 0 || !/^\s*(?:#.*)?$/.test(value.slice(end + 1))) continue; value = value.slice(1, end); }
|
|
22
|
-
else value = value.replace(/\s+#.*$/, '').trim();
|
|
23
|
-
if (/[\x00-\x1f$`]/.test(value)) continue;
|
|
24
|
-
result[match[1]] = credentials.has(match[1]) ? (value ? 'present' : '') : value.slice(0, 2049);
|
|
25
|
-
}
|
|
26
|
-
return result;
|
|
27
|
-
}
|
|
28
|
-
export function loopbackEndpoint(value) {
|
|
29
|
-
try {
|
|
30
|
-
const url = new URL(value);
|
|
31
|
-
if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname) || url.username || url.password || url.search || url.hash || !['/', '/v1', '/v1/'].includes(url.pathname)) return null;
|
|
32
|
-
return url.origin.replace('://localhost', '://127.0.0.1');
|
|
33
|
-
} catch { return null; }
|
|
34
|
-
}
|
|
35
|
-
async function boundedFile(root, name) {
|
|
36
|
-
const path = await realpath(resolve(root, name));
|
|
37
|
-
if (!inside(await realpath(root), path)) throw new Error('Outside project');
|
|
38
|
-
if (!(await stat(path)).isFile()) throw new Error('Unsupported file');
|
|
39
|
-
const handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
|
|
40
|
-
try {
|
|
41
|
-
const info = await handle.stat();
|
|
42
|
-
if (!info.isFile() || info.size > LIMIT) throw new Error('Unsupported file');
|
|
43
|
-
const buffer = Buffer.alloc(LIMIT + 1), { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
|
44
|
-
if (bytesRead > LIMIT) throw new Error('Oversized file');
|
|
45
|
-
return { body: buffer.subarray(0, bytesRead).toString('utf8'), at: info.mtime.toISOString() };
|
|
46
|
-
} finally { await handle.close(); }
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export async function discoverProject(root) {
|
|
50
|
-
const sources = [], inventory = [], runtimes = new Map();
|
|
51
|
-
const local = (values, sourceId) => {
|
|
52
|
-
for (const field of endpoints) {
|
|
53
|
-
const nativeOllama = field === 'OLLAMA_HOST' || field === 'OLLAMA_BASE_URL';
|
|
54
|
-
const value = values[field];
|
|
55
|
-
const url = loopbackEndpoint(nativeOllama && value && !value.includes('://') ? `http://${value}` : value);
|
|
56
|
-
if (!url) continue;
|
|
57
|
-
// A named protocol declaration takes precedence over an API-compatible alias.
|
|
58
|
-
if (runtimes.get(url)?.type === 'ollama' && !nativeOllama) continue;
|
|
59
|
-
runtimes.set(url, { id: `discovered:${url}`, name: nativeOllama ? `Ollama · ${new URL(url).host}` : url, type: nativeOllama ? 'ollama' : 'discover', url, discoverySource: sourceId, discoveryField: field });
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
// Shell-wide endpoint variables are not evidence that this project uses a runtime.
|
|
63
|
-
// Only selected-project declarations and explicit terminal configuration authorize probes.
|
|
64
|
-
for (const name of ['.env', '.env.local', 'package.json', 'requirements.txt']) {
|
|
65
|
-
let file;
|
|
66
|
-
try { file = await boundedFile(root, name); }
|
|
67
|
-
catch (error) {
|
|
68
|
-
if (error.code !== 'ENOENT') sources.push({ id: `project:${name}`, name, kind: 'Project discovery', state: 'unavailable', detail: name, note: 'This supported file could not be read within the project and size limits.', coverage: ['Configuration'], access: 'Selected project' });
|
|
69
|
-
continue;
|
|
70
|
-
}
|
|
71
|
-
const id = `project:${name}`;
|
|
72
|
-
if (name.startsWith('.env')) {
|
|
73
|
-
const declarations = envDeclarations(file.body), identified = configuredServices(declarations);
|
|
74
|
-
local(declarations, id);
|
|
75
|
-
const unknown = identified.unrecognizedEndpoints.filter(field => !loopbackEndpoint(declarations[field]));
|
|
76
|
-
const fields = Object.keys(declarations);
|
|
77
|
-
if (!fields.length) continue;
|
|
78
|
-
for (const item of identified.services) inventory.push({ ...item, sourceId: id, evidence: item.evidence.map(entry => ({ ...entry, sourceId: id })) });
|
|
79
|
-
sources.push({ id, name, kind: 'Project discovery', state: unknown.length ? 'partial' : 'connected', detail: fields.join(' · '), updatedAt: new Date().toISOString(), modifiedAt: file.at, coverage: ['Declared services', 'Local endpoint candidates'], access: 'Selected project', note: 'Literal declarations only; this does not establish that the application loaded this file. Credential contents, expressions, and unrelated settings are discarded. Only recognized HTTPS service identities and safe loopback origins are retained.' + (unknown.length ? ' Some endpoint declarations could not identify a cloud service.' : '') });
|
|
80
|
-
} else {
|
|
81
|
-
let matches;
|
|
82
|
-
try {
|
|
83
|
-
if (name === 'package.json') {
|
|
84
|
-
const parsed = JSON.parse(file.body);
|
|
85
|
-
matches = Object.keys({ ...parsed.dependencies, ...parsed.devDependencies, ...parsed.optionalDependencies }).filter(key => sdks.has(key));
|
|
86
|
-
} else matches = [];
|
|
87
|
-
} catch { sources.push({ id, name, kind: 'Project discovery', state: 'unavailable', detail: name, note: 'The package manifest could not be parsed.', coverage: ['SDK declarations'], access: 'Selected project' }); continue; }
|
|
88
|
-
// requirements use a deliberately small name/version grammar, never URLs or commands.
|
|
89
|
-
if (name === 'requirements.txt') matches = file.body.split(/\r?\n/).map(line => line.match(/^\s*([a-zA-Z0-9_-]+)(?:\s*(?:[<>=!~\[]|$))/)?.[1]).filter(key => sdks.has(key));
|
|
90
|
-
if (matches.length) sources.push({ id, name: 'SDK declarations', kind: 'Project discovery', state: 'detected', detail: `${name} · ${[...new Set(matches)].join(' · ')}`, updatedAt: new Date().toISOString(), modifiedAt: file.at, coverage: ['SDK declarations'], access: 'Selected project', note: 'Supported SDKs are declared here. An SDK does not establish a configured provider, running service, request, or cost.' });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
return { sources, inventory, runtimes: [...runtimes.values()] };
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
export function discoveryCandidates(project, configured = [], ollama, limit = 8) {
|
|
97
|
-
const claimed = new Set([ollama, ...configured.map(item => item.url)].filter(Boolean).map(value => loopbackEndpoint(value) || value));
|
|
98
|
-
return [...new Map([...project.runtimes].map(item => [item.url, item])).values()].filter(item => !claimed.has(item.url)).slice(0, limit);
|
|
99
|
-
}
|
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
import { sampleSnapshot } from './sample.mjs';
|
|
2
|
-
import { WORKLOADS, CATALOG, CONFIGURED_SERVICES, PREFERENCES, preferenceFor, selectReplacement } from './demo-policy.mjs';
|
|
3
|
-
|
|
4
|
-
// Published standard, uncached text-token rates in USD per million tokens.
|
|
5
|
-
// This catalog belongs to the fictional hosted demonstration, never live billing.
|
|
6
|
-
export const PRICE_DATE = '2026-10-10';
|
|
7
|
-
const openai = model => `https://developers.openai.com/api/docs/models/${model}`;
|
|
8
|
-
const anthropic = 'https://platform.claude.com/docs/en/about-claude/pricing';
|
|
9
|
-
const google = 'https://ai.google.dev/gemini-api/docs/pricing';
|
|
10
|
-
const groq = 'https://console.groq.com/docs/models';
|
|
11
|
-
const baseten = 'https://www.baseten.co/pricing/';
|
|
12
|
-
const mistral = 'https://docs.mistral.ai/inference/pricing';
|
|
13
|
-
const rate = (name, provider, input, output, source, creator = provider) => ({ name, provider, input, output, source, creator });
|
|
14
|
-
export const PRICES = [
|
|
15
|
-
rate('GPT-5.4', 'OpenAI', 2.5, 15, openai('gpt-5.4')),
|
|
16
|
-
rate('GPT-5.4 mini', 'OpenAI', .75, 4.5, openai('gpt-5.4-mini')),
|
|
17
|
-
rate('GPT-5.4 nano', 'OpenAI', .2, 1.25, openai('gpt-5.4-nano')),
|
|
18
|
-
rate('Claude Opus 4.6', 'Anthropic', 5, 25, anthropic),
|
|
19
|
-
rate('Claude Sonnet 4.6', 'Anthropic', 3, 15, anthropic),
|
|
20
|
-
rate('Claude Haiku 4.5', 'Anthropic', 1, 5, anthropic),
|
|
21
|
-
rate('Gemini 2.5 Pro', 'Google', 1.25, 10, google),
|
|
22
|
-
rate('Gemini 2.5 Flash', 'Google', .3, 2.5, google),
|
|
23
|
-
rate('Gemini 2.5 Flash-Lite', 'Google', .1, .4, google),
|
|
24
|
-
rate('gpt-oss-120b', 'Groq', .15, .6, groq, 'OpenAI'),
|
|
25
|
-
rate('gpt-oss-20b', 'Groq', .075, .3, groq, 'OpenAI'),
|
|
26
|
-
rate('GLM-5.3', 'Baseten', 1.4, 4.4, baseten, 'Z.ai'),
|
|
27
|
-
rate('GLM-5.3-Flash', 'Baseten', .15, .5, baseten, 'Z.ai'),
|
|
28
|
-
rate('Mistral Medium 3.5', 'Mistral', 1.5, 7.5, mistral),
|
|
29
|
-
rate('Mistral Small 4', 'Mistral', .15, .6, mistral),
|
|
30
|
-
];
|
|
31
|
-
export function tokenCost(price, input, output) {
|
|
32
|
-
if (!price || !Number.isFinite(input) || !Number.isFinite(output)) return null;
|
|
33
|
-
return (input * price.input + output * price.output) / 1_000_000;
|
|
34
|
-
}
|
|
35
|
-
const priceFor = (provider, model) => PRICES.find(price => price.provider === provider && price.name === model);
|
|
36
|
-
const REQUESTS_PER_GROUP = 1000;
|
|
37
|
-
|
|
38
|
-
// A bounded set of explicit synthetic groups, not millions of duplicated rows.
|
|
39
|
-
// Tokens and cost are group totals. The request count is sample-only metadata.
|
|
40
|
-
export function enterpriseSnapshot(now = Date.now()) {
|
|
41
|
-
const data = sampleSnapshot(now);
|
|
42
|
-
data.simulation = { kind: 'enterprise-groups', days: 30, requestsPerGroup: REQUESTS_PER_GROUP, priceDate: PRICE_DATE, prices: PRICES };
|
|
43
|
-
data.project = { name: 'enterprise-sample', scope: 'Fictional enterprise environment' };
|
|
44
|
-
const replacements = new Map([
|
|
45
|
-
['GPT-5', priceFor('OpenAI', 'GPT-5.4')], ['GPT-5 mini', priceFor('OpenAI', 'GPT-5.4 mini')], ['GPT-5 nano', priceFor('OpenAI', 'GPT-5.4 nano')],
|
|
46
|
-
['Claude Sonnet 4.5', priceFor('Anthropic', 'Claude Sonnet 4.6')], ['Claude Opus 4.1', priceFor('Anthropic', 'Claude Opus 4.6')], ['Claude Haiku 3.5', priceFor('Anthropic', 'Claude Haiku 4.5')],
|
|
47
|
-
['DeepSeek R1', priceFor('Baseten', 'GLM-5.3')], ['Mistral Medium 3', priceFor('Mistral', 'Mistral Medium 3.5')],
|
|
48
|
-
]);
|
|
49
|
-
const rename = (item, field) => { const price = replacements.get(item[field]); if (price) { item[field] = price.name; item.modelCreator = price.creator; } };
|
|
50
|
-
data.models.forEach(item => rename(item, 'name'));
|
|
51
|
-
data.deployments.forEach(item => { rename(item, 'model'); if (item.provider === 'Baseten') item.servingMode = 'shared'; });
|
|
52
|
-
data.records.forEach((record, index) => {
|
|
53
|
-
rename(record, 'model');
|
|
54
|
-
record.sampleRequests = REQUESTS_PER_GROUP;
|
|
55
|
-
record.occurredAt = new Date(now - (now - Date.parse(record.occurredAt)) * 30).toISOString();
|
|
56
|
-
const shard = index % 5 + 1;
|
|
57
|
-
record.agentId += `.worker-${shard}`;
|
|
58
|
-
if (record.agent) record.agent += `.worker-${shard}`;
|
|
59
|
-
record.taskId += `-partition-${shard}`;
|
|
60
|
-
record.inputTokens *= REQUESTS_PER_GROUP;
|
|
61
|
-
if (record.status === 'running') { record.status = 'completed'; record.outputTokens = 800; }
|
|
62
|
-
if (record.outputTokens != null) record.outputTokens *= REQUESTS_PER_GROUP;
|
|
63
|
-
record.cacheReadTokens = 0;
|
|
64
|
-
record.cost = tokenCost(priceFor(record.provider, record.model), record.inputTokens, record.outputTokens);
|
|
65
|
-
record.costKind = 'estimated';
|
|
66
|
-
record.requestId = null;
|
|
67
|
-
});
|
|
68
|
-
for (const source of data.sources) {
|
|
69
|
-
if (source.id === 'sample-activity') {
|
|
70
|
-
source.records = 9_600_000;
|
|
71
|
-
source.detail = 'Fictional request groups · 30 days';
|
|
72
|
-
source.note = '9.6 million fictional requests represented by 9,600 groups of 1,000. Tokens and spend are group totals. Published uncached text-token prices are applied where available; other costs are unknown. No application or provider was contacted.';
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
return data;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
export function demoSnapshot(now = Date.now(), stage = 'before', preference = 'default') {
|
|
79
|
-
if (!['before', 'after'].includes(stage)) throw new Error('Unknown comparison stage');
|
|
80
|
-
preference = preferenceFor(preference).value;
|
|
81
|
-
const data = enterpriseSnapshot(now);
|
|
82
|
-
data.project = { name: 'insurance-tech-demo', scope: 'Fictional insurance operations application' };
|
|
83
|
-
const usage = WORKLOADS.map(() => ({ inputTokens: 0, outputTokens: 0 }));
|
|
84
|
-
data.records.forEach((record, index) => {
|
|
85
|
-
record.assignmentId = index % WORKLOADS.length;
|
|
86
|
-
const profile = WORKLOADS[record.assignmentId].tokenProfile;
|
|
87
|
-
if (profile) {
|
|
88
|
-
// Fixed fictional application profiles, independent of stage/preference.
|
|
89
|
-
// Schema-heavy SQL prompts and longer reconciliation scripts have distinct
|
|
90
|
-
// input/output mixes. Both include any billed reasoning output.
|
|
91
|
-
const variation = .85 + (index * 17 % 31) / 100;
|
|
92
|
-
record.inputTokens = Math.round(profile.input * variation) * REQUESTS_PER_GROUP;
|
|
93
|
-
record.outputTokens = Math.round(profile.output * variation) * REQUESTS_PER_GROUP;
|
|
94
|
-
}
|
|
95
|
-
if (!record.outputTokens) record.outputTokens = (80 + index * 37 % 1600) * REQUESTS_PER_GROUP;
|
|
96
|
-
usage[record.assignmentId].inputTokens += record.inputTokens;
|
|
97
|
-
usage[record.assignmentId].outputTokens += record.outputTokens;
|
|
98
|
-
});
|
|
99
|
-
const plans = Object.fromEntries(PREFERENCES.map(({ value }) => [value, WORKLOADS.map((workload, index) => selectReplacement(workload, usage[index], value))]));
|
|
100
|
-
const assignments = plans[preference];
|
|
101
|
-
const signature = pairs => JSON.stringify(pairs.map(pair => pair.after?.id ?? null));
|
|
102
|
-
const equivalentPreferences = Object.keys(plans).filter(value => value !== preference && signature(plans[value]) === signature(assignments));
|
|
103
|
-
const feasible = assignments.every(pair => !pair.unresolved);
|
|
104
|
-
data.simulation = { ...data.simulation, kind: 'comparison-groups', stage, preference, feasible, equivalentPreferences, catalogSize: CATALOG.filter(model => CONFIGURED_SERVICES.includes(model.provider)).length, configuredServices: CONFIGURED_SERVICES, assignments, prices: [...new Map(assignments.flatMap(pair => [pair.before, pair.after]).filter(Boolean).map(model => [model.id, model])).values()] };
|
|
105
|
-
const models = new Map(), deployments = new Map();
|
|
106
|
-
data.records.forEach((record, index) => {
|
|
107
|
-
const assignmentId = record.assignmentId, assignment = assignments[assignmentId], workload = WORKLOADS[assignmentId];
|
|
108
|
-
const price = stage === 'after' ? assignment.after ?? assignment.before : assignment.before, sequence = Math.floor(index / WORKLOADS.length);
|
|
109
|
-
const worker = String(sequence % 20 + 1).padStart(2, '0');
|
|
110
|
-
record.workload = `services/${workload.id.replaceAll('.', '/')}`;
|
|
111
|
-
record.agentId = record.agent = `${workload.id}.worker-${worker}`;
|
|
112
|
-
record.task = workload.tasks[sequence % workload.tasks.length];
|
|
113
|
-
record.taskId = `${workload.id}.${record.task}.batch-${Math.floor(sequence / workload.tasks.length) % 10 + 1}`;
|
|
114
|
-
record.model = price.name; record.modelCreator = price.creator; record.provider = price.provider;
|
|
115
|
-
record.location = 'cloud'; record.status = 'completed'; record.durationMs = null;
|
|
116
|
-
record.cost = tokenCost(price, record.inputTokens, record.outputTokens);
|
|
117
|
-
record.beforeCost = tokenCost(assignment.before, record.inputTokens, record.outputTokens);
|
|
118
|
-
record.afterCost = tokenCost(assignment.after, record.inputTokens, record.outputTokens);
|
|
119
|
-
record.routeKind = ['direct', 'router', 'gateway'][index % 3];
|
|
120
|
-
record.route = record.routeKind === 'direct' ? `${price.provider} API` : record.routeKind === 'router' ? 'insurance.router' : 'insurance-gateway';
|
|
121
|
-
record.path = [{ name: record.route, kind: record.routeKind }];
|
|
122
|
-
if (record.routeKind !== 'direct') record.path.push({ name: `${price.provider} API`, kind: 'direct' });
|
|
123
|
-
record.deploymentId = `demo-deployment-${assignmentId}`;
|
|
124
|
-
if (stage === 'after' && !assignment.after) return;
|
|
125
|
-
models.set(price.id, { id: `demo-model-${price.id}`, name: price.name, provider: price.provider, modelCreator: price.creator, sourceId: 'sample-activity', location: 'cloud', state: 'observed' });
|
|
126
|
-
deployments.set(assignmentId, { id: record.deploymentId, name: workload.id, model: price.name, modelCreator: price.creator, provider: price.provider, location: 'cloud', servingMode: 'shared', state: 'configured', sourceId: 'inventory', poolIds: [`pool-${price.provider.toLowerCase()}`] });
|
|
127
|
-
});
|
|
128
|
-
data.models = [...models.values()]; data.deployments = [...deployments.values()];
|
|
129
|
-
const providers = CONFIGURED_SERVICES;
|
|
130
|
-
data.inventory = [...providers.map(name => ({ name, kind: 'provider', location: 'cloud', state: 'configured', sourceId: 'inventory' })), { name: 'insurance.router', kind: 'router', location: 'cloud', state: 'configured' }, { name: 'insurance-gateway', kind: 'gateway', location: 'cloud', state: 'configured' }];
|
|
131
|
-
data.capacityPools = data.capacityPools.filter(pool => providers.some(name => pool.id === `pool-${name.toLowerCase()}`));
|
|
132
|
-
data.hosts = []; data.runtimeStats = []; delete data.machine;
|
|
133
|
-
data.sources = data.sources.filter(source => ['sample-activity', 'sample-capacity', 'inventory'].includes(source.id));
|
|
134
|
-
data.sources.find(source => source.id === 'sample-capacity').note = 'Fictional provider quota readings, held identical in both stages. They do not establish a change in latency, throughput or available capacity.';
|
|
135
|
-
data.sources.find(source => source.id === 'sample-activity').note += ' Insurance workflow and worker names are fictional declarations. Assignments distinguish qualified replacements, retained baselines and explicit preference overrides. Overrides do not pass all primary criteria. Application quality and latency have not been measured.';
|
|
136
|
-
data.simulation.windows = Object.fromEntries([0, 1, 6, 24, 168, 720].map(hours => {
|
|
137
|
-
const rows = data.records.filter(record => !hours || Date.parse(record.occurredAt) >= now - hours * 3600000);
|
|
138
|
-
const totals = selected => ({ requests: selected.length * REQUESTS_PER_GROUP, assignedRequests: selected.filter(record => assignments[record.assignmentId].after).length * REQUESTS_PER_GROUP, qualifiedRequests: selected.filter(record => assignments[record.assignmentId].after && !assignments[record.assignmentId].override).length * REQUESTS_PER_GROUP, overrideRequests: selected.filter(record => assignments[record.assignmentId].override).length * REQUESTS_PER_GROUP, beforeCost: selected.reduce((n, r) => n + r.beforeCost, 0), afterCost: feasible ? selected.reduce((n, r) => n + r.afterCost, 0) : null });
|
|
139
|
-
return [hours, { ...totals(rows), workloads: assignments.map((pair, index) => {
|
|
140
|
-
const selected = rows.filter(record => record.assignmentId === index);
|
|
141
|
-
return { ...totals(selected), afterCost: pair.after ? selected.reduce((n, r) => n + tokenCost(pair.after, r.inputTokens, r.outputTokens), 0) : null };
|
|
142
|
-
}) }];
|
|
143
|
-
}));
|
|
144
|
-
if (stage === 'after' && !feasible) {
|
|
145
|
-
data.records = data.records.filter(record => assignments[record.assignmentId].after);
|
|
146
|
-
const source = data.sources.find(source => source.id === 'sample-activity');
|
|
147
|
-
source.records = data.records.length * REQUESTS_PER_GROUP;
|
|
148
|
-
source.note += ' Incomplete After plan: only workloads with an assignment are included; preference overrides are identified in comparison metadata. Unresolved workloads remain in comparison metadata, without an After assignment or cost.';
|
|
149
|
-
}
|
|
150
|
-
return data;
|
|
151
|
-
}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
// Hardware is supplied by existing exporters, never discovered on the network.
|
|
2
|
-
const label = value => typeof value === 'string' ? value.slice(0, 160).replace(/[\x00-\x1f]/g, '') : '';
|
|
3
|
-
const number = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
|
|
4
|
-
const percent = value => number(value) !== null && value <= 100 ? value : null;
|
|
5
|
-
const list = (value, max) => Array.isArray(value) ? value.slice(0, max).filter(item => item && typeof item === 'object') : [];
|
|
6
|
-
const unique = items => items.filter(item => item.id && items.filter(other => other.id === item.id).length === 1);
|
|
7
|
-
|
|
8
|
-
export function normalizeHosts(value, now = Date.now()) {
|
|
9
|
-
return unique(list(value?.hosts, 64).map(host => {
|
|
10
|
-
const at = typeof host.observedAt === 'string' ? Date.parse(host.observedAt) : NaN;
|
|
11
|
-
const observedAt = Number.isFinite(at) && at <= now ? new Date(at).toISOString() : null;
|
|
12
|
-
const memoryTotal = number(host.memoryTotal), used = number(host.memoryUsed);
|
|
13
|
-
return {
|
|
14
|
-
id: label(host.id), name: label(host.name) || label(host.hostname) || label(host.id),
|
|
15
|
-
nameField: label(host.name) ? 'name' : label(host.hostname) ? 'hostname' : 'id', hostname: label(host.hostname),
|
|
16
|
-
site: label(host.site), rack: label(host.rack),
|
|
17
|
-
chip: label(host.chip), cores: number(host.cores), platform: label(host.platform), arch: label(host.arch),
|
|
18
|
-
sourceId: 'hardware', observedAt, memoryTotal, memoryUsed: observedAt && used !== null && (memoryTotal === null || used <= memoryTotal) ? used : null,
|
|
19
|
-
cpuPercent: observedAt ? percent(host.cpuPercent) : null,
|
|
20
|
-
accelerators: unique(list(host.accelerators, 16).map(device => {
|
|
21
|
-
const total = number(device.memoryTotal), used = number(device.memoryUsed);
|
|
22
|
-
return { id: label(device.id), name: label(device.name) || 'Accelerator', vendor: label(device.vendor) || null, kind: ['gpu', 'lpu', 'wse', 'npu', 'tpu', 'other'].includes(device.kind) ? device.kind : 'unknown', memoryTotal: total,
|
|
23
|
-
memoryUsed: observedAt && used !== null && (total === null || used <= total) ? used : null,
|
|
24
|
-
utilization: observedAt ? percent(device.utilization) : null };
|
|
25
|
-
})),
|
|
26
|
-
// A catalog or model name alone never establishes residency.
|
|
27
|
-
residencyReported: Boolean(observedAt && Array.isArray(host.loadedModels)),
|
|
28
|
-
loadedModels: observedAt ? unique(list(host.loadedModels, 100).map(model => ({ id: label(model.id), name: label(model.name), modelCreator: label(model.modelCreator) || null, provider: label(model.provider), runtime: label(model.runtime), memoryBytes: number(model.memoryBytes), memoryKind: ['accelerator', 'system', 'unified'].includes(model.memoryKind) ? model.memoryKind : 'unknown', deploymentId: label(model.deploymentId) || null })).filter(model => model.name)) : [],
|
|
29
|
-
history: [],
|
|
30
|
-
};
|
|
31
|
-
}).filter(host => host.id !== 'this-machine'));
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export function hostState(host, interval = 15, now = Date.now()) {
|
|
35
|
-
const at = Date.parse(host.observedAt);
|
|
36
|
-
if (!Number.isFinite(at) || at > now) return 'unknown';
|
|
37
|
-
return now - at > interval * 3000 ? 'stale' : 'sampled';
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Re-reading a file does not add a sample or freshen its measurement time.
|
|
41
|
-
export function retainHostHistory(hosts, histories) {
|
|
42
|
-
for (const id of histories.keys()) if (!hosts.some(host => host.id === id)) histories.delete(id);
|
|
43
|
-
return hosts.map(host => {
|
|
44
|
-
let history = histories.get(host.id) ?? [];
|
|
45
|
-
const lastAt = Date.parse(history.at(-1)?.at);
|
|
46
|
-
const at = Date.parse(host.observedAt);
|
|
47
|
-
if (!Number.isFinite(at) || at < lastAt) history = [];
|
|
48
|
-
if (Number.isFinite(at)) {
|
|
49
|
-
const sample = { at: host.observedAt, cpuPercent: host.cpuPercent, memoryUsed: host.memoryUsed };
|
|
50
|
-
if (at === lastAt) history[history.length - 1] = sample;
|
|
51
|
-
else history.push(sample);
|
|
52
|
-
}
|
|
53
|
-
history = history.slice(-240);
|
|
54
|
-
histories.set(host.id, history);
|
|
55
|
-
return { ...host, history: [...history] };
|
|
56
|
-
});
|
|
57
|
-
}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
// Shared by collector and browser. Dates describe retained request evidence.
|
|
2
|
-
export const ACTIVITY_RANGES = [1, 6, 24, 168, 720, 0];
|
|
3
|
-
export const rangeLabel = hours => ({ 1: '1h', 6: '6h', 24: '24h', 168: '7d', 720: '30d', 0: 'All time' })[hours];
|
|
4
|
-
export const rangeDescription = hours => hours === 0 ? 'All available history' : `Last ${rangeLabel(hours)}`;
|
|
5
|
-
export function historyExtent(records, observedAt) {
|
|
6
|
-
const now = Date.parse(observedAt);
|
|
7
|
-
let first = Infinity, last = -Infinity;
|
|
8
|
-
for (const record of records) {
|
|
9
|
-
const at = Date.parse(record.occurredAt);
|
|
10
|
-
if (!Number.isFinite(at) || at > now) continue;
|
|
11
|
-
first = Math.min(first, at); last = Math.max(last, at);
|
|
12
|
-
}
|
|
13
|
-
return { first: Number.isFinite(first) ? new Date(first).toISOString() : null, last: Number.isFinite(last) ? new Date(last).toISOString() : null };
|
|
14
|
-
}
|
|
15
|
-
export function activityWindow(records, hours, observedAt) {
|
|
16
|
-
const now = Date.parse(observedAt);
|
|
17
|
-
if (hours !== 0) return { start: now - hours * 3600000, end: now };
|
|
18
|
-
const extent = historyExtent(records, observedAt);
|
|
19
|
-
const end = extent.last ? Date.parse(extent.last) : now;
|
|
20
|
-
return { start: extent.first ? Math.min(Date.parse(extent.first), end - 60000) : end - 3600000, end };
|
|
21
|
-
}
|
|
22
|
-
export function windowRecords(records, hours, observedAt) {
|
|
23
|
-
const { start, end } = activityWindow(records, hours, observedAt);
|
|
24
|
-
return records.filter(record => { const at = Date.parse(record.occurredAt); return at >= start && at <= end; });
|
|
25
|
-
}
|
|
26
|
-
export function intervalLabel(ms) {
|
|
27
|
-
const [unit, divisor] = ms >= 86400000 ? ['day', 86400000] : ms >= 3600000 ? ['hour', 3600000] : ms >= 60000 ? ['minute', 60000] : ['second', 1000];
|
|
28
|
-
return `${new Intl.NumberFormat('en', { maximumFractionDigits: 1 }).format(ms / divisor)}-${unit}`;
|
|
29
|
-
}
|
|
30
|
-
export function activityTime(value, span, detail = false) {
|
|
31
|
-
return new Date(value).toLocaleString([], detail ? { year: 'numeric', month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' } : span > 86400000 ? { month: 'short', day: 'numeric', ...(span > 31536000000 ? { year: 'numeric' } : {}) } : { hour: '2-digit', minute: '2-digit' });
|
|
32
|
-
}
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
|
|
3
|
-
const CHECKPOINT_BYTES = 4096;
|
|
4
|
-
const digest = bytes => createHash('sha256').update(bytes).digest('hex');
|
|
5
|
-
|
|
6
|
-
// Keep only hashes of bytes already consumed, never an extra raw-content cache.
|
|
7
|
-
// Check both the beginning and read boundary: copy-truncate can regrow past the
|
|
8
|
-
// saved offset between polls without changing the inode.
|
|
9
|
-
export function checkpointForRead(bytes, start, previous = null) {
|
|
10
|
-
if (!bytes.length) return previous;
|
|
11
|
-
const size = Math.min(bytes.length, CHECKPOINT_BYTES);
|
|
12
|
-
const head = start === 0 ? { start: 0, size, digest: digest(bytes.subarray(0, size)) } : previous?.head;
|
|
13
|
-
const tail = { start: start + bytes.length - size, size, digest: digest(bytes.subarray(bytes.length - size)) };
|
|
14
|
-
return { head, tail };
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export async function checkpointMatches(handle, checkpoint) {
|
|
18
|
-
if (!checkpoint) return true;
|
|
19
|
-
const { head, tail } = checkpoint;
|
|
20
|
-
for (const part of head?.start === tail.start && head.size === tail.size ? [tail] : [head, tail].filter(Boolean)) {
|
|
21
|
-
const buffer = Buffer.alloc(part.size);
|
|
22
|
-
const { bytesRead } = await handle.read(buffer, 0, buffer.length, part.start);
|
|
23
|
-
if (bytesRead !== part.size || digest(buffer) !== part.digest) return false;
|
|
24
|
-
}
|
|
25
|
-
return true;
|
|
26
|
-
}
|