modelmix 5.1.20 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +138 -0
- package/README.md +69 -2
- package/demo/benchmark.js +83 -0
- package/demo/package.json +2 -0
- package/demo/prompts/story.txt +35 -0
- package/demo/prompts/template-engine.txt +41 -0
- package/demo/short.js +2 -0
- package/effort.js +3 -1
- package/index.d.ts +9 -5
- package/index.js +75 -12
- package/lib/model-chain.js +1 -1
- package/lib/parse-json-response.js +14 -0
- package/lib/providers/anthropic.js +3 -1
- package/lib/providers/base.js +65 -17
- package/lib/providers/google.js +13 -2
- package/lib/providers/openai-compatible.js +36 -0
- package/lib/providers/openai.js +52 -8
- package/lib/token-usage.js +5 -0
- package/package.json +5 -2
- package/plugins/benchmark/index.d.ts +112 -0
- package/plugins/benchmark/index.js +575 -0
- package/plugins/benchmark/test/benchmark.test.js +518 -0
- package/plugins/skills/index.d.ts +9 -0
- package/plugins/skills/index.js +107 -0
- package/plugins/skills/test/skills.test.js +182 -0
- package/pnpm-workspace.yaml +2 -2
- package/skills/modelmix/SKILL.md +32 -2
- package/test/deepseek.test.js +273 -1
- package/test/fallback.test.js +57 -0
- package/test/google.test.js +55 -0
- package/test/history.test.js +7 -4
- package/test/json.test.js +29 -1
- package/test/plugins.test.js +150 -1
- package/test/public-api.test.js +2 -0
- package/test/tokens.test.js +75 -3
- package/RLM_PLUGIN_SPEC.md +0 -465
- package/demo/package-lock.json +0 -516
package/.gitignore
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# Logs
|
|
2
|
+
logs
|
|
3
|
+
*.log
|
|
4
|
+
npm-debug.log*
|
|
5
|
+
yarn-debug.log*
|
|
6
|
+
yarn-error.log*
|
|
7
|
+
lerna-debug.log*
|
|
8
|
+
.pnpm-debug.log*
|
|
9
|
+
|
|
10
|
+
# Diagnostic reports (https://nodejs.org/api/report.html)
|
|
11
|
+
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
|
12
|
+
|
|
13
|
+
# Runtime data
|
|
14
|
+
pids
|
|
15
|
+
*.pid
|
|
16
|
+
*.seed
|
|
17
|
+
*.pid.lock
|
|
18
|
+
|
|
19
|
+
# Directory for instrumented libs generated by jscoverage/JSCover
|
|
20
|
+
lib-cov
|
|
21
|
+
|
|
22
|
+
# Coverage directory used by tools like istanbul
|
|
23
|
+
coverage
|
|
24
|
+
*.lcov
|
|
25
|
+
|
|
26
|
+
# nyc test coverage
|
|
27
|
+
.nyc_output
|
|
28
|
+
|
|
29
|
+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
|
30
|
+
.grunt
|
|
31
|
+
|
|
32
|
+
# Bower dependency directory (https://bower.io/)
|
|
33
|
+
bower_components
|
|
34
|
+
|
|
35
|
+
# node-waf configuration
|
|
36
|
+
.lock-wscript
|
|
37
|
+
|
|
38
|
+
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
|
39
|
+
build/Release
|
|
40
|
+
|
|
41
|
+
# Dependency directories
|
|
42
|
+
node_modules/
|
|
43
|
+
jspm_packages/
|
|
44
|
+
|
|
45
|
+
# Snowpack dependency directory (https://snowpack.dev/)
|
|
46
|
+
web_modules/
|
|
47
|
+
|
|
48
|
+
# TypeScript cache
|
|
49
|
+
*.tsbuildinfo
|
|
50
|
+
|
|
51
|
+
# Optional npm cache directory
|
|
52
|
+
.npm
|
|
53
|
+
|
|
54
|
+
# Optional eslint cache
|
|
55
|
+
.eslintcache
|
|
56
|
+
|
|
57
|
+
# Optional stylelint cache
|
|
58
|
+
.stylelintcache
|
|
59
|
+
|
|
60
|
+
# Microbundle cache
|
|
61
|
+
.rpt2_cache/
|
|
62
|
+
.rts2_cache_cjs/
|
|
63
|
+
.rts2_cache_es/
|
|
64
|
+
.rts2_cache_umd/
|
|
65
|
+
|
|
66
|
+
# Optional REPL history
|
|
67
|
+
.node_repl_history
|
|
68
|
+
|
|
69
|
+
# Output of 'npm pack'
|
|
70
|
+
*.tgz
|
|
71
|
+
|
|
72
|
+
# Yarn Integrity file
|
|
73
|
+
.yarn-integrity
|
|
74
|
+
|
|
75
|
+
# dotenv environment variable files
|
|
76
|
+
.env
|
|
77
|
+
.env.development.local
|
|
78
|
+
.env.test.local
|
|
79
|
+
.env.production.local
|
|
80
|
+
.env.local
|
|
81
|
+
|
|
82
|
+
# parcel-bundler cache (https://parceljs.org/)
|
|
83
|
+
.cache
|
|
84
|
+
.parcel-cache
|
|
85
|
+
|
|
86
|
+
# Next.js build output
|
|
87
|
+
.next
|
|
88
|
+
out
|
|
89
|
+
|
|
90
|
+
# Nuxt.js build / generate output
|
|
91
|
+
.nuxt
|
|
92
|
+
dist
|
|
93
|
+
|
|
94
|
+
# Gatsby files
|
|
95
|
+
.cache/
|
|
96
|
+
# Comment in the public line in if your project uses Gatsby and not Next.js
|
|
97
|
+
# https://nextjs.org/blog/next-9-1#public-directory-support
|
|
98
|
+
# public
|
|
99
|
+
|
|
100
|
+
# vuepress build output
|
|
101
|
+
.vuepress/dist
|
|
102
|
+
|
|
103
|
+
# vuepress v2.x temp and cache directory
|
|
104
|
+
.temp
|
|
105
|
+
.cache
|
|
106
|
+
|
|
107
|
+
# Docusaurus cache and generated files
|
|
108
|
+
.docusaurus
|
|
109
|
+
|
|
110
|
+
# Serverless directories
|
|
111
|
+
.serverless/
|
|
112
|
+
|
|
113
|
+
# FuseBox cache
|
|
114
|
+
.fusebox/
|
|
115
|
+
|
|
116
|
+
# DynamoDB Local files
|
|
117
|
+
.dynamodb/
|
|
118
|
+
|
|
119
|
+
# TernJS port file
|
|
120
|
+
.tern-port
|
|
121
|
+
|
|
122
|
+
# Stores VSCode versions used for testing VSCode extensions
|
|
123
|
+
.vscode-test
|
|
124
|
+
|
|
125
|
+
# yarn v2
|
|
126
|
+
.yarn/cache
|
|
127
|
+
.yarn/unplugged
|
|
128
|
+
.yarn/build-state.yml
|
|
129
|
+
.yarn/install-state.gz
|
|
130
|
+
.pnp.*
|
|
131
|
+
|
|
132
|
+
.DS_Store
|
|
133
|
+
demo/jailbreak.mjs
|
|
134
|
+
CLAUDE.md
|
|
135
|
+
demo/jailbreak.js
|
|
136
|
+
/demo/lab
|
|
137
|
+
/demo/results
|
|
138
|
+
/.pnpm-store/
|
package/README.md
CHANGED
|
@@ -53,6 +53,7 @@ OPENAI_API_KEY="sk-proj-..."
|
|
|
53
53
|
OPENROUTER_API_KEY="sk-or-..."
|
|
54
54
|
MOONSHOT_API_KEY="your-moonshot-key..."
|
|
55
55
|
MINIMAX_API_KEY="your-minimax-key..."
|
|
56
|
+
DEEPSEEK_API_KEY="your-deepseek-key..."
|
|
56
57
|
NVIDIA_API_KEY="nvapi-..."
|
|
57
58
|
...
|
|
58
59
|
GEMINI_API_KEY="AIza..."
|
|
@@ -209,7 +210,9 @@ ModelMix provides convenient shorthand methods for quickly accessing different A
|
|
|
209
210
|
| `qwen3827b()` | OpenRouter | qwen/qwen3.8-27b | [\$0.45][15] | [\$3.20][15] |
|
|
210
211
|
| `qwen38flash()` | OpenRouter | qwen/qwen3.8-flash | [\$0.16][19] | [\$0.47][19] |
|
|
211
212
|
| `deepseekV4Flash()` | Fireworks | models/deepseek-v4-flash | [\$0.14][10] | [\$0.28][10] |
|
|
213
|
+
| `deepseekV41Flash()` | OpenRouter | deepseek/deepseek-v4.1-flash | [\$0.15][27] | [\$0.60][27] |
|
|
212
214
|
| `deepseekV4Pro()` | Fireworks | models/deepseek-v4-pro-0813 | [\$1.32][12] | [\$3.96][12] |
|
|
215
|
+
| `deepseekPro()` | OpenRouter | deepseek/deepseek-v4-pro-0813 | [\$0.5808][28] | [\$1.7424][28] |
|
|
213
216
|
| `GLM53()` | OpenRouter | z-ai/glm-5.3 | [\$1.40][16] | [\$4.40][16] |
|
|
214
217
|
| `GLM53Flash()` | OpenRouter | z-ai/glm-5.3-flash | [\$0.075][20] | [\$0.25][20] |
|
|
215
218
|
| `GLM52()` | Together | zai-org/GLM-5.2 | [\$1.40][7] | [\$4.40][7] |
|
|
@@ -262,6 +265,16 @@ OpenRouter fallbacks are disabled globally by default and are appended only with
|
|
|
262
265
|
|
|
263
266
|
[25]: https://openrouter.ai/meta/muse-spark-1.2 "Muse Spark 1.2 on OpenRouter"
|
|
264
267
|
[26]: https://openrouter.ai/meta/muse-spark-1.3-contributor "Muse Spark 1.3 Contributor on OpenRouter"
|
|
268
|
+
[27]: https://openrouter.ai/deepseek/deepseek-v4.1-flash "DeepSeek V4.1 Flash on OpenRouter"
|
|
269
|
+
[28]: https://openrouter.ai/deepseek/deepseek-v4-pro-0813 "DeepSeek V4 Pro 0813 on OpenRouter"
|
|
270
|
+
|
|
271
|
+
`deepseekPro()` uses the pinned OpenRouter model `deepseek/deepseek-v4-pro-0813` and requires `OPENROUTER_API_KEY`. Use `chain('deepseekPro@100')` for maximum reasoning effort. Cost estimates use the listed base rates and $0.05808/M cached input tokens; actual rates may change, including provider and time-based pricing.
|
|
272
|
+
|
|
273
|
+
`deepseekV41Flash()` supports text and image input through OpenRouter and requires `OPENROUTER_API_KEY`. Use `chain('deepseekV41Flash@100')` for maximum reasoning effort. Cost estimates use the listed base rates and $0.015/M cached input tokens; actual OpenRouter pricing varies by provider and time.
|
|
274
|
+
|
|
275
|
+
For direct Fireworks access, use `deepseekV41Flash({ mix: { fireworks: true, openrouter: false } })` with `FIREWORKS_API_KEY`. This selects `accounts/fireworks/models/deepseek-v4p1-flash`, priced at [$0.22 input / $0.007 cached input / $0.66 output per 1M tokens](https://fireworks.ai/models/deepseek-ai/deepseek-v4p1-flash). Set both providers to `true` to try Fireworks first and fall back to OpenRouter.
|
|
276
|
+
|
|
277
|
+
For the native DeepSeek API, use `deepseekV41Flash({ mix: { deepseek: true, openrouter: false } })` with `DEEPSEEK_API_KEY`. It calls `https://api.deepseek.com/chat/completions` with `deepseek-flash`, currently DeepSeek V4.1 Flash. `MixDeepSeek` is also available for explicit `.attach()` calls. Native cost estimates use [peak rates](https://api-docs.deepseek.com/quick_start/pricing/): $0.30 input / $0.006 cached input / $1.20 output per 1M tokens; actual off-peak charges are half. When all three providers are enabled, the order is DeepSeek → Fireworks → OpenRouter.
|
|
265
278
|
|
|
266
279
|
Muse Spark methods ending in `c` select Contributor: prompts and outputs may be used to improve Meta products. Methods without `c` select the standard tier. `museSpark12()` now selects standard; use `museSpark12c()` for the previous Contributor behavior.
|
|
267
280
|
|
|
@@ -755,7 +768,7 @@ Every response from `raw()` now includes a `tokens` object with the following st
|
|
|
755
768
|
cacheSavings: 0.00018432, // USD saved by cache reads
|
|
756
769
|
cacheWritePremium: 0, // Extra USD paid to write this cache entry
|
|
757
770
|
breakEvenHits: 0, // Full future hits needed to recover that premium
|
|
758
|
-
cost: 0.00011568, //
|
|
771
|
+
cost: 0.00011568, // OpenRouter charge when reported; otherwise estimated USD
|
|
759
772
|
costBreakdown: {
|
|
760
773
|
uncachedInput: 0.0000352,
|
|
761
774
|
cachedInput: 0.00002048,
|
|
@@ -780,7 +793,7 @@ console.log(model.lastRaw.tokens);
|
|
|
780
793
|
// Same normalized token and cost structure returned by raw()
|
|
781
794
|
```
|
|
782
795
|
|
|
783
|
-
`thinking` contains internal reasoning tokens when a provider reports them separately;
|
|
796
|
+
`thinking` contains internal reasoning tokens when a provider reports them separately; `output` excludes those tokens. Cost calculation bills `output + thinking` once at the output rate. OpenAI Chat/Responses and Anthropic include reasoning in their native output totals; native Grok reports it separately, while Gemini uses `thoughtsTokenCount`. `cost` uses OpenRouter's reported charge when available (including zero); otherwise it uses catalog pricing. `costBreakdown` and cache savings remain catalog estimates and can differ from the actual charge. `cached` aggregates cache reads reported by the provider, while `cacheWrite` aggregates cache writes. Anthropic additionally exposes `cacheWrite5m` and `cacheWrite1h` because those writes cost 1.25× and 2× the normal input rate, respectively. `cacheSavings` compares cache reads with the normal input rate, `cacheWritePremium` compares writes with that rate, and `breakEvenHits` estimates how many complete future hits recover the current write premium. For Anthropic, `input` is normalized to include uncached input, cache reads, and cache writes. Missing usage or pricing categories return `0`. The `speed` field is the generation speed measured in output tokens per second (integer).
|
|
784
797
|
|
|
785
798
|
## 🧠 Prompt Caching
|
|
786
799
|
|
|
@@ -987,6 +1000,60 @@ Supported policies are `'inherit'`, `'none'`, `{ include: [...] }`, and `{ exclu
|
|
|
987
1000
|
|
|
988
1001
|
Child `systemFile` templates use the same EJS engine, `assign()` data contract, and relative Markdown includes as ordinary ModelMix templates. Use either `system` or `systemFile`, not both.
|
|
989
1002
|
|
|
1003
|
+
Plugins may append `{ tool, callback }` entries to `context.request.tools` using the same definitions as `addTools()`. These tools are available for that execution and its tool continuations, alongside registered local/MCP tools. They do not change the instance's tool registry. Duplicate names are rejected before calling a provider. Child invocations rebuild their tools through the selected plugins or explicit `tools` input.
|
|
1004
|
+
|
|
1005
|
+
When plugins add tools, native `options.tools` entries are combined with registered and plugin tools; duplicate function names are rejected. OpenAI Responses converts function definitions and preserves tool calls, results, and accompanying reasoning across continuations.
|
|
1006
|
+
|
|
1007
|
+
### Skills plugin
|
|
1008
|
+
|
|
1009
|
+
The included plugin loads local [Agent Skills](https://agentskills.io/specification). Pass explicit skill directories or `SKILL.md` files:
|
|
1010
|
+
|
|
1011
|
+
```javascript
|
|
1012
|
+
import { ModelMix } from 'modelmix';
|
|
1013
|
+
import { skills } from 'modelmix/plugins/skills/index.js';
|
|
1014
|
+
|
|
1015
|
+
const model = ModelMix.new()
|
|
1016
|
+
.gpt6astra()
|
|
1017
|
+
.opus5()
|
|
1018
|
+
.use(await skills({ paths: ['./skills/writing', './skills/research/SKILL.md'] }))
|
|
1019
|
+
.addText('Use the writing skill to improve this paragraph: ...');
|
|
1020
|
+
|
|
1021
|
+
console.log(await model.message());
|
|
1022
|
+
```
|
|
1023
|
+
|
|
1024
|
+
`skills()` is asynchronous. Each file must have YAML frontmatter with non-empty string `name` and `description` fields; duplicate names and malformed files fail during loading. Relative paths resolve from `process.cwd()`.
|
|
1025
|
+
|
|
1026
|
+
Only names and descriptions are appended to the system prompt. A model supporting tool calls can select a skill with `read_skill({ name })`, then load supporting UTF-8 text files with `read_skill({ name, path: 'references/style.md' })`. Full instructions are returned literally, including any EJS syntax. Existing system instructions and tools are preserved; `read_skill` is reserved while this plugin runs.
|
|
1027
|
+
|
|
1028
|
+
Skill metadata and `SKILL.md` content are snapshots taken when creating the plugin; recreate it to reload edits. Supporting files are read on demand. Reads must stay inside the registered skill directory, including resolved symlinks. Files are returned in full, so callers should register appropriately sized, trusted skills. The plugin does not execute scripts, install tools, or grant permissions from `allowed-tools` metadata. Skills requiring additional capabilities need tools supplied by the application.
|
|
1029
|
+
|
|
1030
|
+
### Benchmark plugin
|
|
1031
|
+
|
|
1032
|
+
The included benchmark plugin derives task-specific criteria, runs each configured model independently, and uses the other distinct models as anonymous judges. Model specifications use the same `shortcut@effort` syntax as `chain()`:
|
|
1033
|
+
|
|
1034
|
+
```javascript
|
|
1035
|
+
const { ModelMix } = require('modelmix');
|
|
1036
|
+
const { benchmark } = require('modelmix/plugins/benchmark');
|
|
1037
|
+
|
|
1038
|
+
const report = await ModelMix.new()
|
|
1039
|
+
.use(benchmark({
|
|
1040
|
+
criteriaModel: 'gpt56luna@20',
|
|
1041
|
+
models: ['gpt56luna@20', 'sonnet5@20', 'gemini38flash@20']
|
|
1042
|
+
}))
|
|
1043
|
+
.addText('Your benchmark task')
|
|
1044
|
+
.json();
|
|
1045
|
+
```
|
|
1046
|
+
|
|
1047
|
+
Each criterion is scored from 0 to 10 with equal weight. The report contains the original task, criteria, responses, individual evaluations, averages, errors, elapsed time, token usage, and available cost estimates. Failed responses or evaluations are recorded while the remaining models continue; criteria-generation failures and cancellation stop the run. The first version accepts text tasks, runs sequentially, and does not support streaming.
|
|
1048
|
+
|
|
1049
|
+
Set `config.debug: 1` to print progress and failure reasons. Failed model outputs retain their text and provider finish reason in `errors[].error.details`; outputs marked as truncated are excluded from scoring. Adjust `options.max_tokens` to allow enough room for reasoning and output.
|
|
1050
|
+
|
|
1051
|
+
Pass `mix` to `benchmark()` to select providers for the criteria model, participants, and judges using each shortcut's provider flags. For example, `mix: { deepseek: true, openrouter: false }` routes `deepseekV41Flash` directly to the official DeepSeek API using `DEEPSEEK_API_KEY`. Omitting `mix` preserves each shortcut's default providers.
|
|
1052
|
+
|
|
1053
|
+
JSON parsing accepts a Markdown code block or a lone closing triple-backtick delimiter after valid JSON. Other extra content, malformed JSON, and invalid evaluation scores remain errors.
|
|
1054
|
+
|
|
1055
|
+
Run `node demo/benchmark.js` for a complete five-model comparison. It allows up to 32,768 output tokens per call, prints intermediate progress, a final ranking and error details, then saves each response as Markdown together with the full JSON report under `demo/results/`.
|
|
1056
|
+
|
|
990
1057
|
### Recursive Language Model plugin
|
|
991
1058
|
|
|
992
1059
|
The separately publishable `@modelmix/rlm` workspace package keeps document parsing, planner prompts, and `isolated-vm` out of the core `modelmix` dependency tree. It requires Node.js 22 or newer.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { ModelMix } from '../index.js';
|
|
2
|
+
import benchmarkPlugin from '../plugins/benchmark/index.js';
|
|
3
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
|
|
7
|
+
try { process.loadEnvFile(); } catch {}
|
|
8
|
+
|
|
9
|
+
const { benchmark } = benchmarkPlugin;
|
|
10
|
+
|
|
11
|
+
const task = await readFile(process.argv[2] || new URL('./prompts/story.txt', import.meta.url), 'utf8');
|
|
12
|
+
|
|
13
|
+
const models = [
|
|
14
|
+
// 'opus50@20',
|
|
15
|
+
'gpt6astra@0',
|
|
16
|
+
'gpt6astra@20',
|
|
17
|
+
'gpt56sol@40',
|
|
18
|
+
// 'gemini38flash@20',
|
|
19
|
+
// 'grok46@20',
|
|
20
|
+
'deepseekV41Flash@60'
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
console.log(`Running benchmark: ${models.length} models, 26 sequential model calls.`);
|
|
24
|
+
|
|
25
|
+
const report = await ModelMix.new({
|
|
26
|
+
config: { debug: 1 },
|
|
27
|
+
options: { max_tokens: 65536 }
|
|
28
|
+
})
|
|
29
|
+
.use(benchmark({
|
|
30
|
+
criteriaModel: 'opus50@20',
|
|
31
|
+
models,
|
|
32
|
+
mix: { deepseek: true, openrouter: false }
|
|
33
|
+
}))
|
|
34
|
+
.assign({ task })
|
|
35
|
+
.addText('<%- task %>')
|
|
36
|
+
.json();
|
|
37
|
+
|
|
38
|
+
const runId = new Date().toISOString().replace(/[:.]/g, '-');
|
|
39
|
+
const demoDirectory = path.dirname(fileURLToPath(import.meta.url));
|
|
40
|
+
const resultsDirectory = path.join(demoDirectory, 'results', runId);
|
|
41
|
+
await mkdir(resultsDirectory, { recursive: true });
|
|
42
|
+
|
|
43
|
+
for (const result of report.results) {
|
|
44
|
+
if (result.response === null) continue;
|
|
45
|
+
const filename = result.id.replace(/[^A-Za-z0-9_-]/g, '_');
|
|
46
|
+
const contents = `# ${result.id}\n\nScore: ${result.score === null ? 'N/A' : Math.round(result.score * 10)}/100\n\nEstimated generation cost (USD): ${result.responseMetrics?.cost ?? 'N/A'}\n\n---\n\n${result.response}\n`;
|
|
47
|
+
await writeFile(path.join(resultsDirectory, `${filename}.md`), contents, 'utf8');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
await writeFile(
|
|
51
|
+
path.join(resultsDirectory, 'report.json'),
|
|
52
|
+
`${JSON.stringify(report, null, 2)}\n`,
|
|
53
|
+
'utf8'
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
const ranking = report.results
|
|
57
|
+
.map(result => ({
|
|
58
|
+
model: result.id,
|
|
59
|
+
score: result.score,
|
|
60
|
+
cost: result.responseMetrics?.cost?.toFixed(2) ?? 'N/A',
|
|
61
|
+
timeSeconds: Number.isFinite(result.responseMetrics?.elapsedMs)
|
|
62
|
+
? (result.responseMetrics.elapsedMs / 1000).toFixed(2)
|
|
63
|
+
: 'N/A',
|
|
64
|
+
evaluations: `${result.evaluationCount.valid}/${result.evaluationCount.expected}`
|
|
65
|
+
}))
|
|
66
|
+
.sort((left, right) => {
|
|
67
|
+
if (left.score === right.score) return 0;
|
|
68
|
+
if (left.score === null) return 1;
|
|
69
|
+
if (right.score === null) return -1;
|
|
70
|
+
return right.score - left.score;
|
|
71
|
+
})
|
|
72
|
+
.map(row => ({ ...row, score: row.score === null ? null : Math.round(row.score * 10) }));
|
|
73
|
+
|
|
74
|
+
console.table(ranking);
|
|
75
|
+
if (report.errors.length > 0) {
|
|
76
|
+
console.table(report.errors.map(({ stage, participant, judge, error }) => ({
|
|
77
|
+
stage,
|
|
78
|
+
participant,
|
|
79
|
+
judge,
|
|
80
|
+
error: error.details?.error?.message ?? error.message
|
|
81
|
+
})));
|
|
82
|
+
}
|
|
83
|
+
console.log(`Full responses and report saved to ${resultsDirectory}`);
|
package/demo/package.json
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
"main": "index.js",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"test": "echo \"Error: no test specified\" && exit 1",
|
|
9
|
+
"benchmark": "node benchmark.js",
|
|
10
|
+
"benchmark:template-engine": "node benchmark.js prompts/template-engine.txt",
|
|
9
11
|
"rlm-basic": "node rlm-basic.js",
|
|
10
12
|
"rlm-simple": "node rlm-simple.js",
|
|
11
13
|
"repl-powers": "node repl-powers.js"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
Escribí un cuento breve (600–900 palabras), de cualquier género y ambientación, que cumpla estas restricciones de construcción:
|
|
2
|
+
|
|
3
|
+
ESTRUCTURA
|
|
4
|
+
|
|
5
|
+
- Circular: el protagonista termina en el mismo lugar físico donde empieza, pero al final ese lugar significa lo contrario (refugio → trampa, punto de partida → castigo, etc.).
|
|
6
|
+
- El protagonista enfrenta una prueba o rito con una regla explícita. El criterio real de evaluación es otro, nunca se enuncia, y solo se revela en la última línea de diálogo.
|
|
7
|
+
- El protagonista cumple la tarea visible, pagando un costo moral o físico real, y aun así fracasa en la prueba verdadera.
|
|
8
|
+
- El final muestra la inminencia de la consecuencia, no la consecuencia. Cortar justo antes.
|
|
9
|
+
- La consecuencia final es simétrica: lo que el protagonista dañó durante la prueba vuelve contra él.
|
|
10
|
+
|
|
11
|
+
SIEMBRA
|
|
12
|
+
|
|
13
|
+
- Un personaje marginal y sin peso aparente (un loco, un borracho, un niño, una voz anónima) anuncia el final con una frase críptica. El protagonista lo ignora.
|
|
14
|
+
- Al menos dos detalles que parecen ambientales o caprichosos deben resultar funcionales al final (por ejemplo, algo que impide al protagonista orientarse, o una marca física que lo delata).
|
|
15
|
+
- El momento en que el protagonista falla debe estar a la vista del lector, sin ser señalado como decisivo. Debe resignificarse en la relectura.
|
|
16
|
+
- Una frase o instrucción repetida al principio y al final que enmarque el recorrido.
|
|
17
|
+
|
|
18
|
+
PERSONAJES
|
|
19
|
+
|
|
20
|
+
- Protagonista sin nombre, designado por su función o rol.
|
|
21
|
+
- Antagonista/evaluador con apodo, lacónico, no explica nada y tiene un rasgo físico que oculta (algo que nunca se saca o nunca muestra).
|
|
22
|
+
- La crueldad o arbitrariedad del evaluador se muestra con acciones, sin juicio del narrador.
|
|
23
|
+
- El protagonista enfrenta una elección concreta dentro de la prueba (no si hacerlo, sino cómo o a quién), que lo hace más responsable.
|
|
24
|
+
- Hacia el final, el protagonista hace preguntas que asumen su éxito y no reciben respuesta.
|
|
25
|
+
|
|
26
|
+
VOZ Y TONO
|
|
27
|
+
|
|
28
|
+
- Narrador en primera persona, en pasado, con tono de anécdota oral o leyenda urbana; registro coloquial.
|
|
29
|
+
- El primer párrafo resume la premisa sin rodeos. El suspenso no depende de ocultar de qué trata.
|
|
30
|
+
- Contraste entre un marco cotidiano o burocrático y un contenido brutal o extraño, contado con naturalidad.
|
|
31
|
+
- Al menos una secuencia de diálogo con repetición y escalada (la misma orden dicha varias veces, subiendo de intensidad).
|
|
32
|
+
- Remate: una línea de diálogo muy breve (dos o tres palabras) que explica el fracaso, seguida de la revelación espacial y la amenaza.
|
|
33
|
+
- Economía: descripción mínima, acceso limitado a la mente del protagonista, sin moraleja explícita.
|
|
34
|
+
|
|
35
|
+
No imites la trama de ningún cuento existente. Inventá la prueba, el mundo y la consecuencia desde cero.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Develop a minimalist template engine as a Node.js library using ES modules and no external dependencies.
|
|
2
|
+
|
|
3
|
+
The complete implementation must fit within 200 physical lines, including comments and blank lines. Do not minify the code or artificially pack statements onto single lines. Write code and identifiers in English, and explanations in Spanish.
|
|
4
|
+
|
|
5
|
+
Implement these three features:
|
|
6
|
+
|
|
7
|
+
1. Compilation to JavaScript
|
|
8
|
+
- `compile(template, options)` returns a reusable function that accepts data and produces a string.
|
|
9
|
+
- Support `<% code %>` for executing JavaScript, `<%= expression %>` for HTML-escaped output, and `<%- expression %>` for unescaped output.
|
|
10
|
+
- Allow conditionals and loops to open and close across separate tags, with text between them.
|
|
11
|
+
- Correctly preserve quotes, backslashes, and line breaks in literal text.
|
|
12
|
+
2. Errors linked to the original template
|
|
13
|
+
- Runtime errors must report the filename, original line number, and a short template excerpt.
|
|
14
|
+
- Preserve the original error as `cause`.
|
|
15
|
+
- For errors inside includes, identify the file where the failure actually occurred.
|
|
16
|
+
- For syntax errors, report at least the filename. Do not invent a line number if it cannot be determined accurately.
|
|
17
|
+
3. Nested includes
|
|
18
|
+
- Provide an `include(path, data)` function inside templates.
|
|
19
|
+
- Resolve each relative path from the file containing the include call.
|
|
20
|
+
- Includes inherit parent data and may override values locally without mutating the parent data.
|
|
21
|
+
- Support multiple nesting levels.
|
|
22
|
+
- Detect inclusion cycles and report the chain of files involved.
|
|
23
|
+
- Provide clear errors for missing files.
|
|
24
|
+
|
|
25
|
+
Public API:
|
|
26
|
+
|
|
27
|
+
- `compile(template, options)`
|
|
28
|
+
- `render(template, data, options)`
|
|
29
|
+
- `renderFile(filename, data)`
|
|
30
|
+
|
|
31
|
+
Keep everything synchronous. Require explicit property access through the `data` object inside templates, without `with`. Do not add caching, layouts, plugins, configurable delimiters, or browser support.
|
|
32
|
+
|
|
33
|
+
Templates are trusted code. Explicitly document that the engine provides no sandbox and must not execute templates supplied by untrusted users.
|
|
34
|
+
|
|
35
|
+
Deliver only:
|
|
36
|
+
|
|
37
|
+
- The complete implementation in a single file.
|
|
38
|
+
- The actual line count of the implementation file.
|
|
39
|
+
- A brief explanation of limitations.
|
|
40
|
+
|
|
41
|
+
Do not include tests, examples, sample templates, or demonstration code. Prioritize clarity and correctness over additional features.
|
package/demo/short.js
CHANGED
|
@@ -28,6 +28,8 @@ const mmix = await ModelMix.new(setup)
|
|
|
28
28
|
.museSpark12() // (fallback 17) OpenRouter meta/muse-spark-1.2
|
|
29
29
|
.museSpark13c() // (fallback 18) OpenRouter meta/muse-spark-1.3-contributor
|
|
30
30
|
.gpt6astra() // (fallback 19) OpenAI gpt-6-astra
|
|
31
|
+
.deepseekV41Flash({ mix: { deepseek: true, fireworks: true, openrouter: true } }) // (fallback 20 + provider fallback) DeepSeek/Fireworks/OpenRouter V4.1 Flash
|
|
32
|
+
.deepseekPro() // (fallback 21) OpenRouter deepseek/deepseek-v4-pro-0813
|
|
31
33
|
.addText("What's your name?");
|
|
32
34
|
|
|
33
35
|
console.log(await mmix.message());
|
package/effort.js
CHANGED
|
@@ -125,6 +125,7 @@ const PROVIDER_FAMILY_BY_CLASS = {
|
|
|
125
125
|
MixKimi: 'openai',
|
|
126
126
|
MixMiniMax: 'openai',
|
|
127
127
|
MixMiMo: 'openai',
|
|
128
|
+
MixDeepSeek: 'openai',
|
|
128
129
|
MixGrok: 'openai',
|
|
129
130
|
MixGroq: 'openai',
|
|
130
131
|
MixTogether: 'openai',
|
|
@@ -232,7 +233,8 @@ function isGemini25(modelKey) {
|
|
|
232
233
|
function isDeepSeekV4(modelKey) {
|
|
233
234
|
if (typeof modelKey !== 'string') return false;
|
|
234
235
|
const key = modelKey.toLowerCase();
|
|
235
|
-
return key
|
|
236
|
+
return key === 'deepseek-flash' || key === '~deepseek/deepseek-pro-latest'
|
|
237
|
+
|| key.includes('deepseek-v4') || key.includes('deepseek_v4');
|
|
236
238
|
}
|
|
237
239
|
|
|
238
240
|
function isMiniMax(modelKey) {
|
package/index.d.ts
CHANGED
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Type definitions for modelmix
|
|
3
|
-
* @see https://github.com/clasen/ModelMix
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
1
|
export type MessageRole = 'user' | 'assistant' | 'system' | 'tool' | string;
|
|
7
2
|
|
|
8
3
|
export type DebugLevel = 0 | 1 | 2 | 3 | 4;
|
|
@@ -87,6 +82,7 @@ export interface ModelMixMixFlags {
|
|
|
87
82
|
moonshot?: boolean;
|
|
88
83
|
minimax?: boolean;
|
|
89
84
|
mimo?: boolean;
|
|
85
|
+
deepseek?: boolean;
|
|
90
86
|
[key: string]: boolean | undefined;
|
|
91
87
|
}
|
|
92
88
|
|
|
@@ -161,6 +157,7 @@ export interface TokenCostBreakdown {
|
|
|
161
157
|
|
|
162
158
|
export interface TokenUsage {
|
|
163
159
|
input: number;
|
|
160
|
+
/** Generated tokens excluding separately reported reasoning. */
|
|
164
161
|
output: number;
|
|
165
162
|
/** Internal reasoning tokens billed at the output rate when reported separately. */
|
|
166
163
|
thinking: number;
|
|
@@ -177,7 +174,9 @@ export interface TokenUsage {
|
|
|
177
174
|
cacheWritePremium: number;
|
|
178
175
|
/** Full future cache hits needed to recover the current write premium. */
|
|
179
176
|
breakEvenHits: number;
|
|
177
|
+
/** USD charged by OpenRouter when available, otherwise the catalog estimate. */
|
|
180
178
|
cost: number;
|
|
179
|
+
/** Catalog-based estimate; its total can differ from the reported charge in cost. */
|
|
181
180
|
costBreakdown: TokenCostBreakdown;
|
|
182
181
|
speed?: number;
|
|
183
182
|
}
|
|
@@ -254,6 +253,7 @@ export interface PluginExecutionContext {
|
|
|
254
253
|
request: {
|
|
255
254
|
system: string;
|
|
256
255
|
messages: ChatMessage[];
|
|
256
|
+
tools: ToolWithCallback[];
|
|
257
257
|
options: ModelMixOptions;
|
|
258
258
|
config: ModelMixConfig;
|
|
259
259
|
outputMode: ModelMixOutputMode;
|
|
@@ -527,7 +527,10 @@ export declare class ModelMix {
|
|
|
527
527
|
mimo25(args?: ModelAttachArgs): this;
|
|
528
528
|
mimo25pro(args?: ModelAttachArgs): this;
|
|
529
529
|
deepseekV4Pro(args?: ModelAttachArgs): this;
|
|
530
|
+
/** Uses deepseek/deepseek-v4-pro-0813 through OpenRouter. */
|
|
531
|
+
deepseekPro(args?: ModelAttachArgs): this;
|
|
530
532
|
deepseekV4Flash(args?: ModelAttachArgs): this;
|
|
533
|
+
deepseekV41Flash(args?: ModelAttachArgs): this;
|
|
531
534
|
GLM52(args?: ModelAttachArgs): this;
|
|
532
535
|
GLM53(args?: ModelAttachArgs): this;
|
|
533
536
|
GLM53Flash(args?: ModelAttachArgs): this;
|
|
@@ -624,6 +627,7 @@ export declare class MixKimi extends MixOpenAI {}
|
|
|
624
627
|
export declare class MixAnthropic extends MixCustom {}
|
|
625
628
|
export declare class MixMiniMax extends MixOpenAI {}
|
|
626
629
|
export declare class MixMiMo extends MixOpenAI {}
|
|
630
|
+
export declare class MixDeepSeek extends MixOpenAI {}
|
|
627
631
|
export declare class MixPerplexity extends MixCustom {}
|
|
628
632
|
export declare class MixOllama extends MixCustom {}
|
|
629
633
|
export declare class MixGrok extends MixOpenAI {}
|