@lawaty/lacode 2.1.0-beta.0 → 2.2.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/index.js +60 -9
- package/dist/pool.js +53 -8
- package/docs/CONFIGURATION.md +3 -1
- package/package.json +2 -1
- package/presets/README.md +6 -6
- package/scripts/pool-selection.mjs +187 -0
- package/scripts/refresh-pool.mjs +78 -61
- package/scripts/score-models.mjs +93 -38
package/CHANGELOG.md
CHANGED
|
@@ -9,6 +9,19 @@ commit as the changelog entry it describes.
|
|
|
9
9
|
|
|
10
10
|
## [Unreleased]
|
|
11
11
|
|
|
12
|
+
## [2.2.0-beta.0] — context-aware pool routing
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- Pool: context-aware routing. `pool.txt` lines carry a `c:<context>` token, and the router prefers the smallest model whose context fits the task prompt, so large-context models are reserved for large tasks.
|
|
17
|
+
- Pool: `npm run score` probes providers with no direct HTTP endpoint (or an unexpanded `${...}` URL) through the local opencode session, and considers catalogue-priced models on providers other than `opencode`/`opencode-go`, whose credits cover them.
|
|
18
|
+
- Pool: a rate-limit reset or a `402 out of credits` answer schedules a transient retry; a transient failure keeps the model's previous score until the retry.
|
|
19
|
+
|
|
20
|
+
### Changed
|
|
21
|
+
|
|
22
|
+
- Pool: the context floor is now 100k. Slots are ordered as `opencode-go` (weight 4), small-context providers (weight 2), large-context providers (weight 1), and at most one `opencode` model last.
|
|
23
|
+
- Pool: small-context models are preferred for ordinary work; large-context models are used when the estimated task size needs them or when small slots are full.
|
|
24
|
+
|
|
12
25
|
## [2.1.0-beta.0] — every provider counts, and the timer installs itself
|
|
13
26
|
|
|
14
27
|
### Added
|
package/dist/index.js
CHANGED
|
@@ -109,7 +109,14 @@ function parseSlots(parsed, origin) {
|
|
|
109
109
|
problems.push(`${at} (${model}): bucket must be a non-empty name; using the model id`);
|
|
110
110
|
}
|
|
111
111
|
const notify = coerceNotify(entry.notify, at, model, problems);
|
|
112
|
-
|
|
112
|
+
const rawContext = entry?.context;
|
|
113
|
+
let context;
|
|
114
|
+
if (rawContext !== void 0) {
|
|
115
|
+
const parsedContext = typeof rawContext === "number" ? rawContext : Number(rawContext);
|
|
116
|
+
if (Number.isFinite(parsedContext) && parsedContext > 0) context = parsedContext;
|
|
117
|
+
else problems.push(`${at} (${model}): context must be a positive number, got ${JSON.stringify(rawContext)}; ignoring it`);
|
|
118
|
+
}
|
|
119
|
+
slots.push({ index: slots.length + 1, model, weight, ...bucket ? { bucket } : {}, notify, ...context === void 0 ? {} : { context } });
|
|
113
120
|
});
|
|
114
121
|
if (slots.length > SLOTS_WARN_AT) {
|
|
115
122
|
problems.push(`${origin}: ${slots.length} slots is unusually large, but every entry is kept (the pool has no slot cap)`);
|
|
@@ -129,18 +136,26 @@ function parseTextSlots(text, origin) {
|
|
|
129
136
|
let weight;
|
|
130
137
|
let bucket;
|
|
131
138
|
let notify;
|
|
139
|
+
let context;
|
|
132
140
|
for (const token of rest) {
|
|
133
141
|
if (token.startsWith("notify:")) {
|
|
134
142
|
if (notify === void 0) notify = token.slice("notify:".length);
|
|
135
143
|
continue;
|
|
136
144
|
}
|
|
145
|
+
if (token.startsWith("c:")) {
|
|
146
|
+
if (context === void 0) {
|
|
147
|
+
const parsedContext = Number(token.slice(2));
|
|
148
|
+
if (Number.isFinite(parsedContext) && parsedContext > 0) context = parsedContext;
|
|
149
|
+
}
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
137
152
|
if (token.startsWith("b:")) {
|
|
138
153
|
if (bucket === void 0) bucket = token.slice(2);
|
|
139
154
|
} else if (weight === void 0) {
|
|
140
155
|
weight = token;
|
|
141
156
|
}
|
|
142
157
|
}
|
|
143
|
-
entries.push({ model, ...weight === void 0 ? {} : { weight }, ...bucket === void 0 ? {} : { bucket }, ...notify === void 0 ? {} : { notify } });
|
|
158
|
+
entries.push({ model, ...weight === void 0 ? {} : { weight }, ...bucket === void 0 ? {} : { bucket }, ...notify === void 0 ? {} : { notify }, ...context === void 0 ? {} : { context } });
|
|
144
159
|
}
|
|
145
160
|
return parseSlots({ slots: entries }, origin);
|
|
146
161
|
}
|
|
@@ -713,20 +728,50 @@ function snapshot(s) {
|
|
|
713
728
|
}
|
|
714
729
|
var loadOf = (counts, slot) => counts.get(slot) ?? 0;
|
|
715
730
|
var underCeiling = (slot, counts) => loadOf(counts, slot.index) < slotCeiling(slot);
|
|
716
|
-
function pickSlot(s, snap) {
|
|
731
|
+
function pickSlot(s, snap, neededContext) {
|
|
717
732
|
const slots = s.slots.length > 0 ? s.slots : FALLBACK_SLOTS;
|
|
718
733
|
const configured = slots.filter((slot) => providerReady(s, slot.model));
|
|
719
734
|
const usable = configured.filter((slot) => isAvailable(s, slot.model, snap.cooling));
|
|
720
735
|
const pool = usable.length > 0 ? usable : configured.length > 0 ? configured : slots;
|
|
721
|
-
|
|
722
|
-
|
|
736
|
+
if (neededContext === void 0) {
|
|
737
|
+
for (const slot of usable) {
|
|
738
|
+
if (loadOf(snap.counts, slot.index) < slot.weight) return slot;
|
|
739
|
+
}
|
|
740
|
+
return pool.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, pool[0]);
|
|
741
|
+
}
|
|
742
|
+
const adequate = usable.filter((slot) => slot.context == null || slot.context >= neededContext);
|
|
743
|
+
const adequateKnown = adequate.filter((slot) => slot.context != null);
|
|
744
|
+
const candidates = adequate.length > 0 ? adequate : usable;
|
|
745
|
+
const fallbackPool = adequate.length > 0 ? adequate : pool;
|
|
746
|
+
if (adequateKnown.length === 0) {
|
|
747
|
+
const known = usable.filter((slot) => slot.context != null);
|
|
748
|
+
if (known.length > 0) {
|
|
749
|
+
const largest = Math.max(...known.map((slot) => slot.context));
|
|
750
|
+
const largestSlots = known.filter((slot) => slot.context === largest);
|
|
751
|
+
for (const slot of largestSlots) {
|
|
752
|
+
if (loadOf(snap.counts, slot.index) < slot.weight) return slot;
|
|
753
|
+
}
|
|
754
|
+
return largestSlots.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, largestSlots[0]);
|
|
755
|
+
}
|
|
756
|
+
for (const slot of candidates) {
|
|
757
|
+
if (loadOf(snap.counts, slot.index) < slot.weight) return slot;
|
|
758
|
+
}
|
|
759
|
+
return fallbackPool.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, fallbackPool[0]);
|
|
723
760
|
}
|
|
724
|
-
|
|
761
|
+
const underWeight = candidates.filter((slot) => loadOf(snap.counts, slot.index) < slot.weight);
|
|
762
|
+
if (underWeight.length > 0) {
|
|
763
|
+
const known = underWeight.filter((slot) => slot.context != null);
|
|
764
|
+
if (known.length === 0) return underWeight[0];
|
|
765
|
+
return known.reduce((best, slot) => slot.context < best.context ? slot : best, known[0]);
|
|
766
|
+
}
|
|
767
|
+
return candidates.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, candidates[0]);
|
|
725
768
|
}
|
|
726
|
-
function poolPass(s, snap) {
|
|
769
|
+
function poolPass(s, snap, neededContext) {
|
|
727
770
|
const slots = s.slots.length > 0 ? s.slots : FALLBACK_SLOTS;
|
|
728
771
|
const usable = slots.filter((slot) => isAvailable(s, slot.model, snap.cooling));
|
|
729
|
-
|
|
772
|
+
const candidates = neededContext === void 0 ? usable : usable.filter((slot) => slot.context == null || slot.context >= neededContext);
|
|
773
|
+
const pool = candidates.length > 0 ? candidates : usable;
|
|
774
|
+
if (pool.some((slot) => loadOf(snap.counts, slot.index) < slot.weight)) return 1;
|
|
730
775
|
return "overflow";
|
|
731
776
|
}
|
|
732
777
|
function agentFor(base, slot) {
|
|
@@ -2964,6 +3009,11 @@ var reportText = (report) => {
|
|
|
2964
3009
|
`summary: ${answered} answered, ${benched} benched, ${cooling} still cooling`
|
|
2965
3010
|
].join("\n");
|
|
2966
3011
|
};
|
|
3012
|
+
var CONTEXT_HEADROOM_TOKENS = 2e4;
|
|
3013
|
+
function estimateContextTokens(prompt) {
|
|
3014
|
+
if (typeof prompt !== "string" || prompt.length === 0) return void 0;
|
|
3015
|
+
return Math.ceil(prompt.length / 4) + CONTEXT_HEADROOM_TOKENS;
|
|
3016
|
+
}
|
|
2967
3017
|
var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
|
|
2968
3018
|
const opts = options ?? {};
|
|
2969
3019
|
const notify = opts.notify ?? ((title, message) => void notifyDesktop(title, message));
|
|
@@ -3887,7 +3937,8 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
|
|
|
3887
3937
|
refreshCatalog(s, serverUrl);
|
|
3888
3938
|
const snap = snapshot(s);
|
|
3889
3939
|
if (BASES.includes(requested)) {
|
|
3890
|
-
const
|
|
3940
|
+
const neededContext = estimateContextTokens(output.args?.prompt);
|
|
3941
|
+
const slot = pickSlot(s, snap, neededContext);
|
|
3891
3942
|
const variant = agentFor(requested, slot.index);
|
|
3892
3943
|
if (s.known.size > 0 && !s.known.has(variant)) {
|
|
3893
3944
|
await warnOnce(`variant ${variant} missing from config; spawn left unchanged`);
|
package/dist/pool.js
CHANGED
|
@@ -109,7 +109,14 @@ function parseSlots(parsed, origin) {
|
|
|
109
109
|
problems.push(`${at} (${model}): bucket must be a non-empty name; using the model id`);
|
|
110
110
|
}
|
|
111
111
|
const notify = coerceNotify(entry.notify, at, model, problems);
|
|
112
|
-
|
|
112
|
+
const rawContext = entry?.context;
|
|
113
|
+
let context;
|
|
114
|
+
if (rawContext !== void 0) {
|
|
115
|
+
const parsedContext = typeof rawContext === "number" ? rawContext : Number(rawContext);
|
|
116
|
+
if (Number.isFinite(parsedContext) && parsedContext > 0) context = parsedContext;
|
|
117
|
+
else problems.push(`${at} (${model}): context must be a positive number, got ${JSON.stringify(rawContext)}; ignoring it`);
|
|
118
|
+
}
|
|
119
|
+
slots.push({ index: slots.length + 1, model, weight, ...bucket ? { bucket } : {}, notify, ...context === void 0 ? {} : { context } });
|
|
113
120
|
});
|
|
114
121
|
if (slots.length > SLOTS_WARN_AT) {
|
|
115
122
|
problems.push(`${origin}: ${slots.length} slots is unusually large, but every entry is kept (the pool has no slot cap)`);
|
|
@@ -129,18 +136,26 @@ function parseTextSlots(text, origin) {
|
|
|
129
136
|
let weight;
|
|
130
137
|
let bucket;
|
|
131
138
|
let notify;
|
|
139
|
+
let context;
|
|
132
140
|
for (const token of rest) {
|
|
133
141
|
if (token.startsWith("notify:")) {
|
|
134
142
|
if (notify === void 0) notify = token.slice("notify:".length);
|
|
135
143
|
continue;
|
|
136
144
|
}
|
|
145
|
+
if (token.startsWith("c:")) {
|
|
146
|
+
if (context === void 0) {
|
|
147
|
+
const parsedContext = Number(token.slice(2));
|
|
148
|
+
if (Number.isFinite(parsedContext) && parsedContext > 0) context = parsedContext;
|
|
149
|
+
}
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
137
152
|
if (token.startsWith("b:")) {
|
|
138
153
|
if (bucket === void 0) bucket = token.slice(2);
|
|
139
154
|
} else if (weight === void 0) {
|
|
140
155
|
weight = token;
|
|
141
156
|
}
|
|
142
157
|
}
|
|
143
|
-
entries.push({ model, ...weight === void 0 ? {} : { weight }, ...bucket === void 0 ? {} : { bucket }, ...notify === void 0 ? {} : { notify } });
|
|
158
|
+
entries.push({ model, ...weight === void 0 ? {} : { weight }, ...bucket === void 0 ? {} : { bucket }, ...notify === void 0 ? {} : { notify }, ...context === void 0 ? {} : { context } });
|
|
144
159
|
}
|
|
145
160
|
return parseSlots({ slots: entries }, origin);
|
|
146
161
|
}
|
|
@@ -713,20 +728,50 @@ function snapshot(s) {
|
|
|
713
728
|
}
|
|
714
729
|
var loadOf = (counts, slot) => counts.get(slot) ?? 0;
|
|
715
730
|
var underCeiling = (slot, counts) => loadOf(counts, slot.index) < slotCeiling(slot);
|
|
716
|
-
function pickSlot(s, snap) {
|
|
731
|
+
function pickSlot(s, snap, neededContext) {
|
|
717
732
|
const slots = s.slots.length > 0 ? s.slots : FALLBACK_SLOTS;
|
|
718
733
|
const configured = slots.filter((slot) => providerReady(s, slot.model));
|
|
719
734
|
const usable = configured.filter((slot) => isAvailable(s, slot.model, snap.cooling));
|
|
720
735
|
const pool = usable.length > 0 ? usable : configured.length > 0 ? configured : slots;
|
|
721
|
-
|
|
722
|
-
|
|
736
|
+
if (neededContext === void 0) {
|
|
737
|
+
for (const slot of usable) {
|
|
738
|
+
if (loadOf(snap.counts, slot.index) < slot.weight) return slot;
|
|
739
|
+
}
|
|
740
|
+
return pool.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, pool[0]);
|
|
741
|
+
}
|
|
742
|
+
const adequate = usable.filter((slot) => slot.context == null || slot.context >= neededContext);
|
|
743
|
+
const adequateKnown = adequate.filter((slot) => slot.context != null);
|
|
744
|
+
const candidates = adequate.length > 0 ? adequate : usable;
|
|
745
|
+
const fallbackPool = adequate.length > 0 ? adequate : pool;
|
|
746
|
+
if (adequateKnown.length === 0) {
|
|
747
|
+
const known = usable.filter((slot) => slot.context != null);
|
|
748
|
+
if (known.length > 0) {
|
|
749
|
+
const largest = Math.max(...known.map((slot) => slot.context));
|
|
750
|
+
const largestSlots = known.filter((slot) => slot.context === largest);
|
|
751
|
+
for (const slot of largestSlots) {
|
|
752
|
+
if (loadOf(snap.counts, slot.index) < slot.weight) return slot;
|
|
753
|
+
}
|
|
754
|
+
return largestSlots.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, largestSlots[0]);
|
|
755
|
+
}
|
|
756
|
+
for (const slot of candidates) {
|
|
757
|
+
if (loadOf(snap.counts, slot.index) < slot.weight) return slot;
|
|
758
|
+
}
|
|
759
|
+
return fallbackPool.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, fallbackPool[0]);
|
|
760
|
+
}
|
|
761
|
+
const underWeight = candidates.filter((slot) => loadOf(snap.counts, slot.index) < slot.weight);
|
|
762
|
+
if (underWeight.length > 0) {
|
|
763
|
+
const known = underWeight.filter((slot) => slot.context != null);
|
|
764
|
+
if (known.length === 0) return underWeight[0];
|
|
765
|
+
return known.reduce((best, slot) => slot.context < best.context ? slot : best, known[0]);
|
|
723
766
|
}
|
|
724
|
-
return
|
|
767
|
+
return candidates.reduce((best, slot) => loadOf(snap.counts, slot.index) < loadOf(snap.counts, best.index) ? slot : best, candidates[0]);
|
|
725
768
|
}
|
|
726
|
-
function poolPass(s, snap) {
|
|
769
|
+
function poolPass(s, snap, neededContext) {
|
|
727
770
|
const slots = s.slots.length > 0 ? s.slots : FALLBACK_SLOTS;
|
|
728
771
|
const usable = slots.filter((slot) => isAvailable(s, slot.model, snap.cooling));
|
|
729
|
-
|
|
772
|
+
const candidates = neededContext === void 0 ? usable : usable.filter((slot) => slot.context == null || slot.context >= neededContext);
|
|
773
|
+
const pool = candidates.length > 0 ? candidates : usable;
|
|
774
|
+
if (pool.some((slot) => loadOf(snap.counts, slot.index) < slot.weight)) return 1;
|
|
730
775
|
return "overflow";
|
|
731
776
|
}
|
|
732
777
|
function agentFor(base, slot) {
|
package/docs/CONFIGURATION.md
CHANGED
|
@@ -7,7 +7,7 @@ The plugin options, where the model pool gets its slots, and exactly what LaCode
|
|
|
7
7
|
The slot list is the first of these that yields usable slots:
|
|
8
8
|
|
|
9
9
|
1. plugin options — `models` in the [tuple](../README.md#configure)
|
|
10
|
-
2. `~/.config/lacode/pool.txt` — flat `model [weight] [b:<bucket>] [notify:on|off]` per line, preferred
|
|
10
|
+
2. `~/.config/lacode/pool.txt` — flat `model [weight] [b:<bucket>] [c:<context>] [notify:on|off]` per line, preferred
|
|
11
11
|
3. `~/.config/lacode/pool.json` (its parent directory is created for you)
|
|
12
12
|
4. the bundled `presets/free-tier.json`
|
|
13
13
|
5. a built-in fallback, so routing always works
|
|
@@ -33,6 +33,8 @@ A slot may also declare a shared quota bucket with `b:<name>` on the line (or a
|
|
|
33
33
|
the sibling would spend the same exhausted window; without one, the model is its
|
|
34
34
|
own bucket. `pool_status` shows each shared bucket's effective capacity.
|
|
35
35
|
|
|
36
|
+
`c:<context>` is the model's context window in tokens; the router uses it to prefer the smallest model that fits the task, and it is optional for hand-written lines.
|
|
37
|
+
|
|
36
38
|
Each slot gets a `notify` flag that controls the desktop "rate-limit reset"
|
|
37
39
|
notice raised when a slot that was cooling for a quota/rate-limit reason becomes
|
|
38
40
|
usable again. The default is **provider-based**: on for `opencode` and
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lawaty/lacode",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.2.0-beta.0",
|
|
4
4
|
"description": "Ultimate opencode setup for Go users — cost-controlled pooled subagents, hang recovery, and notifications for opencode.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -51,6 +51,7 @@
|
|
|
51
51
|
"README.md",
|
|
52
52
|
"docs",
|
|
53
53
|
"scripts/score-models.mjs",
|
|
54
|
+
"scripts/pool-selection.mjs",
|
|
54
55
|
"scripts/refresh-pool.mjs"
|
|
55
56
|
],
|
|
56
57
|
"scripts": {
|
package/presets/README.md
CHANGED
|
@@ -36,15 +36,15 @@ change here takes effect without a restart. Plugin CODE still needs one.
|
|
|
36
36
|
in place, no JSON and no objects:
|
|
37
37
|
|
|
38
38
|
```text
|
|
39
|
-
# provider/model-id weight
|
|
40
|
-
opencode/big-pickle 3
|
|
41
|
-
opencode-go/beam-free 2
|
|
42
|
-
opencode/deepseek-v4.1-flash-free 1
|
|
43
|
-
opencode/glm-5.3-flash-free 1
|
|
39
|
+
# provider/model-id weight c:<context>
|
|
40
|
+
opencode/big-pickle 3 c:200000
|
|
41
|
+
opencode-go/beam-free 2 c:200000
|
|
42
|
+
opencode/deepseek-v4.1-flash-free 1 c:200000
|
|
43
|
+
opencode/glm-5.3-flash-free 1 c:200000
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
`#` starts a comment (whole-line or trailing), blank lines are ignored, and the
|
|
47
|
-
weight is optional (default 1). The rules are exactly the same as the JSON form
|
|
47
|
+
weight is optional (default 1). `c:<context>` is the model's context window in tokens; the router uses it to prefer the smallest model that fits the task, and it is optional for hand-written lines. The rules are exactly the same as the JSON form
|
|
48
48
|
below: any number of slots, a model may repeat on another line, split on the
|
|
49
49
|
first slash only. The loader accepts either shape in the same file, so a leading
|
|
50
50
|
`{` means JSON and anything else means the flat form.
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// Pure pool selection: which scored models become slots, and when a limited
|
|
2
|
+
// model is worth retrying. Imported by scripts/score-models.mjs; refresh-pool
|
|
3
|
+
// consumes the resulting `selection` and does not repeat this policy.
|
|
4
|
+
|
|
5
|
+
export const MIN_CONTEXT = 100_000
|
|
6
|
+
export const SMALL_CONTEXT_MAX = 200_000
|
|
7
|
+
export const MIN_SCORE = 60
|
|
8
|
+
export const POOL_SIZE = 10
|
|
9
|
+
export const MAX_ZEN = 1
|
|
10
|
+
export const LARGE_RESERVE = 3
|
|
11
|
+
export const MAX_PER_PROVIDER = 2
|
|
12
|
+
export const MAX_PER_PROVIDER_LARGE = 1
|
|
13
|
+
export const DEFAULT_QUOTA_RETRY_MS = 15 * 60 * 1000
|
|
14
|
+
export const CREDIT_RETRY_MS = 6 * 60 * 60 * 1000
|
|
15
|
+
export const MAX_RETRY_DELAY_MS = 24 * 60 * 60 * 1000
|
|
16
|
+
export const STATED_COOLDOWN_MAX_MS = 6 * 60 * 60 * 1000
|
|
17
|
+
|
|
18
|
+
export const isTransientFailure = (row) =>
|
|
19
|
+
row?.ok === false &&
|
|
20
|
+
/timeout|timed out|rate-limit|no answer|no response|unreachable|ECONN|ETIMEDOUT|fetch failed|socket hang up|credit|402/i.test(
|
|
21
|
+
String(row.reason ?? ""),
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
const isLimitText = (text) =>
|
|
25
|
+
/rate[\s_-]?limit|usage limit|too many requests|quota|freelimiterror|overloaded|limits?[\s_-]?(?:reached|exceeded)|\b429\b/i.test(
|
|
26
|
+
String(text ?? ""),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
const isDailyLimit = (text) => /per[\s-]?day|daily|free-models-per-day|requests?\s+per\s+day/i.test(String(text ?? ""))
|
|
30
|
+
|
|
31
|
+
const isCreditText = /out of credits|insufficient credits|credit balance|requires more credits|\b402\b/i
|
|
32
|
+
|
|
33
|
+
const bounded = (ms) => (Number.isFinite(ms) && ms > 0 && ms <= STATED_COOLDOWN_MAX_MS ? ms : undefined)
|
|
34
|
+
|
|
35
|
+
export const statedCooldownMs = (text) => {
|
|
36
|
+
const value = String(text ?? "")
|
|
37
|
+
const hm = /resets?\s+in\s+(\d+)\s*(?:hour|hr|h)s?\s*(\d+)?\s*(?:minute|min|m)?s?/i.exec(value)
|
|
38
|
+
if (hm) return bounded((Number(hm[1]) * 60 + (hm[2] ? Number(hm[2]) : 0)) * 60 * 1000)
|
|
39
|
+
const m = /resets?\s+in\s+(\d+)\s*(?:minute|min|m)s?/i.exec(value)
|
|
40
|
+
if (m) return bounded(Number(m[1]) * 60 * 1000)
|
|
41
|
+
const s = /retry[\s-]?in\s+(\d+)\s*(?:second|sec|s)s?/i.exec(value)
|
|
42
|
+
if (s) return bounded(Number(s[1]) * 1000)
|
|
43
|
+
return undefined
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const retryAfterHeaderMs = (value) => {
|
|
47
|
+
if (!value) return undefined
|
|
48
|
+
const text = String(value).trim()
|
|
49
|
+
if (/^\d+$/.test(text)) return bounded(Number(text) * 1000)
|
|
50
|
+
const at = Date.parse(text)
|
|
51
|
+
return Number.isFinite(at) ? bounded(at - Date.now()) : undefined
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function retryAfterMs(text, header, now = Date.now()) {
|
|
55
|
+
const headerMs = retryAfterHeaderMs(header)
|
|
56
|
+
if (headerMs) return now + headerMs
|
|
57
|
+
const stated = statedCooldownMs(text)
|
|
58
|
+
if (stated) return now + stated
|
|
59
|
+
const normalized = String(text ?? "")
|
|
60
|
+
if (isDailyLimit(text)) {
|
|
61
|
+
const d = new Date(now)
|
|
62
|
+
const next = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate() + 1, 0, 0, 0, 0)
|
|
63
|
+
return next > now ? next : now + MAX_RETRY_DELAY_MS
|
|
64
|
+
}
|
|
65
|
+
if (isCreditText.test(normalized)) return now + CREDIT_RETRY_MS
|
|
66
|
+
if (isLimitText(text)) return now + DEFAULT_QUOTA_RETRY_MS
|
|
67
|
+
return undefined
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const rowsOf = (providers) =>
|
|
71
|
+
Array.isArray(providers)
|
|
72
|
+
? providers.flatMap((entry) => (Array.isArray(entry?.rows) ? entry.rows : []))
|
|
73
|
+
: []
|
|
74
|
+
|
|
75
|
+
const providerOf = (model) => String(model ?? "").split("/")[0]
|
|
76
|
+
|
|
77
|
+
export function selectPool({
|
|
78
|
+
providers,
|
|
79
|
+
previousProviders,
|
|
80
|
+
minContext = MIN_CONTEXT,
|
|
81
|
+
minScore = MIN_SCORE,
|
|
82
|
+
poolSize = POOL_SIZE,
|
|
83
|
+
maxZen = MAX_ZEN,
|
|
84
|
+
} = {}) {
|
|
85
|
+
const rankable = (row) =>
|
|
86
|
+
row?.ok === true &&
|
|
87
|
+
typeof row.score === "number" &&
|
|
88
|
+
Number.isFinite(row.score) &&
|
|
89
|
+
row.score >= minScore &&
|
|
90
|
+
(row.context == null || row.context >= minContext)
|
|
91
|
+
const byScore = (a, b) => b.score - a.score || String(a.model).localeCompare(String(b.model))
|
|
92
|
+
|
|
93
|
+
const rows = rowsOf(providers)
|
|
94
|
+
const previousRows = rowsOf(previousProviders)
|
|
95
|
+
const currentByModel = new Map(rows.map((row) => [row.model, row]))
|
|
96
|
+
const erroredProviders = new Set(
|
|
97
|
+
(Array.isArray(providers) ? providers : [])
|
|
98
|
+
.filter((entry) => entry?.error)
|
|
99
|
+
.map((entry) => String(entry.provider ?? "")),
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
// A model the previous run scored is kept when this run could not reach it
|
|
103
|
+
// for a transient reason. A model the provider says is gone is not kept.
|
|
104
|
+
const fallback = previousRows.filter((row) => {
|
|
105
|
+
if (!rankable(row)) return false
|
|
106
|
+
const current = currentByModel.get(row.model)
|
|
107
|
+
if (current) return isTransientFailure(current)
|
|
108
|
+
return erroredProviders.has(providerOf(row.model))
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
const merge = (pick) => {
|
|
112
|
+
const seen = new Set()
|
|
113
|
+
const out = []
|
|
114
|
+
for (const row of [...rows, ...fallback]) {
|
|
115
|
+
if (!pick(row) || !rankable(row) || seen.has(row.model)) continue
|
|
116
|
+
seen.add(row.model)
|
|
117
|
+
out.push(row)
|
|
118
|
+
}
|
|
119
|
+
return out.sort(byScore)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const go = merge((row) => providerOf(row.model) === "opencode-go")
|
|
123
|
+
const other = merge((row) => providerOf(row.model) !== "opencode-go" && providerOf(row.model) !== "opencode")
|
|
124
|
+
const zen = merge((row) => providerOf(row.model) === "opencode")
|
|
125
|
+
|
|
126
|
+
const small = other
|
|
127
|
+
.filter((row) => row.context != null && row.context < SMALL_CONTEXT_MAX)
|
|
128
|
+
.sort(byScore)
|
|
129
|
+
const large = other
|
|
130
|
+
.filter((row) => row.context == null || row.context >= SMALL_CONTEXT_MAX)
|
|
131
|
+
.sort(byScore)
|
|
132
|
+
|
|
133
|
+
const capPerProvider = (rows, max) => {
|
|
134
|
+
const counts = new Map()
|
|
135
|
+
const out = []
|
|
136
|
+
for (const row of rows) {
|
|
137
|
+
const provider = providerOf(row.model)
|
|
138
|
+
const count = counts.get(provider) ?? 0
|
|
139
|
+
if (count >= max) continue
|
|
140
|
+
counts.set(provider, count + 1)
|
|
141
|
+
out.push(row)
|
|
142
|
+
}
|
|
143
|
+
return out
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const cappedSmall = capPerProvider(small, MAX_PER_PROVIDER)
|
|
147
|
+
const cappedLarge = capPerProvider(large, MAX_PER_PROVIDER_LARGE)
|
|
148
|
+
|
|
149
|
+
const remaining = Math.max(0, poolSize - go.length)
|
|
150
|
+
const pickedLarge = cappedLarge.slice(0, Math.min(LARGE_RESERVE, remaining))
|
|
151
|
+
const pickedSmall = cappedSmall.slice(0, Math.max(0, remaining - pickedLarge.length))
|
|
152
|
+
|
|
153
|
+
if (pickedLarge.length + pickedSmall.length < remaining) {
|
|
154
|
+
const need = remaining - pickedLarge.length - pickedSmall.length
|
|
155
|
+
pickedLarge.push(...cappedLarge.slice(pickedLarge.length, pickedLarge.length + need))
|
|
156
|
+
}
|
|
157
|
+
if (pickedLarge.length + pickedSmall.length < remaining) {
|
|
158
|
+
const need = remaining - pickedLarge.length - pickedSmall.length
|
|
159
|
+
pickedSmall.push(...cappedSmall.slice(pickedSmall.length, pickedSmall.length + need))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const zenSlots = Math.max(0, Math.min(maxZen, poolSize - go.length - pickedSmall.length - pickedLarge.length))
|
|
163
|
+
const pickedZen = zen.slice(0, zenSlots)
|
|
164
|
+
|
|
165
|
+
const selection = [
|
|
166
|
+
...go.map((row) => ({ model: row.model, weight: 4, context: row.context ?? null })),
|
|
167
|
+
...pickedSmall.map((row) => ({ model: row.model, weight: 2, context: row.context ?? null })),
|
|
168
|
+
...pickedLarge.map((row) => ({ model: row.model, weight: 1, context: row.context ?? null })),
|
|
169
|
+
...pickedZen.map((row) => ({ model: row.model, weight: 1, bucket: "opencode", context: row.context ?? null })),
|
|
170
|
+
]
|
|
171
|
+
|
|
172
|
+
const warnings = []
|
|
173
|
+
const total = selection.length
|
|
174
|
+
if (total === 0) warnings.push("no scorable model at or above the minimum score is available")
|
|
175
|
+
else if (total < poolSize) warnings.push(`only ${total} scorable model(s) available, wanted ${poolSize}`)
|
|
176
|
+
|
|
177
|
+
const limited = rows.filter((row) => row?.ok === false && Number.isFinite(row.retryAfter))
|
|
178
|
+
const retry =
|
|
179
|
+
limited.length > 0
|
|
180
|
+
? {
|
|
181
|
+
after: new Date(Math.min(...limited.map((row) => row.retryAfter))).toISOString(),
|
|
182
|
+
models: limited.map((row) => row.model),
|
|
183
|
+
}
|
|
184
|
+
: undefined
|
|
185
|
+
|
|
186
|
+
return { selection, warnings, retry }
|
|
187
|
+
}
|
package/scripts/refresh-pool.mjs
CHANGED
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
// 1. every opencode-go model with ok:true, a numeric score at or above the
|
|
10
10
|
// score floor and a context at or above the context floor, sorted by score
|
|
11
11
|
// descending, weight 4
|
|
12
|
-
// 2. the best (10 - n) models from every
|
|
13
|
-
// descending, weight 2
|
|
12
|
+
// 2. the best (10 - n) models from every provider other than opencode,
|
|
13
|
+
// sorted by score descending, weight 2
|
|
14
|
+
// 3. at most one opencode model as a last resort, weight 2, with b:opencode
|
|
14
15
|
//
|
|
15
16
|
// Total = n + min(10 - n, #others), where n = #1.
|
|
16
17
|
//
|
|
@@ -28,6 +29,8 @@ export const DEFAULT_POOL_PATH = join(HOME, ".config", "lacode", "pool.txt")
|
|
|
28
29
|
export const SYSTEMD_USER_DIR = (home = HOME) => join(home, ".config", "systemd", "user")
|
|
29
30
|
export const STABLE_LIB_DIR = (home = HOME) => join(home, ".local", "lib", "lacode")
|
|
30
31
|
export const STABLE_SCRIPT = (home = HOME) => join(STABLE_LIB_DIR(home), "refresh-pool.mjs")
|
|
32
|
+
export const STABLE_SCORER = (home = HOME) => join(STABLE_LIB_DIR(home), "score-models.mjs")
|
|
33
|
+
export const STABLE_SELECTION = (home = HOME) => join(STABLE_LIB_DIR(home), "pool-selection.mjs")
|
|
31
34
|
export const REFRESH_SERVICE_NAME = "lacode-refresh-pool.service"
|
|
32
35
|
export const REFRESH_TIMER_NAME = "lacode-refresh-pool.timer"
|
|
33
36
|
|
|
@@ -36,20 +39,6 @@ const isSlotLine = (line) => {
|
|
|
36
39
|
return stripped.length > 0 && stripped.includes("/")
|
|
37
40
|
}
|
|
38
41
|
|
|
39
|
-
const providerOf = (modelOrLine) => {
|
|
40
|
-
const stripped = String(modelOrLine).split("#")[0].trim()
|
|
41
|
-
const token = stripped.split(/\s+/)[0] || ""
|
|
42
|
-
const slash = token.indexOf("/")
|
|
43
|
-
return slash < 0 ? token : token.slice(0, slash)
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
const numericScore = (row) => typeof row?.score === "number" && Number.isFinite(row.score)
|
|
47
|
-
|
|
48
|
-
// Fallbacks for score files written before the floors were recorded there.
|
|
49
|
-
const MIN_CONTEXT = 200_000
|
|
50
|
-
const MIN_SCORE = 60
|
|
51
|
-
const POOL_SIZE = 10
|
|
52
|
-
|
|
53
42
|
function splitHeaderAndSlots(text) {
|
|
54
43
|
const lines = String(text ?? "").split(/\r?\n/)
|
|
55
44
|
// Drop a single trailing empty element produced by a final newline so we can
|
|
@@ -66,7 +55,7 @@ function splitHeaderAndSlots(text) {
|
|
|
66
55
|
function refreshComment(now, scoresGeneratedAt) {
|
|
67
56
|
const when = now instanceof Date ? now.toISOString() : String(now)
|
|
68
57
|
const gen = scoresGeneratedAt ? ` (scores generated ${scoresGeneratedAt})` : ""
|
|
69
|
-
return `# Refreshed ${when} from model-scores.json${gen}: opencode-go first (weight 4),
|
|
58
|
+
return `# Refreshed ${when} from model-scores.json${gen}: opencode-go first (weight 4), small-context providers (weight 2), large-context providers (weight 1), opencode last (max 1).`
|
|
70
59
|
}
|
|
71
60
|
|
|
72
61
|
function withRefreshComment(headerLines, comment) {
|
|
@@ -74,8 +63,9 @@ function withRefreshComment(headerLines, comment) {
|
|
|
74
63
|
return [...kept, comment]
|
|
75
64
|
}
|
|
76
65
|
|
|
77
|
-
function formatGeneratedLine(model, weight, bucket) {
|
|
66
|
+
function formatGeneratedLine(model, weight, bucket, context) {
|
|
78
67
|
const parts = [String(model), String(weight)]
|
|
68
|
+
if (context != null && Number.isFinite(Number(context))) parts.push(`c:${Math.round(Number(context))}`)
|
|
79
69
|
if (bucket) parts.push(`b:${bucket}`)
|
|
80
70
|
return parts.join(" ")
|
|
81
71
|
}
|
|
@@ -84,56 +74,36 @@ function formatGeneratedLine(model, weight, bucket) {
|
|
|
84
74
|
* Pure builder. Throws on every fail-closed condition; never touches the filesystem.
|
|
85
75
|
*
|
|
86
76
|
* @param {{ scores: any, currentPoolText: string, now?: Date|string }} input
|
|
87
|
-
* @returns {{ text: string, slots: string[], warnings: string[], chosen: {
|
|
77
|
+
* @returns {{ text: string, slots: string[], warnings: string[], chosen: { selected: string[] } }}
|
|
88
78
|
*/
|
|
89
79
|
export function buildPool(input) {
|
|
90
|
-
const warnings = []
|
|
91
80
|
const scores = input?.scores
|
|
92
|
-
if (!scores || typeof scores !== "object" || !Array.isArray(scores.providers)) {
|
|
93
|
-
throw new Error("score file is missing or invalid (expected { providers: [...] })")
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
const minScore = Number(scores.minScore) || MIN_SCORE
|
|
97
|
-
const minContext = Number(scores.minContext) || MIN_CONTEXT
|
|
98
|
-
const rankable = (row) =>
|
|
99
|
-
row?.ok === true &&
|
|
100
|
-
numericScore(row) &&
|
|
101
|
-
row.score >= minScore &&
|
|
102
|
-
(row.context == null || row.context >= minContext)
|
|
103
|
-
const byScore = (a, b) => b.score - a.score || String(a.model).localeCompare(String(b.model))
|
|
104
|
-
|
|
105
|
-
const rows = scores.providers.flatMap((entry) => (Array.isArray(entry?.rows) ? entry.rows : []))
|
|
106
|
-
const goRows = rows.filter((row) => providerOf(row.model) === "opencode-go" && rankable(row)).sort(byScore)
|
|
107
|
-
const others = rows.filter((row) => providerOf(row.model) !== "opencode-go" && rankable(row)).sort(byScore)
|
|
108
|
-
const otherRows = others.slice(0, Math.max(0, POOL_SIZE - goRows.length))
|
|
109
|
-
|
|
110
|
-
if (goRows.length + otherRows.length === 0) {
|
|
111
|
-
throw new Error("no scorable model at or above the minimum score is available; refusing to write pool.txt")
|
|
81
|
+
if (!scores || typeof scores !== "object" || !Array.isArray(scores.providers) || !Array.isArray(scores.selection)) {
|
|
82
|
+
throw new Error("score file is missing or invalid (expected { providers: [...], selection: [...] })")
|
|
112
83
|
}
|
|
113
|
-
|
|
114
|
-
|
|
84
|
+
const selection = scores.selection.filter(
|
|
85
|
+
(entry) =>
|
|
86
|
+
entry &&
|
|
87
|
+
typeof entry.model === "string" &&
|
|
88
|
+
entry.model.includes("/") &&
|
|
89
|
+
typeof entry.weight === "number" &&
|
|
90
|
+
Number.isFinite(entry.weight) &&
|
|
91
|
+
entry.weight > 0,
|
|
92
|
+
)
|
|
93
|
+
if (selection.length === 0) {
|
|
94
|
+
throw new Error("score file has no selected models; refusing to write pool.txt")
|
|
115
95
|
}
|
|
116
|
-
|
|
96
|
+
const warnings = Array.isArray(scores.warnings) ? scores.warnings : []
|
|
117
97
|
const { header } = splitHeaderAndSlots(input?.currentPoolText ?? "")
|
|
118
|
-
const outSlots =
|
|
119
|
-
...goRows.map((row) => formatGeneratedLine(row.model, 4)),
|
|
120
|
-
...otherRows.map((row) =>
|
|
121
|
-
formatGeneratedLine(row.model, 2, providerOf(row.model) === "opencode" ? "opencode" : undefined),
|
|
122
|
-
),
|
|
123
|
-
]
|
|
124
|
-
|
|
98
|
+
const outSlots = selection.map((entry) => formatGeneratedLine(entry.model, entry.weight, entry.bucket, entry.context))
|
|
125
99
|
const comment = refreshComment(input?.now ?? new Date(), scores.generatedAt)
|
|
126
100
|
const outHeader = withRefreshComment(header, comment)
|
|
127
|
-
|
|
128
101
|
const text = [...outHeader, "", ...outSlots, ""].join("\n")
|
|
129
102
|
return {
|
|
130
103
|
text,
|
|
131
104
|
slots: outSlots,
|
|
132
105
|
warnings,
|
|
133
|
-
chosen: {
|
|
134
|
-
opencodeGo: goRows.map((r) => r.model),
|
|
135
|
-
others: otherRows.map((r) => r.model),
|
|
136
|
-
},
|
|
106
|
+
chosen: { selected: selection.map((entry) => entry.model) },
|
|
137
107
|
}
|
|
138
108
|
}
|
|
139
109
|
|
|
@@ -173,6 +143,41 @@ function runScorer(max) {
|
|
|
173
143
|
execFileSync(process.execPath, args, { stdio: "inherit" })
|
|
174
144
|
}
|
|
175
145
|
|
|
146
|
+
export function scheduleRetry(retry, options = {}) {
|
|
147
|
+
const at = Date.parse(retry?.after ?? "")
|
|
148
|
+
if (!Number.isFinite(at)) return { scheduled: false, reason: "no reset time" }
|
|
149
|
+
const now = options.now ?? Date.now
|
|
150
|
+
const nowMs = now()
|
|
151
|
+
const seconds = Math.ceil((at - nowMs) / 1000)
|
|
152
|
+
if (!Number.isFinite(seconds) || seconds <= 0) return { scheduled: false, reason: "reset is not in the future" }
|
|
153
|
+
if (seconds > 24 * 60 * 60) return { scheduled: false, reason: "reset is more than a day away" }
|
|
154
|
+
const exec = options.exec ?? execFileSync
|
|
155
|
+
const scriptPath = options.scriptPath ?? fileURLToPath(import.meta.url)
|
|
156
|
+
const nodePath = options.nodePath ?? process.execPath
|
|
157
|
+
const home = options.home ?? HOME
|
|
158
|
+
const path = `${home}/.opencode/bin:/usr/local/bin:/usr/bin:/bin`
|
|
159
|
+
const unit = `lacode-refresh-pool-retry-${Math.floor(nowMs / 1000).toString(36)}`
|
|
160
|
+
try {
|
|
161
|
+
exec(
|
|
162
|
+
"systemd-run",
|
|
163
|
+
[
|
|
164
|
+
"--user",
|
|
165
|
+
"--collect",
|
|
166
|
+
`--unit=${unit}`,
|
|
167
|
+
`--on-active=${seconds}`,
|
|
168
|
+
`--setenv=HOME=${home}`,
|
|
169
|
+
`--setenv=PATH=${path}`,
|
|
170
|
+
nodePath,
|
|
171
|
+
scriptPath,
|
|
172
|
+
],
|
|
173
|
+
{ stdio: "ignore" },
|
|
174
|
+
)
|
|
175
|
+
return { scheduled: true, seconds, after: retry.after }
|
|
176
|
+
} catch (error) {
|
|
177
|
+
return { scheduled: false, reason: String(error?.message ?? error) }
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
176
181
|
export function renderTimerUnits(input = {}) {
|
|
177
182
|
const nodePath = input.nodePath || process.execPath
|
|
178
183
|
const scriptPath = input.scriptPath || fileURLToPath(import.meta.url)
|
|
@@ -212,6 +217,8 @@ function installTimerUnits(stable = false) {
|
|
|
212
217
|
if (stable) {
|
|
213
218
|
mkdirSync(STABLE_LIB_DIR(HOME), { recursive: true })
|
|
214
219
|
copyFileSync(live, scriptPath)
|
|
220
|
+
copyFileSync(resolve(dirname(live), "score-models.mjs"), STABLE_SCORER(HOME))
|
|
221
|
+
copyFileSync(resolve(dirname(live), "pool-selection.mjs"), STABLE_SELECTION(HOME))
|
|
215
222
|
}
|
|
216
223
|
mkdirSync(dirname(units.servicePath), { recursive: true })
|
|
217
224
|
writeFileAtomic(units.servicePath, units.service)
|
|
@@ -232,7 +239,7 @@ function installTimerUnits(stable = false) {
|
|
|
232
239
|
process.exitCode = 1
|
|
233
240
|
return false
|
|
234
241
|
}
|
|
235
|
-
if (stable) console.log(`copied ${live} to ${
|
|
242
|
+
if (stable) console.log(`copied ${live}, score-models.mjs and pool-selection.mjs to ${STABLE_LIB_DIR(HOME)}`)
|
|
236
243
|
console.log(`installed ${units.servicePath}`)
|
|
237
244
|
console.log(`installed ${units.timerPath}`)
|
|
238
245
|
console.log(`enabled ${REFRESH_TIMER_NAME}`)
|
|
@@ -243,12 +250,13 @@ function usage() {
|
|
|
243
250
|
console.log(`Usage: node scripts/refresh-pool.mjs [options]
|
|
244
251
|
|
|
245
252
|
--from-scores skip scoring, use the existing ~/.config/lacode/model-scores.json
|
|
253
|
+
--no-retry do not schedule a transient retry after a rate-limited run
|
|
246
254
|
--install-timer install and enable a user systemd oneshot service + daily timer
|
|
247
255
|
(add --stable to copy the script to ~/.local/lib/lacode and
|
|
248
256
|
point ExecStart there, so plugin-cache updates do not break it)
|
|
249
257
|
--print print the proposed pool.txt and make no changes (alias: --dry-run)
|
|
250
258
|
--dry-run same as --print
|
|
251
|
-
--max N models per provider passed to the scorer (default:
|
|
259
|
+
--max N models per provider passed to the scorer (default: 50)
|
|
252
260
|
--scores PATH scores file (default: ${DEFAULT_SCORES_PATH})
|
|
253
261
|
--pool PATH pool file to read/compare/write (default: ${DEFAULT_POOL_PATH})
|
|
254
262
|
-h, --help this message`)
|
|
@@ -263,12 +271,13 @@ async function main() {
|
|
|
263
271
|
}
|
|
264
272
|
const fromScores = argv.includes("--from-scores")
|
|
265
273
|
const printOnly = argv.includes("--print") || argv.includes("--dry-run")
|
|
274
|
+
const noRetry = argv.includes("--no-retry")
|
|
266
275
|
const value = (flag, fallback) => {
|
|
267
276
|
const hit = argv.find((a) => a === flag || a.startsWith(`${flag}=`))
|
|
268
277
|
if (!hit) return fallback
|
|
269
278
|
return hit.includes("=") ? hit.slice(hit.indexOf("=") + 1) : argv[argv.indexOf(hit) + 1]
|
|
270
279
|
}
|
|
271
|
-
const max = Math.max(1, Number(value("--max", "
|
|
280
|
+
const max = Math.max(1, Number(value("--max", "50")) || 50)
|
|
272
281
|
const scoresPath = value("--scores", DEFAULT_SCORES_PATH)
|
|
273
282
|
const poolPath = value("--pool", DEFAULT_POOL_PATH)
|
|
274
283
|
|
|
@@ -305,12 +314,20 @@ async function main() {
|
|
|
305
314
|
const newSlotCount = built.slots.filter(isSlotLine).length
|
|
306
315
|
const dropped = scores.providers.flatMap((entry) => (Array.isArray(entry?.excluded) ? entry.excluded : []))
|
|
307
316
|
const summary = [
|
|
308
|
-
`
|
|
309
|
-
`
|
|
310
|
-
...(dropped.length ? [`dropped [${dropped.map((x) => x.model).join(", ")}]`] : []),
|
|
317
|
+
`selected [${built.chosen.selected.join(", ")}]`,
|
|
318
|
+
...(dropped.length ? [`dropped ${dropped.length} model(s) below the context/score floor`] : []),
|
|
311
319
|
...(built.warnings.length ? [`warnings [${built.warnings.join("; ")}]`] : []),
|
|
312
320
|
].join(" | ")
|
|
313
321
|
|
|
322
|
+
if (!printOnly && !noRetry) {
|
|
323
|
+
const scheduled = scheduleRetry(scores.retry)
|
|
324
|
+
if (scheduled.scheduled) {
|
|
325
|
+
console.log(`scheduled retry in ${scheduled.seconds}s for ${scores.retry.models.length} model(s) at ${scheduled.after}`)
|
|
326
|
+
} else if (scores.retry) {
|
|
327
|
+
console.log(`retry not scheduled: ${scheduled.reason}`)
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
314
331
|
if (printOnly) {
|
|
315
332
|
process.stdout.write(built.text)
|
|
316
333
|
console.error(`--print: proposed ${oldSlotCount} -> ${newSlotCount} slots; no changes written. ${summary}`)
|
package/scripts/score-models.mjs
CHANGED
|
@@ -26,8 +26,10 @@
|
|
|
26
26
|
// measured score is below --min-score: those are listed as excluded instead.
|
|
27
27
|
// * It never writes pool.txt. It prints a SUGGESTED table for you to paste; the
|
|
28
28
|
// pool stays yours to decide.
|
|
29
|
-
// *
|
|
30
|
-
//
|
|
29
|
+
// * opencode and opencode-go stay free-only. Every other provider is probed
|
|
30
|
+
// even when its catalogue lists a price, because provider credits cover it;
|
|
31
|
+
// --free-only restores the cost-0 rule everywhere. Non-chat/media/safety/
|
|
32
|
+
// meta-router models are always skipped.
|
|
31
33
|
//
|
|
32
34
|
// OUTPUT
|
|
33
35
|
// a per-provider ranked table, an overall ranking, a suggested pool.txt block,
|
|
@@ -38,24 +40,27 @@ import { execFileSync, spawn } from "node:child_process"
|
|
|
38
40
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
|
|
39
41
|
import { homedir } from "node:os"
|
|
40
42
|
import { dirname, join } from "node:path"
|
|
43
|
+
import { MIN_CONTEXT, MIN_SCORE, retryAfterMs, selectPool } from "./pool-selection.mjs"
|
|
41
44
|
|
|
42
45
|
const AUTH = join(homedir(), ".local", "share", "opencode", "auth.json")
|
|
43
46
|
const OUT = join(homedir(), ".config", "lacode", "model-scores.json")
|
|
44
47
|
// No default provider list: `opencode models --verbose` with no argument returns
|
|
45
48
|
// every provider opencode knows about, and that is what we scan by default.
|
|
46
49
|
const SESSION_PROVIDERS = new Set(["opencode", "opencode-go"])
|
|
50
|
+
const FREE_ONLY_PROVIDERS = new Set(["opencode", "opencode-go"])
|
|
47
51
|
|
|
48
52
|
// ── scoring weights ─────────────────────────────────────────────────────────
|
|
49
53
|
// Mechanical work prizes speed and availability over raw capability. Context is
|
|
50
54
|
// scored on a log curve so 256k vs 1M is a difference, not a cliff.
|
|
51
55
|
const W = { speed: 0.35, context: 0.3, output: 0.15, reliability: 0.2 }
|
|
52
56
|
const MAX_CONTEXT = 1_000_000
|
|
53
|
-
const MIN_CONTEXT = 200_000 // below this, a model is excluded instead of ranked
|
|
54
|
-
const MIN_SCORE = 60 // below this, a probed model is excluded from ranking
|
|
55
57
|
const FAST_MS = 300 // at or below this, speed scores 1.0
|
|
56
58
|
const SLOW_MS = 8_000 // at or above this, speed scores 0
|
|
57
59
|
const PROMPT = "Reply with the single word: ok"
|
|
58
60
|
const MAX_TOKENS = 8
|
|
61
|
+
const PROBE_ATTEMPTS = 2 // one retry for a timeout or a dropped connection
|
|
62
|
+
const isTransientProbe = (reason) =>
|
|
63
|
+
/timeout|timed out|ECONNRESET|ETIMEDOUT|fetch failed|socket hang up|no answer/i.test(String(reason ?? ""))
|
|
59
64
|
|
|
60
65
|
// ── args ────────────────────────────────────────────────────────────────────
|
|
61
66
|
const argv = process.argv.slice(2)
|
|
@@ -70,31 +75,36 @@ const value = (flag, fallback) => {
|
|
|
70
75
|
console.log(`Usage: npm run score -- [options]
|
|
71
76
|
|
|
72
77
|
--providers a,b restrict the scan to these providers (default: all configured)
|
|
73
|
-
--max N probe at most N
|
|
78
|
+
--max N probe at most N candidate models per provider (default: 50)
|
|
74
79
|
--min-context N exclude models whose context limit is below N (default: ${MIN_CONTEXT})
|
|
75
80
|
--min-score N do not rank models scoring below N (default: ${MIN_SCORE})
|
|
76
|
-
--
|
|
81
|
+
--free-only only probe catalogue cost 0/0 models on every provider
|
|
82
|
+
--timeout MS per-request deadline in ms (default: 45000)
|
|
77
83
|
--delay MS pause between requests in ms (default: 300)
|
|
78
84
|
--no-write do not write ~/.config/lacode/model-scores.json
|
|
79
85
|
-h, --help this message
|
|
80
86
|
|
|
81
|
-
|
|
82
|
-
|
|
87
|
+
opencode and opencode-go stay free-only; every other provider is probed even
|
|
88
|
+
when its catalogue lists a price, because provider credits cover it. Use
|
|
89
|
+
--free-only to apply the cost-0 rule everywhere. Non-chat/media generators,
|
|
90
|
+
safety classifiers, and OpenRouter meta-routers are always skipped.
|
|
83
91
|
Credentials come from ~/.local/share/opencode/auth.json; a provider with no
|
|
84
92
|
entry falls back to the "opencode" key (which is how the console-openrouter
|
|
85
93
|
connection authenticates).
|
|
86
|
-
opencode/opencode-go models are probed through a temporary local opencode serve --pure session, while the other providers are probed directly
|
|
94
|
+
opencode/opencode-go models are probed through a temporary local opencode serve --pure session, while the other providers are probed directly.
|
|
95
|
+
Providers with no direct HTTP endpoint (or an unexpanded \${...} URL) are probed through the same local opencode session as opencode/opencode-go.`)
|
|
87
96
|
process.exit(0)
|
|
88
97
|
}
|
|
89
98
|
|
|
90
99
|
const providerArg = value("--providers", null)
|
|
91
100
|
const providers = providerArg ? String(providerArg).split(",").map((s) => s.trim()).filter(Boolean) : null
|
|
92
|
-
const maxPerProvider = Math.max(1, Number(value("--max", "
|
|
101
|
+
const maxPerProvider = Math.max(1, Number(value("--max", "50")) || 50)
|
|
93
102
|
const minContext = Math.max(0, Number(value("--min-context", String(MIN_CONTEXT))) || MIN_CONTEXT)
|
|
94
103
|
const minScore = Math.max(0, Number(value("--min-score", String(MIN_SCORE))) || MIN_SCORE)
|
|
95
|
-
const timeoutMs = Math.max(1_000, Number(value("--timeout", "
|
|
104
|
+
const timeoutMs = Math.max(1_000, Number(value("--timeout", "45000")) || 45_000)
|
|
96
105
|
const delayMs = Math.max(0, Number(value("--delay", "300")) || 300)
|
|
97
106
|
const writeResults = !has("--no-write")
|
|
107
|
+
const freeOnlyFlag = has("--free-only")
|
|
98
108
|
|
|
99
109
|
// ── discovery ───────────────────────────────────────────────────────────────
|
|
100
110
|
/** Parse `opencode models --verbose <provider>`: alternating id line + JSON block. */
|
|
@@ -148,9 +158,10 @@ function isChatModel(model) {
|
|
|
148
158
|
}
|
|
149
159
|
|
|
150
160
|
function buildProviderEntry(provider, rows) {
|
|
161
|
+
const freeOnly = freeOnlyFlag || FREE_ONLY_PROVIDERS.has(provider)
|
|
151
162
|
const models = rows
|
|
152
|
-
.filter(({ model }) => model?.api?.
|
|
153
|
-
.filter(({ model }) => (model.cost?.input ?? 1) === 0 && (model.cost?.output ?? 1) === 0)
|
|
163
|
+
.filter(({ model }) => model?.api?.id)
|
|
164
|
+
.filter(({ model }) => !freeOnly || ((model.cost?.input ?? 1) === 0 && (model.cost?.output ?? 1) === 0))
|
|
154
165
|
.filter(({ model }) => !model.status || model.status === "active")
|
|
155
166
|
.filter(({ model }) => isChatModel(model))
|
|
156
167
|
.map(({ id, model }) => ({
|
|
@@ -372,10 +383,17 @@ async function probeDirect(candidate, provider) {
|
|
|
372
383
|
: response.status === 404
|
|
373
384
|
? "not served"
|
|
374
385
|
: `http ${response.status}`
|
|
375
|
-
|
|
386
|
+
const retryAfter =
|
|
387
|
+
kind === "rate-limited" || /credit|402/i.test(detail)
|
|
388
|
+
? retryAfterMs(body, response.headers.get("retry-after"))
|
|
389
|
+
: undefined
|
|
390
|
+
return { ok: false, ms, reason: `${kind}: ${detail}`, retryAfter }
|
|
376
391
|
}
|
|
377
392
|
// A 200 can still carry an error object (OpenRouter does this).
|
|
378
|
-
if (parsed?.error)
|
|
393
|
+
if (parsed?.error) {
|
|
394
|
+
const reason = `${parsed.error.message ?? "error"} `.trim()
|
|
395
|
+
return { ok: false, ms, reason, retryAfter: retryAfterMs(reason) }
|
|
396
|
+
}
|
|
379
397
|
if (!Array.isArray(parsed?.choices) || parsed.choices.length === 0) {
|
|
380
398
|
return { ok: false, ms, reason: "no choices in response" }
|
|
381
399
|
}
|
|
@@ -409,7 +427,7 @@ async function probeViaSession(candidate, provider) {
|
|
|
409
427
|
if (!createRes.ok) {
|
|
410
428
|
const detail = createdJson?.error?.message ?? createdJson?.message ?? createText.slice(0, 160)
|
|
411
429
|
const ms = Math.round(performance.now() - started)
|
|
412
|
-
return { ok: false, ms, reason: `session create http ${createRes.status}: ${detail}
|
|
430
|
+
return { ok: false, ms, reason: `session create http ${createRes.status}: ${detail}`, retryAfter: retryAfterMs(detail) }
|
|
413
431
|
}
|
|
414
432
|
id = createdJson?.id ?? createdJson?.data?.id ?? null
|
|
415
433
|
if (!id) {
|
|
@@ -435,11 +453,12 @@ async function probeViaSession(candidate, provider) {
|
|
|
435
453
|
const ms = Math.round(performance.now() - (promptStarted ?? started))
|
|
436
454
|
if (!msgRes.ok) {
|
|
437
455
|
const detail = parsed?.error?.message ?? parsed?.message ?? msgText.slice(0, 160)
|
|
438
|
-
return { ok: false, ms, reason: `http ${msgRes.status}: ${detail}
|
|
456
|
+
return { ok: false, ms, reason: `http ${msgRes.status}: ${detail}`, retryAfter: retryAfterMs(detail) }
|
|
439
457
|
}
|
|
440
458
|
const failure = parsed?.error ?? parsed?.info?.error
|
|
441
459
|
if (failure) {
|
|
442
|
-
|
|
460
|
+
const reason = failureText(failure).slice(0, 240)
|
|
461
|
+
return { ok: false, ms, reason, retryAfter: retryAfterMs(reason) }
|
|
443
462
|
}
|
|
444
463
|
if (!parsed?.info) {
|
|
445
464
|
return { ok: false, ms, reason: "no response" }
|
|
@@ -450,7 +469,8 @@ async function probeViaSession(candidate, provider) {
|
|
|
450
469
|
if (error?.name === "TimeoutError") {
|
|
451
470
|
return { ok: false, ms, reason: `timeout after ${timeoutMs}ms` }
|
|
452
471
|
}
|
|
453
|
-
|
|
472
|
+
const reason = failureText(error).slice(0, 240)
|
|
473
|
+
return { ok: false, ms, reason, retryAfter: retryAfterMs(reason) }
|
|
454
474
|
} finally {
|
|
455
475
|
if (id && server) {
|
|
456
476
|
try {
|
|
@@ -460,9 +480,22 @@ async function probeViaSession(candidate, provider) {
|
|
|
460
480
|
}
|
|
461
481
|
}
|
|
462
482
|
|
|
483
|
+
function usesSession(provider, candidate) {
|
|
484
|
+
if (SESSION_PROVIDERS.has(provider)) return true
|
|
485
|
+
const url = String(candidate.url ?? "")
|
|
486
|
+
return url.length === 0 || url.includes("${")
|
|
487
|
+
}
|
|
488
|
+
|
|
463
489
|
async function probe(candidate, provider) {
|
|
464
|
-
if (
|
|
465
|
-
|
|
490
|
+
if (usesSession(provider, candidate)) return probeViaSession(candidate, provider)
|
|
491
|
+
const outcome = await probeDirect(candidate, provider)
|
|
492
|
+
if (outcome.ok) return outcome
|
|
493
|
+
// A provider whose direct URL is misconfigured can still be reachable
|
|
494
|
+
// through opencode's own provider resolution.
|
|
495
|
+
if (/credential|no credential|not served|invalid request path|could not route/i.test(String(outcome.reason ?? ""))) {
|
|
496
|
+
return probeViaSession(candidate, provider)
|
|
497
|
+
}
|
|
498
|
+
return outcome
|
|
466
499
|
}
|
|
467
500
|
|
|
468
501
|
// ── scoring ─────────────────────────────────────────────────────────────────
|
|
@@ -497,7 +530,7 @@ async function main() {
|
|
|
497
530
|
}
|
|
498
531
|
const candidates = entry.models.slice(0, maxPerProvider)
|
|
499
532
|
const excludedNote = entry.excluded.length ? `, ${entry.excluded.length} excluded (context < ${minContext})` : ""
|
|
500
|
-
console.log(`\n${entry.provider}: probing ${candidates.length}
|
|
533
|
+
console.log(`\n${entry.provider}: probing ${candidates.length} candidate model(s)${entry.models.length > candidates.length ? ` of ${entry.models.length}` : ""}${excludedNote}`)
|
|
501
534
|
if (entry.excluded.length) {
|
|
502
535
|
console.log(` excluded — context below ${minContext}:`)
|
|
503
536
|
for (const x of entry.excluded) console.log(` ${pad(x.model.replace(`${entry.provider}/`, ""), 52)} ${padL(x.context, 9)}`)
|
|
@@ -508,7 +541,12 @@ async function main() {
|
|
|
508
541
|
}
|
|
509
542
|
const rows = []
|
|
510
543
|
for (const candidate of candidates) {
|
|
511
|
-
|
|
544
|
+
let outcome = await probe(candidate, entry.provider)
|
|
545
|
+
for (let attempt = 1; attempt < PROBE_ATTEMPTS && !outcome.ok && isTransientProbe(outcome.reason); attempt++) {
|
|
546
|
+
console.log(` retrying ${candidate.model.replace(`${entry.provider}/`, "")} after ${String(outcome.reason).slice(0, 60)}`)
|
|
547
|
+
await sleep(delayMs)
|
|
548
|
+
outcome = await probe(candidate, entry.provider)
|
|
549
|
+
}
|
|
512
550
|
const row = { provider: entry.provider, ...candidate, ...outcome, score: null }
|
|
513
551
|
row.score = score(row)
|
|
514
552
|
if (row.score && row.score.value < minScore) {
|
|
@@ -556,28 +594,41 @@ async function main() {
|
|
|
556
594
|
const all = results.flatMap((e) => e.rows).filter((r) => r.score && r.score.value >= minScore)
|
|
557
595
|
all.sort((a, b) => b.score.value - a.score.value)
|
|
558
596
|
if (all.length) {
|
|
559
|
-
console.log(`\noverall — best
|
|
597
|
+
console.log(`\noverall — best candidate models right now`)
|
|
560
598
|
all.slice(0, 12).forEach((r, i) => {
|
|
561
599
|
console.log(` ${pad(i + 1, 3)}${pad(r.model, 58)}${padL(`${r.ms}ms`, 8)}${padL(r.score.value, 7)}`)
|
|
562
600
|
})
|
|
563
601
|
}
|
|
564
602
|
|
|
565
|
-
//
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
603
|
+
// Previous scores let a transient outage keep the pool it had.
|
|
604
|
+
const previousScores = (() => {
|
|
605
|
+
try {
|
|
606
|
+
return JSON.parse(readFileSync(OUT, "utf8"))
|
|
607
|
+
} catch {
|
|
608
|
+
return null
|
|
609
|
+
}
|
|
610
|
+
})()
|
|
611
|
+
const { selection, warnings, retry } = selectPool({
|
|
612
|
+
providers: results.map((entry) => ({
|
|
613
|
+
provider: entry.provider,
|
|
614
|
+
error: entry.error,
|
|
615
|
+
rows: entry.rows.map((row) => ({ ...row, score: row.score?.value ?? null })),
|
|
616
|
+
excluded: entry.excluded,
|
|
617
|
+
})),
|
|
618
|
+
previousProviders: previousScores?.providers,
|
|
619
|
+
minContext,
|
|
620
|
+
minScore,
|
|
621
|
+
})
|
|
622
|
+
|
|
623
|
+
if (selection.length) {
|
|
577
624
|
console.log(`\nsuggested pool.txt (paste if it matches your judgement — nothing was changed)`)
|
|
578
|
-
console.log(`# scored ${new Date().toISOString()} — opencode-go first (weight 4),
|
|
579
|
-
for (const
|
|
625
|
+
console.log(`# scored ${new Date().toISOString()} — opencode-go first (weight 4), small-context providers (weight 2), large-context providers (weight 1), opencode last (max 1)`)
|
|
626
|
+
for (const entry of selection) {
|
|
627
|
+
const bucket = entry.bucket ? ` b:${entry.bucket}` : ""
|
|
628
|
+
console.log(`${pad(entry.model, 60)}${entry.weight}${bucket}`)
|
|
629
|
+
}
|
|
580
630
|
}
|
|
631
|
+
if (retry) console.log(`\nlimited: ${retry.models.length} model(s); retry after ${retry.after}`)
|
|
581
632
|
|
|
582
633
|
if (writeResults) {
|
|
583
634
|
try {
|
|
@@ -587,6 +638,9 @@ async function main() {
|
|
|
587
638
|
weights: W,
|
|
588
639
|
minContext,
|
|
589
640
|
minScore,
|
|
641
|
+
selection,
|
|
642
|
+
warnings,
|
|
643
|
+
retry,
|
|
590
644
|
providers: results.map((e) => ({
|
|
591
645
|
provider: e.provider,
|
|
592
646
|
error: e.error,
|
|
@@ -599,6 +653,7 @@ async function main() {
|
|
|
599
653
|
ms: r.ms,
|
|
600
654
|
reason: r.reason,
|
|
601
655
|
score: r.score?.value ?? null,
|
|
656
|
+
retryAfter: r.retryAfter ?? null,
|
|
602
657
|
})),
|
|
603
658
|
})),
|
|
604
659
|
}
|