decidekit 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -1
- package/README.md +25 -13
- package/dist/browser.js +1 -1
- package/dist/fallback.js +2 -2
- package/dist/index.js +8 -8
- package/dist/switch.d.ts +4 -4
- package/dist/switch.js +7 -7
- package/dist/typesafe.d.ts +1 -1
- package/dist/typesafe.js +3 -3
- package/package.json +4 -4
package/CHANGELOG.md
CHANGED
|
@@ -47,7 +47,7 @@ The first release. Numbers are on public test splits unless noted; the scripts t
|
|
|
47
47
|
171 s on the main thread; the longest stall in a worker was 18 ms).
|
|
48
48
|
- `decidekit/react`: `useDecider()` and `useDecision()`.
|
|
49
49
|
- `decidekit/switch`: a decider for the switch agent harness.
|
|
50
|
-
- A CLI: `
|
|
50
|
+
- A CLI: `decidekit train` and `decidekit decide`.
|
|
51
51
|
|
|
52
52
|
### Known limits
|
|
53
53
|
|
package/README.md
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
#
|
|
1
|
+
# decidekit
|
|
2
2
|
|
|
3
|
-
[](https://www.npmjs.com/package/decidekit) [](https://www.npmjs.com/package/decidekit) [](https://github.com/debuffjp/decidekit/actions/workflows/ci.yml) [](LICENSE) [](https://discord.gg/jshbDHSpCY)
|
|
4
4
|
|
|
5
|
-
**[Try it in your browser](https://debuff.jp/
|
|
5
|
+
**[Try it in your browser](https://debuff.jp/decidekit/playground/)** · [Watch it learn from an LLM](https://debuff.jp/decidekit/lanes/) · [Launch post](https://debuff.jp/blog/introducing-decidekit) · [Research: when the LLM is wrong](https://debuff.jp/research/learning-from-a-noisy-llm)
|
|
6
6
|
|
|
7
7
|
A decision model for the browser and Node: it picks one of your labels for a piece of text, with confidence you can trust, and learns from an LLM when it isn't sure. Your labels are the type. No server, no API key.
|
|
8
8
|
|
|
9
|
-
A System One model in the browser: like TypeSafe's Jev, but on the user's device, so each decision is free, private and a few milliseconds. Jev leaves unsure cases to your code;
|
|
9
|
+
A System One model in the browser: like TypeSafe's Jev, but on the user's device, so each decision is free, private and a few milliseconds. Jev leaves unsure cases to your code; decidekit can ask an LLM about them and learn from the answers. decidekit is an independent project, not affiliated with TypeSafe.
|
|
10
10
|
|
|
11
11
|
```ts
|
|
12
12
|
import { createDecider } from "decidekit";
|
|
@@ -109,7 +109,7 @@ The [guide](docs/guide-typesafe.md) covers what differs, including what `confide
|
|
|
109
109
|
|
|
110
110
|
## switch
|
|
111
111
|
|
|
112
|
-
`decidekit/switch` makes
|
|
112
|
+
`decidekit/switch` makes decidekit a decision model for [switch](https://github.com/osolmaz/switch), the harness that
|
|
113
113
|
runs a decision model through a task. Zero-shot, it won two of the three WikiRace races from switch's example on a
|
|
114
114
|
laptop CPU. See the [switch guide](docs/guide-switch.md).
|
|
115
115
|
|
|
@@ -132,7 +132,7 @@ const d = await createWorkerDecider({
|
|
|
132
132
|
const r = await d.decide(text, head);
|
|
133
133
|
```
|
|
134
134
|
|
|
135
|
-
Import `loadHead` from `decidekit/browser` too, not from `
|
|
135
|
+
Import `loadHead` from `decidekit/browser` too, not from `decidekit`: that keeps the encoder out of your page bundle.
|
|
136
136
|
|
|
137
137
|
Tested from npm in Vite 7, webpack 5 and Next.js 16 apps; see [docs/bundlers.md](docs/bundlers.md) for the config, including a one-line alias
|
|
138
138
|
that halves the wasm download. Serve the page cross-origin isolated if you can:
|
|
@@ -146,6 +146,17 @@ That lets wasm use several threads: in our runs with all-MiniLM-L6-v2 it cut p95
|
|
|
146
146
|
|
|
147
147
|
Download on first visit: the model (34 MB) and onnxruntime's wasm (23.6 MB, 5.8 MB gzipped), both cached after.
|
|
148
148
|
|
|
149
|
+
## Models
|
|
150
|
+
|
|
151
|
+
decidekit runs the Decide models, each a sentence encoder plus a calibrated head:
|
|
152
|
+
|
|
153
|
+
| Model | Encoder | Download | Reads |
|
|
154
|
+
|---|---|---|---|
|
|
155
|
+
| Decide Small (default) | `Xenova/bge-small-en-v1.5` | 34 MB | English |
|
|
156
|
+
| Decide Multilingual | `Xenova/multilingual-e5-small` | 118 MB | 76% on Japanese and Chinese intents |
|
|
157
|
+
|
|
158
|
+
Pass the encoder id as `model` to `createDecider()`.
|
|
159
|
+
|
|
149
160
|
## Numbers
|
|
150
161
|
|
|
151
162
|
bge-small-en-v1.5 (the default) at q8, Ryzen 7 260 laptop, Node 24 and headless Chrome 156, 2026-11-01. Accuracy
|
|
@@ -192,9 +203,9 @@ Reproduce: `npm run eval:suite` (Node), `npm run bench` then `npm run bench:run`
|
|
|
192
203
|
|
|
193
204
|
## Playground
|
|
194
205
|
|
|
195
|
-
Try it at [debuff.jp/
|
|
206
|
+
Try it at [debuff.jp/decidekit](https://debuff.jp/decidekit/). To run it locally: `npm run playground`, then serve `playground/dist` cross-origin isolated (`node bench/serve.mjs` style headers).
|
|
196
207
|
Try zero-shot labels, the ready-made heads, a head trained in the page and "none of these".
|
|
197
|
-
[Two lanes](https://debuff.jp/
|
|
208
|
+
[Two lanes](https://debuff.jp/decidekit/lanes/) shows `withFallback` live: decidekit starts from twelve label names,
|
|
198
209
|
asks a small LLM when it's unsure, learns from each answer, and replays the first 20 messages at the end.
|
|
199
210
|
|
|
200
211
|
## Ready-made heads
|
|
@@ -213,7 +224,7 @@ and offline apps, and text that can't leave the device. Details, code and poor f
|
|
|
213
224
|
|
|
214
225
|
| | Runs in the browser | Trains in the browser | Typed result | Calibrated confidence |
|
|
215
226
|
|---|---|---|---|---|
|
|
216
|
-
|
|
|
227
|
+
| decidekit | yes (worker) | yes | yes | trained heads |
|
|
217
228
|
| [jimothy](https://www.npmjs.com/package/jimothy) | inference (worker) | no, Node | yes | yes |
|
|
218
229
|
| [Jev](https://typesafe.ai/) (TypeSafe) | no, API | no | yes | yes |
|
|
219
230
|
| transformers.js zero-shot pipeline | yes | no | no | no |
|
|
@@ -221,7 +232,9 @@ and offline apps, and text that can't leave the device. Details, code and poor f
|
|
|
221
232
|
| LLM structured output (AI SDK `Output.choice`) | no, API | n/a | yes | no |
|
|
222
233
|
|
|
223
234
|
jimothy reports 92.4% on BANKING77, above our 90.7%. Pick it if you train in Node and want the higher number. Pick
|
|
224
|
-
|
|
235
|
+
decidekit if training has to happen in the page, from the user's own data.
|
|
236
|
+
|
|
237
|
+
[system-one](https://github.com/lukeramsden/system-one) is a different kind of tool: a typed client for hosted System One models (Jev and Laya). It runs no model itself.
|
|
225
238
|
|
|
226
239
|
## Docs
|
|
227
240
|
|
|
@@ -230,13 +243,12 @@ API reference, how it works and the FAQ.
|
|
|
230
243
|
|
|
231
244
|
## Community
|
|
232
245
|
|
|
233
|
-
Questions and show-and-tell in [Discussions](https://github.com/debuffjp/
|
|
246
|
+
Questions and show-and-tell in [Discussions](https://github.com/debuffjp/decidekit/discussions), chat in the
|
|
234
247
|
[debuff Discord](https://discord.gg/jshbDHSpCY). Results on your own data, good or bad, help most: there's an
|
|
235
248
|
issue template for them. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
236
249
|
|
|
237
250
|
## Status
|
|
238
251
|
|
|
239
|
-
0.1. English with the default model, other languages with `multilingual-e5-small` ([FAQ](docs/faq.md#which-languages-does-it-support)); short texts (about a paragraph). Not done yet: WebGPU in the worker
|
|
240
|
-
fallback to a hosted model below the threshold, and fine-tuning the encoder (the likely route past about 91%).
|
|
252
|
+
0.1. English with the default model, other languages with `multilingual-e5-small` ([FAQ](docs/faq.md#which-languages-does-it-support)); short texts (about a paragraph). Not done yet: WebGPU in the worker and fine-tuning the encoder (the likely route past about 91%).
|
|
241
253
|
|
|
242
254
|
Made by the [debuff](https://debuff.jp) team. MIT.
|
package/dist/browser.js
CHANGED
|
@@ -36,7 +36,7 @@ export async function createWorkerDecider({ onProgress, ...options } = {}, worke
|
|
|
36
36
|
terminate: () => {
|
|
37
37
|
worker.terminate();
|
|
38
38
|
for (const p of pending.values())
|
|
39
|
-
p.reject(new Error("
|
|
39
|
+
p.reject(new Error("decidekit: the worker was terminated"));
|
|
40
40
|
pending.clear();
|
|
41
41
|
},
|
|
42
42
|
};
|
package/dist/fallback.js
CHANGED
|
@@ -25,9 +25,9 @@ export function withFallback(decider, using, options) {
|
|
|
25
25
|
perLabel.set(e.label, (perLabel.get(e.label) ?? 0) + 1);
|
|
26
26
|
function retrain() {
|
|
27
27
|
if (!examples)
|
|
28
|
-
return Promise.reject(new Error("
|
|
28
|
+
return Promise.reject(new Error("decidekit: withFallback() needs `examples` to retrain a head"));
|
|
29
29
|
if (!examples.length && !learned.length)
|
|
30
|
-
return Promise.reject(new Error("
|
|
30
|
+
return Promise.reject(new Error("decidekit: nothing to train on yet: no answers from `ask`"));
|
|
31
31
|
const calibrateOn = trainOptions?.calibrateOn ?? (confident.length >= 50 ? confident : undefined);
|
|
32
32
|
const trainLabels = learned.some((e) => e.label === NONE) ? [...labels, NONE] : labels;
|
|
33
33
|
retraining ??= decider.train([...examples, ...learned], { ...trainOptions, labels: trainLabels, calibrateOn }).then((h) => {
|
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ function isHead(c) {
|
|
|
7
7
|
if (Array.isArray(c) || !("weights" in c))
|
|
8
8
|
return false;
|
|
9
9
|
if (!(c.weights instanceof Float32Array))
|
|
10
|
-
throw new Error("
|
|
10
|
+
throw new Error("decidekit: this head's weights aren't a Float32Array. Read head files with loadHead(), not JSON.parse.");
|
|
11
11
|
return true;
|
|
12
12
|
}
|
|
13
13
|
const readable = (label) => label.replace(/[_-]+/g, " ").replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase();
|
|
@@ -59,13 +59,13 @@ export async function createDecider({ model = "Xenova/bge-small-en-v1.5", device
|
|
|
59
59
|
let d;
|
|
60
60
|
if (isHead(using)) {
|
|
61
61
|
if (using.model !== model)
|
|
62
|
-
throw new Error(`
|
|
62
|
+
throw new Error(`decidekit: this head was trained on ${using.model}, but the decider uses ${model}. Pass { model: "${using.model}" } to createDecider.`);
|
|
63
63
|
d = toDecision(using.labels, softmax(logits(using, q), using.temperature));
|
|
64
64
|
}
|
|
65
65
|
else {
|
|
66
66
|
const labels = (Array.isArray(using) ? using : Object.keys(using));
|
|
67
67
|
if (labels.length < 2)
|
|
68
|
-
throw new Error(`
|
|
68
|
+
throw new Error(`decidekit: decide() needs at least 2 options, got ${labels.length}`);
|
|
69
69
|
// Bare labels are compared as readable text ("card_arrival" as "card arrival"): 2.6 points better on CLINC150's
|
|
70
70
|
// dev split. A described label scores by the better of its name and its description: on the dev splits that
|
|
71
71
|
// beat the description alone by 0.5 (BANKING77) and 3.1 points (CLINC150), and names alone by 7 and 10.
|
|
@@ -98,15 +98,15 @@ export async function createDecider({ model = "Xenova/bge-small-en-v1.5", device
|
|
|
98
98
|
async function train(examples, opts = {}) {
|
|
99
99
|
const bad = examples.findIndex((e) => typeof e?.text !== "string" || typeof e?.label !== "string" || !e.label);
|
|
100
100
|
if (bad >= 0)
|
|
101
|
-
throw new Error(`
|
|
101
|
+
throw new Error(`decidekit: example ${bad} needs a string text and a non-empty string label, got ${JSON.stringify(examples[bad])}`);
|
|
102
102
|
if (!examples.length)
|
|
103
|
-
throw new Error("
|
|
103
|
+
throw new Error("decidekit: train() needs at least 1 example");
|
|
104
104
|
const labels = [...new Set([...(opts.labels ?? []), ...examples.map((e) => e.label)])].sort();
|
|
105
105
|
if (opts.labels && opts.labelNames === false && labels.some((l) => !examples.some((e) => e.label === l))) {
|
|
106
|
-
throw new Error("
|
|
106
|
+
throw new Error("decidekit: labels without examples need labelNames, which is what they start from");
|
|
107
107
|
}
|
|
108
108
|
if (labels.length < 2)
|
|
109
|
-
throw new Error(`
|
|
109
|
+
throw new Error(`decidekit: train() needs at least 2 labels, got ${labels.length}`);
|
|
110
110
|
const y = examples.map((e) => labels.indexOf(e.label));
|
|
111
111
|
const texts = [...new Set(examples.map((e) => e.text))];
|
|
112
112
|
const fresh = texts.filter((t) => !trainCache.has(t));
|
|
@@ -122,7 +122,7 @@ export async function createDecider({ model = "Xenova/bge-small-en-v1.5", device
|
|
|
122
122
|
const cal = opts.calibrateOn;
|
|
123
123
|
const unknown = cal?.find((e) => !labels.includes(e.label));
|
|
124
124
|
if (unknown)
|
|
125
|
-
throw new Error(`
|
|
125
|
+
throw new Error(`decidekit: calibrateOn has label ${JSON.stringify(unknown.label)}, which isn't one of the head's labels`);
|
|
126
126
|
const calibration = cal?.length ? { X: await vectors(cal.map((e) => e.text)), y: cal.map((e) => labels.indexOf(e.label)) } : undefined;
|
|
127
127
|
return { model, ...(await fitHead(X, y, labels, opts, priors, LABEL_NAME_WEIGHT, undefined, SMALL_HEAD_FLOOR[model] ?? 1, calibration)) };
|
|
128
128
|
}
|
package/dist/switch.d.ts
CHANGED
|
@@ -81,22 +81,22 @@ export interface SwitchOptions {
|
|
|
81
81
|
/** Trained heads for questions that repeat, by question id. Questions without one are decided zero-shot. */
|
|
82
82
|
heads?: Readonly<Record<string, Head>>;
|
|
83
83
|
/**
|
|
84
|
-
* A stronger decider to ask when
|
|
84
|
+
* A stronger decider to ask when decidekit isn't sure, such as Jev through switch's pi-ai decider. Its answers to
|
|
85
85
|
* choice questions train a head for that question, so a question that repeats asks it less and less (see `withFallback`).
|
|
86
86
|
*/
|
|
87
87
|
teacher?: SwitchDecider;
|
|
88
88
|
/** Below this confidence, choice questions go to `teacher`. Default 0.9. */
|
|
89
89
|
minConfidence?: number;
|
|
90
90
|
/**
|
|
91
|
-
* What
|
|
91
|
+
* What decidekit reads for a question. Its encoder reads about 256 tokens and drops the rest, so a long state or long
|
|
92
92
|
* instructions can push out what matters; pick it here, such as the target of a search. Default: `textOf`.
|
|
93
93
|
*/
|
|
94
94
|
text?: (state: SwitchRequest["decisionState"], question: Question, id: string) => string;
|
|
95
95
|
}
|
|
96
|
-
/** The text
|
|
96
|
+
/** The text decidekit reads for a question: its instructions, then the state as plain lines rather than JSON. */
|
|
97
97
|
export declare function textOf(instructions: string, state: Json): string;
|
|
98
98
|
/**
|
|
99
|
-
* Wraps a
|
|
99
|
+
* Wraps a decidekit decider as a switch decider. Choice, bool and score questions become decidekit decisions over the
|
|
100
100
|
* question's criteria; images are ignored. Every call is local, so usage costs nothing.
|
|
101
101
|
*/
|
|
102
102
|
export declare function switchDecider(local: Local, options?: SwitchOptions): SwitchDecider;
|
package/dist/switch.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
//
|
|
2
|
-
// task by asking it typed questions. The types below copy the parts of switch's `Decider` that this needs, so
|
|
1
|
+
// decidekit as a decision model for switch (github.com/osolmaz/switch), a harness that runs a decision model through a
|
|
2
|
+
// task by asking it typed questions. The types below copy the parts of switch's `Decider` that this needs, so decidekit
|
|
3
3
|
// doesn't depend on switch: pass the result wherever switch takes a decider.
|
|
4
4
|
import { withFallback } from "./fallback.js";
|
|
5
|
-
/** The text
|
|
5
|
+
/** The text decidekit reads for a question: its instructions, then the state as plain lines rather than JSON. */
|
|
6
6
|
export function textOf(instructions, state) {
|
|
7
7
|
const lines = [];
|
|
8
8
|
const walk = (v, key) => {
|
|
@@ -18,7 +18,7 @@ export function textOf(instructions, state) {
|
|
|
18
18
|
return [instructions, ...lines].filter(Boolean).join("\n");
|
|
19
19
|
}
|
|
20
20
|
/**
|
|
21
|
-
* Wraps a
|
|
21
|
+
* Wraps a decidekit decider as a switch decider. Choice, bool and score questions become decidekit decisions over the
|
|
22
22
|
* question's criteria; images are ignored. Every call is local, so usage costs nothing.
|
|
23
23
|
*/
|
|
24
24
|
export function switchDecider(local, options = {}) {
|
|
@@ -48,7 +48,7 @@ export function switchDecider(local, options = {}) {
|
|
|
48
48
|
const text = textFor(state, q, id);
|
|
49
49
|
if (q.type === "choice") {
|
|
50
50
|
const keys = Object.keys(q.criteria);
|
|
51
|
-
//
|
|
51
|
+
// decidekit compares two or more options; a single one is the answer.
|
|
52
52
|
if (keys.length === 1)
|
|
53
53
|
return { type: "choice", choice: keys[0], probabilities: { [keys[0]]: 1 }, confidence: 1 };
|
|
54
54
|
if (teacher)
|
|
@@ -66,14 +66,14 @@ export function switchDecider(local, options = {}) {
|
|
|
66
66
|
return { type: "score", score: Number(d.choice), confidence: d.confidence };
|
|
67
67
|
}
|
|
68
68
|
return {
|
|
69
|
-
model: `
|
|
69
|
+
model: `decidekit/${local.model}`,
|
|
70
70
|
// The encoder reads about 256 tokens and cuts the rest, so a long state still fits; tell switch the window is big
|
|
71
71
|
// so it doesn't trim the choices to fit.
|
|
72
72
|
window: 1_000_000,
|
|
73
73
|
charsPerToken: 4,
|
|
74
74
|
async decide(request) {
|
|
75
75
|
const started = performance.now();
|
|
76
|
-
const base = { api: "
|
|
76
|
+
const base = { api: "decidekit", provider: "decidekit", model: `decidekit/${local.model}`, timestamp: Date.now(), tooLong: false };
|
|
77
77
|
try {
|
|
78
78
|
const entries = await Promise.all(Object.entries(request.decisionQuestions).map(async ([id, q]) => [id, await answer(id, q, request.decisionState)]));
|
|
79
79
|
const input = Math.round(JSON.stringify(request.decisionState).length / 4);
|
package/dist/typesafe.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export interface Decides {
|
|
|
5
5
|
}
|
|
6
6
|
export interface ChoiceQuestion<L extends string = string> {
|
|
7
7
|
type: "choice";
|
|
8
|
-
/** Kept for compatibility and ignored:
|
|
8
|
+
/** Kept for compatibility and ignored: decidekit compares the state with the options, it doesn't read a question. */
|
|
9
9
|
instructions: string;
|
|
10
10
|
/** Option names to descriptions (`null` for none), as in TypeSafe's API, or a trained head. */
|
|
11
11
|
criteria: Readonly<Record<L, string | null>> | Head<L>;
|
package/dist/typesafe.js
CHANGED
|
@@ -2,7 +2,7 @@ export function choice(instructions, criteria) {
|
|
|
2
2
|
return { type: "choice", instructions, criteria };
|
|
3
3
|
}
|
|
4
4
|
const unsupported = (type) => () => {
|
|
5
|
-
throw new Error(`
|
|
5
|
+
throw new Error(`decidekit: ${type} questions aren't supported locally, only choice. Ask a yes/no as a choice between two described options.`);
|
|
6
6
|
};
|
|
7
7
|
export const noul = unsupported("noul");
|
|
8
8
|
export const score = unsupported("score");
|
|
@@ -15,7 +15,7 @@ function stateText(state) {
|
|
|
15
15
|
return v;
|
|
16
16
|
if (typeof v === "number" || typeof v === "boolean")
|
|
17
17
|
return String(v);
|
|
18
|
-
throw new Error("
|
|
18
|
+
throw new Error("decidekit: state values must be text, numbers or booleans");
|
|
19
19
|
}).join("\n");
|
|
20
20
|
}
|
|
21
21
|
function options(criteria) {
|
|
@@ -41,7 +41,7 @@ export function typesafeClient(decider) {
|
|
|
41
41
|
const n = Object.keys(probabilities).length;
|
|
42
42
|
answers[id] = { type: "choice", choice, confidence: Math.max(0, (n * confidence - 1) / (n - 1)), probabilities };
|
|
43
43
|
}
|
|
44
|
-
return { model: "
|
|
44
|
+
return { model: "decidekit", answers: answers };
|
|
45
45
|
},
|
|
46
46
|
};
|
|
47
47
|
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "decidekit",
|
|
3
|
-
"version": "0.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"description": "A System One model in the browser: typed, calibrated text decisions on the user's device. Learns from an LLM when it isn't sure.",
|
|
5
5
|
"license": "MIT",
|
|
6
|
-
"repository": "github:debuffjp/
|
|
6
|
+
"repository": "github:debuffjp/decidekit",
|
|
7
7
|
"keywords": [
|
|
8
8
|
"decision model",
|
|
9
9
|
"system one",
|
|
@@ -77,8 +77,8 @@
|
|
|
77
77
|
"engines": {
|
|
78
78
|
"node": ">=22.18"
|
|
79
79
|
},
|
|
80
|
-
"homepage": "https://debuff.jp/
|
|
81
|
-
"bugs": "https://github.com/debuffjp/
|
|
80
|
+
"homepage": "https://debuff.jp/decidekit/",
|
|
81
|
+
"bugs": "https://github.com/debuffjp/decidekit/issues",
|
|
82
82
|
"bin": {
|
|
83
83
|
"decidekit": "./dist/cli.js"
|
|
84
84
|
},
|