@byokit/decide 0.1.0

package/README.md

@@ -0,0 +1,52 @@
+# @byokit/decide
+
+Typed questions in, a typed answer with confidence out, and an abstain below a floor, so your app takes its safe
+default (ask the person) instead of guessing. Backends: your own `rules`, and [Jev](https://openrouter.ai/docs/guides/community/jev)
+over TypeSafe's API or OpenRouter.
+
+```ts
+import { decide, jev, rules } from '@byokit/decide';
+
+const backends = [
+  rules((s) => (/^(thanks|thank you)\b/i.test(s.text) ? 'chat' : undefined)), // the obvious cases, free
+  jev({ key: hostConfig.jevKey }),            // or jev({ key: openRouterKey, via: 'openrouter' })
+];
+const { intent } = await decide({ text: 'can you check if the plumber replied?' }, {
+  intent: { kind: 'choice', options: { task: 'Something new to do', followup: 'About an earlier job', chat: 'Just talking' } },
+}, { privacy: 'may-leave', backends });
+
+if (intent.abstained) askThePerson(); else route(intent.answer);
+// { answer: 'followup', confidence: 0.91, probabilities: {...}, abstained: false, by: 'jev', ms: 214 }
+```
+
+- **Questions**: `choice` (options with a one-line description each), `yesno`, and `score` (an ordered rubric, lowest
+  first; the answer is the level's index).
+- **The floors are code, not a prompt** (ported from firstmate's dispatch resolver): a 0.6 floor on the answer's
+  confidence by default (`floor` per question); a choice option can declare its own floor (`floors`), checked against its
+  own probability, and a pick under it falls to the most probable other option that clears its own; a tie abstains.
+  An answer whose probabilities are missing an option, out of range or don't sum to 1 is an abstain, never an error.
+- **Backends** are tried in order for the questions still unanswered. A backend that fails or takes longer than
+  `timeoutMs` (default 5 s) answers nothing. `privacy: 'stays-here'` skips every backend the state would leave the
+  device for (Jev), so private text never goes to one.
+- **Keys**: `jev()` takes the key your host read from its own environment or config. The kit never reads an environment
+  variable, and the key goes only into the one request header. Never ship a key inside an app: keep it on the home
+  computer and let paired devices ask it.
+
+## Evals
+
+Each decision gets a labelled file, `evals/<decision>.jsonl`: a header `{ decision, question, note }`, then one case per
+line, `{ state, expect, jev, ms }`. `expect` is the right answer, a list of right answers, or `null` when only an abstain
+is right; `jev` is a Jev-shaped answer, replayed offline so CI never calls a model. The included example is hand-made,
+not a live recording.
+
+```sh
+npx --package=@byokit/decide byokit-eval evals/intent.jsonl                    # replay stored answers
+npx --package=@byokit/decide byokit-eval evals/intent.jsonl --floor 0.7        # try another floor
+TYPESAFE_API_KEY=… npx --package=@byokit/decide byokit-eval evals/intent.jsonl --live typesafe --record
+```
+
+Agreement counts right answers and correctly expected abstentions; abstentions are also reported separately. Clear-but-wrong
+(answered, and wrong) is the number that must stay near 0; the command exits 1 when its rate is above
+`--max-clear-wrong` (default 0). Set floors from the eval, not by guessing. `evaluate()` in `@byokit/decide/eval` runs the
+same report over any backends, including your rules. `--record` keeps previous answers when a live refresh fails and
+marks a partially refreshed file as such. `evals/example-urgent.jsonl` shows the format.

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+// byokit-eval <file.jsonl>... [--floor N] [--max-clear-wrong RATE] [--live typesafe|openrouter [--record]]
+// Offline by default: replays stored Jev-shaped answers. --live asks Jev with the caller's TYPESAFE_API_KEY
+// (or OPENROUTER_API_KEY with --live openrouter); --record writes successful refreshes back into the file.
+// Exits 1 when a file's clear-but-wrong rate is above --max-clear-wrong (default 0).
+import { readFileSync, writeFileSync } from 'node:fs';
+import { parseArgs } from 'node:util';
+import { decide, jev } from "./index.js";
+import { evaluate, format, parse, replay, summary } from "./eval.js";
+const { values: o, positionals: files } = (() => {
+    try {
+        return parseArgs({
+            allowPositionals: true,
+            options: { floor: { type: 'string' }, 'max-clear-wrong': { type: 'string', default: '0' }, live: { type: 'string' }, record: { type: 'boolean' } },
+        });
+    }
+    catch {
+        console.error('invalid arguments');
+        process.exit(2);
+    }
+})();
+const validRate = (value) => value.trim() !== '' && Number.isFinite(Number(value)) && Number(value) >= 0 && Number(value) <= 1;
+if (!files.length || (o.live && o.live !== 'typesafe' && o.live !== 'openrouter') ||
+    (o.floor !== undefined && !validRate(o.floor)) || !validRate(o['max-clear-wrong'])) {
+    console.error('usage: byokit-eval <file.jsonl>... [--floor N] [--max-clear-wrong RATE] [--live typesafe|openrouter [--record]]');
+    process.exit(2);
+}
+const via = o.live;
+const key = via && process.env[via === 'typesafe' ? 'TYPESAFE_API_KEY' : 'OPENROUTER_API_KEY'];
+if (via && !key) {
+    console.error(`--live ${via} needs ${via === 'typesafe' ? 'TYPESAFE_API_KEY' : 'OPENROUTER_API_KEY'} set for this one command`);
+    process.exit(2);
+}
+let failed = false;
+for (const path of files) {
+    const f = parse(readFileSync(path, 'utf8'));
+    const q = o.floor ? { ...f.question, floor: Number(o.floor) } : f.question;
+    let ask = replay(q);
+    let refreshed = 0;
+    if (via && key) {
+        let last;
+        // Keeps Jev's own answer for --record; the key stays inside jev().
+        const keep = async (url, init) => {
+            const res = await fetch(url, init);
+            last = res.ok ? (await res.clone().json())?.answers?.[f.decision] : undefined;
+            return res;
+        };
+        const backend = jev({ key, via, fetch: keep });
+        ask = async (c) => {
+            last = undefined;
+            const a = (await decide(c.state, { [f.decision]: q }, { privacy: 'may-leave', backends: [backend] }))[f.decision];
+            if (o.record && a.probabilities) {
+                Object.assign(c, { jev: last, ms: a.ms });
+                refreshed++;
+            }
+            return a;
+        };
+    }
+    const r = await evaluate(f.cases, ask);
+    console.log(summary(`${path}: ${f.decision}`, via ? `jev via ${via}` : 'jev, recorded', r));
+    if (via && o.record && refreshed)
+        writeFileSync(path, format({ ...f, note: refreshed === f.cases.length
+                ? `answers recorded live from Jev via ${via}, ${new Date().toISOString().slice(0, 10)}`
+                : `partial live refresh ${refreshed}/${f.cases.length} via ${via}; ${f.note ?? 'previous provenance unknown'}` }));
+    if (r.cases && r.clearWrong / r.cases > Number(o['max-clear-wrong']))
+        failed = true;
+}
+process.exit(failed ? 1 : 0);

package/dist/eval.d.ts

@@ -0,0 +1,39 @@
+import { type Answer, type Question } from './index.ts';
+export type Case = {
+    state: unknown;
+    expect: string | boolean | number | null | Array<string | boolean | number>;
+    jev?: unknown;
+    ms?: number;
+};
+/** `note` says where the stored answers came from. */
+export type EvalFile = {
+    decision: string;
+    question: Question;
+    note?: string;
+    cases: Case[];
+};
+export type Report = {
+    cases: number;
+    agree: number;
+    abstained: number;
+    /** Answered, and not a right answer: the number that must stay near 0. */
+    clearWrong: number;
+    ms: {
+        min: number;
+        median: number;
+        max: number;
+    };
+    wrong: Array<{
+        line: number;
+        expect: Case['expect'];
+        got: Answer['answer'];
+        confidence: number;
+    }>;
+};
+export declare function parse(text: string): EvalFile;
+export declare function format(f: EvalFile): string;
+/** Scores `ask` over the cases, in order. */
+export declare function evaluate(cases: Case[], ask: (c: Case) => Promise<Answer>): Promise<Report>;
+/** A stored Jev-shaped answer through the same floors a live answer takes. */
+export declare function replay(q: Question): (c: Case) => Promise<Answer>;
+export declare function summary(name: string, by: string, r: Report): string;

package/dist/eval.js

@@ -0,0 +1,50 @@
+// An eval file is JSONL: a header line `{ decision, question, note? }`, then one labelled case per line,
+// `{ state, expect, jev?, ms? }`. `expect` is the right answer, a list of right answers, or null when only an abstain is
+// right. `jev` is a stored Jev-shaped answer replayed offline (CI never calls a model); --live --record refreshes it.
+import { resolve } from "./index.js";
+import { raw } from "./jev.js";
+export function parse(text) {
+    const [head, ...rest] = text.split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l));
+    if (!head?.decision || !head.question)
+        throw new Error('the first line needs decision and question');
+    return { ...head, cases: rest };
+}
+export function format(f) {
+    const { cases, ...head } = f;
+    return [head, ...cases].map((l) => JSON.stringify(l)).join('\n') + '\n';
+}
+/** Scores `ask` over the cases, in order. */
+export async function evaluate(cases, ask) {
+    const r = { cases: cases.length, agree: 0, abstained: 0, clearWrong: 0, ms: { min: 0, median: 0, max: 0 }, wrong: [] };
+    const ms = [];
+    for (const [i, c] of cases.entries()) {
+        const a = await ask(c);
+        ms.push(a.ms);
+        const right = c.expect === null ? [] : Array.isArray(c.expect) ? c.expect : [c.expect];
+        if (a.abstained) {
+            r.abstained++;
+            if (c.expect === null)
+                r.agree++;
+        }
+        else if (right.includes(a.answer))
+            r.agree++;
+        else
+            r.clearWrong++, r.wrong.push({ line: i + 2, expect: c.expect, got: a.answer, confidence: a.confidence });
+    }
+    ms.sort((a, b) => a - b);
+    if (ms.length)
+        r.ms = { min: ms[0], median: ms[Math.floor((ms.length - 1) / 2)], max: ms[ms.length - 1] };
+    return r;
+}
+/** A stored Jev-shaped answer through the same floors a live answer takes. */
+export function replay(q) {
+    return async (c) => ({ ...resolve(q, raw(q, c.jev)), by: 'jev (recorded)', ms: c.ms ?? 0 });
+}
+export function summary(name, by, r) {
+    const pct = (n) => `${r.cases ? Math.round((n / r.cases) * 100) : 0}%`;
+    return [
+        `${name} (${by}): ${r.cases} cases`,
+        `  agree ${r.agree}/${r.cases}   clear-but-wrong ${r.clearWrong} (${pct(r.clearWrong)})   abstained ${r.abstained} (${pct(r.abstained)})   ms min/median/max ${r.ms.min}/${r.ms.median}/${r.ms.max}`,
+        ...r.wrong.map((w) => `  wrong: line ${w.line} expected ${JSON.stringify(w.expect)}, got ${JSON.stringify(w.got)} at ${w.confidence}`),
+    ].join('\n');
+}

package/dist/index.d.ts

@@ -0,0 +1,59 @@
+export { jev } from './jev.ts';
+export type Question = 
+/** Pick one option; `floors` holds an option's own floor, checked against that option's probability. */
+{
+    kind: 'choice';
+    options: Record<string, string>;
+    instructions?: string;
+    floor?: number;
+    floors?: Record<string, number>;
+} | {
+    kind: 'yesno';
+    question: string;
+    yes?: string;
+    no?: string;
+    floor?: number;
+}
+/** An ordered rubric, lowest first. The answer is the most probable level's index. */
+ | {
+    kind: 'score';
+    levels: string[];
+    instructions?: string;
+    floor?: number;
+};
+export type Answer = {
+    /** null when abstained: the app takes its safe default (ask a person). */
+    answer: string | boolean | number | null;
+    confidence: number;
+    probabilities?: Record<string, number>;
+    abstained: boolean;
+    /** Why it abstained, or which runner-up it fell to. For logs, not for people. */
+    reason?: string;
+    by: string;
+    ms: number;
+};
+/** A backend's answer before the floors: every option's probability, keyed as options (choice), 'true'/'false'
+ * (yesno) or level indexes (score). A missing or malformed one is an abstain. */
+export type Raw = {
+    probabilities: Record<string, number>;
+    confidence?: number;
+    pick?: string;
+};
+export type Backend = {
+    name: string;
+    /** Whether the state leaves this device. Such a backend is skipped for `privacy: 'stays-here'`. */
+    leaves: boolean;
+    ask(state: unknown, questions: Record<string, Question>, signal: AbortSignal): Promise<Record<string, Raw | undefined>>;
+};
+export type Options = {
+    privacy: 'stays-here' | 'may-leave';
+    backends: Backend[];
+    timeoutMs?: number;
+};
+export declare const FLOOR = 0.6;
+/** Asks each backend in order for the questions still unanswered; a failed or slow backend answers nothing. */
+export declare function decide(state: unknown, questions: Record<string, Question>, opts: Options): Promise<Record<string, Answer>>;
+/** The floors on one raw answer. Exported for apps that hold a recorded answer. */
+export declare function resolve(q: Question, raw: Raw | undefined): Omit<Answer, 'by' | 'ms'>;
+/** The app's own function as a backend: return the answer when the case is obvious, undefined otherwise. Stays here. */
+export declare function rules(fn: (state: any, name: string, q: Question) => string | boolean | number | undefined): Backend;

package/dist/index.js

@@ -0,0 +1,97 @@
+// Typed questions in, a typed answer with confidence out, abstaining below a floor. The floor, the per-option floors,
+// the runner-up and the tie are code, never a prompt: ported from firstmate's bin/fm-dispatch-resolve.sh.
+export { jev } from "./jev.js";
+export const FLOOR = 0.6;
+/** Asks each backend in order for the questions still unanswered; a failed or slow backend answers nothing. */
+export async function decide(state, questions, opts) {
+    const out = Object.create(null);
+    const open = () => Object.fromEntries(Object.entries(questions).filter(([k]) => !Object.hasOwn(out, k) || out[k].abstained));
+    for (const b of opts.backends) {
+        if (b.leaves && opts.privacy !== 'may-leave')
+            continue;
+        const todo = open();
+        if (!Object.keys(todo).length)
+            break;
+        const t0 = Date.now();
+        let raws = {};
+        let failed = '';
+        const controller = new AbortController();
+        let timer;
+        const deadline = new Promise((_, reject) => {
+            timer = setTimeout(() => { controller.abort(); reject(new Error('timed out')); }, opts.timeoutMs ?? 5000);
+        });
+        try {
+            raws = await Promise.race([b.ask(state, todo, controller.signal), deadline]);
+        }
+        catch (e) {
+            failed = `${b.name} failed: ${e.message}`;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        const ms = Date.now() - t0;
+        for (const [k, q] of Object.entries(todo)) {
+            const raw = Object.hasOwn(raws, k) ? raws[k] : undefined;
+            const a = { ...resolve(q, raw), by: b.name, ms };
+            if (!raw && failed)
+                a.reason = failed;
+            if (!Object.hasOwn(out, k) || !a.abstained || a.probabilities)
+                out[k] = a;
+        }
+    }
+    for (const k of Object.keys(questions))
+        if (!Object.hasOwn(out, k))
+            out[k] = { answer: null, confidence: 0, abstained: true, reason: 'no backend answered', by: 'none', ms: 0 };
+    return out;
+}
+/** The floors on one raw answer. Exported for apps that hold a recorded answer. */
+export function resolve(q, raw) {
+    const keys = q.kind === 'choice' ? Object.keys(q.options) : q.kind === 'yesno' ? ['true', 'false'] : q.levels.map((_, i) => String(i));
+    const p = raw?.probabilities;
+    const ok = p && Object.keys(p).length === keys.length && keys.every((k) => Object.hasOwn(p, k) && typeof p[k] === 'number' && p[k] >= 0 && p[k] <= 1)
+        && Math.abs(keys.reduce((s, k) => s + p[k], 0) - 1) <= 0.01
+        && (raw.confidence === undefined || (raw.confidence >= 0 && raw.confidence <= 1))
+        && (raw.pick === undefined || keys.includes(raw.pick));
+    if (!ok)
+        return { answer: null, confidence: 0, abstained: true, reason: raw ? 'malformed answer' : 'no answer' };
+    const ranked = [...keys].sort((a, b) => p[b] - p[a]);
+    const picked = raw.pick ?? ranked[0];
+    const confidence = raw.confidence ?? p[picked];
+    const floor = q.floor ?? FLOOR;
+    const own = (k) => (q.kind === 'choice' && q.floors && Object.hasOwn(q.floors, k) ? q.floors[k] : undefined) ?? floor;
+    const typed = (k) => (q.kind === 'choice' ? k : q.kind === 'yesno' ? k === 'true' : Number(k));
+    const done = (k, reason) => ({ answer: typed(k), confidence: k === picked ? confidence : p[k], probabilities: p, abstained: false, ...(reason && { reason }) });
+    const abstain = (reason) => ({ answer: null, confidence, probabilities: p, abstained: true, reason });
+    if (p[ranked[0]] === p[ranked[1]])
+        return abstain('tie');
+    // Without a declared floor on the pick, the one floor applies to the answer's confidence, exactly as firstmate's.
+    if (!(q.kind === 'choice' && q.floors && Object.hasOwn(q.floors, picked)))
+        return confidence >= floor ? done(picked) : abstain(`confidence ${confidence} below floor ${floor}`);
+    if (p[picked] >= own(picked))
+        return done(picked);
+    // A runner-up never needs weaker support than it would as the pick: it must clear its own floor.
+    const clear = ranked.filter((k) => k !== picked && p[k] >= own(k));
+    if (!clear.length)
+        return abstain(`${picked} probability ${p[picked]} below its floor ${own(picked)}; no other option clears its own`);
+    if (clear.length > 1 && p[clear[0]] === p[clear[1]])
+        return abstain('runner-up tie');
+    return done(clear[0], `fell to ${clear[0]}: ${picked} probability ${p[picked]} below its floor ${own(picked)}`);
+}
+/** The app's own function as a backend: return the answer when the case is obvious, undefined otherwise. Stays here. */
+export function rules(fn) {
+    return {
+        name: 'rules',
+        leaves: false,
+        async ask(state, questions) {
+            const out = Object.create(null);
+            for (const [k, q] of Object.entries(questions)) {
+                const a = fn(state, k, q);
+                if (a === undefined)
+                    continue;
+                const keys = q.kind === 'choice' ? Object.keys(q.options) : q.kind === 'yesno' ? ['true', 'false'] : q.levels.map((_, i) => String(i));
+                out[k] = { probabilities: Object.fromEntries(keys.map((o) => [o, o === String(a) ? 1 : 0])) };
+            }
+            return out;
+        },
+    };
+}

package/dist/jev.d.ts

@@ -0,0 +1,8 @@
+import type { Backend, Question, Raw } from './index.ts';
+export declare function jev(opts: {
+    key: string;
+    via?: 'typesafe' | 'openrouter';
+    fetch?: typeof fetch;
+}): Backend;
+/** Jev's answer as a Raw; anything off-shape is undefined, which the floors treat as an abstain. */
+export declare function raw(q: Question, a: any): Raw | undefined;

package/dist/jev.js

@@ -0,0 +1,41 @@
+const BASE = { typesafe: 'https://api.typesafe.ai', openrouter: 'https://openrouter.ai/api' };
+export function jev(opts) {
+    const { key, via = 'typesafe', fetch: f = globalThis.fetch } = opts;
+    if (!key)
+        throw new Error('jev needs a key');
+    return {
+        name: 'jev',
+        leaves: true,
+        async ask(state, questions, signal) {
+            const res = await f(`${BASE[via]}/v1/systemone`, {
+                method: 'POST',
+                signal,
+                headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
+                body: JSON.stringify({ model: 'jev-latest', state, questions: Object.fromEntries(Object.entries(questions).map(([k, q]) => [k, wire(q)])) }),
+            });
+            if (!res.ok)
+                throw new Error(`http ${res.status}`);
+            const answers = (await res.json())?.answers ?? {};
+            return Object.fromEntries(Object.entries(questions).map(([k, q]) => [k, raw(q, Object.hasOwn(answers, k) ? answers[k] : undefined)]));
+        },
+    };
+}
+function wire(q) {
+    if (q.kind === 'choice')
+        return { type: 'choice', instructions: q.instructions ?? 'Which option fits the state?', criteria: q.options };
+    if (q.kind === 'yesno')
+        return { type: 'noul', instructions: q.question, ...(q.yes && q.no && { criteria: { true: q.yes, false: q.no } }) };
+    return { type: 'score', instructions: q.instructions ?? 'Where does the state fall on this scale?', criteria: q.levels };
+}
+/** Jev's answer as a Raw; anything off-shape is undefined, which the floors treat as an abstain. */
+export function raw(q, a) {
+    if (!a || typeof a !== 'object')
+        return undefined;
+    if (q.kind === 'yesno')
+        return typeof a.noul === 'number' ? { probabilities: { true: a.noul, false: 1 - a.noul } } : undefined;
+    if (typeof a.probabilities !== 'object' || typeof a.confidence !== 'number')
+        return undefined;
+    if (q.kind === 'choice')
+        return typeof a.choice === 'string' ? { probabilities: a.probabilities, confidence: a.confidence, pick: a.choice } : undefined;
+    return { probabilities: a.probabilities, confidence: a.confidence };
+}

package/evals/example-urgent.jsonl

@@ -0,0 +1,6 @@
+{"decision":"urgent","question":{"kind":"yesno","question":"Does this message need an answer today?","yes":"Something is time-sensitive today","no":"It can wait"},"note":"example: hand-made answers in Jev's shape, not recorded; refresh with byokit-eval --live typesafe --record"}
+{"state":"The school called, Maya has a fever, can someone pick her up before 2?","expect":true,"jev":{"type":"noul","noul":0.97},"ms":180}
+{"state":"When you get a chance, send me that lasagna recipe.","expect":false,"jev":{"type":"noul","noul":0.04},"ms":165}
+{"state":"Dinner at grandma's is still on for Sunday, right?","expect":false,"jev":{"type":"noul","noul":0.31},"ms":172}
+{"state":"The plumber is here and wants to know if he can turn the water off now.","expect":true,"jev":{"type":"noul","noul":0.93},"ms":190}
+{"state":"Can you look at the car insurance renewal sometime?","expect":[true,false],"jev":{"type":"noul","noul":0.52},"ms":201}

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@byokit/decide",
+  "version": "0.1.0",
+  "description": "Typed questions in, a typed answer with confidence out, abstaining below a floor. Rules and Jev backends, and an eval runner.",
+  "license": "Apache-2.0",
+  "type": "module",
+  "repository": { "type": "git", "url": "git+https://github.com/umeranjum17/byokit.git", "directory": "packages/decide" },
+  "engines": { "node": ">=22.18" },
+  "exports": {
+    ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
+    "./eval": { "types": "./dist/eval.d.ts", "default": "./dist/eval.js" }
+  },
+  "bin": { "byokit-eval": "dist/cli.js" },
+  "files": ["dist", "evals"],
+  "scripts": { "prepack": "tsc -b" },
+  "publishConfig": { "access": "public" }
+}