@gentbajko/slopify 3.25.8 → 4.0.0

package/dist/slices/model-checks/repo.js

@@ -0,0 +1,120 @@
+import { z } from "zod";
+import { providerFamilies } from "../../kernel/ports/model.js";
+import { isLocalCliProvider, providerById, sharedKeyOf, } from "../settings/model.js";
+import { readSetting, writeSetting } from "../settings/repo.js";
+import { autoKey, autoModes, checkVerdicts, verdictSources, } from "./model.js";
+import { mergeVerdict } from "./rules.js";
+const checkRow = z.object({
+    provider: z.string(),
+    family: z.enum(providerFamilies),
+    model: z.string(),
+    account: z.string(),
+    listed: z.number().nullable(),
+    listed_at: z.string().nullable(),
+    verdict: z.enum(checkVerdicts).nullable(),
+    verdict_source: z.enum(verdictSources).nullable(),
+    sentence: z.string().nullable(),
+    checked_at: z.string().nullable(),
+    cost: z.number(),
+    first_works_at: z.string().nullable(),
+    notice_seen: z.number(),
+});
+const generationRow = z.object({ credential_generation: z.string() });
+function toRow(raw) {
+    const row = checkRow.parse(raw);
+    return {
+        provider: row.provider,
+        family: row.family,
+        model: row.model,
+        account: row.account,
+        listed: row.listed === null ? null : row.listed === 1,
+        listedAt: row.listed_at,
+        verdict: row.verdict,
+        verdictSource: row.verdict_source,
+        sentence: row.sentence,
+        checkedAt: row.checked_at,
+        cost: row.cost,
+        firstWorksAt: row.first_works_at,
+        noticeSeen: row.notice_seen === 1,
+    };
+}
+export function readChecks(db, provider) {
+    const rows = provider === undefined
+        ? db.prepare("SELECT * FROM model_checks ORDER BY provider, family, model").all()
+        : db
+            .prepare("SELECT * FROM model_checks WHERE provider = ? ORDER BY family, model")
+            .all(provider);
+    return rows.map(toRow);
+}
+function readOne(db, key) {
+    const raw = db
+        .prepare("SELECT * FROM model_checks WHERE provider = ? AND family = ? AND model = ?")
+        .get(key.provider, key.family, key.model);
+    return raw === undefined ? undefined : toRow(raw);
+}
+// A row of another key or login is started over before anything is written to it.
+function ensure(db, key, account) {
+    const found = readOne(db, key);
+    if (found !== undefined && found.account === account)
+        return found;
+    db.prepare("INSERT INTO model_checks (provider, family, model, account) VALUES (?, ?, ?, ?)" +
+        " ON CONFLICT(provider, family, model) DO UPDATE SET account = excluded.account," +
+        " listed = NULL, listed_at = NULL, verdict = NULL, verdict_source = NULL, sentence = NULL," +
+        " checked_at = NULL, cost = 0, first_works_at = NULL, notice_seen = 0").run(key.provider, key.family, key.model, account);
+    return undefined;
+}
+export function writeListed(db, write) {
+    const update = db.prepare("UPDATE model_checks SET listed = ?, listed_at = ? WHERE provider = ? AND family = ? AND model = ?");
+    for (const [model, listed] of write.models) {
+        ensure(db, { provider: write.provider, family: write.family, model }, write.account);
+        update.run(listed ? 1 : 0, write.at, write.provider, write.family, model);
+    }
+}
+export function writeVerdict(db, write) {
+    const previous = ensure(db, write, write.account);
+    const merged = mergeVerdict(previous, write);
+    const firstWorks = previous?.firstWorksAt ?? (merged.verdict === "works" ? merged.checkedAt : null);
+    // A model a run already used is no news to the person who chose it.
+    const seen = (previous?.noticeSeen ?? false) ||
+        (previous?.firstWorksAt == null && firstWorks !== null && write.source === "run");
+    db.prepare("UPDATE model_checks SET verdict = ?, verdict_source = ?, sentence = ?, checked_at = ?," +
+        " cost = cost + ?, first_works_at = ?, notice_seen = ?" +
+        " WHERE provider = ? AND family = ? AND model = ?").run(merged.verdict, merged.verdictSource, merged.sentence, merged.checkedAt, write.cost, firstWorks, seen ? 1 : 0, write.provider, write.family, write.model);
+    const stored = readOne(db, write);
+    if (stored === undefined)
+        throw new Error("model_checks row vanished during its write");
+    return stored;
+}
+export function markNoticesSeen(db, keys) {
+    if (keys === "all") {
+        db.prepare("UPDATE model_checks SET notice_seen = 1 WHERE first_works_at IS NOT NULL").run();
+        return;
+    }
+    const update = db.prepare("UPDATE model_checks SET notice_seen = 1 WHERE provider = ? AND family = ? AND model = ?");
+    for (const key of keys)
+        update.run(key.provider, key.family, key.model);
+}
+// Whose models a row describes. A keyed provider's is its key's generation (a new key starts
+// over), the shared key's for Gemini speech without its own; a CLI's and the system voice's is
+// the provider itself, since the CLI's login is not Slopify's to tell apart. Undefined: no key.
+export function accountOf(db, provider) {
+    if (isLocalCliProvider(provider) || providerById(provider).auth !== "key")
+        return provider;
+    const generation = (id) => {
+        const raw = db
+            .prepare("SELECT credential_generation FROM provider_keys WHERE provider = ?")
+            .get(id);
+        return raw === undefined
+            ? undefined
+            : `${id}:${generationRow.parse(raw).credential_generation}`;
+    };
+    const shared = sharedKeyOf[provider];
+    return generation(provider) ?? (shared === undefined ? undefined : generation(shared));
+}
+export function readAuto(db) {
+    const value = readSetting(db, autoKey);
+    return autoModes.find((mode) => mode === value) ?? "free";
+}
+export function writeAuto(db, mode) {
+    writeSetting(db, autoKey, mode);
+}

package/dist/slices/model-checks/rules.js

@@ -0,0 +1,112 @@
+import { modelStandings, } from "./model.js";
+// The newer of two decisive answers wins, whichever it is; an answer older than the stored one
+// (a slow call finishing late) changes nothing; "failed" tells nothing about access, so it
+// never replaces a decisive one.
+export function mergeVerdict(previous, next) {
+    const decisive = previous?.verdict === "works" || previous?.verdict === "no-access";
+    if (previous?.checkedAt != null && Date.parse(next.at) < Date.parse(previous.checkedAt))
+        return previous;
+    if (next.verdict === "failed" && decisive && previous !== undefined)
+        return previous;
+    return {
+        verdict: next.verdict,
+        verdictSource: next.source,
+        sentence: next.sentence,
+        checkedAt: next.at,
+    };
+}
+// A row learned with another key or login says nothing about this one.
+export function standingOf(row, account, retiring) {
+    const extra = retiring === undefined ? {} : { retiring };
+    if (row === undefined || account === undefined || row.account !== account)
+        return { standing: "known", checkedAt: null, sentence: null, source: null, ...extra };
+    const standing = row.verdict === "works" || row.verdict === "no-access"
+        ? row.verdict
+        : row.listed === true
+            ? "listed"
+            : "known";
+    return {
+        standing,
+        checkedAt: row.checkedAt,
+        sentence: row.sentence,
+        source: row.verdict === null ? null : row.verdictSource,
+        ...extra,
+    };
+}
+const order = (view) => modelStandings.indexOf(view?.standing ?? "known");
+// The picker's list. By default only models that work for you, plus the one already chosen
+// (never hidden, whatever its standing); before anything works, the listed models, then the
+// known ones, so a picker is never empty before the first check.
+export function pickerEntries(models, standings, selected, showAll) {
+    const viewOf = (model) => standings !== undefined && Object.hasOwn(standings, model.id) ? standings[model.id] : undefined;
+    const all = models
+        .map((model, index) => ({ model, view: viewOf(model), index }))
+        .sort((a, b) => order(a.view) - order(b.view) || a.index - b.index);
+    const has = (standing) => all.some((entry) => (entry.view?.standing ?? "known") === standing);
+    const shown = has("works") ? "works" : has("listed") ? "listed" : "known";
+    const keep = all.filter((entry) => showAll || entry.model.id === selected || (entry.view?.standing ?? "known") === shown);
+    return {
+        entries: keep.map(({ model, view }) => ({ model, view })),
+        hidden: all.length - keep.length,
+        fallback: shown !== "works",
+    };
+}
+const monthNames = [
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+];
+export function ago(at, now) {
+    const minutes = Math.max(0, Math.round((now.getTime() - Date.parse(at)) / 60_000));
+    if (minutes < 1)
+        return "just now";
+    if (minutes < 60)
+        return `${String(minutes)} min ago`;
+    const hours = Math.round(minutes / 60);
+    if (hours < 24)
+        return `${String(hours)} h ago`;
+    const days = Math.round(hours / 24);
+    return days === 1 ? "yesterday" : `${String(days)} days ago`;
+}
+function shortDate(day) {
+    const [, month, date] = day.split("-");
+    const name = monthNames[Number(month) - 1];
+    return name === undefined ? day : `${String(Number(date))} ${name}`;
+}
+// The badge beside a model. Retiring outranks the rest: it is the one that needs doing.
+export function badgeOf(view, now) {
+    if (view?.retiring !== undefined)
+        return `⚠ retiring ${shortDate(view.retiring.on)}${view.retiring.replacement === null ? "" : ` → ${view.retiring.replacement.name}`}`;
+    if (view === undefined)
+        return "? not checked";
+    if (view.standing === "works")
+        return view.checkedAt === null
+            ? "✓ works"
+            : `✓ works · ${view.source === "run" ? "used" : "checked"} ${ago(view.checkedAt, now)}`;
+    if (view.standing === "no-access")
+        return "✕ not on your plan";
+    return view.standing === "listed" ? "? listed for you · not checked" : "? not checked";
+}
+// Models that started working for you since you last looked: Settings → Models' New for you.
+export function newForYou(rows) {
+    return rows
+        .filter((row) => row.verdict === "works" && row.firstWorksAt !== null && !row.noticeSeen)
+        .sort((a, b) => (b.firstWorksAt ?? "").localeCompare(a.firstWorksAt ?? ""));
+}
+// A check estimate in words: "free", "less than a cent", "about $0.04".
+export function dollarsText(dollars, unpriced = 0) {
+    const base = dollars <= 0 ? "free" : dollars < 0.01 ? "less than a cent" : `about $${dollars.toFixed(2)}`;
+    if (unpriced === 0)
+        return base;
+    const more = `${String(unpriced)} model${unpriced === 1 ? "" : "s"} with no listed price`;
+    return dollars <= 0 ? `${more} (price unknown)` : `${base}, plus ${more}`;
+}

package/dist/slices/model-checks/run.js

@@ -0,0 +1,143 @@
+import { errorText } from "../../kernel/errors.js";
+import { isLocalCliProvider, providerIds } from "../settings/model.js";
+import { readSetting, writeSetting } from "../settings/repo.js";
+import { automaticAllowed, costOf, isVideoTarget } from "./cost.js";
+import { baselineKey, keyOfModel, } from "./model.js";
+import { markNoticesSeen, writeVerdict } from "./repo.js";
+// ceiling: an automatic run checks at most this many models of one provider (decision 7).
+export const automaticPerProvider = 25;
+const isProvider = (id) => providerIds.includes(id);
+export function planChecks(input, scope) {
+    const manual = scope.trigger === "manual";
+    const rowOf = new Map(input.rows.map((row) => [keyOfModel(row), row]));
+    const skipped = [];
+    const picked = [];
+    const perProvider = new Map();
+    const offered = input.offered.filter((model) => (scope.provider === undefined || model.provider === scope.provider) &&
+        (scope.models === undefined || scope.models.includes(model.model)));
+    // Automatic runs take what is in use first, then what already worked, then what is new.
+    const rank = (model) => {
+        const row = rowOf.get(keyOfModel(model));
+        if (input.chosen.has(keyOfModel(model)))
+            return 0;
+        return row?.verdict === "works" ? 1 : 2;
+    };
+    const ordered = manual ? offered : [...offered].sort((a, b) => rank(a) - rank(b));
+    for (const model of ordered) {
+        if (!isProvider(model.provider))
+            continue;
+        const account = input.accounts(model.provider);
+        // An automatic run says nothing about what it was never going to check.
+        if (account === undefined) {
+            if (manual)
+                skipped.push({ model, reason: "No key is saved for this provider." });
+            continue;
+        }
+        if (isVideoTarget(input.catalogue, model)) {
+            if (manual)
+                skipped.push({ model, reason: "Image-to-video models are listed, never checked." });
+            continue;
+        }
+        const cost = costOf(input.catalogue, model);
+        if (!manual) {
+            const row = rowOf.get(keyOfModel(model));
+            const sameAccount = row !== undefined && row.account === account;
+            const inScope = input.chosen.has(keyOfModel(model)) ||
+                (sameAccount && row.verdict === "works") ||
+                (sameAccount && row.listed === true && row.verdict === null);
+            if (!inScope)
+                continue;
+            if (!automaticAllowed(input.catalogue, model, cost)) {
+                skipped.push({ model, reason: "It costs money, so only Check models runs it." });
+                continue;
+            }
+            const count = perProvider.get(model.provider) ?? 0;
+            if (count >= automaticPerProvider)
+                continue;
+            perProvider.set(model.provider, count + 1);
+        }
+        picked.push({ ...model, account, cost });
+    }
+    return {
+        trigger: scope.trigger,
+        targets: picked,
+        totalDollars: picked.reduce((sum, target) => sum + (target.cost.dollars ?? 0), 0),
+        unpriced: picked.filter((target) => target.cost.dollars === null).length,
+        planAccounts: [
+            ...new Set(picked.filter((t) => isLocalCliProvider(t.provider)).map((t) => t.provider)),
+        ],
+        skipped,
+    };
+}
+// A plan that costs money runs only with the person's confirmed estimate: the dollars they
+// were shown must cover the plan's.
+export function needsConfirmation(plan, confirmedDollars) {
+    if (plan.totalDollars <= 0 && plan.unpriced === 0)
+        return false;
+    return confirmedDollars === undefined || confirmedDollars + 1e-9 < plan.totalDollars;
+}
+export async function runChecks(deps, plan, options = {}) {
+    if (plan.trigger === "manual" && needsConfirmation(plan, options.confirmedDollars))
+        return {
+            ok: false,
+            reason: "estimate",
+            totalDollars: plan.totalDollars,
+            unpriced: plan.unpriced,
+        };
+    const signal = options.signal ?? new AbortController().signal;
+    const tally = { checked: 0, works: 0, noAccess: 0, failed: 0, skipped: 0, spent: 0 };
+    const byProvider = new Map();
+    for (const target of plan.targets)
+        byProvider.set(target.provider, [...(byProvider.get(target.provider) ?? []), target]);
+    await Promise.all([...byProvider.values()].map(async (targets) => {
+        for (const target of targets) {
+            if (signal.aborted)
+                return;
+            let outcome;
+            try {
+                outcome = await deps.check(target, signal);
+            }
+            catch (error) {
+                if (signal.aborted)
+                    return;
+                deps.log.write("warn", "model-checks.check", {
+                    detail: `${target.provider} ${target.model}: ${errorText(error)}`,
+                });
+                tally.failed += 1;
+                continue;
+            }
+            if (outcome.kind === "skipped") {
+                tally.skipped += 1;
+                continue;
+            }
+            tally.checked += 1;
+            tally.spent += outcome.spent;
+            if (outcome.verdict === "works")
+                tally.works += 1;
+            else if (outcome.verdict === "no-access")
+                tally.noAccess += 1;
+            else
+                tally.failed += 1;
+            writeVerdict(deps.db, {
+                ...target,
+                verdict: outcome.verdict,
+                source: "check",
+                sentence: outcome.sentence,
+                at: deps.clock.now().toISOString(),
+                cost: outcome.spent,
+            });
+            // The rest of this provider's checks would fail the same way (a rejected key, a
+            // used-up limit), and each might cost something.
+            if (outcome.stopProvider) {
+                tally.skipped += targets.length - targets.indexOf(target) - 1;
+                return;
+            }
+        }
+    }));
+    // What works at the first run is what the person already had: none of it is news.
+    if (readSetting(deps.db, baselineKey) === undefined) {
+        markNoticesSeen(deps.db, "all");
+        writeSetting(deps.db, baselineKey, deps.clock.now().toISOString());
+    }
+    return { ok: true, summary: tally };
+}

package/dist/slices/model-checks/service.js

@@ -0,0 +1,198 @@
+import { errorText } from "../../kernel/errors.js";
+import { choiceSites } from "../model-upkeep/usage.js";
+import { isUncataloguedProvider, providerById, providerIds, providers, } from "../settings/model.js";
+import { readSetting, writeSetting } from "../settings/repo.js";
+import { keyOfModel, weeklyAtKey, } from "./model.js";
+import { accountOf, readAuto, readChecks } from "./repo.js";
+import { newForYou, standingOf } from "./rules.js";
+import { needsConfirmation, planChecks, runChecks, } from "./run.js";
+// ceiling: the weekly look runs once seven days have passed since the last one.
+export const weeklyIntervalMs = 7 * 24 * 60 * 60_000;
+function familyOfSlot(slot) {
+    return slot === "llm" ? "llm" : slot === "audio" ? "tts" : "image";
+}
+export function chosenModels(db) {
+    const chosen = new Set();
+    for (const site of choiceSites(db))
+        for (const [slot, choice] of Object.entries(site.choices)) {
+            if (choice === undefined || choice.provider === "" || choice.model.trim() === "")
+                continue;
+            chosen.add(keyOfModel({ provider: choice.provider, family: familyOfSlot(slot), model: choice.model }));
+        }
+    return chosen;
+}
+export function createModelChecks(deps) {
+    const running = new Set();
+    const stopping = new AbortController();
+    const offered = async (provider) => {
+        const { family } = providerById(provider);
+        if (isUncataloguedProvider(provider)) {
+            const models = await deps.providerModels(provider, family).catch(() => []);
+            return models.map((model) => ({ provider, family, model: model.id }));
+        }
+        return deps
+            .catalogue()[family].filter((row) => row.provider === provider && row.enabled && !row.deprecated)
+            .map((row) => ({ provider, family, model: row.id }));
+    };
+    const withAccount = () => providerIds.filter((id) => accountOf(deps.db, id) !== undefined);
+    // Named models of a CLI are taken as named: asking the CLI for its list would start it, and
+    // a picker asks this for its one chosen model whenever it opens.
+    const offeredFor = (provider, models) => {
+        if (models === undefined || !isUncataloguedProvider(provider))
+            return offered(provider);
+        const { family } = providerById(provider);
+        return Promise.resolve(models.map((model) => ({ provider, family, model })));
+    };
+    const plan = async (scope) => {
+        const scoped = scope.provider === undefined ? withAccount() : [scope.provider];
+        return planChecks({
+            catalogue: deps.catalogue(),
+            rows: readChecks(deps.db),
+            offered: (await Promise.all(scoped.map((id) => offeredFor(id, scope.models)))).flat(),
+            accounts: (id) => accountOf(deps.db, id),
+            chosen: chosenModels(deps.db),
+        }, scope);
+    };
+    // A provider is taken before the first await, so two starts that arrive together can't both
+    // pass the running check and pay for the same checks twice.
+    const reserve = (ids) => {
+        let held = [...ids];
+        for (const id of held)
+            running.add(id);
+        const release = () => {
+            for (const id of held)
+                running.delete(id);
+            held = [];
+        };
+        // Once planned, only the providers it checks stay taken.
+        const keepFor = (checks) => {
+            const owned = held.filter((id) => checks.targets.some((t) => t.provider === id));
+            for (const id of held)
+                if (!owned.includes(id))
+                    running.delete(id);
+            held = owned;
+        };
+        return { release, keepFor };
+    };
+    const launch = (checks, release) => runChecks({ db: deps.db, clock: deps.clock, log: deps.log, check: deps.check }, checks, 
+    // A manual plan's estimate was confirmed in `start`; an automatic one is free or cheap.
+    { confirmedDollars: checks.totalDollars, signal: stopping.signal })
+        .then((answer) => answer.ok
+        ? answer.summary
+        : { checked: 0, works: 0, noAccess: 0, failed: 0, skipped: 0, spent: 0 })
+        .finally(release);
+    const start = async (scope, confirmedDollars) => {
+        const scoped = scope.provider === undefined ? withAccount() : [scope.provider];
+        const busy = scoped.filter((id) => running.has(id));
+        if (busy.length > 0)
+            return { ok: false, reason: "running", providers: busy };
+        const { release, keepFor } = reserve(scoped);
+        let checks;
+        try {
+            checks = await plan(scope);
+        }
+        catch (error) {
+            release();
+            throw error;
+        }
+        keepFor(checks);
+        if (scope.trigger === "manual" && needsConfirmation(checks, confirmedDollars)) {
+            release();
+            return { ok: false, reason: "estimate", plan: checks };
+        }
+        return { ok: true, plan: checks, done: launch(checks, release) };
+    };
+    const automatic = async (trigger, provider) => {
+        const scoped = provider === undefined ? withAccount() : [provider];
+        for (const id of scoped)
+            await deps.listing.refresh(id, true).catch((error) => {
+                deps.log.write("warn", "model-checks.listing", {
+                    detail: `${id}: ${errorText(error)}`,
+                });
+            });
+        if (readAuto(deps.db) === "off")
+            return;
+        const free = scoped.filter((id) => !running.has(id));
+        if (free.length === 0)
+            return;
+        const { release, keepFor } = reserve(free);
+        let kept;
+        try {
+            const checks = await plan(provider === undefined ? { trigger } : { trigger, provider });
+            kept = {
+                ...checks,
+                targets: checks.targets.filter((t) => free.some((id) => id === t.provider)),
+            };
+            keepFor(kept);
+            if (trigger === "weekly")
+                writeSetting(deps.db, weeklyAtKey, deps.clock.now().toISOString());
+        }
+        catch (error) {
+            release();
+            throw error;
+        }
+        if (kept.targets.length > 0)
+            await launch(kept, release);
+        else
+            release();
+    };
+    return {
+        offered,
+        plan,
+        start,
+        automatic,
+        weeklyDue: () => {
+            if (readAuto(deps.db) === "off")
+                return false;
+            const last = readSetting(deps.db, weeklyAtKey);
+            return (last === undefined || deps.clock.now().getTime() - Date.parse(last) >= weeklyIntervalMs);
+        },
+        standings: (provider, models) => {
+            const { family } = providerById(provider);
+            const account = accountOf(deps.db, provider);
+            const rows = new Map(readChecks(deps.db, provider)
+                .filter((row) => row.family === family)
+                .map((row) => [row.model, row]));
+            return Object.fromEntries(models.map((model) => [
+                model.id,
+                standingOf(rows.get(model.id), account, deps.retiring?.({ provider, family, model: model.id })),
+            ]));
+        },
+        overview: () => {
+            const rows = readChecks(deps.db);
+            const catalogue = deps.catalogue();
+            const nameOf = (key) => catalogue[key.family].find((m) => m.provider === key.provider && m.id === key.model)
+                ?.name ?? key.model;
+            const states = providers.map((provider) => {
+                const account = accountOf(deps.db, provider.id);
+                const own = rows.filter((row) => row.provider === provider.id && row.account === account);
+                const times = own.flatMap((row) => (row.checkedAt === null ? [] : [row.checkedAt])).sort();
+                return {
+                    provider: provider.id,
+                    family: provider.family,
+                    running: running.has(provider.id),
+                    lastCheckedAt: times.at(-1) ?? null,
+                    spent: own.reduce((sum, row) => sum + row.cost, 0),
+                    works: own.filter((row) => row.verdict === "works").length,
+                    noAccess: own.filter((row) => row.verdict === "no-access").length,
+                    listed: own.filter((row) => row.listed === true).length,
+                };
+            });
+            const current = rows.filter((row) => providerIds.includes(row.provider) &&
+                row.account === accountOf(deps.db, row.provider));
+            return {
+                auto: readAuto(deps.db),
+                providers: states,
+                newForYou: newForYou(current).map((row) => ({
+                    provider: row.provider,
+                    family: row.family,
+                    model: row.model,
+                    name: nameOf(row),
+                    firstWorksAt: row.firstWorksAt ?? "",
+                })),
+            };
+        },
+        running: () => [...running],
+        stop: () => stopping.abort(),
+    };
+}