@columenlabs/lumen-ai 0.1.71 → 0.1.73

package/dist/assistant/state.js

@@ -0,0 +1,138 @@
+import { keyHint } from './models.js';
+import { APP_SCOPE } from './store.js';
+/** USD a month when nobody set a cap. */
+export const DEFAULT_CAP = 10;
+/** `2026-10` in the app's time zone. */
+export function monthIn(timeZone, at = new Date()) {
+    return new Intl.DateTimeFormat('en-CA', { timeZone, year: 'numeric', month: '2-digit' })
+        .format(at)
+        .slice(0, 7);
+}
+export function keyState(d, secret) {
+    if (!d)
+        return { set: false };
+    return {
+        set: true,
+        hint: keyHint(secret(d)),
+        saved_at: d.saved_at,
+        ...(d.test ? { test: d.test } : {}),
+    };
+}
+export class State {
+    docs;
+    o;
+    constructor(docs, o) {
+        this.docs = docs;
+        this.o = o;
+    }
+    /** USD a month when nobody set a cap. */
+    get defaultCap() {
+        return this.o.defaultCap;
+    }
+    month() {
+        return monthIn(this.o.timeZone);
+    }
+    person(id) {
+        return this.docs.read(id, 'settings').then((d) => d ?? {});
+    }
+    setPerson(id, change) {
+        return this.docs.update(id, 'settings', (d) => change(d ?? {}));
+    }
+    app() {
+        return this.docs.read(APP_SCOPE, 'app').then((d) => d ?? {});
+    }
+    setApp(change) {
+        return this.docs.update(APP_SCOPE, 'app', (d) => change(d ?? {}));
+    }
+    appKey() {
+        return this.docs.read(APP_SCOPE, 'key');
+    }
+    /** The app's key: the stored one, else the app's fallback (a Worker secret). */
+    async appApiKey() {
+        const d = await this.appKey();
+        if (d)
+            return { key: d.api_key, fromEnv: false };
+        const env = await this.o.appKey?.();
+        return env ? { key: env, fromEnv: true } : null;
+    }
+    ownKey(id) {
+        return this.docs.read(id, 'key');
+    }
+    scrapeql() {
+        return this.docs.read(APP_SCOPE, 'scrapeql');
+    }
+    async appCap() {
+        return (await this.app()).cap_usd ?? this.o.defaultCap;
+    }
+    spend(id, month = this.month()) {
+        return this.docs.read(id, `spend:${month}`).then((d) => d ?? { rows: [] });
+    }
+    /** Adds an answer's use to this month's spend, by label and payer. */
+    async addSpend(id, payer, parts) {
+        if (!parts.some((p) => p.calls))
+            return;
+        await this.docs.update(id, `spend:${this.month()}`, (d) => {
+            const rows = [...(d?.rows ?? [])];
+            for (const p of parts) {
+                if (!p.calls)
+                    continue;
+                const i = rows.findIndex((r) => r.label === p.label && r.payer === payer);
+                const prev = i >= 0 ? rows[i] : undefined;
+                const row = prev
+                    ? {
+                        ...prev,
+                        calls: prev.calls + p.calls,
+                        in: prev.in + p.in,
+                        read: prev.read + p.read,
+                        out: prev.out + p.out,
+                        usd: prev.usd + p.usd,
+                    }
+                    : { ...p, payer };
+                if (i >= 0)
+                    rows[i] = row;
+                else
+                    rows.push(row);
+            }
+            return { rows };
+        });
+    }
+    /** Who pays the person's next answer, or why nobody can. */
+    async paying(id) {
+        const [person, app, own, spend, appCap] = await Promise.all([
+            this.person(id),
+            this.appApiKey(),
+            this.ownKey(id),
+            this.spend(id),
+            this.appCap(),
+        ]);
+        const spent = (p) => spend.rows.filter((r) => r.payer === p).reduce((a, r) => a + r.usd, 0);
+        const sources = {
+            app: app ? { payer: 'app', apiKey: app.key, cap: appCap, left: appCap - spent('app') } : null,
+            own: own
+                ? {
+                    payer: 'own',
+                    apiKey: own.api_key,
+                    cap: own.cap_usd ?? this.o.defaultCap,
+                    left: (own.cap_usd ?? this.o.defaultCap) - spent('own'),
+                }
+                : null,
+        };
+        const first = person.pay ?? 'app';
+        const order = first === 'own' ? ['own', 'app'] : ['app', 'own'];
+        for (const p of order) {
+            const s = sources[p];
+            if (s && s.left > 0)
+                return { paying: s, wait: null };
+        }
+        if (!sources.app && !sources.own)
+            return { paying: null, wait: 'not_connected' };
+        // Both at their caps, or only one set and at its cap: name the one the person pays with first.
+        const capped = sources[first] ? first : first === 'own' ? 'app' : 'own';
+        return { paying: null, wait: capped === 'own' ? 'own_cap' : 'app_cap' };
+    }
+}
+export const WAIT_TEXT = {
+    not_connected: 'No Claude key is set: an admin adds one in Settings ▸ Model providers.',
+    app_cap: "You reached this month's cap on the app's key. Add your own key, or wait for next month.",
+    own_cap: 'Your own key reached its monthly cap: raise it in Settings ▸ Spend.',
+};

package/dist/assistant/store.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Where the assistant keeps things: chats, each person's settings, keys and spend, and the app's own
+ * settings (its key, the cap per person, the ScrapeQL token). One table of documents over any SQLite,
+ * a Durable Object's own (`doSql`) or D1 (`d1Sql`); an app with another store implements
+ * `AssistantStore` itself.
+ *
+ * Every value and title goes through the app's `Sealer` before it is stored, so chats and keys are
+ * as private as the app's other sealed data. Without a sealer they are stored as written.
+ */
+/** Seals text before it is stored and opens it after. The app's own (the hub's USERS_KEK sealing). */
+export interface Sealer {
+    seal(plain: string): Promise<string>;
+    open(sealed: string): Promise<string>;
+}
+/** One stored document: `scope` is a person's id or `APP_SCOPE`, `key` names it. */
+export interface StoredDoc {
+    key: string;
+    /** A short line kept apart from the value, so a list never reads every value (a chat's title). */
+    meta: string | null;
+    value: string;
+    /** Last written, epoch ms. */
+    at: number;
+}
+export interface AssistantStore {
+    /** Creates the table. Safe to call often. */
+    init(): Promise<void>;
+    get(scope: string, key: string): Promise<StoredDoc | null>;
+    put(scope: string, key: string, value: string, meta?: string | null): Promise<void>;
+    remove(scope: string, key: string): Promise<void>;
+    /** Keys starting with `prefix`, newest first, without their values. */
+    list(scope: string, prefix: string, limit: number): Promise<Omit<StoredDoc, 'value'>[]>;
+    /** Drops documents starting with `prefix` (any scope) last written before `before`. */
+    purge(prefix: string, before: number): Promise<number>;
+}
+/** The scope of what belongs to the app, not one person. */
+export declare const APP_SCOPE = "_app";
+type Row = Record<string, unknown>;
+export interface SqlRunner {
+    all(sql: string, ...args: unknown[]): Promise<Row[]>;
+    run(sql: string, ...args: unknown[]): Promise<void>;
+}
+/** A Durable Object's SQLite (`ctx.storage.sql`). */
+export declare function doSql(sql: {
+    exec(q: string, ...args: unknown[]): {
+        toArray(): unknown[];
+    };
+}): SqlRunner;
+/** A D1 database binding. */
+export declare function d1Sql(db: {
+    prepare(q: string): {
+        bind(...a: unknown[]): {
+            all(): Promise<{
+                results: unknown[];
+            }>;
+            run(): Promise<unknown>;
+        };
+    };
+}): SqlRunner;
+export declare function sqlAssistantStore(sql: SqlRunner): AssistantStore;
+/** JSON documents over a store, sealed when the app gives a sealer. */
+export declare class Docs {
+    private readonly store;
+    private readonly sealer?;
+    constructor(store: AssistantStore, sealer?: Sealer | undefined);
+    private seal;
+    private open;
+    read<T>(scope: string, key: string): Promise<T | null>;
+    write<T>(scope: string, key: string, value: T, meta?: string): Promise<void>;
+    /** Read, change and write back; `change` returning null removes it. */
+    update<T>(scope: string, key: string, change: (prev: T | null) => T | null): Promise<T | null>;
+    remove(scope: string, key: string): Promise<void>;
+    list(scope: string, prefix: string, limit: number): Promise<{
+        key: string;
+        meta: string | null;
+        at: number;
+    }[]>;
+}
+export {};

package/dist/assistant/store.js

@@ -0,0 +1,120 @@
+/**
+ * Where the assistant keeps things: chats, each person's settings, keys and spend, and the app's own
+ * settings (its key, the cap per person, the ScrapeQL token). One table of documents over any SQLite,
+ * a Durable Object's own (`doSql`) or D1 (`d1Sql`); an app with another store implements
+ * `AssistantStore` itself.
+ *
+ * Every value and title goes through the app's `Sealer` before it is stored, so chats and keys are
+ * as private as the app's other sealed data. Without a sealer they are stored as written.
+ */
+/** The scope of what belongs to the app, not one person. */
+export const APP_SCOPE = '_app';
+/** A Durable Object's SQLite (`ctx.storage.sql`). */
+export function doSql(sql) {
+    return {
+        all: async (q, ...a) => sql.exec(q, ...a).toArray(),
+        run: async (q, ...a) => {
+            sql.exec(q, ...a);
+        },
+    };
+}
+/** A D1 database binding. */
+export function d1Sql(db) {
+    return {
+        all: async (q, ...a) => (await db
+            .prepare(q)
+            .bind(...a)
+            .all()).results,
+        run: async (q, ...a) => {
+            await db
+                .prepare(q)
+                .bind(...a)
+                .run();
+        },
+    };
+}
+/** The table name, so an app's own tables never clash with it. */
+const TABLE = 'assistant_docs';
+export function sqlAssistantStore(sql) {
+    let ready = null;
+    const init = () => {
+        ready ??= sql.run(`CREATE TABLE IF NOT EXISTS ${TABLE} (scope TEXT NOT NULL, key TEXT NOT NULL, meta TEXT, value TEXT NOT NULL, at INTEGER NOT NULL, PRIMARY KEY (scope, key))`);
+        return ready;
+    };
+    return {
+        init,
+        async get(scope, key) {
+            await init();
+            const [r] = await sql.all(`SELECT key, meta, value, at FROM ${TABLE} WHERE scope = ? AND key = ?`, scope, key);
+            return r
+                ? {
+                    key: String(r.key),
+                    meta: r.meta == null ? null : String(r.meta),
+                    value: String(r.value),
+                    at: Number(r.at),
+                }
+                : null;
+        },
+        async put(scope, key, value, meta = null) {
+            await init();
+            await sql.run(`INSERT INTO ${TABLE} (scope, key, meta, value, at) VALUES (?, ?, ?, ?, ?) ON CONFLICT (scope, key) DO UPDATE SET meta = excluded.meta, value = excluded.value, at = excluded.at`, scope, key, meta, value, Date.now());
+        },
+        async remove(scope, key) {
+            await init();
+            await sql.run(`DELETE FROM ${TABLE} WHERE scope = ? AND key = ?`, scope, key);
+        },
+        async list(scope, prefix, limit) {
+            await init();
+            const rows = await sql.all(`SELECT key, meta, at FROM ${TABLE} WHERE scope = ? AND substr(key, 1, ?) = ? ORDER BY at DESC LIMIT ?`, scope, prefix.length, prefix, limit);
+            return rows.map((r) => ({
+                key: String(r.key),
+                meta: r.meta == null ? null : String(r.meta),
+                at: Number(r.at),
+            }));
+        },
+        async purge(prefix, before) {
+            await init();
+            const [r] = await sql.all(`SELECT count(*) AS n FROM ${TABLE} WHERE substr(key, 1, ?) = ? AND at < ?`, prefix.length, prefix, before);
+            await sql.run(`DELETE FROM ${TABLE} WHERE substr(key, 1, ?) = ? AND at < ?`, prefix.length, prefix, before);
+            return Number(r?.n ?? 0);
+        },
+    };
+}
+/** JSON documents over a store, sealed when the app gives a sealer. */
+export class Docs {
+    store;
+    sealer;
+    constructor(store, sealer) {
+        this.store = store;
+        this.sealer = sealer;
+    }
+    seal = (s) => (this.sealer ? this.sealer.seal(s) : Promise.resolve(s));
+    open = (s) => (this.sealer ? this.sealer.open(s) : Promise.resolve(s));
+    async read(scope, key) {
+        const d = await this.store.get(scope, key);
+        return d ? JSON.parse(await this.open(d.value)) : null;
+    }
+    async write(scope, key, value, meta) {
+        await this.store.put(scope, key, await this.seal(JSON.stringify(value)), meta === undefined ? null : await this.seal(meta));
+    }
+    /** Read, change and write back; `change` returning null removes it. */
+    async update(scope, key, change) {
+        const next = change(await this.read(scope, key));
+        if (next === null)
+            await this.store.remove(scope, key);
+        else
+            await this.write(scope, key, next);
+        return next;
+    }
+    remove(scope, key) {
+        return this.store.remove(scope, key);
+    }
+    async list(scope, prefix, limit) {
+        const rows = await this.store.list(scope, prefix, limit);
+        return Promise.all(rows.map(async (r) => ({
+            key: r.key,
+            meta: r.meta == null ? null : await this.open(r.meta),
+            at: r.at,
+        })));
+    }
+}

package/dist/assistant/tools.d.ts

@@ -0,0 +1,49 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+/** One tool as the model sees it. `read` false is a write, which only ever drafts. */
+export interface ToolDef {
+    name: string;
+    title?: string;
+    description?: string;
+    /** JSON Schema of its input (an object). */
+    inputSchema: Record<string, unknown>;
+    read: boolean;
+}
+/** Tools opened for one answer: what they are, how to call one, and how to let them go. */
+export interface OpenTools {
+    tools: ToolDef[];
+    call(name: string, args: unknown): Promise<unknown>;
+    close(): Promise<void>;
+}
+/** Opens a set of tools for one answer (or one save). */
+export type ToolSource = () => Promise<OpenTools>;
+/** A tool that answered with an error: its text is what the model reads. */
+export declare class ToolError extends Error {
+}
+/** Waits for `work`, or fails after `ms` saying what timed out. */
+export declare function withDeadline<T>(what: string, ms: number, work: Promise<T>): Promise<T>;
+/**
+ * An MCP server built for the caller and connected in process. A server connects once, so pass a
+ * function that builds a new one. A tool is a read when its `readOnlyHint` is set.
+ */
+export declare function mcpServerTools(build: () => McpServer | Promise<McpServer>): ToolSource;
+/**
+ * An MCP server the app serves as a fetch handler (`(Request) => Response`), called in process
+ * without a network hop. Use it for a server built on another MCP SDK.
+ */
+export declare function mcpFetchTools(handler: (req: Request) => Promise<Response>, url?: string): ToolSource;
+/** A remote MCP server over Streamable HTTP, with the headers it needs (an Access service token). */
+export declare function remoteMcpTools(o: {
+    url: string;
+    headers?: Record<string, string>;
+    fetch?: typeof fetch;
+}): ToolSource;
+/** A tool written as a plain function. */
+export interface PlainTool extends ToolDef {
+    run(args: unknown): Promise<unknown>;
+}
+/** Plain functions as tools, no MCP server needed. */
+export declare function plainTools(tools: PlainTool[]): ToolSource;
+/** A tool name the model may use: letters, digits, `_` and `-`, at most 64. */
+export declare function toolName(group: string, name: string): string;
+/** Rows in a list result: an array, or the first array field of an object. */
+export declare function rowCount(v: unknown): number | undefined;

package/dist/assistant/tools.js

@@ -0,0 +1,110 @@
+/**
+ * The assistant's tools: the app's own, from any MCP server (in process, served as a fetch handler,
+ * or remote) or plain functions. Each tool is a read or a write; the person sets each group of
+ * tools Off, Read or Read + draft, and a write called while its group allows drafts only makes a
+ * card in the chat. It runs when the person presses Save.
+ */
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';
+/** A tool that answered with an error: its text is what the model reads. */
+export class ToolError extends Error {
+}
+/** Waits for `work`, or fails after `ms` saying what timed out. */
+export function withDeadline(what, ms, work) {
+    let timer;
+    const late = new Promise((_, fail) => {
+        timer = setTimeout(() => fail(new Error(`${what} took over ${ms / 1000} s`)), ms);
+    });
+    return Promise.race([work, late]).finally(() => clearTimeout(timer));
+}
+/** An MCP client over `transport`: its tools, called as JSON. */
+async function clientTools(connect, done = async () => undefined) {
+    const client = new Client({ name: 'lumen-assistant', version: '1.0.0' });
+    await withDeadline('connecting to tools', 20_000, connect(client));
+    const { tools } = await withDeadline('listing tools', 20_000, client.listTools());
+    return {
+        tools: tools.map((t) => ({
+            name: t.name,
+            ...(t.title ? { title: t.title } : {}),
+            ...(t.description ? { description: t.description } : {}),
+            inputSchema: t.inputSchema,
+            read: t.annotations?.readOnlyHint === true,
+        })),
+        async call(name, args) {
+            const r = await client.callTool({
+                name,
+                arguments: (args ?? {}),
+            });
+            const first = r.content?.find((c) => c.type === 'text');
+            const text = first?.text ?? '';
+            if (r.isError)
+                throw new ToolError(text.slice(0, 500) || `${name} failed`);
+            try {
+                return JSON.parse(text);
+            }
+            catch {
+                return text;
+            }
+        },
+        async close() {
+            await client.close().catch(() => undefined);
+            await done().catch(() => undefined);
+        },
+    };
+}
+/**
+ * An MCP server built for the caller and connected in process. A server connects once, so pass a
+ * function that builds a new one. A tool is a read when its `readOnlyHint` is set.
+ */
+export function mcpServerTools(build) {
+    return async () => {
+        const server = await build();
+        const [a, b] = InMemoryTransport.createLinkedPair();
+        await server.connect(a);
+        return clientTools((c) => c.connect(b), () => server.close());
+    };
+}
+/**
+ * An MCP server the app serves as a fetch handler (`(Request) => Response`), called in process
+ * without a network hop. Use it for a server built on another MCP SDK.
+ */
+export function mcpFetchTools(handler, url = 'https://assistant.local/mcp') {
+    return () => clientTools((c) => c.connect(new StreamableHTTPClientTransport(new URL(url), {
+        fetch: (input, init) => handler(new Request(input, init)),
+    })));
+}
+/** A remote MCP server over Streamable HTTP, with the headers it needs (an Access service token). */
+export function remoteMcpTools(o) {
+    return () => clientTools((c) => c.connect(new StreamableHTTPClientTransport(new URL(o.url), {
+        requestInit: { headers: o.headers ?? {} },
+        ...(o.fetch ? { fetch: o.fetch } : {}),
+    })));
+}
+/** Plain functions as tools, no MCP server needed. */
+export function plainTools(tools) {
+    return async () => ({
+        tools: tools.map(({ run: _run, ...t }) => t),
+        async call(name, args) {
+            const t = tools.find((x) => x.name === name);
+            if (!t)
+                throw new ToolError(`No tool ${name}`);
+            return t.run(args);
+        },
+        close: async () => undefined,
+    });
+}
+/** A tool name the model may use: letters, digits, `_` and `-`, at most 64. */
+export function toolName(group, name) {
+    return `${group}__${name}`.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 64);
+}
+/** Rows in a list result: an array, or the first array field of an object. */
+export function rowCount(v) {
+    if (Array.isArray(v))
+        return v.length;
+    if (v && typeof v === 'object')
+        for (const x of Object.values(v))
+            if (Array.isArray(x))
+                return x.length;
+    return undefined;
+}

package/dist/assistant/web.d.ts

@@ -0,0 +1,45 @@
+/**
+ * The web: `fetch`, a plain GET from the Worker (no browser, 10 s, 2 MB), tried first for APIs, JSON
+ * and plain pages; and ScrapeQL's own MCP tools (`search`, `scrape`, `extract`, `ask`) over its API
+ * for pages that need a real browser or after a fetch fails. ScrapeQL sits behind Cloudflare Access,
+ * so the app reaches it with one Access service token an admin pastes in Settings.
+ */
+import { type ToolSource } from './tools.js';
+export interface WebFetchOptions {
+    /** Hosts never fetched (the app's own, behind sign-in), each with its subdomains. */
+    blockHosts?: string[];
+    userAgent?: string;
+    fetch?: typeof fetch;
+}
+/** One GET, never thrown: what it got, or why not. */
+export declare function webFetch(raw: string, headers?: Record<string, string>, o?: WebFetchOptions): Promise<{
+    ok: boolean;
+    status: number;
+    url: string;
+    error: string;
+} | {
+    ok: boolean;
+    status: number;
+    url: string;
+    content_type: string;
+    json: {} | null;
+    truncated: boolean;
+    error?: string | undefined;
+} | {
+    ok: boolean;
+    status: number;
+    url: string;
+    content_type: string;
+    text: string;
+    truncated: boolean;
+    error?: string | undefined;
+}>;
+/** `fetch` as a tool, at most `PARALLEL` at once per answer. */
+export declare function webFetchTools(o?: WebFetchOptions): ToolSource;
+/** A Cloudflare Access service token: its Client ID and Client Secret. */
+export interface AccessToken {
+    client_id: string;
+    client_secret: string;
+}
+/** ScrapeQL's MCP tools over its API, with an Access service token. Every one of them reads. */
+export declare function scrapeqlTools(url: string, token: AccessToken, f?: typeof fetch): ToolSource;