@pmndrs/pine 0.0.0-stage → 0.1.0

package/src/store.d.ts

@@ -0,0 +1,89 @@
+import { DatabaseSync } from "node:sqlite";
+import type { PineEvent, PublishOptions, Task, TaskStatus } from "./types.ts";
+import { type Logger } from "./util.ts";
+export declare function openDb(dataDir: string): DatabaseSync;
+type Handler = (event: PineEvent) => void | Promise<void>;
+/** Append-only event log with in-process subscriptions. Everything that happens goes through here. */
+export declare class Bus {
+    private db;
+    private log;
+    private subs;
+    private pending;
+    /** Applied to every payload before it is stored or delivered. */
+    redact?: <T>(payload: T) => T;
+    constructor(db: DatabaseSync, log: Logger);
+    /**
+     * Take a secret out of what is already stored. A token the owner typed into a conversation
+     * ("here is the bot's token") was in events and tasks before anyone knew it was one.
+     */
+    scrub(secret: string): void;
+    publish(topic: string, payload?: unknown, opts?: PublishOptions): PineEvent;
+    /** Handlers run asynchronously after publish() returns; a throwing handler is logged, never rethrown. */
+    subscribe(pattern: string | string[], handler: Handler, name?: string): () => void;
+    /** Resolve with the next event matching `pattern` (and `filter`), or reject after `timeoutMs`. */
+    waitFor(pattern: string, filter?: (e: PineEvent) => boolean, timeoutMs?: number): Promise<PineEvent>;
+    /** Wait until every handler started so far has finished. */
+    drain(): Promise<void>;
+    get(id: number): PineEvent | undefined;
+    query(q?: {
+        topic?: string;
+        after?: number;
+        limit?: number;
+    }): PineEvent[];
+    prune(olderThanMs: number): number;
+    private fromRow;
+}
+/** Shared world state: small JSON values under slash-separated keys. Every write is also an event on `state/<key>`. */
+export declare class State {
+    private db;
+    private bus;
+    constructor(db: DatabaseSync, bus: Bus);
+    get<T = unknown>(key: string): T | undefined;
+    set(key: string, value: unknown, opts?: {
+        source?: string;
+        cause?: PineEvent | null;
+    }): void;
+    list(patterns?: string | string[]): Array<{
+        key: string;
+        value: unknown;
+        ts: number;
+        source: string;
+    }>;
+}
+/** Private key-value storage for sources and the daemon (cursors, last-run times). No events. */
+export declare class Kv {
+    private db;
+    constructor(db: DatabaseSync);
+    get<T = unknown>(key: string): T | undefined;
+    set(key: string, value: unknown): void;
+    /** Remove every entry whose key starts with a prefix. */
+    forget(prefix: string): void;
+    /** The keys that start with a prefix. */
+    keys(prefix: string): string[];
+}
+export declare class TaskStore {
+    private db;
+    constructor(db: DatabaseSync);
+    insert(t: Omit<Task, "id">): Task;
+    update(id: number, patch: Partial<Pick<Task, "status" | "started" | "finished" | "result" | "error" | "session" | "meta">>): void;
+    get(id: number): Task | undefined;
+    list(q?: {
+        status?: TaskStatus;
+        agent?: string;
+        session?: string;
+        limit?: number;
+    }): Task[];
+    /** Named sessions of an agent that have had tasks, most recently used first. */
+    sessions(agent: string): Array<{
+        session: string;
+        tasks: number;
+        last: number;
+    }>;
+    /** Delete everything said in these conversations of an agent, with all the work they started. */
+    forget(agent: string, sessions: string[]): number;
+    /** Give a conversation's tasks another conversation name. */
+    rename(agent: string, from: string, to: string): void;
+    byStatus(status: TaskStatus): Task[];
+    private fromRow;
+}
+export {};

package/src/store.js

@@ -0,0 +1,251 @@
+import fs from "node:fs";
+import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { errText, topicMatches } from "./util.js";
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS events (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, ts INTEGER NOT NULL, topic TEXT NOT NULL, source TEXT NOT NULL,
+  payload TEXT, cause INTEGER, hops INTEGER NOT NULL DEFAULT 0, reply TEXT, trust TEXT NOT NULL DEFAULT 'untrusted'
+);
+CREATE INDEX IF NOT EXISTS events_topic ON events(topic, id);
+CREATE INDEX IF NOT EXISTS events_ts ON events(ts);
+CREATE TABLE IF NOT EXISTS state (key TEXT PRIMARY KEY, value TEXT, ts INTEGER NOT NULL, source TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY, value TEXT);
+CREATE TABLE IF NOT EXISTS tasks (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, agent TEXT NOT NULL, session TEXT NOT NULL, kind TEXT NOT NULL, prompt TEXT NOT NULL,
+  status TEXT NOT NULL, priority INTEGER NOT NULL DEFAULT 0, created INTEGER NOT NULL, started INTEGER, finished INTEGER,
+  result TEXT, error TEXT, event INTEGER, hops INTEGER NOT NULL DEFAULT 0, parent INTEGER, depth INTEGER NOT NULL DEFAULT 0,
+  reply TEXT, meta TEXT
+);
+CREATE INDEX IF NOT EXISTS tasks_status ON tasks(status, id);
+`;
+export function openDb(dataDir) {
+    fs.mkdirSync(dataDir, { recursive: true });
+    const db = new DatabaseSync(path.join(dataDir, "pine.db"));
+    db.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA busy_timeout = 5000;");
+    db.exec(SCHEMA);
+    return db;
+}
+const parse = (s) => (typeof s === "string" && s.length ? JSON.parse(s) : undefined);
+/** Append-only event log with in-process subscriptions. Everything that happens goes through here. */
+export class Bus {
+    db;
+    log;
+    subs = new Set();
+    pending = new Set();
+    /** Applied to every payload before it is stored or delivered. */
+    redact;
+    constructor(db, log) {
+        this.db = db;
+        this.log = log;
+    }
+    /**
+     * Take a secret out of what is already stored. A token the owner typed into a conversation
+     * ("here is the bot's token") was in events and tasks before anyone knew it was one.
+     */
+    scrub(secret) {
+        if (secret.length < 8)
+            return;
+        this.db.prepare("UPDATE events SET payload = replace(payload, ?, '[redacted]') WHERE instr(payload, ?) > 0").run(secret, secret);
+        for (const col of ["prompt", "result", "error", "meta"])
+            this.db.prepare(`UPDATE tasks SET ${col} = replace(${col}, ?, '[redacted]') WHERE instr(${col}, ?) > 0`).run(secret, secret);
+    }
+    publish(topic, payload = null, opts = {}) {
+        if (!topic || /[+#]/.test(topic))
+            throw new Error(`invalid topic: ${JSON.stringify(topic)}`);
+        const cause = opts.cause ?? undefined;
+        const event = {
+            id: 0,
+            ts: Date.now(),
+            topic,
+            source: opts.source ?? "pine",
+            payload,
+            cause: cause?.id || undefined,
+            hops: (cause?.hops ?? 0) + (opts.hop ? 1 : 0),
+            reply: opts.reply,
+            trust: opts.trust ?? "untrusted",
+        };
+        if (this.redact)
+            event.payload = this.redact(event.payload);
+        if (opts.persist !== false) {
+            const r = this.db
+                .prepare("INSERT INTO events (ts, topic, source, payload, cause, hops, reply, trust) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+                .run(event.ts, topic, event.source, JSON.stringify(event.payload ?? null), event.cause ?? null, event.hops, event.reply ? JSON.stringify(event.reply) : null, event.trust);
+            event.id = Number(r.lastInsertRowid);
+        }
+        for (const sub of this.subs) {
+            if (!topicMatches(sub.pattern, topic))
+                continue;
+            const p = Promise.resolve()
+                .then(() => sub.handler(event))
+                .catch((e) => this.log.error(`subscriber ${sub.name} failed on ${topic}`, errText(e)))
+                .finally(() => this.pending.delete(p));
+            this.pending.add(p);
+        }
+        return event;
+    }
+    /** Handlers run asynchronously after publish() returns; a throwing handler is logged, never rethrown. */
+    subscribe(pattern, handler, name = "anonymous") {
+        const entries = (Array.isArray(pattern) ? pattern : [pattern]).map((p) => ({ pattern: p, handler, name }));
+        for (const e of entries)
+            this.subs.add(e);
+        return () => {
+            for (const e of entries)
+                this.subs.delete(e);
+        };
+    }
+    /** Resolve with the next event matching `pattern` (and `filter`), or reject after `timeoutMs`. */
+    waitFor(pattern, filter = () => true, timeoutMs = 10_000) {
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                off();
+                reject(new Error(`no event on ${pattern} within ${timeoutMs} ms`));
+            }, timeoutMs);
+            const off = this.subscribe(pattern, (e) => {
+                if (!filter(e))
+                    return;
+                clearTimeout(timer);
+                off();
+                resolve(e);
+            });
+        });
+    }
+    /** Wait until every handler started so far has finished. */
+    async drain() {
+        while (this.pending.size)
+            await Promise.allSettled([...this.pending]);
+    }
+    get(id) {
+        const row = this.db.prepare("SELECT * FROM events WHERE id = ?").get(id);
+        return row ? this.fromRow(row) : undefined;
+    }
+    query(q = {}) {
+        const limit = Math.min(q.limit ?? 100, 1000);
+        // Patterns are matched in JS; a literal prefix narrows the scan.
+        const prefix = q.topic ? q.topic.split(/[+#]/)[0] : "";
+        const rows = q.after !== undefined
+            ? this.db.prepare("SELECT * FROM events WHERE id > ? AND topic LIKE ? ESCAPE '\\' ORDER BY id ASC LIMIT ?").all(q.after, `${prefix.replace(/[\\%_]/g, "\\$&")}%`, limit * 20)
+            : this.db.prepare("SELECT * FROM events WHERE topic LIKE ? ESCAPE '\\' ORDER BY id DESC LIMIT ?").all(`${prefix.replace(/[\\%_]/g, "\\$&")}%`, limit * 20);
+        const events = rows.map((r) => this.fromRow(r)).filter((e) => !q.topic || topicMatches(q.topic, e.topic)).slice(0, limit);
+        return q.after !== undefined ? events : events.reverse();
+    }
+    prune(olderThanMs) {
+        return Number(this.db.prepare("DELETE FROM events WHERE ts < ?").run(Date.now() - olderThanMs).changes);
+    }
+    fromRow(r) {
+        return { id: r.id, ts: r.ts, topic: r.topic, source: r.source, payload: parse(r.payload) ?? null, cause: r.cause ?? undefined, hops: r.hops, reply: parse(r.reply), trust: r.trust === "owner" ? "owner" : "untrusted" };
+    }
+}
+/** Shared world state: small JSON values under slash-separated keys. Every write is also an event on `state/<key>`. */
+export class State {
+    db;
+    bus;
+    constructor(db, bus) {
+        this.db = db;
+        this.bus = bus;
+    }
+    get(key) {
+        const row = this.db.prepare("SELECT value FROM state WHERE key = ?").get(key);
+        return row ? parse(row.value) : undefined;
+    }
+    set(key, value, opts = {}) {
+        if (!key || /[+#]/.test(key))
+            throw new Error(`invalid state key: ${JSON.stringify(key)}`);
+        const previous = this.get(key);
+        if (value === undefined || value === null)
+            this.db.prepare("DELETE FROM state WHERE key = ?").run(key);
+        else
+            this.db.prepare("INSERT INTO state (key, value, ts, source) VALUES (?, ?, ?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value, ts = excluded.ts, source = excluded.source").run(key, JSON.stringify(value), Date.now(), opts.source ?? "pine");
+        if (JSON.stringify(previous) !== JSON.stringify(value ?? undefined))
+            this.bus.publish(`state/${key}`, { value: value ?? null, previous: previous ?? null }, { source: opts.source ?? "pine", cause: opts.cause, hop: Boolean(opts.cause) });
+    }
+    list(patterns = "#") {
+        const list = Array.isArray(patterns) ? patterns : [patterns];
+        return this.db.prepare("SELECT * FROM state ORDER BY key").all()
+            .filter((r) => list.some((p) => topicMatches(p, r.key)))
+            .map((r) => ({ key: r.key, value: parse(r.value), ts: r.ts, source: r.source }));
+    }
+}
+/** Private key-value storage for sources and the daemon (cursors, last-run times). No events. */
+export class Kv {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    get(key) {
+        const row = this.db.prepare("SELECT value FROM kv WHERE key = ?").get(key);
+        return row ? parse(row.value) : undefined;
+    }
+    set(key, value) {
+        this.db.prepare("INSERT INTO kv (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, JSON.stringify(value));
+    }
+    /** Remove every entry whose key starts with a prefix. */
+    forget(prefix) {
+        this.db.prepare("DELETE FROM kv WHERE substr(key, 1, ?) = ?").run(prefix.length, prefix);
+    }
+    /** The keys that start with a prefix. */
+    keys(prefix) {
+        return this.db.prepare("SELECT key FROM kv WHERE substr(key, 1, ?) = ? ORDER BY key").all(prefix.length, prefix).map((r) => r.key);
+    }
+}
+export class TaskStore {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    insert(t) {
+        const r = this.db
+            .prepare("INSERT INTO tasks (agent, session, kind, prompt, status, priority, created, event, hops, parent, depth, reply, meta) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
+            .run(t.agent, t.session, t.kind, t.prompt, t.status, t.priority, t.created, t.event ?? null, t.hops, t.parent ?? null, t.depth, t.reply ? JSON.stringify(t.reply) : null, JSON.stringify(t.meta ?? {}));
+        return { ...t, id: Number(r.lastInsertRowid) };
+    }
+    update(id, patch) {
+        const cols = Object.keys(patch);
+        if (!cols.length)
+            return;
+        const values = cols.map((c) => (c === "meta" ? JSON.stringify(patch.meta) : (patch[c] ?? null)));
+        this.db.prepare(`UPDATE tasks SET ${cols.map((c) => `${c} = ?`).join(", ")} WHERE id = ?`).run(...values, id);
+    }
+    get(id) {
+        const row = this.db.prepare("SELECT * FROM tasks WHERE id = ?").get(id);
+        return row ? this.fromRow(row) : undefined;
+    }
+    list(q = {}) {
+        const where = [];
+        const args = [];
+        if (q.status)
+            where.push("status = ?"), args.push(q.status);
+        if (q.agent)
+            where.push("agent = ?"), args.push(q.agent);
+        if (q.session)
+            where.push("session = ?"), args.push(q.session);
+        const sql = `SELECT * FROM tasks ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY id DESC LIMIT ?`;
+        return this.db.prepare(sql).all(...args, Math.min(q.limit ?? 50, 1000)).map((r) => this.fromRow(r));
+    }
+    /** Named sessions of an agent that have had tasks, most recently used first. */
+    sessions(agent) {
+        return this.db.prepare("SELECT session, COUNT(*) AS tasks, MAX(created) AS last FROM tasks WHERE agent = ? AND session NOT LIKE '\\_t%' ESCAPE '\\' GROUP BY session ORDER BY last DESC").all(agent).map((r) => ({ session: r.session, tasks: r.tasks, last: r.last }));
+    }
+    /** Delete everything said in these conversations of an agent, with all the work they started. */
+    forget(agent, sessions) {
+        if (!sessions.length)
+            return 0;
+        const marks = sessions.map(() => "?").join(", ");
+        const sql = `WITH RECURSIVE gone(id) AS (SELECT id FROM tasks WHERE agent = ? AND session IN (${marks}) UNION SELECT t.id FROM tasks t JOIN gone g ON t.parent = g.id) DELETE FROM tasks WHERE id IN (SELECT id FROM gone)`;
+        return Number(this.db.prepare(sql).run(agent, ...sessions).changes);
+    }
+    /** Give a conversation's tasks another conversation name. */
+    rename(agent, from, to) {
+        this.db.prepare("UPDATE tasks SET session = ? WHERE agent = ? AND session = ?").run(to, agent, from);
+    }
+    byStatus(status) {
+        return this.db.prepare("SELECT * FROM tasks WHERE status = ? ORDER BY id ASC").all(status).map((r) => this.fromRow(r));
+    }
+    fromRow(r) {
+        return {
+            id: r.id, agent: r.agent, session: r.session, kind: r.kind, prompt: r.prompt, status: r.status, priority: r.priority, created: r.created,
+            started: r.started ?? undefined, finished: r.finished ?? undefined, result: r.result ?? undefined, error: r.error ?? undefined,
+            event: r.event ?? undefined, hops: r.hops, parent: r.parent ?? undefined, depth: r.depth, reply: parse(r.reply), meta: parse(r.meta) ?? {},
+        };
+    }
+}

package/src/timers.d.ts

@@ -0,0 +1,83 @@
+import { type PineConfig, type ScheduleConfig } from "./config.ts";
+import type { Router } from "./router.ts";
+import type { Bus, Kv } from "./store.ts";
+import { type Logger } from "./util.ts";
+/** Five fields: minute hour day-of-month month day-of-week (0 or 7 is Sunday). Local time. */
+export declare function parseCron(expr: string): (d: Date) => boolean;
+/** A moment as it is written into a file and shown to people: local time, to the minute or second. */
+export declare const localTime: (ms: number) => string;
+/** When a schedule fires, in words. */
+export declare const whenText: (s: Pick<ScheduleConfig, "cron" | "every" | "at">) => string;
+export interface ScheduleInfo {
+    name: string;
+    /** "global", or the agent it belongs to. */
+    scope: string;
+    /** The conversation it wakes, for an agent's own. */
+    session?: string;
+    when: string;
+    task?: string;
+    /** The next time it fires, if that can be said. */
+    next?: number;
+    topic: string;
+}
+/**
+ * The scheduler: everything in pine that happens because of the time.
+ *
+ * A schedule fires at set times (cron), every so often, or once. Firing publishes an event on
+ * `timer/<name>` (`timer/<name>@<agent>` for an agent's own). A schedule with a task also wakes
+ * whoever it is for with that task: one conversation of an agent, an agent's main conversation,
+ * or every agent. Schedules are made by the owner, in the files or by asking an agent, so what
+ * they start counts as the owner's wish.
+ */
+export declare class Scheduler {
+    private cfg;
+    private bus;
+    private kv;
+    private router?;
+    private log;
+    private handles;
+    private routes;
+    private crons;
+    constructor(cfg: PineConfig, bus: Bus, kv: Kv, log: Logger, router?: Router);
+    /** Every schedule with its key: those for all agents by name, an agent's own as name@agent. */
+    private all;
+    private match;
+    start(): void;
+    stop(): void;
+    /** A schedule with a task wakes whoever it is for: the routes that do so, rebuilt whenever a schedule or the list of agents changes. */
+    private wire;
+    /** Set the clocks: one look every quarter minute for the cron times, an interval for each "every", a timeout for each "at". */
+    private arm;
+    private fire;
+    /** The next time a schedule fires. For an interval, counted from now. */
+    private next;
+    private info;
+    list(): ScheduleInfo[];
+    /** What one agent has: its own schedules, in whichever conversation, and those for all agents that carry a task. */
+    available(agentId: string): ScheduleInfo[];
+    /** Check a schedule that is to be added, before anybody is asked: its name, and that its times can be read. Returns it as it goes into the file. */
+    check(raw: {
+        name: string;
+        task: string;
+        cron?: string;
+        every?: string;
+        at?: string | number;
+        session?: string;
+    }): Record<string, unknown>;
+    private file;
+    /**
+     * Add a schedule while pine runs: to one agent's agent.json, or to pine.json for all agents.
+     * The file is written and read back through the normal loader, so it is checked like one that
+     * was there at the start; if it does not pass, the file is put back as it was.
+     */
+    add(agentId: string | undefined, entry: Record<string, unknown>): ScheduleInfo;
+    /** Take a schedule out of the configuration, leaving the file as it was before it was added. */
+    private drop;
+    remove(agentId: string | undefined, name: string): boolean;
+    /** A conversation was removed: what was scheduled for it alone goes with it. */
+    closeSession(agentId: string, session: string): string[];
+    /** A conversation got another name: its schedules follow it. */
+    renameSession(agentId: string, from: string, to: string): void;
+    /** Agents were added or removed: whom a schedule for all agents wakes has changed. */
+    agentsChanged(): void;
+}