@ithousecall/chat 0.0.0-stage → 0.2.1

package/dist/docs/drive.d.ts

@@ -0,0 +1,20 @@
+import type { DocsSource } from "./source.ts";
+export type DriveFile = {
+    id: string;
+    name: string;
+    mimeType: string;
+    modifiedTime: string;
+    webViewLink: string;
+};
+/** The Drive calls the source makes, so its logic can be tested without Google. */
+export interface DriveApi {
+    list(query: string, pageToken: string | undefined, signal: AbortSignal): Promise<{
+        files: DriveFile[];
+        nextPageToken?: string;
+    }>;
+    exportMarkdown(fileId: string, signal: AbortSignal): Promise<string>;
+    download(fileId: string, signal: AbortSignal): Promise<string>;
+}
+export declare function googleDriveApi(keyFile: string): DriveApi;
+/** Google Docs and Markdown or text files in one Drive folder and everything below it. */
+export declare function driveSource(name: string, folderId: string, api: DriveApi): DocsSource;

package/dist/docs/drive.js

@@ -0,0 +1,85 @@
+import { drive } from "@googleapis/drive";
+import { GoogleAuth } from "google-auth-library";
+const FOLDER = "application/vnd.google-apps.folder";
+const GOOGLE_DOC = "application/vnd.google-apps.document";
+const TEXT = new Set(["text/markdown", "text/plain", "text/x-markdown"]);
+function text(data) {
+    if (typeof data !== "string")
+        throw new TypeError("Drive did not return text");
+    return data;
+}
+export function googleDriveApi(keyFile) {
+    const client = drive({
+        version: "v3",
+        auth: new GoogleAuth({ keyFile, scopes: ["https://www.googleapis.com/auth/drive.readonly"] }),
+    });
+    return {
+        list: async (q, pageToken, signal) => {
+            const { data } = await client.files.list({
+                q,
+                pageSize: 1000,
+                fields: "nextPageToken, files(id, name, mimeType, modifiedTime, webViewLink)",
+                supportsAllDrives: true,
+                includeItemsFromAllDrives: true,
+                ...(pageToken === undefined ? {} : { pageToken }),
+            }, { signal });
+            const files = (data.files ?? []).flatMap((file) => file.id && file.name && file.mimeType && file.modifiedTime && file.webViewLink
+                ? [
+                    {
+                        id: file.id,
+                        name: file.name,
+                        mimeType: file.mimeType,
+                        modifiedTime: file.modifiedTime,
+                        webViewLink: file.webViewLink,
+                    },
+                ]
+                : []);
+            return data.nextPageToken ? { files, nextPageToken: data.nextPageToken } : { files };
+        },
+        exportMarkdown: async (fileId, signal) => text((await client.files.export({ fileId, mimeType: "text/markdown" }, { responseType: "text", signal })).data),
+        download: async (fileId, signal) => text((await client.files.get({ fileId, alt: "media", supportsAllDrives: true }, { responseType: "text", signal }))
+            .data),
+    };
+}
+/** Google Docs and Markdown or text files in one Drive folder and everything below it. */
+export function driveSource(name, folderId, api) {
+    const mimeTypes = new Map();
+    return {
+        name,
+        list: async (signal) => {
+            const docs = [];
+            const folders = [folderId];
+            const seen = new Set(folders);
+            for (let folder = folders.pop(); folder !== undefined; folder = folders.pop()) {
+                let pageToken;
+                do {
+                    const page = await api.list(`'${folder}' in parents and trashed = false`, pageToken, signal);
+                    for (const file of page.files) {
+                        if (file.mimeType === FOLDER) {
+                            if (!seen.has(file.id))
+                                folders.push(file.id);
+                            seen.add(file.id);
+                        }
+                        else if (file.mimeType === GOOGLE_DOC || TEXT.has(file.mimeType)) {
+                            mimeTypes.set(file.id, file.mimeType);
+                            docs.push({
+                                id: file.id,
+                                title: file.name.replace(/\.(md|markdown|txt)$/i, ""),
+                                url: file.webViewLink,
+                                updatedAt: file.modifiedTime,
+                            });
+                        }
+                    }
+                    pageToken = page.nextPageToken;
+                } while (pageToken !== undefined);
+            }
+            return docs;
+        },
+        fetch: async (id, signal) => {
+            const mimeType = mimeTypes.get(id);
+            if (mimeType === undefined)
+                throw new Error(`Drive file ${id} was not listed`);
+            return mimeType === GOOGLE_DOC ? api.exportMarkdown(id, signal) : api.download(id, signal);
+        },
+    };
+}

package/dist/docs/html.d.ts

@@ -0,0 +1,2 @@
+/** Confluence storage format (XHTML with ac:/ri: macros) to Markdown: code macros become fences, macro parameters and images go. */
+export declare function storageToMarkdown(storage: string): string;

package/dist/docs/html.js

@@ -0,0 +1,13 @@
+import TurndownService from "turndown";
+import { escapeHtml } from "../format.js";
+const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
+const CODE_MACRO = /<ac:structured-macro[^>]*ac:name="code"[^>]*>[\s\S]*?<ac:plain-text-body><!\[CDATA\[([\s\S]*?)\]\]><\/ac:plain-text-body>[\s\S]*?<\/ac:structured-macro>/g;
+/** Confluence storage format (XHTML with ac:/ri: macros) to Markdown: code macros become fences, macro parameters and images go. */
+export function storageToMarkdown(storage) {
+    const html = storage
+        .replaceAll(CODE_MACRO, (_match, code) => `<pre><code>${escapeHtml(code)}</code></pre>`)
+        .replaceAll(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_match, text) => escapeHtml(text))
+        .replaceAll(/<ac:parameter[^>]*>[\s\S]*?<\/ac:parameter>/g, "")
+        .replaceAll(/<ac:image[\s\S]*?<\/ac:image>/g, "");
+    return turndown.turndown(html);
+}

package/dist/docs/index.d.ts

@@ -0,0 +1,31 @@
+import type { DocMeta } from "./source.ts";
+export type SearchHit = {
+    docKey: string;
+    title: string;
+    url: string;
+    heading: string;
+    snippet: string;
+};
+type IndexedDoc = {
+    docKey: string;
+    title: string;
+    url: string;
+    text: string;
+};
+/** Each meaningful word of the query as a quoted FTS5 term, OR-joined, so punctuation cannot break the syntax. */
+export declare function ftsQuery(query: string): string;
+export declare class DocsIndex {
+    #private;
+    constructor(path: string);
+    close(): void;
+    /** `updatedAt` of every indexed document of a source, by document ID. */
+    versions(source: string): Map<string, string>;
+    upsert(source: string, doc: DocMeta, body: string): void;
+    remove(source: string, id: string): void;
+    /** Drop every document of a source that is no longer configured; returns how many. */
+    removeSourcesExcept(sources: readonly string[]): number;
+    search(query: string, limit: number): SearchHit[];
+    read(docKey: string): IndexedDoc | undefined;
+    recordSync(source: string, error: string | null): void;
+}
+export {};

package/dist/docs/index.js

@@ -0,0 +1,146 @@
+import { DatabaseSync } from "node:sqlite";
+import { chunkMarkdown } from "./chunk.js";
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS documents (
+  key TEXT PRIMARY KEY,
+  source TEXT NOT NULL,
+  title TEXT NOT NULL,
+  url TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  text TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS documents_source ON documents (source);
+CREATE TABLE IF NOT EXISTS chunks (
+  id INTEGER PRIMARY KEY,
+  doc_key TEXT NOT NULL,
+  heading TEXT NOT NULL,
+  text TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS chunks_doc_key ON chunks (doc_key);
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(text, heading, content='chunks', content_rowid='id');
+CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+  INSERT INTO chunks_fts (rowid, text, heading) VALUES (new.id, new.text, new.heading);
+END;
+CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+  INSERT INTO chunks_fts (chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
+END;
+CREATE TABLE IF NOT EXISTS sync_state (
+  source TEXT PRIMARY KEY,
+  last_sync_at TEXT,
+  last_error TEXT
+);
+`;
+const STOPWORDS = new Set("a an and are as at be but by can could do does for from get how i if in is it me my of on or our should so that the this to was we what when where which who why will with would you your".split(" "));
+/** Each meaningful word of the query as a quoted FTS5 term, OR-joined, so punctuation cannot break the syntax. */
+export function ftsQuery(query) {
+    const words = (query.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).filter((word) => !STOPWORDS.has(word));
+    return [...new Set(words)].map((word) => `"${word}"`).join(" OR ");
+}
+function text(value) {
+    if (typeof value !== "string")
+        throw new TypeError(`expected a string column, got ${typeof value}`);
+    return value;
+}
+export class DocsIndex {
+    #db;
+    constructor(path) {
+        this.#db = new DatabaseSync(path);
+        this.#db.exec("PRAGMA journal_mode = WAL");
+        this.#db.exec(SCHEMA);
+    }
+    close() {
+        this.#db.close();
+    }
+    /** `updatedAt` of every indexed document of a source, by document ID. */
+    versions(source) {
+        const rows = this.#db.prepare("SELECT key, updated_at FROM documents WHERE source = ?").all(source);
+        const prefix = `${source}:`;
+        return new Map(rows.map((row) => [text(row.key).slice(prefix.length), text(row.updated_at)]));
+    }
+    upsert(source, doc, body) {
+        const key = `${source}:${doc.id}`;
+        this.#transaction(() => {
+            this.#removeKey(key);
+            this.#db
+                .prepare("INSERT INTO documents (key, source, title, url, updated_at, text) VALUES (?, ?, ?, ?, ?, ?)")
+                .run(key, source, doc.title, doc.url, doc.updatedAt, body);
+            const insert = this.#db.prepare("INSERT INTO chunks (doc_key, heading, text) VALUES (?, ?, ?)");
+            for (const chunk of chunkMarkdown(doc.title, body))
+                insert.run(key, chunk.heading, chunk.text);
+        });
+    }
+    remove(source, id) {
+        this.#transaction(() => this.#removeKey(`${source}:${id}`));
+    }
+    /** Drop every document of a source that is no longer configured; returns how many. */
+    removeSourcesExcept(sources) {
+        const keep = JSON.stringify(sources);
+        let removed = 0;
+        this.#transaction(() => {
+            this.#db
+                .prepare("DELETE FROM chunks WHERE doc_key IN (SELECT key FROM documents WHERE source NOT IN (SELECT value FROM json_each(?)))")
+                .run(keep);
+            removed = Number(this.#db.prepare("DELETE FROM documents WHERE source NOT IN (SELECT value FROM json_each(?))").run(keep)
+                .changes);
+            this.#db.prepare("DELETE FROM sync_state WHERE source NOT IN (SELECT value FROM json_each(?))").run(keep);
+        });
+        return removed;
+    }
+    search(query, limit) {
+        const match = ftsQuery(query);
+        if (match === "")
+            return [];
+        const rows = this.#db
+            .prepare(`SELECT c.doc_key, d.title, d.url, c.heading, snippet(chunks_fts, 0, '', '', '…', 40) AS snippet
+         FROM chunks_fts
+         JOIN chunks c ON c.id = chunks_fts.rowid
+         JOIN documents d ON d.key = c.doc_key
+         WHERE chunks_fts MATCH ?
+         ORDER BY bm25(chunks_fts, 1.0, 2.0)
+         LIMIT 200`)
+            .all(match);
+        const best = new Map();
+        for (const row of rows) {
+            const docKey = text(row.doc_key);
+            if (best.has(docKey))
+                continue;
+            best.set(docKey, {
+                docKey,
+                title: text(row.title),
+                url: text(row.url),
+                heading: text(row.heading),
+                snippet: text(row.snippet).replaceAll(/\s+/g, " ").trim(),
+            });
+            if (best.size === limit)
+                break;
+        }
+        return [...best.values()];
+    }
+    read(docKey) {
+        const row = this.#db.prepare("SELECT key, title, url, text FROM documents WHERE key = ?").get(docKey);
+        if (row === undefined)
+            return undefined;
+        return { docKey: text(row.key), title: text(row.title), url: text(row.url), text: text(row.text) };
+    }
+    recordSync(source, error) {
+        this.#db
+            .prepare(`INSERT INTO sync_state (source, last_sync_at, last_error) VALUES (?, ?, ?)
+         ON CONFLICT (source) DO UPDATE SET last_sync_at = excluded.last_sync_at, last_error = excluded.last_error`)
+            .run(source, new Date().toISOString(), error);
+    }
+    #removeKey(key) {
+        this.#db.prepare("DELETE FROM chunks WHERE doc_key = ?").run(key);
+        this.#db.prepare("DELETE FROM documents WHERE key = ?").run(key);
+    }
+    #transaction(work) {
+        this.#db.exec("BEGIN");
+        try {
+            work();
+            this.#db.exec("COMMIT");
+        }
+        catch (error) {
+            this.#db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+}

package/dist/docs/local.d.ts

@@ -0,0 +1,3 @@
+import type { DocsSource } from "./source.ts";
+/** Markdown and text files under one folder; the folder is the scope. */
+export declare function localSource(name: string, root: string, urlBase: string | undefined): DocsSource;

package/dist/docs/local.js

@@ -0,0 +1,39 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { basename, extname, join, relative, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+const EXTENSIONS = new Set([".md", ".markdown", ".txt"]);
+async function walk(dir) {
+    const entries = await readdir(dir, { withFileTypes: true });
+    const nested = await Promise.all(entries
+        .filter((entry) => !entry.name.startsWith("."))
+        .map(async (entry) => {
+        const path = join(dir, entry.name);
+        if (entry.isDirectory())
+            return walk(path);
+        return entry.isFile() && EXTENSIONS.has(extname(entry.name).toLowerCase()) ? [path] : [];
+    }));
+    return nested.flat();
+}
+function title(text, path) {
+    return /^#[ \t]+(.+?)[ \t]*#*[ \t]*$/m.exec(text)?.[1] ?? basename(path, extname(path));
+}
+/** Markdown and text files under one folder; the folder is the scope. */
+export function localSource(name, root, urlBase) {
+    return {
+        name,
+        list: async () => {
+            const files = (await walk(root)).sort();
+            return Promise.all(files.map(async (path) => {
+                const id = relative(root, path).split(sep).join("/");
+                const [text, info] = await Promise.all([readFile(path, "utf8"), stat(path)]);
+                return {
+                    id,
+                    title: title(text, path),
+                    url: urlBase === undefined ? pathToFileURL(path).href : `${urlBase.replace(/\/+$/, "")}/${encodeURI(id)}`,
+                    updatedAt: info.mtime.toISOString(),
+                };
+            }));
+        },
+        fetch: (id) => readFile(join(root, ...id.split("/")), "utf8"),
+    };
+}

package/dist/docs/source.d.ts

@@ -0,0 +1,14 @@
+export type DocMeta = {
+    id: string;
+    title: string;
+    url: string;
+    updatedAt: string;
+};
+/** One place docs come from. Documents are keyed `${name}:${id}` in the index. */
+export interface DocsSource {
+    readonly name: string;
+    /** Every document in scope, metadata only. */
+    list(signal: AbortSignal): Promise<DocMeta[]>;
+    /** One document's text as Markdown or plain text. */
+    fetch(id: string, signal: AbortSignal): Promise<string>;
+}

package/dist/docs/source.js

@@ -0,0 +1 @@
+export {};

package/dist/docs/sync.d.ts

@@ -0,0 +1,13 @@
+import type { DocsIndex } from "./index.ts";
+import type { DocsSource } from "./source.ts";
+type SyncResult = {
+    indexed: number;
+    removed: number;
+    failed: number;
+};
+/** Bring the index in line with one source: fetch only changed documents, drop the ones that left its scope. */
+export declare function syncSource(index: DocsIndex, source: DocsSource, signal: AbortSignal): Promise<SyncResult>;
+export declare function syncAll(index: DocsIndex, sources: readonly DocsSource[], signal: AbortSignal): Promise<void>;
+/** Sync now and every `intervalMinutes`; the returned stop function resolves once no sync is running. */
+export declare function startDocsSync(index: DocsIndex, sources: readonly DocsSource[], intervalMinutes: number): () => Promise<void>;
+export {};

package/dist/docs/sync.js

@@ -0,0 +1,68 @@
+/** Bring the index in line with one source: fetch only changed documents, drop the ones that left its scope. */
+export async function syncSource(index, source, signal) {
+    const docs = await source.list(signal);
+    const known = index.versions(source.name);
+    const result = { indexed: 0, removed: 0, failed: 0 };
+    for (const doc of docs) {
+        if (known.get(doc.id) === doc.updatedAt)
+            continue;
+        try {
+            index.upsert(source.name, doc, await source.fetch(doc.id, signal));
+            result.indexed++;
+        }
+        catch (error) {
+            if (signal.aborted)
+                throw error;
+            result.failed++;
+            console.error(`[docs] ${source.name}: could not fetch ${doc.title} (${doc.id})`, error);
+        }
+    }
+    const listed = new Set(docs.map((doc) => doc.id));
+    for (const id of known.keys()) {
+        if (listed.has(id))
+            continue;
+        index.remove(source.name, id);
+        result.removed++;
+    }
+    return result;
+}
+export async function syncAll(index, sources, signal) {
+    const removed = index.removeSourcesExcept(sources.map((source) => source.name));
+    if (removed > 0)
+        console.log(`[docs] removed ${removed} documents of sources no longer configured`);
+    for (const source of sources) {
+        try {
+            const result = await syncSource(index, source, signal);
+            index.recordSync(source.name, result.failed === 0 ? null : `${result.failed} documents failed to fetch`);
+            console.log(`[docs] ${source.name}: ${result.indexed} indexed, ${result.removed} removed, ${result.failed} failed`);
+        }
+        catch (error) {
+            if (signal.aborted)
+                return;
+            index.recordSync(source.name, error instanceof Error ? error.message : String(error));
+            console.error(`[docs] ${source.name}: sync failed`, error);
+        }
+    }
+}
+/** Sync now and every `intervalMinutes`; the returned stop function resolves once no sync is running. */
+export function startDocsSync(index, sources, intervalMinutes) {
+    const controller = new AbortController();
+    let busy = true;
+    const sync = () => {
+        busy = true;
+        return syncAll(index, sources, controller.signal).finally(() => {
+            busy = false;
+        });
+    };
+    let running = sync();
+    const timer = setInterval(() => {
+        if (!busy)
+            running = sync();
+    }, intervalMinutes * 60_000);
+    timer.unref();
+    return async () => {
+        clearInterval(timer);
+        controller.abort();
+        await running;
+    };
+}

package/dist/docs/tools.d.ts

@@ -0,0 +1,3 @@
+import { type Extension } from "@earendil-works/pi-durable";
+import type { DocsIndex } from "./index.ts";
+export declare function docsExtension(index: DocsIndex): Extension;

package/dist/docs/tools.js

@@ -0,0 +1,59 @@
+import { Type } from "@earendil-works/pi-ai";
+import { defineExtension, section } from "@earendil-works/pi-durable";
+import { defineTool } from "../extensions/tool.js";
+const READ_LIMIT = 30_000;
+function formatHits(hits) {
+    if (hits.length === 0)
+        return "No documents matched. Try other words, or say the docs do not cover this.";
+    const lines = hits.map((hit, index) => [
+        `${index + 1}. ${hit.title}${hit.heading.includes(" > ") ? ` (${hit.heading.split(" > ").slice(1).join(" > ")})` : ""}`,
+        `   doc: ${hit.docKey}`,
+        `   url: ${hit.url}`,
+        `   ${hit.snippet}`,
+    ].join("\n"));
+    return lines.join("\n\n");
+}
+export function docsExtension(index) {
+    return defineExtension({
+        name: "docs",
+        tools: [
+            defineTool({
+                name: "search_docs",
+                description: "Search the company's IT documentation (runbooks, how-tos, policies). Returns the best matching documents with a snippet and link.",
+                parameters: Type.Object({
+                    query: Type.String({ description: "Keywords describing the problem or task" }),
+                    limit: Type.Optional(Type.Integer({ minimum: 1, maximum: 10, description: "How many documents, default 5" })),
+                }),
+                execute: async (args) => ({
+                    output: [{ type: "text", text: formatHits(index.search(args.query, args.limit ?? 5)) }],
+                }),
+            }),
+            defineTool({
+                name: "read_doc",
+                description: "Read one document found with search_docs, by its doc key.",
+                parameters: Type.Object({
+                    doc: Type.String({ description: "The doc key from search_docs, like runbooks:vpn.md" }),
+                }),
+                execute: async (args) => {
+                    const doc = index.read(args.doc);
+                    if (doc === undefined) {
+                        return {
+                            output: [{ type: "text", text: `No document ${args.doc}. Use search_docs first.` }],
+                            isError: true,
+                        };
+                    }
+                    const truncated = doc.text.length > READ_LIMIT;
+                    const body = truncated
+                        ? `${doc.text.slice(0, READ_LIMIT)}\n\n[truncated; the full document is at the link]`
+                        : doc.text;
+                    return { output: [{ type: "text", text: `# ${doc.title}\nurl: ${doc.url}\n\n${body}` }] };
+                },
+            }),
+        ],
+        sections: [
+            section("docs", () => "Before answering a how-to or troubleshooting question, call search_docs, and read_doc when a snippet is not enough. " +
+                "Base the answer on what the docs say and cite each document you used as a Markdown link [title](url). " +
+                "If the docs do not cover it, say so instead of guessing."),
+        ],
+    });
+}

package/dist/extensions/approvals.d.ts

@@ -0,0 +1,26 @@
+import type { Static, TSchema } from "@earendil-works/pi-ai";
+import { type Harness, type ToolRegistration } from "@earendil-works/pi-durable";
+import { type ApprovalDecision, type ApprovalFeedback, type ApprovalOutcome, type ApprovalPrompt, type ChatAdapter, type ChatUser, type Platform } from "../chat/adapter.ts";
+type ApprovalSeed = {
+    prompt: ApprovalPrompt;
+    conversationId: number;
+    createdAt: number;
+};
+export type Approval = ApprovalSeed & {
+    status: "pending" | ApprovalOutcome;
+    decidedBy: ChatUser | null;
+};
+type ApprovalTransport = {
+    adapters: ReadonlyMap<Platform, ChatAdapter>;
+    timeoutMs: number;
+};
+/** Write tools ask for approval through these adapters; set once at startup. */
+export declare function setApprovalTransport(next: ApprovalTransport): void;
+/**
+ * A tool that runs only after its requester, the author of the message that led to the call, confirms it in chat.
+ * Replay-safe: after a restart the call reruns, finds its approval and posted prompt, and keeps waiting.
+ */
+export declare function withApproval<P extends TSchema>(tool: ToolRegistration<P>, summarize: (args: Static<P>) => string): ToolRegistration<P>;
+/** Records the requester's decision; anyone else's click, or a late one, changes nothing. */
+export declare function resolveApproval(harness: Harness, decision: ApprovalDecision): Promise<ApprovalFeedback>;
+export {};