@qearlyao/familiar 0.2.5 → 0.3.0

package/dist/web/multipart.js

@@ -0,0 +1,80 @@
+export function isWebUploadAttachment(value) {
+    return !!value && typeof value === "object" && Buffer.isBuffer(value.buffer);
+}
+export async function readRawBody(request, maxBytes) {
+    const chunks = [];
+    let total = 0;
+    for await (const chunk of request) {
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+        total += buffer.length;
+        if (total > maxBytes)
+            throw new Error("Request body too large");
+        chunks.push(buffer);
+    }
+    return Buffer.concat(chunks);
+}
+export function multipartBoundary(contentType) {
+    const header = Array.isArray(contentType) ? contentType.find((value) => value.includes("boundary=")) : contentType;
+    const match = header?.match(/boundary=(?:"([^"]+)"|([^;]+))/i);
+    if (!match?.[1] && !match?.[2])
+        throw new Error("Missing multipart boundary");
+    return match[1] ?? match[2] ?? "";
+}
+export function parseContentDisposition(header) {
+    const parts = header.split(";").map((part) => part.trim());
+    const values = {};
+    for (const part of parts.slice(1)) {
+        const [key, rawValue] = part.split("=");
+        if (!key || rawValue === undefined)
+            continue;
+        values[key.toLowerCase()] = rawValue.replace(/^"|"$/g, "");
+    }
+    return values;
+}
+export async function readMultipartBody(request, contentType) {
+    const boundary = multipartBoundary(contentType);
+    const raw = await readRawBody(request, 32 * 1024 * 1024);
+    const binary = raw.toString("binary");
+    const marker = `--${boundary}`;
+    const attachments = [];
+    const body = { text: "" };
+    for (const section of binary.split(marker).slice(1)) {
+        if (!section || section === "--\r\n" || section === "--")
+            continue;
+        const trimmed = section.replace(/^\r\n/, "").replace(/\r\n--$/, "");
+        const headerEnd = trimmed.indexOf("\r\n\r\n");
+        if (headerEnd < 0)
+            continue;
+        const headerText = trimmed.slice(0, headerEnd);
+        let contentBinary = trimmed.slice(headerEnd + 4);
+        if (contentBinary.endsWith("\r\n"))
+            contentBinary = contentBinary.slice(0, -2);
+        const headers = Object.fromEntries(headerText.split("\r\n").map((line) => {
+            const colon = line.indexOf(":");
+            return colon >= 0
+                ? [line.slice(0, colon).trim().toLowerCase(), line.slice(colon + 1).trim()]
+                : [line.toLowerCase(), ""];
+        }));
+        const disposition = parseContentDisposition(headers["content-disposition"] ?? "");
+        const name = disposition.name;
+        if (!name)
+            continue;
+        if (name === "text" || name === "channelKey" || name === "clientId") {
+            body[name] = Buffer.from(contentBinary, "binary").toString("utf8");
+            continue;
+        }
+        if (name !== "attachments")
+            continue;
+        const buffer = Buffer.from(contentBinary, "binary");
+        if (buffer.length === 0)
+            continue;
+        attachments.push({
+            name: disposition.filename,
+            mimeType: headers["content-type"],
+            size: buffer.length,
+            buffer,
+        });
+    }
+    body.attachments = attachments;
+    return body;
+}

package/dist/web/payloads.js

@@ -0,0 +1,34 @@
+import { supportedThinkingLevels } from "../models.js";
+import { isRecord } from "../util/guards.js";
+export function commandArgs(command, args) {
+    if (!isRecord(args))
+        return "";
+    if (command === "model")
+        return typeof args.model === "string" ? args.model : "";
+    if (command === "thinking")
+        return typeof args.level === "string" ? args.level : "";
+    if (command === "channel-trigger")
+        return typeof args.trigger === "string" ? args.trigger : "";
+    return "";
+}
+export function agentSettingsPayload(familiarAgent, channelKey, personaName) {
+    const { model } = familiarAgent.resolveChannelModel(channelKey);
+    return {
+        model: familiarAgent.getModel(channelKey),
+        thinking: familiarAgent.getThinkingLevel(channelKey),
+        supportedThinking: supportedThinkingLevels(model),
+        persona: { name: personaName },
+    };
+}
+export function sessionDto(session) {
+    return {
+        key: session.key,
+        label: session.label,
+        service: session.channel.service,
+        scope: session.channel.scope,
+        channelId: session.channel.channelId,
+        channelName: session.channel.channelName,
+        threadId: session.channel.threadId,
+        isDefault: session.isDefault,
+    };
+}

package/dist/{web-static.js → web/static.js}

@@ -1,12 +1,14 @@
 import { createReadStream, existsSync } from "node:fs";
 import { lstat, realpath, stat } from "node:fs/promises";
-import { extname, join, relative, resolve } from "node:path";
+import { dirname, extname, join, relative, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { attachmentsDir, browserScreenshotsDir, generatedAttachmentsDir } from "./generated-media.js";
-import { sendText } from "./web-http.js";
-function getProjectRoot() {
-    return resolve(fileURLToPath(import.meta.url), "../..");
-}
+import { attachmentsDir, browserScreenshotsDir, generatedAttachmentsDir } from "../generated-media.js";
+import { sendText } from "./http.js";
+const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+const DIST_DIR = resolve(PROJECT_ROOT, "web/dist");
+// Only successful realpaths are cached: the attachment root dirs are created
+// lazily on first write, so a missing root must stay retryable.
+const rootRealPathCache = new Map();
 function mimeType(path) {
     const extension = extname(path).toLowerCase();
     if (extension === ".html")
@@ -36,19 +38,18 @@ function mimeType(path) {
     return "application/octet-stream";
 }
 export async function serveStatic(response, requestPath) {
-    const distDir = resolve(getProjectRoot(), "web/dist");
-    if (!existsSync(distDir))
+    if (!existsSync(DIST_DIR))
         return false;
     const pathname = decodeURIComponent(requestPath.split("?")[0] || "/");
-    const candidate = resolve(distDir, pathname === "/" ? "index.html" : pathname.slice(1));
-    if (!candidate.startsWith(distDir)) {
+    const candidate = resolve(DIST_DIR, pathname === "/" ? "index.html" : pathname.slice(1));
+    if (!candidate.startsWith(DIST_DIR)) {
         sendText(response, 403, "Forbidden");
         return true;
     }
     let filePath = candidate;
     const fileStat = await stat(filePath).catch(() => undefined);
     if (!fileStat?.isFile())
-        filePath = join(distDir, "index.html");
+        filePath = join(DIST_DIR, "index.html");
     const stream = createReadStream(filePath);
     response.writeHead(200, { "content-type": mimeType(filePath) });
     stream.pipe(response);
@@ -88,9 +89,13 @@ export async function serveAttachment(config, response, requestPath, rangeHeader
         { root: browserScreenshotsDir(config), relativePath: relativePath.replace(/^screenshot[\\/]/, "") },
     ];
     for (const { root, relativePath: candidateRelativePath } of candidates) {
-        const rootRealPath = await realpath(root).catch(() => undefined);
-        if (!rootRealPath)
-            continue;
+        let rootRealPath = rootRealPathCache.get(root);
+        if (rootRealPath === undefined) {
+            rootRealPath = await realpath(root).catch(() => undefined);
+            if (!rootRealPath)
+                continue;
+            rootRealPathCache.set(root, rootRealPath);
+        }
         const filePath = resolve(root, candidateRelativePath);
         const rel = relative(root, filePath);
         if (rel.startsWith("..") || rel.startsWith("/") || rel.startsWith("\\") || rel === "") {

package/dist/web/stream.js

@@ -0,0 +1,69 @@
+import { isRecord } from "../util/guards.js";
+import { acceptWebSocket, decodeFrames } from "./events.js";
+export function attachWebSocketStream(server, options) {
+    const { authorize, eventHub, getRuntime, abort, retry, deleteLatest } = options;
+    const runtimeActions = { abort, retry, delete: deleteLatest };
+    const handleStreamMessage = (raw, client) => {
+        const message = JSON.parse(raw);
+        if (!isRecord(message) || typeof message.type !== "string")
+            return;
+        if (message.type === "hello") {
+            if (!client.channelKey)
+                return;
+            eventHub.replay(client, client.channelKey, typeof message.lastEventId === "string" ? message.lastEventId : null);
+            return;
+        }
+        const action = runtimeActions[message.type];
+        if (action) {
+            void getRuntime(client.channelKey)
+                .then(action)
+                .catch((error) => console.error(`WebSocket ${message.type} runtime lookup failed`, error));
+        }
+    };
+    server.on("upgrade", (request, socket) => {
+        const netSocket = socket;
+        const url = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`);
+        if (url.pathname !== "/api/web/stream" || !authorize(request, url.pathname)) {
+            netSocket.write("HTTP/1.1 401 Unauthorized\r\n\r\n");
+            netSocket.destroy();
+            return;
+        }
+        const requestedChannelKey = url.searchParams.get("channelKey") || undefined;
+        void getRuntime(requestedChannelKey)
+            .then((runtime) => {
+            if (netSocket.destroyed)
+                return;
+            if (!acceptWebSocket(request, netSocket))
+                return;
+            netSocket.setNoDelay(true);
+            const client = { socket: netSocket, channelKey: runtime.channelKey, authed: false };
+            const dispose = eventHub.registerClient(client);
+            let frameBuffer = Buffer.alloc(0);
+            netSocket.on("data", (chunk) => {
+                try {
+                    frameBuffer = Buffer.concat([frameBuffer, chunk]);
+                    const decoded = decodeFrames(frameBuffer);
+                    frameBuffer = decoded.remaining;
+                    if (decoded.close)
+                        netSocket.destroy();
+                    for (const raw of decoded.messages) {
+                        handleStreamMessage(raw, client);
+                    }
+                }
+                catch (error) {
+                    console.error("WebSocket frame handling failed", error);
+                    netSocket.destroy();
+                }
+            });
+            netSocket.on("close", dispose);
+            netSocket.on("error", dispose);
+        })
+            .catch((error) => {
+            console.error("WebSocket runtime lookup failed", error);
+            if (!netSocket.destroyed) {
+                netSocket.write("HTTP/1.1 503 Service Unavailable\r\n\r\n");
+                netSocket.destroy();
+            }
+        });
+    });
+}

package/dist/web-tools/cache.js

@@ -0,0 +1,42 @@
+import { MAX_CACHE_CHARS_PER_PAGE } from "./types.js";
+export class PageCache {
+    ttlMs;
+    capacity;
+    entries = new Map();
+    constructor(options = {}) {
+        this.ttlMs = options.ttlMs ?? 5 * 60 * 1000;
+        this.capacity = options.capacity ?? 20;
+    }
+    get(url) {
+        const entry = this.entries.get(url);
+        if (!entry)
+            return undefined;
+        if (Date.now() - entry.fetchedAt > this.ttlMs) {
+            this.entries.delete(url);
+            return undefined;
+        }
+        entry.lastAccessed = Date.now();
+        this.entries.delete(url);
+        this.entries.set(url, entry);
+        return entry;
+    }
+    set(url, content, provider) {
+        if (content.length > MAX_CACHE_CHARS_PER_PAGE)
+            return;
+        if (this.entries.has(url))
+            this.entries.delete(url);
+        const now = Date.now();
+        this.entries.set(url, {
+            content,
+            provider,
+            fetchedAt: now,
+            lastAccessed: now,
+        });
+        while (this.entries.size > this.capacity) {
+            const oldest = this.entries.keys().next().value;
+            if (!oldest)
+                break;
+            this.entries.delete(oldest);
+        }
+    }
+}

package/dist/web-tools/config.js

@@ -0,0 +1,16 @@
+export function readEnvKey(name) {
+    const value = process.env[name];
+    return value?.trim() ? value.trim() : undefined;
+}
+export function loadWebConfig() {
+    return {
+        apiKeys: {
+            BRAVE_API_KEY: readEnvKey("BRAVE_API_KEY"),
+            TAVILY_API_KEY: readEnvKey("TAVILY_API_KEY"),
+            EXA_API_KEY: readEnvKey("EXA_API_KEY"),
+            JINA_API_KEY: readEnvKey("JINA_API_KEY"),
+            TINYFISH_API_KEY: readEnvKey("TINYFISH_API_KEY"),
+        },
+        warnings: [],
+    };
+}

package/dist/web-tools/fetch-providers.js

@@ -0,0 +1,119 @@
+import { isRecord } from "../util/guards.js";
+import { fetchJson, fetchText } from "./http.js";
+import { FETCH_TIMEOUT_MS, MAX_RESPONSE_BYTES, ProviderError } from "./types.js";
+export function createJinaProvider(apiKey) {
+    return {
+        name: "jina",
+        async fetch(url, signal) {
+            const target = `https://r.jina.ai/${url}`;
+            const headers = buildJinaHeaders(apiKey, "application/json");
+            try {
+                const jsonContent = await fetchJinaContent(target, headers, signal, true);
+                if (jsonContent)
+                    return jsonContent;
+            }
+            catch (error) {
+                if (!shouldFallbackToText(error)) {
+                    throw error;
+                }
+            }
+            const textContent = await fetchJinaContent(target, buildJinaHeaders(apiKey, "text/plain"), signal, false);
+            if (textContent)
+                return textContent;
+            throw new ProviderError("jina", "jina returned an empty response.", false);
+        },
+    };
+}
+export function createTinyfishProvider(apiKey) {
+    const trimmed = apiKey.trim();
+    return {
+        name: "tinyfish",
+        async fetch(url, signal) {
+            const response = await fetchJson("tinyfish", "https://api.fetch.tinyfish.ai", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "X-API-Key": trimmed,
+                },
+                body: JSON.stringify({
+                    urls: [url],
+                    format: "markdown",
+                }),
+                signal,
+                timeoutMs: FETCH_TIMEOUT_MS,
+                maxBytes: MAX_RESPONSE_BYTES.fetch,
+                validate: parseTinyfishResponse,
+            });
+            return response.content;
+        },
+    };
+}
+export function parseTinyfishResponse(value) {
+    if (!isRecord(value)) {
+        throw new ProviderError("tinyfish", "TinyFish returned unexpected response shape.", false);
+    }
+    const results = value.results;
+    if (Array.isArray(results)) {
+        const first = results[0];
+        if (isRecord(first)) {
+            const content = typeof first.content === "string"
+                ? first.content
+                : typeof first.markdown === "string"
+                    ? first.markdown
+                    : typeof first.text === "string"
+                        ? first.text
+                        : "";
+            if (content.trim())
+                return { content: content.replaceAll(/\r\n/g, "\n").trim() };
+        }
+    }
+    const errors = Array.isArray(value.errors) ? value.errors : undefined;
+    const firstError = errors?.find((entry) => isRecord(entry));
+    if (isRecord(firstError)) {
+        const message = typeof firstError.message === "string"
+            ? firstError.message
+            : typeof firstError.error === "string"
+                ? firstError.error
+                : "TinyFish failed to fetch the page.";
+        throw new ProviderError("tinyfish", message, false);
+    }
+    throw new ProviderError("tinyfish", "TinyFish returned no page content.", false);
+}
+export function buildJinaHeaders(apiKey, accept) {
+    const headers = {
+        Accept: accept,
+        "X-Retain-Images": "none",
+    };
+    if (apiKey?.trim())
+        headers.Authorization = `Bearer ${apiKey.trim()}`;
+    return headers;
+}
+export async function fetchJinaContent(targetUrl, headers, signal, preferJson) {
+    const responseText = await fetchText("jina", targetUrl, {
+        headers,
+        signal,
+        timeoutMs: FETCH_TIMEOUT_MS,
+        maxBytes: MAX_RESPONSE_BYTES.fetch,
+    });
+    if (preferJson) {
+        try {
+            const parsed = JSON.parse(responseText);
+            if (isRecord(parsed) && isRecord(parsed.data)) {
+                if (typeof parsed.data.content === "string" && parsed.data.content.trim()) {
+                    return parsed.data.content.replaceAll(/\r\n/g, "\n").trim();
+                }
+                if (typeof parsed.data.markdown === "string" && parsed.data.markdown.trim()) {
+                    return parsed.data.markdown.replaceAll(/\r\n/g, "\n").trim();
+                }
+            }
+        }
+        catch {
+            return undefined;
+        }
+        return undefined;
+    }
+    return responseText.replaceAll(/\r\n/g, "\n").trim() || undefined;
+}
+export function shouldFallbackToText(error) {
+    return error instanceof ProviderError && (error.status === 406 || error.status === 415);
+}

package/dist/web-tools/format.js

@@ -0,0 +1,88 @@
+import { FETCH_DEFAULT_MAX_CHARS, SEARCH_OUTPUT_BUDGET, WEB_UNTRUSTED_PREFIX, } from "./types.js";
+import { normalizeIsoDate } from "./util.js";
+export function collectSearchNotes(requested, served, notes = []) {
+    if (requested !== served) {
+        notes.push(`Depth: requested ${requested}, served ${served}`);
+    }
+    return [...new Set(notes)];
+}
+export function buildSearchDocument(args) {
+    const lines = [`## Search Results (via ${args.provider}, ${args.depth})`];
+    if (args.notes?.length) {
+        lines.push("", ...args.notes);
+    }
+    if (args.freshness || args.domains?.length || args.appliedFilters) {
+        const notes = [];
+        if (args.freshness)
+            notes.push(`Freshness: ${args.freshness}`);
+        if (args.domains?.length)
+            notes.push(`Domains: ${args.domains.join(", ")}`);
+        lines.push("", ...notes);
+    }
+    for (const [index, result] of args.results.entries()) {
+        lines.push("", `### ${index + 1}. ${result.title}`, `URL: ${result.url}`);
+        const published = normalizeIsoDate(result.publishedAt);
+        if (published)
+            lines.push(`Published: ${published.slice(0, 10)}`);
+        lines.push(`Snippet: ${result.snippet || "[No snippet available]"}`);
+        if (result.content) {
+            lines.push("", "Content:", result.content);
+        }
+    }
+    return lines.join("\n");
+}
+export function formatFetchContent(url, provider, chunk) {
+    const header = `## Content from ${url} (via ${provider})`;
+    if (chunk.offset >= chunk.totalChars) {
+        return prefixUntrustedWebContent([
+            header,
+            "",
+            `[Offset ${chunk.offset} is beyond the end of the document. Total content length: ${chunk.totalChars} characters.]`,
+        ].join("\n"));
+    }
+    const lines = [
+        header,
+        "",
+        `[Showing chars ${chunk.offset}-${chunk.offset + chunk.returnedChars - 1} of ${chunk.totalChars}]`,
+        "",
+        chunk.text,
+    ];
+    if (chunk.hasMore && chunk.nextOffset !== undefined) {
+        lines.push("", `[More content available. Next chunk: fetch_web(url="${url}", offset=${chunk.nextOffset})]`);
+    }
+    return prefixUntrustedWebContent(lines.join("\n"));
+}
+export function prefixUntrustedWebContent(text) {
+    return `${WEB_UNTRUSTED_PREFIX}\n\n${text}`;
+}
+export function paginateContent(content, offset, maxChars = FETCH_DEFAULT_MAX_CHARS) {
+    const totalChars = content.length;
+    if (offset >= totalChars) {
+        return { text: "", offset, returnedChars: 0, totalChars, hasMore: false };
+    }
+    const safeMaxChars = Math.max(1, Math.min(maxChars, 20_000));
+    const end = Math.min(offset + safeMaxChars, totalChars);
+    const text = content.slice(offset, end).trim();
+    return {
+        text,
+        offset,
+        returnedChars: text.length,
+        totalChars,
+        nextOffset: end < totalChars ? end : undefined,
+        hasMore: end < totalChars,
+    };
+}
+export function formatSearchResults(args) {
+    const notes = collectSearchNotes(args.requestedDepth, args.servedDepth, [...(args.notes ?? [])]);
+    const document = buildSearchDocument({
+        provider: args.provider,
+        depth: args.servedDepth,
+        freshness: args.freshness,
+        domains: args.domains,
+        results: args.results,
+        appliedFilters: args.appliedFilters,
+        notes,
+    });
+    const output = prefixUntrustedWebContent(document);
+    return output.length > SEARCH_OUTPUT_BUDGET ? `${output.slice(0, SEARCH_OUTPUT_BUDGET - 3).trimEnd()}...` : output;
+}

package/dist/web-tools/http.js

@@ -0,0 +1,81 @@
+import { ProviderError } from "./types.js";
+export function buildRequestSignal(signal, timeoutMs) {
+    return AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
+}
+export async function readBoundedBody(response, maxBytes) {
+    if (!response.body)
+        return "";
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    const chunks = [];
+    let totalBytes = 0;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        if (!value)
+            continue;
+        totalBytes += value.byteLength;
+        if (totalBytes > maxBytes) {
+            throw new Error(`Response exceeded size limit of ${maxBytes} bytes.`);
+        }
+        chunks.push(decoder.decode(value, { stream: true }));
+    }
+    chunks.push(decoder.decode());
+    return chunks.join("");
+}
+export function createHttpError(provider, response) {
+    return new ProviderError(provider, `${provider} request failed: ${response.status} ${response.statusText}`.trim(), response.status >= 500 || response.status === 408 || response.status === 429, response.status);
+}
+// Pass through aborts and already-typed ProviderErrors raw; wrap everything else
+// (network failures, JSON.parse/validate throws) as a retryable ProviderError.
+function rethrowAsProviderError(provider, error, signal) {
+    if (error instanceof ProviderError)
+        throw error;
+    if (signal.aborted)
+        throw error;
+    throw new ProviderError(provider, error instanceof Error ? `${provider} request failed: ${error.message}` : `${provider} request failed.`, true, undefined, error);
+}
+async function performFetch(provider, url, options) {
+    try {
+        const response = await fetch(url, {
+            method: options.method ?? "GET",
+            headers: options.headers,
+            body: options.body,
+            redirect: "error",
+            signal: buildRequestSignal(options.signal, options.timeoutMs),
+        });
+        if (!response.ok)
+            throw createHttpError(provider, response);
+        return await readBoundedBody(response, options.maxBytes);
+    }
+    catch (error) {
+        rethrowAsProviderError(provider, error, options.signal);
+    }
+}
+export async function fetchJson(provider, url, options) {
+    try {
+        const body = await performFetch(provider, url, {
+            method: options.method,
+            headers: options.headers,
+            body: options.body,
+            signal: options.signal,
+            timeoutMs: options.timeoutMs,
+            maxBytes: options.maxBytes,
+        });
+        const parsed = body ? JSON.parse(body) : null;
+        return options.validate(parsed);
+    }
+    catch (error) {
+        rethrowAsProviderError(provider, error, options.signal);
+    }
+}
+export async function fetchText(provider, url, options) {
+    return await performFetch(provider, url, {
+        method: "GET",
+        headers: options.headers,
+        signal: options.signal,
+        timeoutMs: options.timeoutMs,
+        maxBytes: options.maxBytes,
+    });
+}

package/dist/web-tools/routing.js

@@ -0,0 +1,29 @@
+export function searchProviderOrder(depth, args) {
+    if (depth === "thorough")
+        return ["tavily", "exa", "brave"];
+    if (args.domains?.length)
+        return ["tavily", "exa", "brave"];
+    return ["brave", "tavily", "exa"];
+}
+export function canServe(provider, depth) {
+    if (depth === "thorough")
+        return provider.capabilities.has("search") && provider.capabilities.has("content");
+    return provider.capabilities.has("search");
+}
+export function canServeSearchArgs(provider, args) {
+    if (!canServe(provider, args.depth))
+        return false;
+    if ((args.domains?.length ?? 0) > 1 && !provider.capabilities.has("domainFilter"))
+        return false;
+    return true;
+}
+export function resolveSearchProviders(args, searchProviders) {
+    const providers = [];
+    for (const name of searchProviderOrder(args.depth, args)) {
+        const candidate = searchProviders[name];
+        if (candidate && canServeSearchArgs(candidate, args) && !providers.includes(candidate)) {
+            providers.push(candidate);
+        }
+    }
+    return providers;
+}