@steel-dev/atlas 0.1.0

package/dist/providers/search.js

@@ -0,0 +1,433 @@
+import { generateText } from "ai";
+import { sleep } from "../async.js";
+import { isZaiModelId } from "../defaults.js";
+import { readEnv } from "../env.js";
+import { errorMessage } from "../errors.js";
+import { normalizeUrlForSource } from "../url.js";
+export function safeDomain(url) {
+    try {
+        return new URL(url).hostname.replace(/^www\./, "");
+    }
+    catch {
+        return url;
+    }
+}
+function toResult(index, url, title, snippet) {
+    if (typeof url !== "string" || !/^https?:\/\//i.test(url))
+        return null;
+    return {
+        position: index + 1,
+        title: typeof title === "string" && title.trim() ? title.trim() : url,
+        url,
+        snippet,
+        domain: safeDomain(url),
+    };
+}
+async function readErrorBody(resp) {
+    try {
+        const text = (await resp.text()).trim();
+        return text.length > 200 ? `${text.slice(0, 200)}…` : text;
+    }
+    catch {
+        return "";
+    }
+}
+class SearchProviderError extends Error {
+    statusCode;
+    retryAfterMs;
+    constructor(message, statusCode, retryAfterMs) {
+        super(message);
+        this.name = "SearchProviderError";
+        this.statusCode = statusCode;
+        if (retryAfterMs !== undefined)
+            this.retryAfterMs = retryAfterMs;
+    }
+}
+function parseRetryAfterMs(headers) {
+    const raw = headers.get("retry-after");
+    if (!raw)
+        return undefined;
+    const seconds = Number(raw);
+    if (Number.isFinite(seconds))
+        return Math.max(0, seconds * 1_000);
+    const at = Date.parse(raw);
+    if (Number.isFinite(at))
+        return Math.max(0, at - Date.now());
+    return undefined;
+}
+async function searchHttpError(label, resp) {
+    const body = await readErrorBody(resp);
+    return new SearchProviderError(`${label}: HTTP ${resp.status}: ${body}`, resp.status, parseRetryAfterMs(resp.headers));
+}
+const SEARCH_RETRY_MAX_ATTEMPTS = 5;
+const SEARCH_RETRY_BASE_DELAY_MS = 500;
+const SEARCH_RETRY_MAX_DELAY_MS = 15_000;
+function classifySearchRetry(err) {
+    if (err instanceof Error &&
+        (err.name === "AbortError" || err.name === "TimeoutError")) {
+        return { retryable: false };
+    }
+    if (err instanceof SearchProviderError) {
+        const status = err.statusCode;
+        const retryable = status === 408 || status === 409 || status === 429 || status >= 500;
+        if (!retryable)
+            return { retryable: false };
+        return {
+            retryable: true,
+            ...(err.retryAfterMs !== undefined
+                ? { retryAfterMs: err.retryAfterMs }
+                : {}),
+        };
+    }
+    const message = (err instanceof Error ? err.message : String(err)).toLowerCase();
+    return /rate limit|too many requests|overloaded|concurrent connections|timeout|timed out|econnreset|etimedout|eai_again|socket hang up|fetch failed|network error/.test(message)
+        ? { retryable: true }
+        : { retryable: false };
+}
+function searchBackoffDelayMs(attempt, retryAfterMs) {
+    const exponential = Math.min(SEARCH_RETRY_MAX_DELAY_MS, SEARCH_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1));
+    const jittered = exponential / 2 + Math.random() * (exponential / 2);
+    return Math.min(SEARCH_RETRY_MAX_DELAY_MS, Math.max(retryAfterMs ?? 0, jittered));
+}
+async function searchWithRetry(provider, q) {
+    let tries = 0;
+    for (;;) {
+        tries++;
+        try {
+            return await provider.search(q);
+        }
+        catch (err) {
+            const { retryable, retryAfterMs } = classifySearchRetry(err);
+            if (!retryable ||
+                tries >= SEARCH_RETRY_MAX_ATTEMPTS ||
+                q.signal?.aborted) {
+                throw err;
+            }
+            await sleep(searchBackoffDelayMs(tries, retryAfterMs), q.signal);
+        }
+    }
+}
+function clampLimit(limit, max) {
+    return Math.min(Math.max(1, Math.floor(limit ?? 8)), max);
+}
+export function tavily(opts = {}) {
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_TAVILY_API_KEY", "TAVILY_API_KEY");
+    if (!apiKey) {
+        throw new Error("tavily() requires an apiKey (or set ATLAS_TAVILY_API_KEY / TAVILY_API_KEY)");
+    }
+    const endpoint = `${(opts.baseUrl ?? "https://api.tavily.com").replace(/\/+$/, "")}/search`;
+    return {
+        id: "tavily",
+        async search({ query, maxResults, signal }) {
+            const resp = await fetch(endpoint, {
+                method: "POST",
+                signal: signal ?? null,
+                headers: {
+                    "content-type": "application/json",
+                    authorization: `Bearer ${apiKey}`,
+                },
+                body: JSON.stringify({
+                    query,
+                    max_results: clampLimit(maxResults, 20),
+                }),
+            });
+            if (!resp.ok) {
+                throw await searchHttpError("tavily", resp);
+            }
+            const data = (await resp.json());
+            const rows = Array.isArray(data.results) ? data.results : [];
+            const results = [];
+            for (const row of rows) {
+                if (!row || typeof row !== "object")
+                    continue;
+                const record = row;
+                const snippet = typeof record.content === "string"
+                    ? record.content.slice(0, 500).trim()
+                    : "";
+                const result = toResult(results.length, record.url, record.title, snippet);
+                if (result)
+                    results.push(result);
+            }
+            return results;
+        },
+    };
+}
+export function exa(opts = {}) {
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_EXA_API_KEY", "EXA_API_KEY");
+    if (!apiKey) {
+        throw new Error("exa() requires an apiKey (or set ATLAS_EXA_API_KEY / EXA_API_KEY)");
+    }
+    const endpoint = `${(opts.baseUrl ?? "https://api.exa.ai").replace(/\/+$/, "")}/search`;
+    return {
+        id: "exa",
+        async search({ query, maxResults, signal }) {
+            const resp = await fetch(endpoint, {
+                method: "POST",
+                signal: signal ?? null,
+                headers: {
+                    "content-type": "application/json",
+                    "x-api-key": apiKey,
+                },
+                body: JSON.stringify({
+                    query,
+                    numResults: clampLimit(maxResults, 100),
+                    ...(opts.type ? { type: opts.type } : {}),
+                    contents: { highlights: { numSentences: 3, highlightsPerUrl: 2 } },
+                }),
+            });
+            if (!resp.ok) {
+                throw await searchHttpError("exa", resp);
+            }
+            const data = (await resp.json());
+            const rows = Array.isArray(data.results) ? data.results : [];
+            const results = [];
+            for (const row of rows) {
+                if (!row || typeof row !== "object")
+                    continue;
+                const record = row;
+                const highlights = Array.isArray(record.highlights)
+                    ? record.highlights.filter((h) => typeof h === "string")
+                    : [];
+                const snippet = highlights.join(" … ").trim() ||
+                    (typeof record.text === "string"
+                        ? record.text.slice(0, 500).trim()
+                        : "");
+                const result = toResult(results.length, record.url, record.title, snippet);
+                if (result)
+                    results.push(result);
+            }
+            return results;
+        },
+    };
+}
+export function brave(opts = {}) {
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_BRAVE_API_KEY", "BRAVE_API_KEY");
+    if (!apiKey) {
+        throw new Error("brave() requires an apiKey (or set ATLAS_BRAVE_API_KEY / BRAVE_API_KEY)");
+    }
+    const base = `${(opts.baseUrl ?? "https://api.search.brave.com").replace(/\/+$/, "")}/res/v1/web/search`;
+    return {
+        id: "brave",
+        async search({ query, maxResults, signal }) {
+            const params = new URLSearchParams({
+                q: query,
+                count: String(clampLimit(maxResults, 20)),
+                extra_snippets: "true",
+            });
+            if (opts.country)
+                params.set("country", opts.country);
+            if (opts.searchLang)
+                params.set("search_lang", opts.searchLang);
+            const resp = await fetch(`${base}?${params.toString()}`, {
+                signal: signal ?? null,
+                headers: {
+                    accept: "application/json",
+                    "x-subscription-token": apiKey,
+                },
+            });
+            if (!resp.ok) {
+                throw await searchHttpError("brave", resp);
+            }
+            const data = (await resp.json());
+            const rows = Array.isArray(data.web?.results) ? data.web.results : [];
+            const results = [];
+            for (const row of rows) {
+                if (!row || typeof row !== "object")
+                    continue;
+                const record = row;
+                const extra = Array.isArray(record.extra_snippets)
+                    ? record.extra_snippets.filter((s) => typeof s === "string")
+                    : [];
+                const snippet = (typeof record.description === "string" ? record.description : "") ||
+                    extra.join(" … ");
+                const result = toResult(results.length, record.url, record.title, snippet.trim());
+                if (result)
+                    results.push(result);
+            }
+            return results;
+        },
+    };
+}
+export function nativeModelSearch(opts) {
+    return {
+        id: "model-native",
+        async search({ query, maxResults, signal }) {
+            const tools = await nativeSearchTools(opts.model);
+            const limit = clampLimit(maxResults, 10);
+            const result = await generateText({
+                model: opts.model,
+                prompt: `Search the web for: ${query}\n\n` +
+                    `Run one web search and list the ${limit} most relevant distinct result pages. ` +
+                    "Do not answer the question; just surface the sources. " +
+                    "Write one line per result in the form `<plain url> :: <one-sentence summary of what the page contains>`, with no markdown links.",
+                tools,
+                maxOutputTokens: 1_500,
+                abortSignal: signal,
+            });
+            const snippets = snippetsByUrl(result.text);
+            const seen = new Set();
+            const results = [];
+            for (const source of result.sources) {
+                if (source.sourceType !== "url")
+                    continue;
+                const key = normalizeUrlForSource(source.url);
+                if (seen.has(key))
+                    continue;
+                seen.add(key);
+                const parsed = toResult(results.length, source.url, source.title, snippets.get(key) ?? "");
+                if (parsed)
+                    results.push(parsed);
+                if (results.length >= limit)
+                    break;
+            }
+            return results;
+        },
+    };
+}
+function snippetsByUrl(text) {
+    const map = new Map();
+    for (const line of text.split("\n")) {
+        const match = /(https?:\/\/\S+?)[)\]>.,]*\s*::\s*(\S.*)/.exec(line);
+        if (!match)
+            continue;
+        const key = normalizeUrlForSource(match[1]);
+        if (!map.has(key))
+            map.set(key, match[2].trim().slice(0, 500));
+    }
+    return map;
+}
+async function importProvider(pkg, load) {
+    try {
+        return await load();
+    }
+    catch (err) {
+        throw new Error(`nativeModelSearch needs "${pkg}" installed for this provider's server-side search; ` +
+            `install it (npm install ${pkg}) or configure a search adapter (tavily(), exa(), brave()). ` +
+            `Original error: ${errorMessage(err)}`);
+    }
+}
+async function nativeSearchTools(model) {
+    const provider = model.provider.toLowerCase();
+    if (isZaiModelId(model.modelId)) {
+        throw new Error(`nativeModelSearch: provider "${model.provider}" has no known server-side search tool; configure a search adapter (tavily(), exa(), brave())`);
+    }
+    if (provider.includes("anthropic")) {
+        const { anthropic } = await importProvider("@ai-sdk/anthropic", () => import("@ai-sdk/anthropic"));
+        return {
+            web_search: anthropic.tools.webSearch_20250305({ maxUses: 1 }),
+        };
+    }
+    if (provider.includes("openai")) {
+        const { openai } = await importProvider("@ai-sdk/openai", () => import("@ai-sdk/openai"));
+        return { web_search: openai.tools.webSearch({}) };
+    }
+    if (provider.includes("google")) {
+        const { google } = await importProvider("@ai-sdk/google", () => import("@ai-sdk/google"));
+        return { google_search: google.tools.googleSearch({}) };
+    }
+    throw new Error(`nativeModelSearch: provider "${model.provider}" has no known server-side search tool; configure a search adapter (tavily(), exa(), brave())`);
+}
+const RRF_K = 60;
+export function openUrlsOf(meta) {
+    const raw = meta?.openUrls;
+    if (!Array.isArray(raw))
+        return [];
+    const out = [];
+    for (const url of raw) {
+        if (typeof url === "string" && /^https?:\/\//i.test(url))
+            out.push(url);
+    }
+    return out;
+}
+function mergeMeta(existing, incoming, preferIncoming) {
+    const openUrls = [
+        ...new Set([...openUrlsOf(existing), ...openUrlsOf(incoming)]),
+    ];
+    const base = preferIncoming
+        ? { ...existing, ...incoming }
+        : { ...incoming, ...existing };
+    if (openUrls.length === 0) {
+        return Object.keys(base).length > 0 ? base : undefined;
+    }
+    return { ...base, openUrls };
+}
+export function mergeSearchResults(lists, limit) {
+    const byUrl = new Map();
+    for (const list of lists) {
+        for (const result of list.results) {
+            const key = normalizeUrlForSource(result.url);
+            const score = 1 / (RRF_K + result.position);
+            const existing = byUrl.get(key);
+            if (!existing) {
+                byUrl.set(key, {
+                    title: result.title,
+                    url: result.url,
+                    snippet: result.snippet,
+                    provider: list.provider,
+                    providerRank: result.position,
+                    providers: [list.provider],
+                    score,
+                    ...(result.meta ? { meta: result.meta } : {}),
+                });
+                continue;
+            }
+            existing.score += score;
+            if (!existing.providers.includes(list.provider)) {
+                existing.providers.push(list.provider);
+            }
+            const betterRank = result.position < existing.providerRank;
+            if (betterRank) {
+                existing.title = result.title;
+                existing.url = result.url;
+                existing.snippet = result.snippet || existing.snippet;
+                existing.provider = list.provider;
+                existing.providerRank = result.position;
+            }
+            const meta = mergeMeta(existing.meta, result.meta, betterRank);
+            if (meta)
+                existing.meta = meta;
+        }
+    }
+    return [...byUrl.values()]
+        .sort((a, b) => b.score - a.score || a.providerRank - b.providerRank)
+        .slice(0, limit);
+}
+export function combineSearchProviders(providers) {
+    return {
+        providers,
+        async run(q) {
+            const warnings = [];
+            const lists = await Promise.all(providers.map(async (provider) => {
+                try {
+                    return {
+                        provider: provider.id,
+                        results: await searchWithRetry(provider, q),
+                    };
+                }
+                catch (err) {
+                    warnings.push(`${provider.id}: ${errorMessage(err)}`);
+                    return { provider: provider.id, results: [] };
+                }
+            }));
+            return {
+                merged: mergeSearchResults(lists, clampLimit(q.maxResults, 20)),
+                warnings,
+            };
+        },
+    };
+}
+export function defaultSearchProviders(model) {
+    const providers = [];
+    if (readEnv("ATLAS_TAVILY_API_KEY", "TAVILY_API_KEY")) {
+        providers.push(tavily());
+    }
+    if (readEnv("ATLAS_EXA_API_KEY", "EXA_API_KEY")) {
+        providers.push(exa());
+    }
+    if (readEnv("ATLAS_BRAVE_API_KEY", "BRAVE_API_KEY")) {
+        providers.push(brave());
+    }
+    if (providers.length > 0)
+        return providers;
+    return [nativeModelSearch({ model })];
+}

package/dist/providers/store.d.ts

@@ -0,0 +1,48 @@
+import { type RunTrace } from "../trace.js";
+export interface JournalEntry {
+    seq: number;
+    kind: "meta" | "event" | "call" | "io" | "trace";
+    type?: string;
+    callKey?: string;
+    data: unknown;
+}
+export interface RunSummary {
+    runId: string;
+    question?: string;
+    status?: string;
+}
+export interface RunStore {
+    append(runId: string, entries: JournalEntry[]): Promise<void>;
+    read(runId: string): AsyncIterable<JournalEntry>;
+    list(): AsyncIterable<RunSummary>;
+}
+export declare function memoryStore(): RunStore;
+export declare function fileStore(dir: string): RunStore;
+export declare class JournalWriter {
+    private readonly store;
+    private readonly runId;
+    private seq;
+    private pending;
+    private flushing;
+    constructor(store: RunStore, runId: string);
+    meta(data: unknown): void;
+    event(type: string, data: unknown): void;
+    call(callKey: string, data: unknown): void;
+    io(callKey: string, data: unknown): void;
+    trace(subKind: string, data: unknown): void;
+    private push;
+    private scheduleFlush;
+    flush(): Promise<void>;
+}
+export declare class ReplayCache {
+    private readonly byKey;
+    private hits;
+    add(callKey: string, data: unknown): void;
+    take(callKey: string): unknown | undefined;
+    values(prefix: string): unknown[];
+    get replayedCalls(): number;
+    get size(): number;
+}
+export declare function loadReplayCache(store: RunStore, runId: string): Promise<ReplayCache>;
+export declare function loadTrace(store: RunStore, runId: string): Promise<RunTrace | null>;
+export declare function loadRunMeta(store: RunStore, runId: string): Promise<Record<string, unknown> | null>;

package/dist/providers/store.js

@@ -0,0 +1,217 @@
+import { appendFile, mkdir, readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { TRACE_SCHEMA_VERSION, } from "../trace.js";
+export function memoryStore() {
+    const runs = new Map();
+    return {
+        async append(runId, entries) {
+            const existing = runs.get(runId) ?? [];
+            existing.push(...entries);
+            runs.set(runId, existing);
+        },
+        async *read(runId) {
+            for (const entry of runs.get(runId) ?? [])
+                yield entry;
+        },
+        async *list() {
+            for (const [runId, entries] of runs) {
+                yield summarize(runId, entries);
+            }
+        },
+    };
+}
+export function fileStore(dir) {
+    const fileFor = (runId) => join(dir, `${runId.replace(/[^\w.-]/g, "_")}.jsonl`);
+    return {
+        async append(runId, entries) {
+            if (entries.length === 0)
+                return;
+            await mkdir(dir, { recursive: true });
+            const lines = entries.map((entry) => JSON.stringify(entry)).join("\n");
+            await appendFile(fileFor(runId), `${lines}\n`, "utf8");
+        },
+        async *read(runId) {
+            let raw;
+            try {
+                raw = await readFile(fileFor(runId), "utf8");
+            }
+            catch {
+                return;
+            }
+            for (const line of raw.split("\n")) {
+                const trimmed = line.trim();
+                if (!trimmed)
+                    continue;
+                try {
+                    yield JSON.parse(trimmed);
+                }
+                catch { }
+            }
+        },
+        async *list() {
+            let names;
+            try {
+                names = await readdir(dir);
+            }
+            catch {
+                return;
+            }
+            for (const name of names) {
+                if (!name.endsWith(".jsonl"))
+                    continue;
+                yield { runId: name.slice(0, -".jsonl".length) };
+            }
+        },
+    };
+}
+function summarize(runId, entries) {
+    const meta = entries.find((entry) => entry.kind === "meta");
+    const metaData = (meta?.data ?? {});
+    const last = [...entries]
+        .reverse()
+        .find((entry) => entry.kind === "event" &&
+        (entry.type === "run.completed" || entry.type === "run.error"));
+    return {
+        runId,
+        ...(metaData.question ? { question: metaData.question } : {}),
+        status: last
+            ? last.type === "run.completed"
+                ? "completed"
+                : "failed"
+            : "incomplete",
+    };
+}
+export class JournalWriter {
+    store;
+    runId;
+    seq = 0;
+    pending = [];
+    flushing = Promise.resolve();
+    constructor(store, runId) {
+        this.store = store;
+        this.runId = runId;
+    }
+    meta(data) {
+        this.push({ seq: this.seq++, kind: "meta", data });
+    }
+    event(type, data) {
+        this.push({ seq: this.seq++, kind: "event", type, data });
+    }
+    call(callKey, data) {
+        this.push({ seq: this.seq++, kind: "call", callKey, data });
+    }
+    io(callKey, data) {
+        this.push({ seq: this.seq++, kind: "io", callKey, data });
+    }
+    trace(subKind, data) {
+        this.push({ seq: this.seq++, kind: "trace", type: subKind, data });
+    }
+    push(entry) {
+        this.pending.push(entry);
+        this.scheduleFlush();
+    }
+    scheduleFlush() {
+        this.flushing = this.flushing.then(async () => {
+            if (this.pending.length === 0)
+                return;
+            const batch = this.pending;
+            this.pending = [];
+            try {
+                await this.store.append(this.runId, batch);
+            }
+            catch {
+                return;
+            }
+        });
+    }
+    async flush() {
+        await this.flushing;
+        if (this.pending.length > 0) {
+            const batch = this.pending;
+            this.pending = [];
+            try {
+                await this.store.append(this.runId, batch);
+            }
+            catch {
+                return;
+            }
+        }
+    }
+}
+export class ReplayCache {
+    byKey = new Map();
+    hits = 0;
+    add(callKey, data) {
+        const queue = this.byKey.get(callKey) ?? [];
+        queue.push(data);
+        this.byKey.set(callKey, queue);
+    }
+    take(callKey) {
+        const queue = this.byKey.get(callKey);
+        if (!queue || queue.length === 0)
+            return undefined;
+        this.hits++;
+        return queue.shift();
+    }
+    values(prefix) {
+        const found = [];
+        for (const [key, queue] of this.byKey) {
+            if (key.startsWith(prefix))
+                found.push(...queue);
+        }
+        return found;
+    }
+    get replayedCalls() {
+        return this.hits;
+    }
+    get size() {
+        let total = 0;
+        for (const queue of this.byKey.values())
+            total += queue.length;
+        return total;
+    }
+}
+export async function loadReplayCache(store, runId) {
+    const cache = new ReplayCache();
+    for await (const entry of store.read(runId)) {
+        if ((entry.kind === "call" || entry.kind === "io") && entry.callKey) {
+            cache.add(entry.callKey, entry.data);
+        }
+    }
+    return cache;
+}
+export async function loadTrace(store, runId) {
+    const spans = [];
+    const steps = [];
+    let digest;
+    let found = false;
+    for await (const entry of store.read(runId)) {
+        if (entry.kind !== "trace")
+            continue;
+        found = true;
+        if (entry.type === "span")
+            spans.push(entry.data);
+        else if (entry.type === "step")
+            steps.push(entry.data);
+        else if (entry.type === "digest")
+            digest = entry.data;
+    }
+    if (!found)
+        return null;
+    return {
+        schemaVersion: TRACE_SCHEMA_VERSION,
+        mode: steps.length > 0 ? "full" : "spans",
+        spans,
+        steps,
+        ...(digest ? { digest } : {}),
+        degraded: Boolean(digest?.degraded),
+    };
+}
+export async function loadRunMeta(store, runId) {
+    for await (const entry of store.read(runId)) {
+        if (entry.kind === "meta") {
+            return (entry.data ?? {});
+        }
+    }
+    return null;
+}