@steel-dev/atlas 0.1.0

package/dist/providers/domain/openalex.js

@@ -0,0 +1,128 @@
+import { readEnv } from "../../env.js";
+import { errorMessage } from "../../errors.js";
+import { safeDomain, } from "../search.js";
+import { buildContent, clampLimit, collapse, fetchJson } from "./shared.js";
+const ENDPOINT = "https://api.openalex.org/works";
+export function openalex(opts = {}) {
+    const defaultLimit = clampLimit(opts.defaultLimit ?? 5);
+    const email = opts.email ?? readEnv("ATLAS_OPENALEX_EMAIL");
+    const sort = opts.sort === "date"
+        ? "publication_date:desc"
+        : opts.sort === "citations"
+            ? "cited_by_count:desc"
+            : undefined;
+    return {
+        id: "openalex",
+        async search({ query, maxResults, signal }) {
+            const q = query.trim();
+            if (!q)
+                return [];
+            const params = new URLSearchParams({
+                search: q,
+                per_page: String(clampLimit(maxResults ?? defaultLimit)),
+            });
+            if (sort)
+                params.set("sort", sort);
+            if (email)
+                params.set("mailto", email);
+            let data;
+            try {
+                data = await fetchJson(`${ENDPOINT}?${params.toString()}`, signal);
+            }
+            catch (err) {
+                throw new Error(`openalex: request failed: ${errorMessage(err)}`);
+            }
+            return toResults(data);
+        },
+    };
+}
+function toResults(data) {
+    const results = data && typeof data === "object"
+        ? data.results
+        : undefined;
+    if (!Array.isArray(results))
+        return [];
+    const out = [];
+    for (const row of results) {
+        const w = (row ?? {});
+        const title = collapse(String(w.title ?? w.display_name ?? ""));
+        const url = workUrl(w);
+        if (!title || !url)
+            continue;
+        const abstract = reconstructAbstract(w.abstract_inverted_index);
+        const authors = Array.isArray(w.authorships)
+            ? w.authorships
+                .map((a) => collapse(String(a?.author?.display_name ?? "")))
+                .filter(Boolean)
+                .slice(0, 12)
+            : [];
+        const venue = collapse(String(w.primary_location?.source?.display_name ?? ""));
+        const year = w.publication_year ? String(w.publication_year) : "";
+        const meta = [];
+        if (venue && year)
+            meta.push(`${venue} (${year})`);
+        else if (venue)
+            meta.push(venue);
+        else if (year)
+            meta.push(`(${year})`);
+        if (typeof w.cited_by_count === "number")
+            meta.push(`Cited by ${w.cited_by_count}`);
+        out.push({
+            position: out.length + 1,
+            title,
+            url,
+            snippet: collapse([meta.join(" · "), abstract].filter(Boolean).join(" — ")),
+            domain: safeDomain(url),
+            meta: {
+                openUrls: openAccessUrls(w),
+                fallbackText: buildContent({ title, authors, meta, abstract }),
+            },
+        });
+    }
+    return out;
+}
+function openAccessUrls(w) {
+    const out = [];
+    const push = (u) => {
+        if (typeof u === "string" && /^https?:\/\//.test(u))
+            out.push(u);
+    };
+    const best = w.best_oa_location;
+    push(best?.pdf_url);
+    push(best?.landing_page_url);
+    push(w.open_access?.oa_url);
+    const locs = w.oa_locations;
+    if (Array.isArray(locs)) {
+        for (const loc of locs) {
+            const l = loc;
+            push(l?.pdf_url);
+            push(l?.landing_page_url);
+        }
+    }
+    return out;
+}
+function workUrl(w) {
+    const doi = typeof w.doi === "string" ? w.doi : "";
+    if (doi)
+        return /^https?:\/\//.test(doi)
+            ? doi
+            : `https://doi.org/${doi.replace(/^doi:/, "")}`;
+    const landing = w.primary_location?.landing_page_url;
+    if (typeof landing === "string" && /^https?:\/\//.test(landing))
+        return landing;
+    const id = w.id;
+    return typeof id === "string" && /^https?:\/\//.test(id) ? id : "";
+}
+function reconstructAbstract(inv) {
+    if (!inv || typeof inv !== "object")
+        return "";
+    const slots = [];
+    for (const [word, positions] of Object.entries(inv)) {
+        if (!Array.isArray(positions))
+            continue;
+        for (const p of positions)
+            if (typeof p === "number")
+                slots[p] = word;
+    }
+    return collapse(slots.filter((s) => typeof s === "string").join(" "));
+}

package/dist/providers/domain/pubmed.d.ts

@@ -0,0 +1,8 @@
+import { type SearchProvider } from "../search.js";
+export interface PubmedOptions {
+    defaultLimit?: number;
+    sort?: "relevance" | "date";
+    apiKey?: string;
+    email?: string;
+}
+export declare function pubmed(opts?: PubmedOptions): SearchProvider;

package/dist/providers/domain/pubmed.js

@@ -0,0 +1,123 @@
+import * as cheerio from "cheerio";
+import { readEnv } from "../../env.js";
+import { errorMessage } from "../../errors.js";
+import { safeDomain, } from "../search.js";
+import { buildContent, clampLimit, collapse, fetchJson, fetchText, } from "./shared.js";
+const EUTILS = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils";
+export function pubmed(opts = {}) {
+    const defaultLimit = clampLimit(opts.defaultLimit ?? 5);
+    const sort = opts.sort === "date" ? "pub_date" : "relevance";
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_NCBI_API_KEY");
+    const email = opts.email ?? readEnv("ATLAS_NCBI_EMAIL");
+    const common = () => {
+        const p = { tool: "atlas" };
+        if (email)
+            p.email = email;
+        if (apiKey)
+            p.api_key = apiKey;
+        return p;
+    };
+    return {
+        id: "pubmed",
+        async search({ query, maxResults, signal }) {
+            const q = query.trim();
+            if (!q)
+                return [];
+            const limit = clampLimit(maxResults ?? defaultLimit);
+            let ids;
+            try {
+                const params = new URLSearchParams({
+                    db: "pubmed",
+                    term: q,
+                    retmode: "json",
+                    retmax: String(limit),
+                    sort,
+                    ...common(),
+                });
+                const data = await fetchJson(`${EUTILS}/esearch.fcgi?${params.toString()}`, signal);
+                ids = extractIds(data);
+            }
+            catch (err) {
+                throw new Error(`pubmed: search failed: ${errorMessage(err)}`);
+            }
+            if (ids.length === 0)
+                return [];
+            let xml;
+            try {
+                const params = new URLSearchParams({
+                    db: "pubmed",
+                    id: ids.join(","),
+                    rettype: "abstract",
+                    retmode: "xml",
+                    ...common(),
+                });
+                xml = await fetchText(`${EUTILS}/efetch.fcgi?${params.toString()}`, signal, "application/xml");
+            }
+            catch (err) {
+                throw new Error(`pubmed: fetch failed: ${errorMessage(err)}`);
+            }
+            return toResults(xml);
+        },
+    };
+}
+function extractIds(data) {
+    const idlist = data && typeof data === "object"
+        ? data.esearchresult?.idlist
+        : undefined;
+    return Array.isArray(idlist)
+        ? idlist.filter((x) => typeof x === "string")
+        : [];
+}
+function toResults(xml) {
+    const $ = cheerio.load(xml, { xml: true });
+    const out = [];
+    $("PubmedArticle").each((_, el) => {
+        const art = $(el);
+        const pmid = art.find("MedlineCitation > PMID").first().text().trim();
+        const title = collapse(art.find("ArticleTitle").first().text());
+        if (!pmid || !title)
+            return;
+        const abstract = art
+            .find("Abstract > AbstractText")
+            .map((_, t) => {
+            const label = $(t).attr("Label");
+            const text = collapse($(t).text());
+            return label && text ? `${label}: ${text}` : text;
+        })
+            .get()
+            .filter(Boolean)
+            .join("\n");
+        const authors = art
+            .find("AuthorList > Author")
+            .map((_, a) => {
+            const last = collapse($(a).find("LastName").first().text());
+            const fore = collapse($(a).find("ForeName").first().text());
+            return [fore, last].filter(Boolean).join(" ");
+        })
+            .get()
+            .filter(Boolean)
+            .slice(0, 12);
+        const journal = collapse(art.find("Journal > Title").first().text());
+        const year = art.find("PubDate > Year").first().text().trim() ||
+            art.find("PubDate > MedlineDate").first().text().trim().slice(0, 4);
+        const venue = journal && year ? `${journal} (${year})` : journal;
+        const url = `https://pubmed.ncbi.nlm.nih.gov/${pmid}/`;
+        out.push({
+            position: out.length + 1,
+            title,
+            url,
+            snippet: collapse([venue, abstract].filter(Boolean).join(" — ")),
+            domain: safeDomain(url),
+            meta: {
+                openUrls: [],
+                fallbackText: buildContent({
+                    title,
+                    authors,
+                    meta: venue ? [venue] : [],
+                    abstract,
+                }),
+            },
+        });
+    });
+    return out;
+}

package/dist/providers/domain/semantic-scholar.d.ts

@@ -0,0 +1,6 @@
+import { type ResearchTool } from "../../custom-tools.js";
+export interface SemanticScholarOptions {
+    defaultLimit?: number;
+    apiKey?: string;
+}
+export declare function semanticScholar(opts?: SemanticScholarOptions): ResearchTool;

package/dist/providers/domain/semantic-scholar.js

@@ -0,0 +1,112 @@
+import { jsonSchema } from "ai";
+import { researchTool, } from "../../custom-tools.js";
+import { readEnv } from "../../env.js";
+import { errorMessage } from "../../errors.js";
+import { buildContent, clampLimit, collapse, manifest, USER_AGENT, } from "./shared.js";
+const ENDPOINT = "https://api.semanticscholar.org/graph/v1/paper/search";
+const FIELDS = "title,abstract,authors,year,venue,citationCount,tldr,externalIds,url";
+const RATE_LIMITED = "rate-limited";
+export function semanticScholar(opts = {}) {
+    const defaultLimit = clampLimit(opts.defaultLimit ?? 5);
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_S2_API_KEY");
+    return researchTool({
+        description: "Search Semantic Scholar, an AI-powered index of scientific papers across all fields. Returns abstracts, one-line TL;DR summaries and citation counts as cited sources.",
+        inputSchema: jsonSchema({
+            type: "object",
+            properties: {
+                query: { type: "string", description: "Search query" },
+            },
+            required: ["query"],
+            additionalProperties: false,
+        }),
+        async execute(input, ctx) {
+            const query = String(input.query ?? "").trim();
+            if (!query)
+                return "semantic-scholar: empty query";
+            const params = new URLSearchParams({
+                query,
+                limit: String(defaultLimit),
+                fields: FIELDS,
+            });
+            let data;
+            try {
+                data = await search(`${ENDPOINT}?${params.toString()}`, apiKey, ctx.signal);
+            }
+            catch (err) {
+                const message = errorMessage(err);
+                if (message === RATE_LIMITED)
+                    return "semantic-scholar: rate limited — set ATLAS_S2_API_KEY (or pass { apiKey }) for higher limits.";
+                return `semantic-scholar: request failed: ${message}`;
+            }
+            return manifest("semantic-scholar", query, ingest(data, ctx));
+        },
+    });
+}
+async function search(url, apiKey, signal) {
+    const headers = {
+        "user-agent": USER_AGENT,
+        accept: "application/json",
+    };
+    if (apiKey)
+        headers["x-api-key"] = apiKey;
+    const resp = await fetch(url, { signal, headers });
+    if (resp.status === 429)
+        throw new Error(RATE_LIMITED);
+    if (!resp.ok)
+        throw new Error(`HTTP ${resp.status} ${resp.statusText}`.trim());
+    return resp.json();
+}
+function ingest(data, ctx) {
+    const rows = data && typeof data === "object"
+        ? data.data
+        : undefined;
+    if (!Array.isArray(rows))
+        return [];
+    const titles = [];
+    for (const row of rows) {
+        const p = (row ?? {});
+        const title = collapse(String(p.title ?? ""));
+        const url = paperUrl(p);
+        if (!title || !url)
+            continue;
+        const authors = Array.isArray(p.authors)
+            ? p.authors
+                .map((a) => collapse(String(a?.name ?? "")))
+                .filter(Boolean)
+                .slice(0, 12)
+            : [];
+        const venue = collapse(String(p.venue ?? ""));
+        const year = p.year ? String(p.year) : "";
+        const tldr = collapse(String(p.tldr?.text ?? ""));
+        const abstract = collapse(String(p.abstract ?? ""));
+        const meta = [];
+        if (venue && year)
+            meta.push(`${venue} (${year})`);
+        else if (venue)
+            meta.push(venue);
+        else if (year)
+            meta.push(`(${year})`);
+        if (typeof p.citationCount === "number")
+            meta.push(`Cited by ${p.citationCount}`);
+        if (tldr)
+            meta.push(`TL;DR: ${tldr}`);
+        ctx.addSource({
+            url,
+            title,
+            content: buildContent({ title, authors, meta, abstract }),
+        });
+        titles.push(title);
+    }
+    return titles;
+}
+function paperUrl(p) {
+    const doi = p.externalIds?.DOI;
+    if (typeof doi === "string" && doi.trim())
+        return `https://doi.org/${doi.replace(/^doi:/i, "").trim()}`;
+    if (typeof p.url === "string" && /^https?:\/\//.test(p.url))
+        return p.url;
+    const id = p.paperId;
+    return typeof id === "string" && id
+        ? `https://www.semanticscholar.org/paper/${id}`
+        : "";
+}

package/dist/providers/domain/shared.d.ts

@@ -0,0 +1,12 @@
+export declare const USER_AGENT = "atlas-research/0.1 (+https://github.com/steel-experiments/atlas)";
+export declare function collapse(text: string): string;
+export declare function clampLimit(n: number, max?: number): number;
+export declare function fetchText(url: string, signal: AbortSignal | undefined, accept: string): Promise<string>;
+export declare function fetchJson(url: string, signal: AbortSignal | undefined): Promise<unknown>;
+export declare function buildContent(parts: {
+    title: string;
+    authors?: string[];
+    meta?: string[];
+    abstract?: string;
+}): string;
+export declare function manifest(tool: string, query: string, titles: string[]): string;

package/dist/providers/domain/shared.js

@@ -0,0 +1,39 @@
+export const USER_AGENT = "atlas-research/0.1 (+https://github.com/steel-experiments/atlas)";
+export function collapse(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+export function clampLimit(n, max = 25) {
+    if (!Number.isFinite(n))
+        return 5;
+    return Math.min(Math.max(1, Math.floor(n)), max);
+}
+export async function fetchText(url, signal, accept) {
+    const resp = await fetch(url, {
+        signal,
+        headers: { "user-agent": USER_AGENT, accept },
+    });
+    if (!resp.ok)
+        throw new Error(`HTTP ${resp.status} ${resp.statusText}`.trim());
+    return resp.text();
+}
+export async function fetchJson(url, signal) {
+    return JSON.parse(await fetchText(url, signal, "application/json"));
+}
+export function buildContent(parts) {
+    const lines = [parts.title];
+    if (parts.authors?.length)
+        lines.push(`Authors: ${parts.authors.join(", ")}`);
+    for (const m of parts.meta ?? [])
+        if (m)
+            lines.push(m);
+    const body = parts.abstract?.trim();
+    if (body)
+        lines.push("", body);
+    return lines.join("\n");
+}
+export function manifest(tool, query, titles) {
+    if (titles.length === 0)
+        return `${tool}: no results for "${query}"`;
+    const list = titles.map((t) => `- ${t}`).join("\n");
+    return `${tool}: found ${titles.length} result(s) for "${query}"; submitted as sources:\n${list}`;
+}

package/dist/providers/domain/wikipedia.d.ts

@@ -0,0 +1,6 @@
+import { type ResearchTool } from "../../custom-tools.js";
+export interface WikipediaOptions {
+    defaultLimit?: number;
+    lang?: string;
+}
+export declare function wikipedia(opts?: WikipediaOptions): ResearchTool;

package/dist/providers/domain/wikipedia.js

@@ -0,0 +1,71 @@
+import { jsonSchema } from "ai";
+import { researchTool, } from "../../custom-tools.js";
+import { errorMessage } from "../../errors.js";
+import { buildContent, clampLimit, collapse, fetchJson, manifest, } from "./shared.js";
+export function wikipedia(opts = {}) {
+    const defaultLimit = clampLimit(opts.defaultLimit ?? 3, 10);
+    const lang = (opts.lang ?? "en").toLowerCase();
+    const api = `https://${lang}.wikipedia.org/w/api.php`;
+    return researchTool({
+        description: "Search Wikipedia for encyclopedic background on people, places, organizations, concepts, and events. Returns article introductions as cited sources.",
+        inputSchema: jsonSchema({
+            type: "object",
+            properties: { query: { type: "string", description: "Search query" } },
+            required: ["query"],
+            additionalProperties: false,
+        }),
+        async execute(input, ctx) {
+            const query = String(input.query ?? "").trim();
+            if (!query)
+                return "wikipedia: empty query";
+            const params = new URLSearchParams({
+                action: "query",
+                generator: "search",
+                gsrsearch: query,
+                gsrlimit: String(defaultLimit),
+                prop: "extracts",
+                exintro: "1",
+                explaintext: "1",
+                format: "json",
+                formatversion: "2",
+            });
+            let data;
+            try {
+                data = await fetchJson(`${api}?${params.toString()}`, ctx.signal);
+            }
+            catch (err) {
+                return `wikipedia: request failed: ${errorMessage(err)}`;
+            }
+            return manifest("wikipedia", query, ingest(data, lang, ctx));
+        },
+    });
+}
+function ingest(data, lang, ctx) {
+    const pages = data && typeof data === "object"
+        ? data.query?.pages
+        : undefined;
+    if (!Array.isArray(pages))
+        return [];
+    const rows = pages
+        .map((p) => {
+        const rec = (p ?? {});
+        return {
+            title: String(rec.title ?? ""),
+            extract: collapse(String(rec.extract ?? "")),
+            index: typeof rec.index === "number" ? rec.index : 0,
+        };
+    })
+        .filter((p) => p.title && p.extract)
+        .sort((a, b) => a.index - b.index);
+    const titles = [];
+    for (const row of rows) {
+        const url = `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(row.title.replace(/ /g, "_"))}`;
+        ctx.addSource({
+            url,
+            title: row.title,
+            content: buildContent({ title: row.title, abstract: row.extract }),
+        });
+        titles.push(row.title);
+    }
+    return titles;
+}

package/dist/providers/exa-agent.d.ts

@@ -0,0 +1,9 @@
+import type { Researcher } from "../researcher.js";
+export interface ExaAgentOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    model?: "exa-research-fast" | "exa-research" | "exa-research-pro";
+    timeoutMs?: number;
+    description?: string;
+}
+export declare function exaAgent(opts?: ExaAgentOptions): Researcher;

package/dist/providers/exa-agent.js

@@ -0,0 +1,67 @@
+import { Exa } from "exa-js";
+import { readEnv } from "../env.js";
+const DEFAULT_DESCRIPTION = "Exa's agentic deep-research (exa-research): autonomously searches, reads, and synthesizes a grounded report. Strong on shopping/product comparison, personalized and recency-heavy queries.";
+const DEFAULT_TIMEOUT_MS = 10 * 60_000;
+const URL_RE = /https?:\/\/[^\s)<>\]]+/g;
+function abortError(signal) {
+    return signal.reason instanceof Error
+        ? signal.reason
+        : new DOMException("The operation was aborted", "AbortError");
+}
+async function raceAbort(p, signal) {
+    if (!signal)
+        return p;
+    p.catch(() => { });
+    if (signal.aborted)
+        throw abortError(signal);
+    let onAbort;
+    const abortP = new Promise((_, reject) => {
+        onAbort = () => reject(abortError(signal));
+        signal.addEventListener("abort", onAbort, { once: true });
+    });
+    try {
+        return await Promise.race([p, abortP]);
+    }
+    finally {
+        signal.removeEventListener("abort", onAbort);
+    }
+}
+function sourcesFromText(text) {
+    const seen = new Set();
+    const out = [];
+    for (const match of text.matchAll(URL_RE)) {
+        const url = match[0].replace(/[.,;:]+$/, "");
+        if (!seen.has(url)) {
+            seen.add(url);
+            out.push({ url });
+        }
+    }
+    return out;
+}
+export function exaAgent(opts = {}) {
+    const apiKey = opts.apiKey ?? readEnv("ATLAS_EXA_API_KEY", "EXA_API_KEY");
+    return {
+        description: opts.description ?? DEFAULT_DESCRIPTION,
+        async research(query, ctx) {
+            if (!apiKey) {
+                throw new Error("exa.agent: no Exa API key (set ATLAS_EXA_API_KEY / EXA_API_KEY or pass { apiKey })");
+            }
+            const client = new Exa(apiKey, opts.baseUrl);
+            const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+            const created = (await raceAbort(client.research.create({
+                instructions: query,
+                model: opts.model ?? "exa-research",
+            }), ctx.signal));
+            const result = await raceAbort(client.research.pollUntilFinished(created.researchId, { timeoutMs }), ctx.signal);
+            if (result.status !== "completed") {
+                throw new Error(`exa.agent: research ${result.status}`);
+            }
+            const fields = result;
+            const report = (fields.output?.content ?? "").trim();
+            const sources = sourcesFromText(report);
+            const cost = fields.costDollars?.total;
+            ctx.log(`exa.agent: ${sources.length} sources${cost != null ? `, $${cost}` : ""}`);
+            return { report, sources, ...(cost != null ? { cost } : {}) };
+        },
+    };
+}

package/dist/providers/fetch.d.ts

@@ -0,0 +1,66 @@
+import type { Dispatcher } from "undici";
+import type { SourceExtractionAttempt, SourceExtractionMetadata } from "../sources.js";
+export declare function looksBlocked(text: string | undefined | null): boolean;
+export declare function looksBlockedPage(markdown: string, raw?: string): boolean;
+export interface FetchedPage {
+    finalUrl: string;
+    title: string | null;
+    markdown: string;
+    metadata: SourceExtractionMetadata;
+    renderedWith: string;
+}
+export type FetchAttempt = {
+    ok: true;
+    page: FetchedPage;
+    attempt: SourceExtractionAttempt;
+} | {
+    ok: false;
+    attempt: SourceExtractionAttempt;
+    escalate: boolean;
+};
+export interface FetchRequest {
+    url: string;
+    signal?: AbortSignal | undefined;
+    onRateLimit?: ((retryAfterSeconds: number) => void) | undefined;
+    guardRedirect?: ((url: string) => Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>) | undefined;
+    dispatcher?: Dispatcher | undefined;
+}
+export interface FetchProvider {
+    readonly id: string;
+    fetch(req: FetchRequest): Promise<FetchAttempt>;
+}
+export declare function isLikelyPdfUrl(url: string): boolean;
+export declare function basicFetch(): FetchProvider;
+export interface SteelOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    proxy?: boolean;
+}
+export declare function steel(opts?: SteelOptions): FetchProvider;
+export interface SteelScrapeResponse {
+    content?: {
+        html?: string;
+        markdown?: string;
+    };
+    metadata?: {
+        statusCode?: number;
+        canonical?: string;
+        title?: string;
+    };
+}
+export declare function steelAttemptFromResponse(response: SteelScrapeResponse, url: string): FetchAttempt;
+export declare function exaContents(opts?: {
+    apiKey?: string;
+    baseUrl?: string;
+}): FetchProvider;
+export declare function defaultFetchProviders(): FetchProvider[];
+export interface ChainFetchOutcome {
+    page: FetchedPage | null;
+    attempts: SourceExtractionAttempt[];
+}
+export declare function fetchThroughChain(chain: FetchProvider[], req: FetchRequest): Promise<ChainFetchOutcome>;