@steel-dev/atlas 0.1.0

package/dist/researcher.d.ts

@@ -0,0 +1,20 @@
+export interface ResearcherContext {
+    budget: {
+        maxUSD: number;
+    };
+    readonly signal?: AbortSignal | undefined;
+    log(message: string): void;
+}
+export interface ResearchReport {
+    report: string;
+    sources: {
+        url: string;
+        title?: string;
+    }[];
+    cost?: number;
+}
+export interface Researcher {
+    description: string;
+    research(query: string, ctx: ResearcherContext): Promise<ResearchReport>;
+}
+export declare function researcher(r: Researcher): Researcher;

package/dist/researcher.js

@@ -0,0 +1,3 @@
+export function researcher(r) {
+    return r;
+}

package/dist/robots.d.ts

@@ -0,0 +1,16 @@
+import type { Dispatcher } from "undici";
+export interface RobotsRule {
+    allow: boolean;
+    pattern: string;
+}
+export declare function parseRobots(text: string, agentToken: string): RobotsRule[];
+export declare function robotsAllows(rules: RobotsRule[], path: string): boolean;
+export interface RobotsCache {
+    allows(url: string, signal?: AbortSignal, dispatcher?: Dispatcher): Promise<boolean>;
+}
+export interface RobotsCacheOptions {
+    agentToken: string;
+    userAgent: string;
+    fetchImpl?: typeof fetch;
+}
+export declare function createRobotsCache(opts: RobotsCacheOptions): RobotsCache;

package/dist/robots.js

@@ -0,0 +1,146 @@
+const ROBOTS_FETCH_TIMEOUT_MS = 5_000;
+const ROBOTS_MAX_CHARS = 512_000;
+function robotsMatches(pattern, path) {
+    const anchored = pattern.endsWith("$");
+    const pat = anchored ? pattern.slice(0, -1) : pattern;
+    let s = 0;
+    let p = 0;
+    let starIdx = -1;
+    let matchIdx = 0;
+    while (s < path.length) {
+        if (p < pat.length && pat[p] === "*") {
+            starIdx = p;
+            matchIdx = s;
+            p++;
+        }
+        else if (p < pat.length && pat[p] === path[s]) {
+            p++;
+            s++;
+        }
+        else if (starIdx !== -1) {
+            p = starIdx + 1;
+            matchIdx++;
+            s = matchIdx;
+        }
+        else {
+            return false;
+        }
+        if (!anchored && p === pat.length)
+            return true;
+    }
+    while (p < pat.length && pat[p] === "*")
+        p++;
+    if (anchored)
+        return p === pat.length && s === path.length;
+    return p === pat.length;
+}
+export function parseRobots(text, agentToken) {
+    const token = agentToken.toLowerCase();
+    const groups = [];
+    let current = null;
+    let agentsOpen = false;
+    for (const rawLine of text.slice(0, ROBOTS_MAX_CHARS).split(/\r?\n/)) {
+        const line = rawLine.split("#")[0].trim();
+        if (!line)
+            continue;
+        const separator = line.indexOf(":");
+        if (separator < 0)
+            continue;
+        const field = line.slice(0, separator).trim().toLowerCase();
+        const value = line.slice(separator + 1).trim();
+        if (field === "user-agent") {
+            if (!agentsOpen || !current) {
+                current = { agents: [], rules: [] };
+                groups.push(current);
+                agentsOpen = true;
+            }
+            current.agents.push(value.toLowerCase());
+        }
+        else if (field === "allow" || field === "disallow") {
+            if (!current)
+                continue;
+            agentsOpen = false;
+            if (value) {
+                current.rules.push({
+                    allow: field === "allow",
+                    pattern: value,
+                });
+            }
+        }
+        else {
+            agentsOpen = false;
+        }
+    }
+    let best = null;
+    let bestAgentLength = -1;
+    let wildcard = null;
+    for (const group of groups) {
+        for (const agent of group.agents) {
+            if (agent === "*") {
+                wildcard = [...(wildcard ?? []), ...group.rules];
+            }
+            else if (token.includes(agent) && agent.length > bestAgentLength) {
+                best = group.rules;
+                bestAgentLength = agent.length;
+            }
+        }
+    }
+    return best ?? wildcard ?? [];
+}
+export function robotsAllows(rules, path) {
+    let verdict = true;
+    let matchedLength = -1;
+    for (const rule of rules) {
+        if (!robotsMatches(rule.pattern, path))
+            continue;
+        if (rule.pattern.length > matchedLength ||
+            (rule.pattern.length === matchedLength && rule.allow && !verdict)) {
+            matchedLength = rule.pattern.length;
+            verdict = rule.allow;
+        }
+    }
+    return verdict;
+}
+export function createRobotsCache(opts) {
+    const fetchImpl = opts.fetchImpl ?? fetch;
+    const byOrigin = new Map();
+    function load(origin, signal, dispatcher) {
+        let pending = byOrigin.get(origin);
+        if (!pending) {
+            pending = (async () => {
+                try {
+                    const timeout = AbortSignal.timeout(ROBOTS_FETCH_TIMEOUT_MS);
+                    const response = await fetchImpl(`${origin}/robots.txt`, {
+                        signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+                        headers: { "user-agent": opts.userAgent },
+                        ...(dispatcher ? { dispatcher } : {}),
+                    });
+                    if (!response.ok)
+                        return null;
+                    const text = await response.text();
+                    return parseRobots(text, opts.agentToken);
+                }
+                catch {
+                    return null;
+                }
+            })();
+            byOrigin.set(origin, pending);
+        }
+        return pending;
+    }
+    return {
+        async allows(url, signal, dispatcher) {
+            let parsed;
+            try {
+                parsed = new URL(url);
+            }
+            catch {
+                return true;
+            }
+            const rules = await load(parsed.origin, signal, dispatcher);
+            if (!rules || rules.length === 0)
+                return true;
+            return robotsAllows(rules, parsed.pathname + parsed.search);
+        },
+    };
+}

package/dist/roles.d.ts

@@ -0,0 +1,6 @@
+import type { AgentRole } from "./events.js";
+export interface RoleCapabilities {
+    budgetLine: boolean;
+    customTools: boolean;
+}
+export declare const ROLE_CAPABILITIES: Record<AgentRole, RoleCapabilities>;

package/dist/roles.js

@@ -0,0 +1,4 @@
+export const ROLE_CAPABILITIES = {
+    research: { budgetLine: true, customTools: true },
+    write: { budgetLine: false, customTools: false },
+};

package/dist/run.d.ts

@@ -0,0 +1,65 @@
+import type { FlexibleSchema } from "ai";
+import { type AtlasConfig, type ResearchOptions } from "./config.js";
+import type { Citation, ResearchEvent, RunStats, StopReason } from "./events.js";
+import { type ReplayCache } from "./providers/store.js";
+import type { RunTrace } from "./trace.js";
+export type RunStatus = "running" | "completed" | "failed" | "aborted" | "paused";
+export interface SourceRecord {
+    id: string;
+    url: string;
+    finalUrl: string;
+    title: string;
+    via: string;
+    chars: number;
+    warnings?: string[];
+}
+export interface ResearchResult {
+    runId: string;
+    question: string;
+    report: string;
+    note: string;
+    sources: SourceRecord[];
+    citations: Citation[];
+    unboundCitations: string[];
+    warnings: string[];
+    stats: RunStats;
+    trace?: RunTrace;
+    eventVersion: string;
+}
+export interface ResearchRun {
+    readonly id: string;
+    events(): AsyncIterable<ResearchEvent>;
+    result(): Promise<ResearchResult>;
+    abort(): Promise<void>;
+    pause(): Promise<void>;
+    finish(): Promise<void>;
+    status(): RunStatus;
+    trace(): RunTrace | undefined;
+}
+export interface StartRunOptions {
+    config: AtlasConfig;
+    question: string;
+    options: ResearchOptions;
+    schema?: FlexibleSchema<unknown> | undefined;
+    replay?: ReplayCache | undefined;
+    anchorStartedAt?: number | undefined;
+    now?: (() => number) | undefined;
+}
+export declare function startRun(start: StartRunOptions): ResearchRun;
+interface RepairBalance {
+    citationsUnsupported: number;
+    citationsBound: number;
+}
+export declare function acceptsRepair(before: RepairBalance, after: RepairBalance): boolean;
+export interface StopReasonInputs {
+    finished: boolean;
+    budgetExhausted: boolean;
+    tokensExhausted: boolean;
+    timedOut: boolean;
+}
+export declare function deriveStopReason(inputs: StopReasonInputs): StopReason;
+export type ResumeOptions = Pick<ResearchOptions, "signal">;
+export declare function resumeRun(runId: string, config: AtlasConfig, resume?: ResumeOptions & {
+    now?: () => number;
+}): Promise<ResearchRun>;
+export {};

package/dist/run.js

@@ -0,0 +1,371 @@
+import { randomUUID } from "node:crypto";
+import { resolveRunConfig, } from "./config.js";
+import { assembleRun } from "./context.js";
+import { AtlasError, errorMessage } from "./errors.js";
+import { EventHub } from "./event-hub.js";
+import { EVENT_SCHEMA_VERSION } from "./events.js";
+import { totalFreshTokens } from "./model.js";
+import { runOrchestrated } from "./orchestrate.js";
+import { isoDate } from "./prompts.js";
+import { JournalWriter, loadReplayCache, loadRunMeta, memoryStore, } from "./providers/store.js";
+import { runSpine } from "./spine.js";
+import { extractStructured } from "./structured.js";
+import { computeDigest } from "./trace-digest.js";
+const EXTRACTION_FRACTION = 0.1;
+const EXTRACTION_MIN_USD = 0.02;
+export function startRun(start) {
+    const question = start.question?.trim();
+    if (!question) {
+        throw new AtlasError("research question is required", "config");
+    }
+    const resolved = resolveRunConfig(start.config, start.options);
+    const runId = start.options.runId ?? `run_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
+    const store = start.config.store ?? memoryStore();
+    const hub = new EventHub();
+    const hardController = new AbortController();
+    const stopController = new AbortController();
+    let statusValue = "running";
+    let pauseRequested = false;
+    let deadlineHit = false;
+    const deadlineTimer = resolved.maxDurationMs !== undefined &&
+        Number.isFinite(resolved.maxDurationMs) &&
+        resolved.maxDurationMs > 0
+        ? setTimeout(() => {
+            deadlineHit = true;
+            hardController.abort();
+        }, resolved.maxDurationMs)
+        : undefined;
+    let recorder;
+    const externalSignal = start.options.signal;
+    const onExternalAbort = () => hardController.abort(externalSignal?.reason);
+    if (externalSignal) {
+        if (externalSignal.aborted)
+            hardController.abort(externalSignal.reason);
+        else
+            externalSignal.addEventListener("abort", onExternalAbort, {
+                once: true,
+            });
+    }
+    const resultPromise = (async () => {
+        await Promise.resolve();
+        const journal = new JournalWriter(store, runId);
+        try {
+            const result = await executeRun({
+                runId,
+                question,
+                resolved,
+                config: start.config,
+                schema: start.schema,
+                journal,
+                replay: start.replay,
+                hub,
+                hardSignal: hardController.signal,
+                stopSignal: stopController.signal,
+                now: start.now ?? Date.now,
+                anchorStartedAt: start.anchorStartedAt,
+                captureRecorder: (r) => {
+                    recorder = r;
+                },
+            });
+            statusValue = "completed";
+            return result;
+        }
+        catch (err) {
+            if (pauseRequested) {
+                statusValue = "paused";
+                journal.event("run.paused", { runId });
+                throw new AtlasError("run paused; resume with atlas.resume()", "paused");
+            }
+            if (deadlineHit) {
+                statusValue = "failed";
+                const event = {
+                    type: "run.error",
+                    message: "run exceeded maxDurationMs before completing",
+                    recoverable: false,
+                };
+                hub.emit(event);
+                journal.event(event.type, event);
+                throw new AtlasError("run exceeded maxDurationMs", "timeout");
+            }
+            if (hardController.signal.aborted) {
+                statusValue = "aborted";
+                throw new AtlasError("run aborted", "aborted");
+            }
+            statusValue = "failed";
+            const event = {
+                type: "run.error",
+                message: errorMessage(err),
+                recoverable: false,
+            };
+            hub.emit(event);
+            journal.event(event.type, event);
+            throw err;
+        }
+        finally {
+            if (deadlineTimer)
+                clearTimeout(deadlineTimer);
+            externalSignal?.removeEventListener("abort", onExternalAbort);
+            await journal.flush();
+            hub.close();
+        }
+    })();
+    resultPromise.catch(() => { });
+    return {
+        id: runId,
+        events: () => hub.iterable(),
+        result: () => resultPromise,
+        status: () => statusValue,
+        trace: () => recorder?.snapshot(),
+        abort: async () => {
+            hardController.abort();
+            await resultPromise.catch(() => { });
+        },
+        pause: async () => {
+            pauseRequested = true;
+            hardController.abort();
+            await resultPromise.catch(() => { });
+        },
+        finish: async () => {
+            stopController.abort();
+            await resultPromise.catch(() => { });
+        },
+    };
+}
+async function executeRun(args) {
+    const { resolved, question, runId } = args;
+    const startedAt = args.now();
+    const { rctx, meter, synthesisGrant } = await assembleRun({
+        runId,
+        question,
+        todayISO: isoDate(args.anchorStartedAt ?? startedAt),
+        resolved,
+        config: args.config,
+        journal: args.journal,
+        replay: args.replay,
+        hub: args.hub,
+        hardSignal: args.hardSignal,
+        stopSignal: args.stopSignal,
+        now: args.now,
+        startedAt,
+    });
+    const { emit } = rctx;
+    args.captureRecorder?.(rctx.recorder);
+    args.journal.meta({
+        runId,
+        question,
+        effort: resolved.effort,
+        budgetUSD: resolved.budgetUSD,
+        maxTokens: resolved.maxTokens,
+        ...(resolved.maxDurationMs !== undefined
+            ? { maxDurationMs: resolved.maxDurationMs }
+            : {}),
+        maxSources: resolved.maxSources,
+        ...(resolved.sourceFilter ? { sourceFilter: resolved.sourceFilter } : {}),
+        eventVersion: EVENT_SCHEMA_VERSION,
+        startedAt,
+    });
+    emit({
+        type: "run.started",
+        runId,
+        question,
+        effort: resolved.effort,
+        budgetUSD: resolved.budgetUSD,
+    });
+    if (!rctx.runCodeEnabled) {
+        emit({
+            type: "run_code.unavailable",
+            detail: 'the optional "isolated-vm" sandbox dependency is not installed or failed to build, ' +
+                "so the run_code tool is omitted this run; install it to let agents compute over source text. " +
+                "Research continues without it.",
+        });
+    }
+    synthesisGrant.release();
+    const extractionGrant = args.schema
+        ? meter.grant({
+            fraction: EXTRACTION_FRACTION,
+            minUSD: EXTRACTION_MIN_USD,
+        })
+        : null;
+    const out = Object.keys(resolved.researchers).length > 0
+        ? await runOrchestrated(rctx, resolved.researchers)
+        : await runSpine(rctx, { meter });
+    emit({ type: "report.completed", report: out.report });
+    let structured;
+    if (args.schema) {
+        try {
+            structured = await extractStructured(rctx.bindModel("write", extractionGrant ?? meter), question, out.report, args.schema, args.hardSignal);
+        }
+        finally {
+            extractionGrant?.release();
+        }
+    }
+    const durationMs = args.now() - startedAt;
+    const stats = buildStats({
+        rctx,
+        bound: {
+            citationsBound: out.citations.length,
+            citationsUnsupported: out.unboundCitations.length,
+        },
+        durationMs,
+        finished: args.stopSignal.aborted,
+    });
+    if (rctx.recorder) {
+        rctx.recorder.finalize(computeDigest(rctx.recorder.spans, rctx.recorder.steps, {
+            runId,
+            wallMs: durationMs,
+            costUSD: stats.costUSD,
+            freshTokens: totalFreshTokens(rctx.usage),
+            replayedUSD: rctx.usage.replayedUSD,
+            gateLimitModel: resolved.maxConcurrentModelCalls,
+            gateLimitIo: resolved.maxConcurrentIo,
+        }));
+    }
+    const result = {
+        runId,
+        question,
+        report: out.report,
+        note: out.note,
+        sources: out.sources
+            ? out.sources.map((source, index) => ({
+                id: `source_${index + 1}`,
+                url: source.url,
+                finalUrl: source.url,
+                title: source.title,
+                via: source.via,
+                chars: source.chars ?? 0,
+            }))
+            : rctx.sources.fetchedSources.map((source) => {
+                const document = source.sourceId
+                    ? rctx.sources.byId.get(source.sourceId)
+                    : undefined;
+                return {
+                    id: source.sourceId ?? "",
+                    url: source.url,
+                    finalUrl: document?.metadata.finalUrl ?? source.url,
+                    title: source.title,
+                    via: document?.metadata.method ?? "unknown",
+                    chars: document?.storedChars ?? 0,
+                    ...(document?.metadata.qualityWarnings
+                        ? { warnings: document.metadata.qualityWarnings }
+                        : {}),
+                };
+            }),
+        citations: out.citations,
+        unboundCitations: out.unboundCitations,
+        warnings: out.warnings ?? [],
+        stats,
+        ...(rctx.recorder ? { trace: rctx.recorder.snapshot() } : {}),
+        eventVersion: EVENT_SCHEMA_VERSION,
+    };
+    if (args.schema) {
+        result.object = structured;
+    }
+    emit({ type: "run.completed", stats });
+    return result;
+}
+export function acceptsRepair(before, after) {
+    return (after.citationsUnsupported < before.citationsUnsupported &&
+        after.citationsBound >= before.citationsBound);
+}
+export function deriveStopReason(inputs) {
+    if (inputs.finished)
+        return "finished";
+    if (inputs.budgetExhausted)
+        return "budget";
+    if (inputs.tokensExhausted)
+        return "tokens";
+    if (inputs.timedOut)
+        return "timeout";
+    return "completed";
+}
+function buildStats(opts) {
+    const { rctx } = opts;
+    const tokens = {};
+    for (const [role, roleUsage] of rctx.usage.byRole) {
+        tokens[role] = {
+            input: roleUsage.input + roleUsage.cacheRead + roleUsage.cacheWrite,
+            output: roleUsage.output,
+        };
+    }
+    const budgetExhausted = rctx.meter.exhausted();
+    const tokensExhausted = totalFreshTokens(rctx.usage) >= rctx.config.maxTokens;
+    const timedOut = rctx.deadlineAt !== undefined && rctx.now() >= rctx.deadlineAt;
+    return {
+        effort: rctx.config.effort,
+        searches: rctx.counters.searches,
+        searchCacheHits: rctx.counters.searchCacheHits,
+        modelCacheHits: rctx.counters.modelCacheHits,
+        modelGatePeakWidth: rctx.counters.modelGatePeakWidth,
+        sourcesFetched: rctx.counters.sourcesFetched,
+        sourcesFailed: rctx.counters.sourcesFailed,
+        citationsBound: opts.bound.citationsBound,
+        citationsUnsupported: opts.bound.citationsUnsupported,
+        tokens,
+        costUSD: Math.round(Math.max(0, rctx.meter.totalSpentUSD() - rctx.usage.replayedUSD) *
+            10_000) / 10_000,
+        durationMs: opts.durationMs,
+        budgetExhausted,
+        tokensExhausted,
+        stopReason: deriveStopReason({
+            finished: opts.finished,
+            budgetExhausted,
+            tokensExhausted,
+            timedOut,
+        }),
+    };
+}
+function sourceFilterFromMeta(value) {
+    if (!value || typeof value !== "object")
+        return undefined;
+    const filter = value;
+    const domains = (key) => Array.isArray(filter[key]) &&
+        filter[key].every((domain) => typeof domain === "string")
+        ? { [key]: filter[key] }
+        : {};
+    const restored = {
+        ...domains("includeDomains"),
+        ...domains("excludeDomains"),
+    };
+    return Object.keys(restored).length > 0 ? restored : undefined;
+}
+export async function resumeRun(runId, config, resume = {}) {
+    const store = config.store;
+    if (!store) {
+        throw new AtlasError("resume requires config.store (the store the original run journaled to)", "resume");
+    }
+    const meta = await loadRunMeta(store, runId);
+    if (!meta || typeof meta.question !== "string") {
+        throw new AtlasError(`no journaled run found for "${runId}"`, "resume");
+    }
+    const replay = await loadReplayCache(store, runId);
+    const budget = {
+        ...(typeof meta.budgetUSD === "number" ? { maxUSD: meta.budgetUSD } : {}),
+        ...(typeof meta.maxTokens === "number"
+            ? { maxTokens: meta.maxTokens }
+            : {}),
+        ...(typeof meta.maxDurationMs === "number"
+            ? { maxDurationMs: meta.maxDurationMs }
+            : {}),
+        ...(typeof meta.maxSources === "number"
+            ? { maxSources: meta.maxSources }
+            : {}),
+    };
+    const sources = sourceFilterFromMeta(meta.sourceFilter);
+    const options = {
+        runId,
+        ...(typeof meta.effort === "string"
+            ? { effort: meta.effort }
+            : {}),
+        ...(Object.keys(budget).length > 0 ? { budget } : {}),
+        ...(sources ? { sources } : {}),
+        ...(resume.signal ? { signal: resume.signal } : {}),
+    };
+    return startRun({
+        config,
+        question: meta.question,
+        options,
+        replay,
+        anchorStartedAt: typeof meta.startedAt === "number" ? meta.startedAt : undefined,
+        now: resume.now,
+    });
+}

package/dist/safe-dispatcher.d.ts

@@ -0,0 +1,16 @@
+import { Agent } from "undici";
+import { type SafetyPolicy } from "./safety.js";
+export type SafeDispatcher = Agent;
+type SafeLookupOptions = {
+    all?: boolean;
+    family?: number;
+    hints?: number;
+    verbatim?: boolean;
+};
+type SafeLookupCallback = (err: NodeJS.ErrnoException | null, address: string | Array<{
+    address: string;
+    family: number;
+}>, family?: number) => void;
+export declare function safeLookup(policy: SafetyPolicy): (hostname: string, options: SafeLookupOptions, callback: SafeLookupCallback) => void;
+export declare function createSafeDispatcher(policy: SafetyPolicy): SafeDispatcher;
+export {};

package/dist/safe-dispatcher.js

@@ -0,0 +1,32 @@
+import { lookup as dnsLookup } from "node:dns";
+import { Agent } from "undici";
+import { isPrivateAddress } from "./safety.js";
+export function safeLookup(policy) {
+    return (hostname, options, callback) => {
+        dnsLookup(hostname, { ...options, all: true }, (err, addresses) => {
+            if (err) {
+                callback(err, "", 0);
+                return;
+            }
+            if (!policy.allowPrivateNetworks) {
+                const blocked = addresses.find((entry) => isPrivateAddress(entry.address));
+                if (blocked) {
+                    callback(Object.assign(new Error(`SSRF blocked: ${hostname} resolves to private address ${blocked.address}`), { code: "ESSRFBLOCKED" }), "", 0);
+                    return;
+                }
+            }
+            if (options.all) {
+                callback(null, addresses);
+            }
+            else {
+                callback(null, addresses[0].address, addresses[0].family);
+            }
+        });
+    };
+}
+export function createSafeDispatcher(policy) {
+    const options = {
+        connect: { lookup: safeLookup(policy) },
+    };
+    return new Agent(options);
+}