@steel-dev/atlas 0.1.0

package/dist/trace.js

@@ -0,0 +1,278 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+export const TRACE_SCHEMA_VERSION = "1.0";
+const MAX_FIELD_CHARS = 256 * 1024; // per prompt/message/output field
+const MAX_TOTAL_CHARS = 512 * 1024 * 1024; // whole-run verbatim budget
+/** Maps trace sites back to their source locations. */
+export const SITE_SOURCE = {
+    research: "src/agent.ts:runAgent",
+    write: "src/agent.ts:runAgent",
+    synthesize: "src/spine.ts:synthesizeHolistic",
+    seed: "src/checklist.ts:seedLedger",
+    search: "src/tools.ts:execSearchTool",
+    fetch: "src/tools.ts:fetchSourceDocument",
+    run_code: "src/tools.ts:run_code",
+};
+const als = new AsyncLocalStorage();
+export function currentFrame() {
+    return als.getStore();
+}
+/** Run `fn` under a trace frame merged onto the current one. */
+export function withTraceFrame(recorder, patch, fn) {
+    if (!recorder)
+        return fn();
+    const parent = als.getStore();
+    return als.run({ ...parent, ...patch }, fn);
+}
+function clampText(value) {
+    if (value.length <= MAX_FIELD_CHARS)
+        return { text: value, truncated: false };
+    return {
+        text: `${value.slice(0, MAX_FIELD_CHARS)}…[truncated]`,
+        truncated: true,
+    };
+}
+function safeJson(value) {
+    let raw;
+    try {
+        raw = JSON.stringify(value);
+    }
+    catch {
+        return { text: '"[unserializable]"', truncated: true };
+    }
+    return clampText(raw);
+}
+function systemFromPrompt(prompt) {
+    const parts = [];
+    for (const message of prompt) {
+        if (message.role !== "system")
+            continue;
+        const content = message.content;
+        if (typeof content === "string")
+            parts.push(content);
+    }
+    return clampText(parts.join("\n\n")).text;
+}
+/** Normalize model content blocks into the trace viewer's block vocabulary. */
+export function toAnthropicBlocks(content) {
+    if (!content)
+        return [];
+    const blocks = [];
+    for (const part of content) {
+        switch (part.type) {
+            case "text":
+                blocks.push({ type: "text", text: clampText(part.text).text });
+                break;
+            case "reasoning":
+                blocks.push({
+                    type: "thinking",
+                    thinking: clampText(part.text).text,
+                });
+                break;
+            case "tool-call":
+                blocks.push({
+                    type: "tool_call",
+                    name: part.toolName,
+                    input: parseToolInput(part.input),
+                });
+                break;
+            case "tool-result":
+                blocks.push({ type: "tool_result", toolName: part.toolName });
+                break;
+            default:
+                blocks.push({ type: part.type });
+        }
+    }
+    return blocks;
+}
+function parseToolInput(input) {
+    const { text } = clampText(input);
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return text;
+    }
+}
+export class TraceRecorder {
+    mode;
+    nowFn;
+    startedAt;
+    spanList = [];
+    stepList = [];
+    spanSeq = 0;
+    totalChars = 0;
+    degradedFlag = false;
+    digest;
+    sink;
+    constructor(opts) {
+        this.mode = opts.mode;
+        this.nowFn = opts.now;
+        this.startedAt = opts.startedAt;
+    }
+    now() {
+        return this.nowFn();
+    }
+    get spans() {
+        return this.spanList;
+    }
+    get steps() {
+        return this.stepList;
+    }
+    get degraded() {
+        return this.degradedFlag;
+    }
+    mintSpanId() {
+        return `span_${++this.spanSeq}`;
+    }
+    /** Where trace entries are streamed for plain-run journaling (P7). */
+    setSink(sink) {
+        this.sink = sink;
+    }
+    addSpan(span) {
+        this.spanList.push(span);
+        this.sink?.("span", span.id, span);
+    }
+    recordAgentSpan(rec) {
+        this.addSpan({
+            id: rec.id,
+            ...(rec.parentId ? { parentId: rec.parentId } : {}),
+            kind: "agent",
+            site: rec.site,
+            agentId: rec.agentId,
+            ...(rec.logicalAgentId ? { logicalAgentId: rec.logicalAgentId } : {}),
+            ...(rec.role ? { role: rec.role } : {}),
+            t0: rec.t0,
+            t1: rec.t1,
+            durationMs: Math.max(0, rec.t1 - rec.t0),
+            waitMs: 0,
+            ...(rec.costUSD !== undefined ? { costUSD: rec.costUSD } : {}),
+            status: rec.status,
+            ...(rec.attrs ? { attrs: rec.attrs } : {}),
+        });
+    }
+    recordToolSpan(rec) {
+        const id = this.mintSpanId();
+        this.addSpan({
+            id,
+            ...(rec.parentId ? { parentId: rec.parentId } : {}),
+            kind: rec.kind,
+            site: rec.site,
+            ...(rec.agentId ? { agentId: rec.agentId } : {}),
+            t0: rec.t0,
+            t1: rec.t1,
+            durationMs: Math.max(0, rec.t1 - rec.t0),
+            waitMs: rec.waitMs,
+            computeMs: Math.max(0, rec.t1 - rec.t0 - rec.waitMs),
+            status: rec.status,
+            ...(rec.attrs ? { attrs: rec.attrs } : {}),
+        });
+    }
+    recordModelCall(rec, frame) {
+        const id = this.mintSpanId();
+        const site = frame?.site ?? rec.role ?? "unattributed";
+        this.addSpan({
+            id,
+            ...(frame?.parentSpanId ? { parentId: frame.parentSpanId } : {}),
+            kind: "model",
+            site,
+            ...(frame?.agentId ? { agentId: frame.agentId } : {}),
+            ...(frame?.logicalAgentId
+                ? { logicalAgentId: frame.logicalAgentId }
+                : {}),
+            role: rec.role,
+            t0: rec.t0,
+            t1: rec.t1,
+            durationMs: Math.max(0, rec.t1 - rec.t0),
+            waitMs: rec.waitMs,
+            computeMs: rec.computeMs,
+            ...(rec.retryDelayMs ? { retryDelayMs: rec.retryDelayMs } : {}),
+            ...(rec.tokens ? { tokens: rec.tokens } : {}),
+            ...(rec.costUSD !== undefined ? { costUSD: rec.costUSD } : {}),
+            status: rec.status,
+            attrs: {
+                callKey: rec.callKey,
+                adapter: `${rec.provider}:${rec.modelId}`,
+                ...(rec.finishReason ? { finishReason: rec.finishReason } : {}),
+                ...(rec.replayed ? { replayed: true } : {}),
+                ...(rec.attempts && rec.attempts > 1 ? { attempts: rec.attempts } : {}),
+            },
+        });
+        if (this.mode === "full" && rec.params) {
+            this.recordStep(id, site, rec, frame);
+        }
+    }
+    recordStep(spanId, site, rec, frame) {
+        const params = rec.params;
+        let truncated = false;
+        if (this.degradedFlag) {
+            // Past the verbatim budget: keep the span, drop the heavy I/O.
+            return;
+        }
+        const messagesPacked = safeJson(params.prompt);
+        truncated = truncated || messagesPacked.truncated;
+        const outputBlocks = rec.error ? [] : toAnthropicBlocks(rec.content);
+        const outputPacked = safeJson(outputBlocks);
+        truncated = truncated || outputPacked.truncated;
+        this.totalChars += messagesPacked.text.length + outputPacked.text.length;
+        if (this.totalChars > MAX_TOTAL_CHARS) {
+            this.degradedFlag = true;
+            return;
+        }
+        const toolNames = params.tools
+            ?.map((t) => ("name" in t ? t.name : undefined))
+            .filter((n) => Boolean(n));
+        const responseFormat = params.responseFormat;
+        const outputSchema = responseFormat && responseFormat.type === "json"
+            ? safeJson(responseFormat.schema ?? responseFormat.name ?? "json").text
+            : undefined;
+        const step = {
+            seq: this.stepList.length + 1,
+            atMs: Math.max(0, rec.t0 - this.startedAt),
+            spanId,
+            role: rec.role,
+            adapter: `${rec.provider}:${rec.modelId}`,
+            site,
+            ...(frame?.agentId ? { agentId: frame.agentId } : {}),
+            durationMs: rec.computeMs,
+            waitMs: rec.waitMs,
+            system: systemFromPrompt(params.prompt),
+            messages: params.prompt,
+            ...(toolNames?.length ? { toolNames } : {}),
+            maxTokens: params.maxOutputTokens ?? 0,
+            ...(outputSchema ? { outputSchema } : {}),
+            output: outputBlocks,
+            ...(rec.tokens
+                ? {
+                    inputTokens: rec.tokens.input + rec.tokens.cacheRead + rec.tokens.cacheWrite,
+                }
+                : {}),
+            ...(rec.costUSD !== undefined ? { costUSD: rec.costUSD } : {}),
+            ...(rec.finishReason ? { finishReason: rec.finishReason } : {}),
+            ...(rec.replayed ? { replayed: true } : {}),
+            ...(truncated ? { truncated: true } : {}),
+            ...(rec.error ? { error: rec.error } : {}),
+        };
+        this.stepList.push(step);
+        this.sink?.("step", spanId, step);
+    }
+    finalize(digest) {
+        this.digest = {
+            ...digest,
+            ...(this.degradedFlag ? { degraded: true } : {}),
+        };
+        this.sink?.("digest", "digest", this.digest);
+    }
+    snapshot() {
+        return {
+            schemaVersion: TRACE_SCHEMA_VERSION,
+            mode: this.mode,
+            spans: this.spanList,
+            steps: this.stepList,
+            ...(this.digest ? { digest: this.digest } : {}),
+            degraded: this.degradedFlag,
+        };
+    }
+}
+export function createTraceRecorder(opts) {
+    return new TraceRecorder(opts);
+}

package/dist/trail.d.ts

@@ -0,0 +1,15 @@
+export interface TrailRenderOptions {
+    maxSearches?: number;
+    maxDeadEnds?: number;
+    guidance?: string;
+}
+export declare function trailCapsFor(maxSources: number, fraction?: number): Required<Pick<TrailRenderOptions, "maxSearches" | "maxDeadEnds">>;
+export interface Trail {
+    readonly searchCount: number;
+    readonly deadEndCount: number;
+    recordSearch(query: string, results: number): void;
+    recordDeadEnd(url: string, reason: string): void;
+    isDeadEnd(url: string): string | undefined;
+    render(opts?: TrailRenderOptions): string;
+}
+export declare function createTrail(): Trail;

package/dist/trail.js

@@ -0,0 +1,74 @@
+import { trailKey } from "./search-normalize.js";
+const RENDER_MAX_SEARCHES = 50;
+const RENDER_MAX_DEAD_ENDS = 25;
+const REASON_MAX_CHARS = 120;
+const CAP_FLOOR_SEARCHES = 30;
+const CAP_FLOOR_DEAD_ENDS = 15;
+export function trailCapsFor(maxSources, fraction = 1) {
+    const maxSearches = Math.max(CAP_FLOOR_SEARCHES, Math.ceil(maxSources * fraction));
+    return {
+        maxSearches,
+        maxDeadEnds: Math.max(CAP_FLOOR_DEAD_ENDS, Math.ceil(maxSearches / 2)),
+    };
+}
+export function createTrail() {
+    const searches = new Map();
+    const deadEnds = new Map();
+    return {
+        get searchCount() {
+            return searches.size;
+        },
+        get deadEndCount() {
+            return deadEnds.size;
+        },
+        recordSearch(query, results) {
+            const text = query.trim();
+            if (!text)
+                return;
+            const key = trailKey(text);
+            const existing = searches.get(key);
+            if (existing) {
+                existing.results = Math.max(existing.results, results);
+            }
+            else {
+                searches.set(key, { query: text, results });
+            }
+        },
+        recordDeadEnd(url, reason) {
+            const target = url.trim();
+            if (!target || deadEnds.has(target))
+                return;
+            deadEnds.set(target, reason.trim().slice(0, REASON_MAX_CHARS));
+        },
+        isDeadEnd(url) {
+            return deadEnds.get(url.trim());
+        },
+        render(opts = {}) {
+            const maxSearches = opts.maxSearches ?? RENDER_MAX_SEARCHES;
+            const maxDeadEnds = opts.maxDeadEnds ?? RENDER_MAX_DEAD_ENDS;
+            const guidance = opts.guidance ?? "do not repeat them; vary the terms or angle instead";
+            const lines = [];
+            if (searches.size > 0) {
+                lines.push(`Searches already run (${searches.size} — ${guidance}):`);
+                const entries = [...searches.values()];
+                for (const entry of entries.slice(0, maxSearches)) {
+                    lines.push(`- "${entry.query}" → ${entry.results} result(s)`);
+                }
+                if (entries.length > maxSearches) {
+                    lines.push(`…and ${entries.length - maxSearches} more searches`);
+                }
+            }
+            if (deadEnds.size > 0) {
+                lines.push("Dead-end fetches (no usable content came back):");
+                const entries = [...deadEnds.entries()];
+                for (const [url, reason] of entries.slice(0, maxDeadEnds)) {
+                    lines.push(`- ${url} — ${reason}`);
+                }
+                if (entries.length > maxDeadEnds) {
+                    lines.push(`…and ${entries.length - maxDeadEnds} more dead ends`);
+                }
+            }
+            return lines.join("\n");
+        },
+    };
+}

package/dist/url.d.ts

@@ -0,0 +1 @@
+export declare function normalizeUrlForSource(url: string): string;

package/dist/url.js

@@ -0,0 +1,25 @@
+const TRACKING_QUERY_PARAMS = new Set([
+    "fbclid",
+    "gclid",
+    "igshid",
+    "mc_cid",
+    "mc_eid",
+    "msclkid",
+]);
+export function normalizeUrlForSource(url) {
+    try {
+        const u = new URL(url);
+        u.hash = "";
+        for (const key of [...u.searchParams.keys()]) {
+            const lower = key.toLowerCase();
+            if (lower.startsWith("utm_") || TRACKING_QUERY_PARAMS.has(lower)) {
+                u.searchParams.delete(key);
+            }
+        }
+        u.searchParams.sort();
+        return u.toString();
+    }
+    catch {
+        return url;
+    }
+}

package/package.json

@@ -0,0 +1,107 @@
+{
+  "name": "@steel-dev/atlas",
+  "version": "0.1.0",
+  "description": "Deep research from your terminal or your code. Backed by Steel Browser and model providers.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "atlas": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "dev": "tsx examples/cli.ts",
+    "eval:browsecomp": "tsx evals/browsecomp.ts",
+    "eval:browsecomp:official": "tsx evals/browsecomp.ts --cases https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv",
+    "eval:draco": "tsx evals/draco.ts",
+    "eval:draco:explore": "tsx examples/eval-explorer/cli.ts",
+    "draco:q": "tsx examples/eval-explorer/query.ts",
+    "trace": "tsx examples/trace.ts",
+    "eval:typecheck": "tsc -p evals/tsconfig.json && tsc -p examples/eval-explorer/tsconfig.client.json",
+    "build": "rm -rf dist && tsc -p tsconfig.build.json && node scripts/bundle-cli.mjs",
+    "typecheck": "tsc -p tsconfig.json && tsc -p examples/tsconfig.cli.json",
+    "lint": "biome check .",
+    "lint:fix": "biome check --write .",
+    "format": "biome format --write .",
+    "check:exports": "publint --strict && attw --pack . --ignore-rules cjs-resolves-to-esm",
+    "test": "vitest",
+    "test:run": "vitest run",
+    "prepublishOnly": "npm run build"
+  },
+  "engines": {
+    "node": ">=20.16.0"
+  },
+  "dependencies": {
+    "@ai-sdk/provider": "^3.0.0",
+    "cheerio": "^1.0.0",
+    "exa-js": "^2.14.0",
+    "pdf-parse": "^2.4.5",
+    "steel-sdk": "^0.18.0",
+    "undici": "^7.25.0",
+    "zod": "^4.1.8"
+  },
+  "peerDependencies": {
+    "@ai-sdk/anthropic": "^3.0.0",
+    "@ai-sdk/google": "^3.0.0",
+    "@ai-sdk/openai": "^3.0.0",
+    "ai": "^6.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@ai-sdk/anthropic": {
+      "optional": true
+    },
+    "@ai-sdk/openai": {
+      "optional": true
+    },
+    "@ai-sdk/google": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@ai-sdk/anthropic": "^3.0.81",
+    "@ai-sdk/google": "^3.0.80",
+    "@ai-sdk/openai": "^3.0.67",
+    "@arethetypeswrong/cli": "^0.18.4",
+    "@biomejs/biome": "^2.5.1",
+    "@types/node": "^22.0.0",
+    "ai": "^6.0.194",
+    "esbuild": "^0.25.12",
+    "publint": "^0.3.21",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2",
+    "vitest": "^2.1.8"
+  },
+  "keywords": [
+    "deep-research",
+    "research",
+    "cli",
+    "anthropic",
+    "claude",
+    "openai",
+    "steel",
+    "web-search",
+    "agent",
+    "llm"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/steel-dev/atlas.git"
+  },
+  "optionalDependencies": {
+    "isolated-vm": "^6.1.2"
+  }
+}