@steel-dev/atlas 0.1.0

package/dist/safety.d.ts

@@ -0,0 +1,23 @@
+import type { ResearchEvent } from "./events.js";
+export interface SafetyPolicy {
+    allowFlaggedUrls?: boolean;
+    allowPrivateNetworks?: boolean;
+}
+export declare function quarantine(text: string, source: {
+    sourceId?: string;
+    url?: string;
+}): string;
+export type UrlGuardResult = {
+    ok: true;
+} | {
+    ok: false;
+    reason: string;
+    kind: "scheme" | "ssrf" | "url-entropy";
+};
+export declare function isPrivateAddress(ip: string): boolean;
+export declare function guardRedirect(rawUrl: string, policy: SafetyPolicy): Promise<UrlGuardResult>;
+export declare function guardUrl(rawUrl: string, opts: {
+    policy: SafetyPolicy;
+    seenDomains: Set<string>;
+    emit?: (event: ResearchEvent) => void;
+}): Promise<UrlGuardResult>;

package/dist/safety.js

@@ -0,0 +1,206 @@
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+export function quarantine(text, source) {
+    const tag = [source.sourceId, source.url].filter(Boolean).join(" ");
+    const neutralized = text.replaceAll("<<<", "‹‹‹");
+    return `<<<untrusted-source ${tag}>>>\n${neutralized}\n<<<end-untrusted-source>>>`;
+}
+const MAX_URL_LENGTH = 2048;
+const ENTROPY_MIN_QUERY_LENGTH = 64;
+const ENTROPY_THRESHOLD = 4.6;
+function shannonEntropy(text) {
+    if (text.length === 0)
+        return 0;
+    const counts = new Map();
+    for (const char of text) {
+        counts.set(char, (counts.get(char) ?? 0) + 1);
+    }
+    let entropy = 0;
+    for (const count of counts.values()) {
+        const p = count / text.length;
+        entropy -= p * Math.log2(p);
+    }
+    return entropy;
+}
+function isPrivateIPv4Octets(a, b) {
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 100 && b >= 64 && b <= 127) ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 168));
+}
+function isPrivateIPv4(ip) {
+    const parts = ip.split(".").map(Number);
+    if (parts.length !== 4)
+        return false;
+    return isPrivateIPv4Octets(parts[0], parts[1]);
+}
+function parseIPv6Groups(ip) {
+    let text = ip;
+    const zone = text.indexOf("%");
+    if (zone !== -1)
+        text = text.slice(0, zone);
+    const v4Tail = /^(.*:)(\d+)\.(\d+)\.(\d+)\.(\d+)$/.exec(text);
+    if (v4Tail) {
+        const octets = v4Tail.slice(2, 6).map(Number);
+        if (octets.some((octet) => octet > 255))
+            return undefined;
+        const hi = ((octets[0] << 8) | octets[1]).toString(16);
+        const lo = ((octets[2] << 8) | octets[3]).toString(16);
+        text = `${v4Tail[1]}${hi}:${lo}`;
+    }
+    const halves = text.split("::");
+    if (halves.length > 2)
+        return undefined;
+    const parseHalf = (half) => half === ""
+        ? []
+        : half.split(":").map((group) => Number.parseInt(group, 16));
+    const head = parseHalf(halves[0]);
+    const tail = halves.length === 2 ? parseHalf(halves[1]) : [];
+    const missing = 8 - head.length - tail.length;
+    if (halves.length === 1 && head.length !== 8)
+        return undefined;
+    if (halves.length === 2 && missing < 0)
+        return undefined;
+    const groups = [
+        ...head,
+        ...Array.from({ length: halves.length === 2 ? missing : 0 }, () => 0),
+        ...tail,
+    ];
+    if (groups.length !== 8)
+        return undefined;
+    if (groups.some((group) => Number.isNaN(group) || group < 0 || group > 0xffff)) {
+        return undefined;
+    }
+    return groups;
+}
+function isPrivateIPv6(ip) {
+    const groups = parseIPv6Groups(ip);
+    if (!groups)
+        return true;
+    const [g0, g1, g2, g3, g4, g5, g6, g7] = groups;
+    const embeddedV4 = (hi) => isPrivateIPv4Octets(hi >> 8, hi & 0xff);
+    const leadingZeros = g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0;
+    if (leadingZeros && g5 === 0 && g6 === 0 && (g7 === 0 || g7 === 1)) {
+        return true;
+    }
+    if ((g0 & 0xfe00) === 0xfc00)
+        return true;
+    if ((g0 & 0xffc0) === 0xfe80)
+        return true;
+    if (leadingZeros && g5 === 0xffff)
+        return embeddedV4(g6);
+    if (g0 === 0x64 &&
+        g1 === 0xff9b &&
+        g2 === 0 &&
+        g3 === 0 &&
+        g4 === 0 &&
+        g5 === 0) {
+        return embeddedV4(g6);
+    }
+    if (g0 === 0x2002)
+        return embeddedV4(g1);
+    return false;
+}
+export function isPrivateAddress(ip) {
+    const family = isIP(ip);
+    if (family === 4)
+        return isPrivateIPv4(ip);
+    if (family === 6)
+        return isPrivateIPv6(ip);
+    return false;
+}
+export async function guardRedirect(rawUrl, policy) {
+    let url;
+    try {
+        url = new URL(rawUrl);
+    }
+    catch {
+        return { ok: false, kind: "scheme", reason: "not a valid URL" };
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+        return {
+            ok: false,
+            kind: "scheme",
+            reason: `scheme ${url.protocol} is not allowed`,
+        };
+    }
+    if (url.username || url.password) {
+        return {
+            ok: false,
+            kind: "ssrf",
+            reason: "URLs with embedded credentials are not allowed",
+        };
+    }
+    if (rawUrl.length > MAX_URL_LENGTH) {
+        return {
+            ok: false,
+            kind: "ssrf",
+            reason: `URL exceeds ${MAX_URL_LENGTH} characters`,
+        };
+    }
+    if (!policy.allowPrivateNetworks) {
+        const hostname = url.hostname.replace(/^\[|\]$/g, "");
+        if (isIP(hostname)) {
+            if (isPrivateAddress(hostname)) {
+                return {
+                    ok: false,
+                    kind: "ssrf",
+                    reason: `address ${hostname} is private or reserved`,
+                };
+            }
+        }
+        else {
+            try {
+                const records = await lookup(hostname, { all: true });
+                for (const record of records) {
+                    if (isPrivateAddress(record.address)) {
+                        return {
+                            ok: false,
+                            kind: "ssrf",
+                            reason: `${hostname} resolves to private address ${record.address}`,
+                        };
+                    }
+                }
+            }
+            catch {
+                return {
+                    ok: false,
+                    kind: "ssrf",
+                    reason: `${hostname} did not resolve`,
+                };
+            }
+        }
+    }
+    return { ok: true };
+}
+export async function guardUrl(rawUrl, opts) {
+    const target = await guardRedirect(rawUrl, opts.policy);
+    if (!target.ok)
+        return target;
+    const url = new URL(rawUrl);
+    const domain = url.hostname.toLowerCase();
+    const newDomain = !opts.seenDomains.has(domain);
+    if (newDomain) {
+        const suspect = `${url.search}${url.hash}`.replace(/^[?#]/, "");
+        if (suspect.length >= ENTROPY_MIN_QUERY_LENGTH &&
+            shannonEntropy(suspect) >= ENTROPY_THRESHOLD &&
+            !opts.policy.allowFlaggedUrls) {
+            opts.emit?.({
+                type: "safety.flag",
+                kind: "url-entropy",
+                detail: `high-entropy query string on first-seen domain ${domain}`,
+                url: rawUrl,
+            });
+            return {
+                ok: false,
+                kind: "url-entropy",
+                reason: "high-entropy query string on a never-seen domain (possible data exfiltration); set safety.allowFlaggedUrls to permit",
+            };
+        }
+    }
+    opts.seenDomains.add(domain);
+    return { ok: true };
+}

package/dist/sandbox.d.ts

@@ -0,0 +1,22 @@
+export interface SandboxSource {
+    source_id: string;
+    url: string;
+    title: string;
+    text: string;
+}
+export interface SandboxRequest {
+    code: string;
+    sources: SandboxSource[];
+    timeoutMs: number;
+}
+export interface SandboxOutput {
+    sources_in_scope: number;
+    stdout: string;
+    result?: unknown;
+    error?: string;
+    truncated?: boolean;
+}
+export declare function isRunCodeAvailable(): Promise<boolean>;
+export declare function clampSandboxTimeout(raw: unknown): number;
+export declare function runCodeSandboxed(req: SandboxRequest): Promise<SandboxOutput>;
+export declare function shapeSandboxOutput(payload: SandboxOutput): string;

package/dist/sandbox.js

@@ -0,0 +1,228 @@
+const DEFAULT_TIMEOUT_MS = 5_000;
+const MAX_TIMEOUT_MS = 10_000;
+const MAX_PAYLOAD_BYTES = 8 * 1024 * 1024;
+const MEMORY_LIMIT_MB = 128;
+const TOTAL_OUTPUT_CAP = 8_000;
+const STDOUT_TRUNCATE_MARKER = "... [output truncated]";
+let isolatedVmPromise = null;
+function loadIsolatedVm() {
+    isolatedVmPromise ??= import("isolated-vm").then((mod) => mod.default ??
+        mod, () => null);
+    return isolatedVmPromise;
+}
+export async function isRunCodeAvailable() {
+    return (await loadIsolatedVm()) !== null;
+}
+const RUNNER_SCRIPT = String.raw `
+const documents = Array.isArray(globalThis.sources) ? globalThis.sources : [];
+const __lines = [];
+let __collected = 0;
+let __truncated = false;
+const printable = (value) => {
+  if (typeof value === "string") return value;
+  try {
+    const json = JSON.stringify(value);
+    return json === undefined ? String(value) : json;
+  } catch {
+    return String(value);
+  }
+};
+const print = (...parts) => {
+  if (__truncated) return;
+  const line = parts.map(printable).join(" ");
+  __collected += line.length + 1;
+  if (__collected > 32000) {
+    __truncated = true;
+    return;
+  }
+  __lines.push(line);
+};
+const grep = (pattern, opts) => {
+  const options = opts && typeof opts === "object" ? opts : {};
+  const isRegExp = Object.prototype.toString.call(pattern) === "[object RegExp]";
+  const source = isRegExp ? pattern.source : String(pattern ?? "");
+  if (!source) throw new Error("grep: pattern must be a non-empty string or RegExp");
+  const baseFlags = isRegExp ? pattern.flags : "";
+  const flags = [...new Set([...baseFlags, "g", ...(options.ignore_case === true ? ["i"] : [])])].join("");
+  const regex = new RegExp(source, flags);
+  const wanted = Array.isArray(options.source_ids) && options.source_ids.length > 0
+    ? new Set(options.source_ids.map((id) => String(id ?? "").trim()))
+    : null;
+  const scope = wanted ? documents.filter((d) => wanted.has(d.source_id)) : documents;
+  const maxRaw = Math.floor(Number(options.max));
+  const max = Number.isFinite(maxRaw) && maxRaw > 0 ? Math.min(maxRaw, 200) : 50;
+  const ctxRaw = options.context === true ? 80 : Math.floor(Number(options.context));
+  const contextChars = Number.isFinite(ctxRaw) && ctxRaw > 0 ? Math.min(ctxRaw, 500) : 0;
+  const matches = [];
+  for (const doc of scope) {
+    regex.lastIndex = 0;
+    let found;
+    while (matches.length < max && (found = regex.exec(doc.text)) !== null) {
+      const text = found[0];
+      const ctxStr = contextChars > 0
+        ? doc.text.slice(Math.max(0, found.index - contextChars), Math.min(doc.text.length, found.index + text.length + contextChars))
+        : "";
+      matches.push({
+        source_id: doc.source_id,
+        url: doc.url,
+        offset: found.index,
+        match: text,
+        text: ctxStr || text,
+        context: ctxStr,
+      });
+      if (text === "") regex.lastIndex++;
+    }
+    if (matches.length >= max) break;
+  }
+  return matches;
+};
+globalThis.print = print;
+globalThis.grep = grep;
+globalThis.console = { log: print };
+let __result;
+let __error;
+try {
+  __result = (0, eval)(String(globalThis.__code || ""));
+} catch (err) {
+  const message = err && typeof err.message === "string" ? err.message : String(err);
+  __error = "code threw: " + message;
+}
+let __resultJSON;
+if (__error === undefined && __result !== undefined && typeof __result !== "function" && typeof __result !== "symbol") {
+  try {
+    const json = JSON.stringify(__result);
+    if (json !== undefined && json.length <= 4000) __resultJSON = json;
+  } catch {}
+}
+JSON.stringify({
+  stdout: __lines.join("\n"),
+  ...(__resultJSON !== undefined ? { resultJSON: __resultJSON } : {}),
+  ...(__error !== undefined ? { error: __error } : {}),
+  ...(__truncated ? { truncated: true } : {}),
+});
+`;
+export function clampSandboxTimeout(raw) {
+    const n = Math.floor(Number(raw));
+    if (!Number.isFinite(n) || n <= 0)
+        return DEFAULT_TIMEOUT_MS;
+    return Math.min(n, MAX_TIMEOUT_MS);
+}
+function hostErrorOutput(err, req) {
+    const message = err instanceof Error ? err.message : String(err);
+    if (/timed out|execution timed out|timeout/i.test(message)) {
+        return {
+            sources_in_scope: req.sources.length,
+            stdout: "",
+            error: `code timed out after ${req.timeoutMs}ms`,
+        };
+    }
+    if (/memory limit|isolate was disposed|array buffer allocation/i.test(message)) {
+        return {
+            sources_in_scope: req.sources.length,
+            stdout: "",
+            error: "code exceeded the sandbox memory limit",
+        };
+    }
+    return {
+        sources_in_scope: req.sources.length,
+        stdout: "",
+        error: `sandbox error: ${message}`,
+    };
+}
+export async function runCodeSandboxed(req) {
+    const ivm = await loadIsolatedVm();
+    if (!ivm) {
+        return {
+            sources_in_scope: req.sources.length,
+            stdout: "",
+            error: 'run_code is unavailable: the optional "isolated-vm" dependency is not installed or failed to build. Run `npm install isolated-vm` to enable the sandbox.',
+        };
+    }
+    const documents = req.sources.map((source) => ({
+        source_id: String(source.source_id ?? ""),
+        url: String(source.url ?? ""),
+        title: String(source.title ?? ""),
+        text: String(source.text ?? ""),
+    }));
+    let payloadBytes = 0;
+    try {
+        payloadBytes = Buffer.byteLength(JSON.stringify(documents), "utf8");
+    }
+    catch {
+        payloadBytes = 0;
+    }
+    if (payloadBytes > MAX_PAYLOAD_BYTES) {
+        return {
+            sources_in_scope: documents.length,
+            stdout: "",
+            error: `sources too large for the sandbox (limit ${MAX_PAYLOAD_BYTES} bytes); restrict source_ids`,
+        };
+    }
+    const isolate = new ivm.Isolate({ memoryLimit: MEMORY_LIMIT_MB });
+    try {
+        const context = await isolate.createContext();
+        await context.global.set("sources", new ivm.ExternalCopy(documents).copyInto({ release: true }));
+        await context.global.set("__code", req.code);
+        const script = await isolate.compileScript(RUNNER_SCRIPT);
+        const raw = await script.run(context, {
+            timeout: req.timeoutMs,
+            copy: true,
+        });
+        let summary;
+        try {
+            summary = JSON.parse(String(raw));
+        }
+        catch {
+            return {
+                sources_in_scope: documents.length,
+                stdout: "",
+                error: "sandbox produced no parseable output",
+            };
+        }
+        const output = {
+            sources_in_scope: documents.length,
+            stdout: summary.stdout ?? "",
+        };
+        if (summary.resultJSON !== undefined) {
+            try {
+                output.result = JSON.parse(summary.resultJSON);
+            }
+            catch { }
+        }
+        if (summary.error !== undefined)
+            output.error = summary.error;
+        if (summary.truncated)
+            output.truncated = true;
+        return output;
+    }
+    catch (err) {
+        return hostErrorOutput(err, req);
+    }
+    finally {
+        try {
+            isolate.dispose();
+        }
+        catch { }
+    }
+}
+export function shapeSandboxOutput(payload) {
+    let body = payload;
+    let json = JSON.stringify(body, null, 2);
+    if (json.length <= TOTAL_OUTPUT_CAP)
+        return json;
+    if ("result" in body) {
+        const { result: _dropped, ...rest } = body;
+        body = { ...rest, truncated: true };
+        json = JSON.stringify(body, null, 2);
+        if (json.length <= TOTAL_OUTPUT_CAP)
+            return json;
+    }
+    const overhead = JSON.stringify({ ...body, stdout: "", truncated: true }, null, 2).length;
+    const room = Math.max(0, TOTAL_OUTPUT_CAP - overhead - STDOUT_TRUNCATE_MARKER.length - 4);
+    body = {
+        ...body,
+        stdout: `${body.stdout.slice(0, room)}\n${STDOUT_TRUNCATE_MARKER}`,
+        truncated: true,
+    };
+    return JSON.stringify(body, null, 2);
+}

package/dist/search-normalize.d.ts

@@ -0,0 +1,2 @@
+export declare function canonicalQuery(query: string): string;
+export declare function trailKey(query: string): string;

package/dist/search-normalize.js

@@ -0,0 +1,13 @@
+const TOKEN_SPLIT = /[^\p{L}\p{N}]+/u;
+const BARE_YEAR = /^(?:19|20)\d{2}$/;
+function tokenize(query) {
+    return query.toLowerCase().split(TOKEN_SPLIT).filter(Boolean);
+}
+export function canonicalQuery(query) {
+    return [...new Set(tokenize(query))].sort().join(" ");
+}
+export function trailKey(query) {
+    const tokens = tokenize(query).filter((token) => !BARE_YEAR.test(token));
+    const canonical = [...new Set(tokens)].sort().join(" ");
+    return canonical || query.trim().toLowerCase();
+}

package/dist/source-documents.d.ts

@@ -0,0 +1,77 @@
+import type { HtmlPageMetadata } from "./html-extract.js";
+import type { SourceDiscoveredLink, SourceDocument, SourceExtractionAttempt, SourceExtractionMetadata } from "./sources.js";
+export declare function createSourceDocument(url: string, title: string, markdown: string, metadata: SourceExtractionMetadata, originalChars: number, sourceId: string, canonicalUrl?: string): SourceDocument;
+export declare function extractionMetadataFromPdf(opts: {
+    markdownChars: number;
+    contentType?: string;
+    finalUrl?: string;
+    notes?: string[];
+    attempts?: SourceExtractionAttempt[];
+    qualityWarnings?: string[];
+    discoveredLinks?: SourceDiscoveredLink[];
+}): SourceExtractionMetadata;
+export declare function extractionMetadataFromText(opts: {
+    markdownChars: number;
+    method: "json_direct" | "text_direct" | "xml_direct";
+    contentType?: string;
+    finalUrl?: string;
+    notes?: string[];
+    attempts?: SourceExtractionAttempt[];
+    qualityWarnings?: string[];
+}): SourceExtractionMetadata;
+export declare function extractionMetadataFromCustomTool(opts: {
+    markdownChars: number;
+    toolName: string;
+}): SourceExtractionMetadata;
+export declare function extractionMetadataFromScrape(opts: {
+    markdownChars: number;
+    contentType?: string;
+    finalUrl?: string;
+    notes?: string[];
+    attempts?: SourceExtractionAttempt[];
+    qualityWarnings?: string[];
+    discoveredLinks?: SourceDiscoveredLink[];
+    pageMetadata?: HtmlPageMetadata;
+}): SourceExtractionMetadata;
+export declare function extractionMetadataFromHtml(opts: {
+    markdownChars: number;
+    contentType?: string;
+    finalUrl?: string;
+    notes?: string[];
+    attempts?: SourceExtractionAttempt[];
+    qualityWarnings?: string[];
+    discoveredLinks?: SourceDiscoveredLink[];
+    pageMetadata?: HtmlPageMetadata;
+}): SourceExtractionMetadata;
+export declare function extractionMetadataFromExa(opts: {
+    markdownChars: number;
+    finalUrl?: string;
+    attempts?: SourceExtractionAttempt[];
+    qualityWarnings?: string[];
+}): SourceExtractionMetadata;
+export declare function storeMarkdown(markdown: string): {
+    markdown: string;
+    originalChars: number;
+    truncated: boolean;
+};
+export declare function formatSourceCard(document: SourceDocument, previewChars?: number): string;
+export declare function sourceCardData(document: SourceDocument, previewChars?: number, goal?: string): Record<string, unknown>;
+export declare function formatSourceChunk(document: SourceDocument, chunkIndex: number): string;
+export interface SourcePassage {
+    sourceId: string;
+    title: string;
+    url: string;
+    canonicalUrl: string;
+    chunkIndex: number;
+    start: number;
+    end: number;
+    score: number;
+    snippet: string;
+}
+export declare function rankSourcePassages(documents: SourceDocument[], query: string, maxResults: number): SourcePassage[];
+export declare function searchSourceDocuments(documents: SourceDocument[], query: string, maxResults: number): string;
+export declare function selectExtractionWindow(document: SourceDocument, query: string, maxChars: number): {
+    text: string;
+    truncated: boolean;
+};
+export declare function quoteSource(document: SourceDocument, start: number, end: number): string;