crw-sdk 0.22.0 → 0.24.0

package/README.md

@@ -58,14 +58,16 @@ CRW_LOCAL=1 node app.js
 | `changeTrackingDiff(cur, prev?)` | Diff vs a prior snapshot | HTTP |
 | `close()` | Shut down the local subprocess | — |
 
-¹ Local search needs a SearXNG URL configured on the engine.
+¹ Local search needs a search backend configured on the engine.
 
 ```ts
-// Structured extraction:
-const data = await crw.extract({
+// Structured extraction (async job → per-URL results array):
+const results = await crw.extract({
   urls: ["https://example.com"],
   schema: { type: "object", properties: { title: { type: "string" } } },
 });
+// results: [{ url, status, data, error, llmUsage }]
+for (const r of results) if (r.status === "completed") console.log(r.url, r.data);
 
 // Parse a PDF:
 import { readFileSync } from "node:fs";

package/dist/cjs/client.js

@@ -95,7 +95,7 @@ class CrwClient {
         return result.links ?? [];
     }
     /**
-     * Works in both modes; local mode needs a SearXNG URL configured on the engine.
+     * Works in both modes; local mode needs a search backend configured on the engine.
      */
     async search(query, opts = {}) {
         const { limit = 5, lang, tbs, sources, categories, scrapeOptions, ...rest } = opts;
@@ -111,8 +111,36 @@ class CrwClient {
         if (scrapeOptions)
             args.scrapeOptions = scrapeOptions;
         Object.assign(args, rest);
-        if (this.apiUrl)
-            return this.httpPost("/v1/search", args);
+        if (this.apiUrl) {
+            // `data` IS the results, in whatever shape the caller asked for: a flat
+            // array, a `{ web, news }` object when `sources` is set, or the engine's
+            // `{ results, answer, … }` when you point the client at a self-hosted
+            // server. Return it untouched — reshaping it here silently emptied both the
+            // grouped and the self-hosted cases.
+            //
+            // `answer` / `citations` / `llmUsage` ride BESIDE `data` on the managed API,
+            // and the old `data` unwrap dropped them: you could pay for an AI answer and
+            // have no way to read it. Attach them non-enumerably, so `results.answer`
+            // works while `for…of`, spread and JSON.stringify keep seeing exactly the
+            // array they saw before.
+            const raw = await this.httpRequest("POST", "/v1/search", args, {
+                raw: true,
+            });
+            const data = (raw.data ?? []);
+            if (data !== null && typeof data === "object") {
+                for (const key of ["answer", "citations", "llmUsage"]) {
+                    if (raw[key] !== undefined) {
+                        Object.defineProperty(data, key, {
+                            value: raw[key],
+                            enumerable: false,
+                            configurable: true,
+                            writable: true,
+                        });
+                    }
+                }
+            }
+            return data;
+        }
         return this.localTransport().toolCall("crw_search", args);
     }
     /**
@@ -170,29 +198,45 @@ class CrwClient {
         Object.assign(args, rest);
         return this.localTransport().toolCall("crw_parse_file", args);
     }
-    /** Structured LLM extraction across URLs (HTTP mode only). */
+    /**
+     * Native multi-URL structured extraction (HTTP mode only). Starts an async
+     * `/v1/extract` job and polls until complete, returning a per-URL results
+     * array (`[{ url, status, data, error, llmUsage }]`) in request order.
+     */
     async extract(opts) {
         if (!this.apiUrl)
             throw new errors_js_1.CrwError(httpOnlyHint("extract", "LLM extract job endpoint"));
-        const { urls, prompt, schema, systemPrompt, pollInterval = 2, timeout = 120 } = opts;
+        const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
         const body = { urls: [...urls] };
         if (prompt !== undefined)
             body.prompt = prompt;
         if (schema !== undefined)
             body.schema = schema;
-        if (systemPrompt !== undefined)
-            body.systemPrompt = systemPrompt;
-        const start = await this.httpRequest("POST", "/v2/extract", body, { raw: true });
+        if (basis !== undefined)
+            body.basis = basis;
+        if (llmApiKey !== undefined)
+            body.llmApiKey = llmApiKey;
+        if (llmProvider !== undefined)
+            body.llmProvider = llmProvider;
+        if (llmModel !== undefined)
+            body.llmModel = llmModel;
+        const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true });
         const jobId = start.id;
-        if (!jobId)
-            throw new errors_js_1.CrwError(`extract did not return job ID: ${JSON.stringify(start)}`);
+        // The managed API answers synchronously: the extraction is already finished and
+        // the payload is in this first response, with no job to poll. Only the async
+        // path hands back an `id`. Demanding one made every managed extract() throw.
+        if (!jobId) {
+            if (Array.isArray(start.results))
+                return start.results;
+            throw new errors_js_1.CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`);
+        }
         const deadline = Date.now() + timeout * 1000;
         for (;;) {
             if (Date.now() > deadline)
                 throw new errors_js_1.CrwTimeoutError(`Extract ${jobId} timed out after ${timeout}s`);
-            const status = await this.httpRequest("GET", `/v2/extract/${jobId}`, undefined, { raw: true, checkSuccess: false });
+            const status = await this.httpRequest("GET", `/v1/extract/${jobId}`, undefined, { raw: true, checkSuccess: false });
             if (status.status === "completed")
-                return status.data ?? {};
+                return status.results ?? [];
             if (status.status === "failed")
                 throw new errors_js_1.CrwError(`Extract failed: ${status.error ?? "unknown"}`);
             await sleep(pollInterval * 1000);

package/dist/esm/client.d.ts

@@ -17,7 +17,7 @@ export declare class CrwClient {
     crawl(url: string, opts?: CrawlOptions): Promise<CrawlResult>;
     map(url: string, opts?: MapOptions): Promise<string[]>;
     /**
-     * Works in both modes; local mode needs a SearXNG URL configured on the engine.
+     * Works in both modes; local mode needs a search backend configured on the engine.
      */
     search(query: string, opts?: SearchOptions): Promise<SearchResult>;
     /**
@@ -37,7 +37,11 @@ export declare class CrwClient {
      * Parse a document (PDF) into markdown / structured JSON. Works in both modes.
      */
     parseFile(content: Uint8Array, opts?: ParseFileOptions): Promise<ParseResult>;
-    /** Structured LLM extraction across URLs (HTTP mode only). */
+    /**
+     * Native multi-URL structured extraction (HTTP mode only). Starts an async
+     * `/v1/extract` job and polls until complete, returning a per-URL results
+     * array (`[{ url, status, data, error, llmUsage }]`) in request order.
+     */
     extract(opts: ExtractOptions): Promise<ExtractResult>;
     /** Scrape many URLs in one async batch job (HTTP mode only). */
     batchScrape(urls: string[], opts?: BatchScrapeOptions): Promise<BatchResult>;

package/dist/esm/client.js

@@ -92,7 +92,7 @@ export class CrwClient {
         return result.links ?? [];
     }
     /**
-     * Works in both modes; local mode needs a SearXNG URL configured on the engine.
+     * Works in both modes; local mode needs a search backend configured on the engine.
      */
     async search(query, opts = {}) {
         const { limit = 5, lang, tbs, sources, categories, scrapeOptions, ...rest } = opts;
@@ -108,8 +108,36 @@ export class CrwClient {
         if (scrapeOptions)
             args.scrapeOptions = scrapeOptions;
         Object.assign(args, rest);
-        if (this.apiUrl)
-            return this.httpPost("/v1/search", args);
+        if (this.apiUrl) {
+            // `data` IS the results, in whatever shape the caller asked for: a flat
+            // array, a `{ web, news }` object when `sources` is set, or the engine's
+            // `{ results, answer, … }` when you point the client at a self-hosted
+            // server. Return it untouched — reshaping it here silently emptied both the
+            // grouped and the self-hosted cases.
+            //
+            // `answer` / `citations` / `llmUsage` ride BESIDE `data` on the managed API,
+            // and the old `data` unwrap dropped them: you could pay for an AI answer and
+            // have no way to read it. Attach them non-enumerably, so `results.answer`
+            // works while `for…of`, spread and JSON.stringify keep seeing exactly the
+            // array they saw before.
+            const raw = await this.httpRequest("POST", "/v1/search", args, {
+                raw: true,
+            });
+            const data = (raw.data ?? []);
+            if (data !== null && typeof data === "object") {
+                for (const key of ["answer", "citations", "llmUsage"]) {
+                    if (raw[key] !== undefined) {
+                        Object.defineProperty(data, key, {
+                            value: raw[key],
+                            enumerable: false,
+                            configurable: true,
+                            writable: true,
+                        });
+                    }
+                }
+            }
+            return data;
+        }
         return this.localTransport().toolCall("crw_search", args);
     }
     /**
@@ -167,29 +195,45 @@ export class CrwClient {
         Object.assign(args, rest);
         return this.localTransport().toolCall("crw_parse_file", args);
     }
-    /** Structured LLM extraction across URLs (HTTP mode only). */
+    /**
+     * Native multi-URL structured extraction (HTTP mode only). Starts an async
+     * `/v1/extract` job and polls until complete, returning a per-URL results
+     * array (`[{ url, status, data, error, llmUsage }]`) in request order.
+     */
     async extract(opts) {
         if (!this.apiUrl)
             throw new CrwError(httpOnlyHint("extract", "LLM extract job endpoint"));
-        const { urls, prompt, schema, systemPrompt, pollInterval = 2, timeout = 120 } = opts;
+        const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
         const body = { urls: [...urls] };
         if (prompt !== undefined)
             body.prompt = prompt;
         if (schema !== undefined)
             body.schema = schema;
-        if (systemPrompt !== undefined)
-            body.systemPrompt = systemPrompt;
-        const start = await this.httpRequest("POST", "/v2/extract", body, { raw: true });
+        if (basis !== undefined)
+            body.basis = basis;
+        if (llmApiKey !== undefined)
+            body.llmApiKey = llmApiKey;
+        if (llmProvider !== undefined)
+            body.llmProvider = llmProvider;
+        if (llmModel !== undefined)
+            body.llmModel = llmModel;
+        const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true });
         const jobId = start.id;
-        if (!jobId)
-            throw new CrwError(`extract did not return job ID: ${JSON.stringify(start)}`);
+        // The managed API answers synchronously: the extraction is already finished and
+        // the payload is in this first response, with no job to poll. Only the async
+        // path hands back an `id`. Demanding one made every managed extract() throw.
+        if (!jobId) {
+            if (Array.isArray(start.results))
+                return start.results;
+            throw new CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`);
+        }
         const deadline = Date.now() + timeout * 1000;
         for (;;) {
             if (Date.now() > deadline)
                 throw new CrwTimeoutError(`Extract ${jobId} timed out after ${timeout}s`);
-            const status = await this.httpRequest("GET", `/v2/extract/${jobId}`, undefined, { raw: true, checkSuccess: false });
+            const status = await this.httpRequest("GET", `/v1/extract/${jobId}`, undefined, { raw: true, checkSuccess: false });
             if (status.status === "completed")
-                return status.data ?? {};
+                return status.results ?? [];
             if (status.status === "failed")
                 throw new CrwError(`Extract failed: ${status.error ?? "unknown"}`);
             await sleep(pollInterval * 1000);

package/dist/esm/types.d.ts

@@ -25,9 +25,53 @@ export interface ScrapeOptions {
     waitFor?: number;
     /** JSON Schema for structured LLM extraction (auto-adds the `json` format). */
     jsonSchema?: Json;
+    /**
+     * Return per-field evidence alongside `json`. Every top-level scalar property
+     * of `jsonSchema` comes back as a {@link Basis}. Requires `jsonSchema`.
+     */
+    basis?: boolean;
     /** Any other engine scrape option, passed through verbatim. */
     [key: string]: unknown;
 }
+/** How well an extracted field is attributed to its source. */
+export type FieldStatus = "supported" | "unverified" | "unsupported" | "notFound";
+/** The citation backing an extracted value. Every field is server-produced. */
+export interface EvidenceCitation {
+    /** The document the engine fetched. Never a model-supplied url. */
+    url: string;
+    title?: string;
+    /**
+     * Verbatim span from the canonical source text. Absent when the span could
+     * not be established (`status: "unverified"`).
+     */
+    excerpt?: string;
+    /** `sha256:<hex>` of the exact text sent to the model. */
+    sourceHash: string;
+    /** Which text `sourceHash` covers. `"llmInput"` for extraction basis. */
+    sourceTextKind: string;
+}
+/**
+ * Per-field extraction evidence. Emitted only for top-level scalar schema
+ * properties, and only when the request set `basis: true`.
+ *
+ * `status` is honest: a field whose attribution could not be verified is marked
+ * `unverified`/`unsupported` rather than given a fabricated citation.
+ * `citations` is empty exactly when `status` is `unsupported` or `notFound`;
+ * `value` is `null` exactly when `status` is `notFound`.
+ */
+export interface Basis {
+    basisVersion: number;
+    field: string;
+    value: unknown | null;
+    status: FieldStatus;
+    confidence?: "low" | "medium" | "high";
+    citations: EvidenceCitation[];
+}
+/** A coded reason a field's attribution was downgraded. Never upstream text. */
+export interface BasisWarning {
+    field: string;
+    code: string;
+}
 export interface CrawlOptions {
     maxDepth?: number;
     maxPages?: number;
@@ -60,7 +104,15 @@ export interface ExtractOptions {
     urls: string[];
     prompt?: string;
     schema?: Json;
-    systemPrompt?: string;
+    /**
+     * Return per-field evidence: each result carries a `basis` array, one
+     * {@link Basis} per top-level scalar schema property. Requires `schema`.
+     */
+    basis?: boolean;
+    /** BYOK: use your own LLM key/provider/model instead of the server's. */
+    llmApiKey?: string;
+    llmProvider?: string;
+    llmModel?: string;
     pollInterval?: number;
     timeout?: number;
 }
@@ -80,7 +132,8 @@ export type ScrapeResult = Json;
 export type CrawlResult = Json[];
 export type SearchResult = Json | Json[];
 export type ParseResult = Json;
-export type ExtractResult = Json;
+/** Native `/v1/extract` returns one result object per URL, in request order. */
+export type ExtractResult = Json[];
 export type BatchResult = Json[];
 export type Capabilities = Json;
 export type DiffResult = Json;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crw-sdk",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
   "license": "MIT",
   "homepage": "https://github.com/us/crw",