crw-sdk 0.23.0 → 0.24.0

package/README.md

@@ -58,7 +58,7 @@ CRW_LOCAL=1 node app.js
 | `changeTrackingDiff(cur, prev?)` | Diff vs a prior snapshot | HTTP |
 | `close()` | Shut down the local subprocess | — |
 
-¹ Local search needs a SearXNG URL configured on the engine.
+¹ Local search needs a search backend configured on the engine.
 
 ```ts
 // Structured extraction (async job → per-URL results array):

package/dist/cjs/client.js

@@ -95,7 +95,7 @@ class CrwClient {
         return result.links ?? [];
     }
     /**
-     * Works in both modes; local mode needs a SearXNG URL configured on the engine.
+     * Works in both modes; local mode needs a search backend configured on the engine.
      */
     async search(query, opts = {}) {
         const { limit = 5, lang, tbs, sources, categories, scrapeOptions, ...rest } = opts;
@@ -111,8 +111,36 @@ class CrwClient {
         if (scrapeOptions)
             args.scrapeOptions = scrapeOptions;
         Object.assign(args, rest);
-        if (this.apiUrl)
-            return this.httpPost("/v1/search", args);
+        if (this.apiUrl) {
+            // `data` IS the results, in whatever shape the caller asked for: a flat
+            // array, a `{ web, news }` object when `sources` is set, or the engine's
+            // `{ results, answer, … }` when you point the client at a self-hosted
+            // server. Return it untouched — reshaping it here silently emptied both the
+            // grouped and the self-hosted cases.
+            //
+            // `answer` / `citations` / `llmUsage` ride BESIDE `data` on the managed API,
+            // and the old `data` unwrap dropped them: you could pay for an AI answer and
+            // have no way to read it. Attach them non-enumerably, so `results.answer`
+            // works while `for…of`, spread and JSON.stringify keep seeing exactly the
+            // array they saw before.
+            const raw = await this.httpRequest("POST", "/v1/search", args, {
+                raw: true,
+            });
+            const data = (raw.data ?? []);
+            if (data !== null && typeof data === "object") {
+                for (const key of ["answer", "citations", "llmUsage"]) {
+                    if (raw[key] !== undefined) {
+                        Object.defineProperty(data, key, {
+                            value: raw[key],
+                            enumerable: false,
+                            configurable: true,
+                            writable: true,
+                        });
+                    }
+                }
+            }
+            return data;
+        }
         return this.localTransport().toolCall("crw_search", args);
     }
     /**
@@ -178,12 +206,14 @@ class CrwClient {
     async extract(opts) {
         if (!this.apiUrl)
             throw new errors_js_1.CrwError(httpOnlyHint("extract", "LLM extract job endpoint"));
-        const { urls, prompt, schema, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
+        const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
         const body = { urls: [...urls] };
         if (prompt !== undefined)
             body.prompt = prompt;
         if (schema !== undefined)
             body.schema = schema;
+        if (basis !== undefined)
+            body.basis = basis;
         if (llmApiKey !== undefined)
             body.llmApiKey = llmApiKey;
         if (llmProvider !== undefined)
@@ -192,8 +222,14 @@ class CrwClient {
             body.llmModel = llmModel;
         const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true });
         const jobId = start.id;
-        if (!jobId)
-            throw new errors_js_1.CrwError(`extract did not return job ID: ${JSON.stringify(start)}`);
+        // The managed API answers synchronously: the extraction is already finished and
+        // the payload is in this first response, with no job to poll. Only the async
+        // path hands back an `id`. Demanding one made every managed extract() throw.
+        if (!jobId) {
+            if (Array.isArray(start.results))
+                return start.results;
+            throw new errors_js_1.CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`);
+        }
         const deadline = Date.now() + timeout * 1000;
         for (;;) {
             if (Date.now() > deadline)

package/dist/esm/client.d.ts

@@ -17,7 +17,7 @@ export declare class CrwClient {
     crawl(url: string, opts?: CrawlOptions): Promise<CrawlResult>;
     map(url: string, opts?: MapOptions): Promise<string[]>;
     /**
-     * Works in both modes; local mode needs a SearXNG URL configured on the engine.
+     * Works in both modes; local mode needs a search backend configured on the engine.
      */
     search(query: string, opts?: SearchOptions): Promise<SearchResult>;
     /**

package/dist/esm/client.js

@@ -92,7 +92,7 @@ export class CrwClient {
         return result.links ?? [];
     }
     /**
-     * Works in both modes; local mode needs a SearXNG URL configured on the engine.
+     * Works in both modes; local mode needs a search backend configured on the engine.
      */
     async search(query, opts = {}) {
         const { limit = 5, lang, tbs, sources, categories, scrapeOptions, ...rest } = opts;
@@ -108,8 +108,36 @@ export class CrwClient {
         if (scrapeOptions)
             args.scrapeOptions = scrapeOptions;
         Object.assign(args, rest);
-        if (this.apiUrl)
-            return this.httpPost("/v1/search", args);
+        if (this.apiUrl) {
+            // `data` IS the results, in whatever shape the caller asked for: a flat
+            // array, a `{ web, news }` object when `sources` is set, or the engine's
+            // `{ results, answer, … }` when you point the client at a self-hosted
+            // server. Return it untouched — reshaping it here silently emptied both the
+            // grouped and the self-hosted cases.
+            //
+            // `answer` / `citations` / `llmUsage` ride BESIDE `data` on the managed API,
+            // and the old `data` unwrap dropped them: you could pay for an AI answer and
+            // have no way to read it. Attach them non-enumerably, so `results.answer`
+            // works while `for…of`, spread and JSON.stringify keep seeing exactly the
+            // array they saw before.
+            const raw = await this.httpRequest("POST", "/v1/search", args, {
+                raw: true,
+            });
+            const data = (raw.data ?? []);
+            if (data !== null && typeof data === "object") {
+                for (const key of ["answer", "citations", "llmUsage"]) {
+                    if (raw[key] !== undefined) {
+                        Object.defineProperty(data, key, {
+                            value: raw[key],
+                            enumerable: false,
+                            configurable: true,
+                            writable: true,
+                        });
+                    }
+                }
+            }
+            return data;
+        }
         return this.localTransport().toolCall("crw_search", args);
     }
     /**
@@ -175,12 +203,14 @@ export class CrwClient {
     async extract(opts) {
         if (!this.apiUrl)
             throw new CrwError(httpOnlyHint("extract", "LLM extract job endpoint"));
-        const { urls, prompt, schema, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
+        const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
         const body = { urls: [...urls] };
         if (prompt !== undefined)
             body.prompt = prompt;
         if (schema !== undefined)
             body.schema = schema;
+        if (basis !== undefined)
+            body.basis = basis;
         if (llmApiKey !== undefined)
             body.llmApiKey = llmApiKey;
         if (llmProvider !== undefined)
@@ -189,8 +219,14 @@ export class CrwClient {
             body.llmModel = llmModel;
         const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true });
         const jobId = start.id;
-        if (!jobId)
-            throw new CrwError(`extract did not return job ID: ${JSON.stringify(start)}`);
+        // The managed API answers synchronously: the extraction is already finished and
+        // the payload is in this first response, with no job to poll. Only the async
+        // path hands back an `id`. Demanding one made every managed extract() throw.
+        if (!jobId) {
+            if (Array.isArray(start.results))
+                return start.results;
+            throw new CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`);
+        }
         const deadline = Date.now() + timeout * 1000;
         for (;;) {
             if (Date.now() > deadline)

package/dist/esm/types.d.ts

@@ -25,9 +25,53 @@ export interface ScrapeOptions {
     waitFor?: number;
     /** JSON Schema for structured LLM extraction (auto-adds the `json` format). */
     jsonSchema?: Json;
+    /**
+     * Return per-field evidence alongside `json`. Every top-level scalar property
+     * of `jsonSchema` comes back as a {@link Basis}. Requires `jsonSchema`.
+     */
+    basis?: boolean;
     /** Any other engine scrape option, passed through verbatim. */
     [key: string]: unknown;
 }
+/** How well an extracted field is attributed to its source. */
+export type FieldStatus = "supported" | "unverified" | "unsupported" | "notFound";
+/** The citation backing an extracted value. Every field is server-produced. */
+export interface EvidenceCitation {
+    /** The document the engine fetched. Never a model-supplied url. */
+    url: string;
+    title?: string;
+    /**
+     * Verbatim span from the canonical source text. Absent when the span could
+     * not be established (`status: "unverified"`).
+     */
+    excerpt?: string;
+    /** `sha256:<hex>` of the exact text sent to the model. */
+    sourceHash: string;
+    /** Which text `sourceHash` covers. `"llmInput"` for extraction basis. */
+    sourceTextKind: string;
+}
+/**
+ * Per-field extraction evidence. Emitted only for top-level scalar schema
+ * properties, and only when the request set `basis: true`.
+ *
+ * `status` is honest: a field whose attribution could not be verified is marked
+ * `unverified`/`unsupported` rather than given a fabricated citation.
+ * `citations` is empty exactly when `status` is `unsupported` or `notFound`;
+ * `value` is `null` exactly when `status` is `notFound`.
+ */
+export interface Basis {
+    basisVersion: number;
+    field: string;
+    value: unknown | null;
+    status: FieldStatus;
+    confidence?: "low" | "medium" | "high";
+    citations: EvidenceCitation[];
+}
+/** A coded reason a field's attribution was downgraded. Never upstream text. */
+export interface BasisWarning {
+    field: string;
+    code: string;
+}
 export interface CrawlOptions {
     maxDepth?: number;
     maxPages?: number;
@@ -60,6 +104,11 @@ export interface ExtractOptions {
     urls: string[];
     prompt?: string;
     schema?: Json;
+    /**
+     * Return per-field evidence: each result carries a `basis` array, one
+     * {@link Basis} per top-level scalar schema property. Requires `schema`.
+     */
+    basis?: boolean;
     /** BYOK: use your own LLM key/provider/model instead of the server's. */
     llmApiKey?: string;
     llmProvider?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crw-sdk",
-  "version": "0.23.0",
+  "version": "0.24.0",
   "description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
   "license": "MIT",
   "homepage": "https://github.com/us/crw",