crw-sdk 0.39.1 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -57,7 +57,8 @@ CRW_LOCAL=1 node app.js
57
57
  | `batchScrape(urls, opts?)` | Scrape many URLs (async) | HTTP |
58
58
  | `capabilities()` | Feature-detect the engine | HTTP |
59
59
  | `changeTrackingDiff(cur, prev?)` | Diff vs a prior snapshot | HTTP |
60
- | `close()` | Shut down the local subprocess | — |
60
+ | `findData(opts?)` / `getData(provider, capability, options?)` | Structured data from named public APIs | HTTP |
61
+ | `close()` | Shut down local subprocesses (returns a promise) | both |
61
62
 
62
63
  ¹ Local search needs a search backend configured on the engine.
63
64
 
@@ -83,6 +84,29 @@ import { readFileSync } from "node:fs";
83
84
  const doc = await crw.parseFile(readFileSync("invoice.pdf"), { formats: ["markdown"] });
84
85
  ```
85
86
 
87
+ ## Data sources
88
+
89
+ Structured records from named public APIs (papers, packages, registries). Find a
90
+ source, then call it with the options it lists:
91
+
92
+ ```ts
93
+ const { items, nextOffset } = await crw.findData({ query: "packages" });
94
+ // items: [{ provider, capability, name, requiredOptions, optionalOptions, creditsCost, licence, ... }]
95
+
96
+ try {
97
+ const result = await crw.getData("depsdev", "package", { system: "npm", name: "react" });
98
+ console.log(result.data, result.provenance.licence);
99
+ } finally {
100
+ await crw.close();
101
+ }
102
+ ```
103
+
104
+ A few sources run on your own machine rather than on the API. For those,
105
+ `getData()` starts a local `crw-mcp` (found via `CRW_BINARY` or `PATH`; install with
106
+ `npm i -g crw-mcp`) and calls it with your API URL and key. That `crw-mcp` child
107
+ stays alive for later calls until you call `close()`.
108
+ Data sources are not available with `CRW_LOCAL=1`.
109
+
86
110
  ## Parity
87
111
 
88
112
  This SDK mirrors the Python [`crw`](https://pypi.org/project/crw/) client method-for-method,
@@ -20,7 +20,11 @@ function httpOnlyHint(name, reason) {
20
20
  return (`${name}() requires HTTP mode (${reason}). It is not available with CRW_LOCAL=1. ` +
21
21
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
22
22
  }
23
+ const DATA_LOCAL_HINT = "data sources need the fastCRW API (set apiUrl/apiKey); not available in CRW_LOCAL mode";
24
+ const SLUG = /^[a-z0-9-]+$/;
23
25
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
26
+ /** A body's machine-readable `errorCode`, when it is a string. */
27
+ const bodyErrorCode = (body) => (typeof body.errorCode === "string" ? body.errorCode : undefined);
24
28
  /**
25
29
  * Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
26
30
  * It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
@@ -61,6 +65,7 @@ class CrwClient {
61
65
  apiUrl;
62
66
  apiKey;
63
67
  local = null;
68
+ dataProxy = null;
64
69
  /**
65
70
  * CRW is cloud-first. With no arguments the client targets the managed cloud
66
71
  * (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
@@ -417,10 +422,114 @@ class CrwClient {
417
422
  Object.assign(body, rest);
418
423
  return this.httpPost("/v1/change-tracking/diff", body);
419
424
  }
420
- /** Shut down the local subprocess if running. */
421
- close() {
422
- this.local?.close();
425
+ /** Find data sources (`GET /v1/data/find`). Page with `offset: result.nextOffset`. */
426
+ async findData(opts = {}) {
427
+ if (!this.apiUrl)
428
+ throw new errors_js_1.CrwError(DATA_LOCAL_HINT);
429
+ const q = new URLSearchParams();
430
+ for (const key of ["query", "category", "provider"]) {
431
+ const value = opts[key]?.trim();
432
+ if (value)
433
+ q.set(key, key === "provider" ? value.toLowerCase() : value);
434
+ }
435
+ if (opts.limit !== undefined)
436
+ q.set("limit", String(opts.limit));
437
+ if (opts.offset !== undefined)
438
+ q.set("offset", String(opts.offset));
439
+ const qs = q.toString();
440
+ const reply = await this.httpRequest("GET", `/v1/data/find${qs ? `?${qs}` : ""}`, undefined, {
441
+ raw: true,
442
+ });
443
+ const tools = Array.isArray(reply.items) ? reply.items : [];
444
+ const list = (v) => (Array.isArray(v) ? v : []);
445
+ const items = tools.map((t) => {
446
+ const options = (t.options ?? {});
447
+ const provenance = (t.provenance ?? {});
448
+ return {
449
+ provider: t.provider,
450
+ capability: t.capability,
451
+ name: t.name ?? null,
452
+ description: t.description ?? null,
453
+ category: t.category ?? null,
454
+ creditsCost: t.creditsCost ?? null,
455
+ requiredOptions: list(options.required),
456
+ optionalOptions: list(options.optional),
457
+ keyRequired: t.keyRequired === true,
458
+ licence: provenance.licence ?? "",
459
+ };
460
+ });
461
+ const result = { items, total: reply.total ?? null };
462
+ if (typeof reply.nextOffset === "number")
463
+ result.nextOffset = reply.nextOffset;
464
+ return result;
465
+ }
466
+ /**
467
+ * Call one data source capability (`GET /v1/data/<provider>/<capability>`).
468
+ * A source that runs on your machine instead of the API is run through a
469
+ * local `crw-mcp` (CRW_BINARY or PATH) that talks to the same API with your key.
470
+ */
471
+ async getData(provider, capability, options = {}) {
472
+ if (!this.apiUrl)
473
+ throw new errors_js_1.CrwError(DATA_LOCAL_HINT);
474
+ for (const [field, value] of [["provider", provider], ["capability", capability]]) {
475
+ if (!SLUG.test(value)) {
476
+ throw new errors_js_1.CrwError(`${field} must be a lowercase id like the ones findData returns, got ${JSON.stringify(value)}`);
477
+ }
478
+ }
479
+ const q = new URLSearchParams();
480
+ for (const [key, value] of Object.entries(options)) {
481
+ if (value === null || value === undefined)
482
+ continue;
483
+ // Same rule as the Python SDK, both MCPs and the API: scalars only.
484
+ if (!["string", "number", "boolean"].includes(typeof value)) {
485
+ throw new errors_js_1.CrwError(`[invalid_options] Option "${key}" must be a string, number or boolean`);
486
+ }
487
+ q.set(key, String(value));
488
+ }
489
+ const qs = q.toString();
490
+ const path = `/v1/data/${provider}/${capability}${qs ? `?${qs}` : ""}`;
491
+ let body;
492
+ try {
493
+ body = await this.httpRequest("GET", path, undefined, { raw: true });
494
+ }
495
+ catch (err) {
496
+ if (err instanceof errors_js_1.CrwApiError && err.statusCode === 409 && err.errorCode === "client_execution_required") {
497
+ const proxy = await this.dataProxyTransport();
498
+ return (await proxy.toolCall("crw_get_data", { provider, capability, options }));
499
+ }
500
+ throw err;
501
+ }
502
+ const p = (body.provenance ?? {});
503
+ const field = (v) => v ?? null;
504
+ return {
505
+ provider: body.provider,
506
+ capability: body.capability,
507
+ creditsUsed: body.credits_used ?? null,
508
+ provenance: {
509
+ source: field(p.source),
510
+ upstreamUrl: field(p.upstream_url),
511
+ licence: field(p.licence),
512
+ licenceUrl: field(p.licence_url),
513
+ attributionText: field(p.attribution_text),
514
+ retrievedAt: field(p.retrieved_at),
515
+ },
516
+ data: body.data ?? null,
517
+ };
518
+ }
519
+ /**
520
+ * Shut down any local subprocess (CRW_LOCAL engine or data-source proxy).
521
+ * Each gets up to three seconds to exit on its own (a data proxy may still be
522
+ * sending usage counts); await it to know they are gone.
523
+ */
524
+ async close() {
525
+ const local = this.local;
526
+ const proxy = this.dataProxy;
423
527
  this.local = null;
528
+ this.dataProxy = null;
529
+ await Promise.all([
530
+ local?.close(),
531
+ proxy?.then((t) => t.close(), () => { }),
532
+ ]);
424
533
  }
425
534
  // --- local (subprocess) mode ---
426
535
  localTransport() {
@@ -443,6 +552,24 @@ class CrwClient {
443
552
  await sleep(pollInterval * 1000);
444
553
  }
445
554
  }
555
+ /**
556
+ * A proxy-mode `crw-mcp`, spawned on first use and kept until close(). The
557
+ * promise is cached so concurrent calls share one process; a failed start is
558
+ * forgotten so the next call retries.
559
+ */
560
+ dataProxyTransport() {
561
+ if (!this.dataProxy) {
562
+ const transport = new local_js_1.LocalTransport((0, local_js_1.proxyChildEnv)(this.apiUrl, this.apiKey));
563
+ const ready = transport.initialize().then(() => transport, (err) => {
564
+ void transport.close();
565
+ if (this.dataProxy === ready)
566
+ this.dataProxy = null;
567
+ throw err;
568
+ });
569
+ this.dataProxy = ready;
570
+ }
571
+ return this.dataProxy;
572
+ }
446
573
  // --- HTTP mode ---
447
574
  async httpRequest(method, path, body, { raw = false, checkSuccess = true, headers: extraHeaders, } = {}) {
448
575
  if (this.apiUrl === null)
@@ -457,7 +584,7 @@ class CrwClient {
457
584
  const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined });
458
585
  const result = await this.readJson(resp);
459
586
  if (checkSuccess && result.success === false) {
460
- throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status);
587
+ throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
461
588
  }
462
589
  if (raw)
463
590
  return result;
@@ -472,8 +599,9 @@ class CrwClient {
472
599
  headers.Authorization = `Bearer ${this.apiKey}`;
473
600
  const resp = await fetch(url, { method: "POST", headers, body: form });
474
601
  const result = await this.readJson(resp);
475
- if (result.success === false)
476
- throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status);
602
+ if (result.success === false) {
603
+ throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
604
+ }
477
605
  return result.data ?? result;
478
606
  }
479
607
  /** Parse the JSON body; surface a non-2xx body's `error` as CrwApiError. */
@@ -490,7 +618,7 @@ class CrwClient {
490
618
  }
491
619
  if (!resp.ok) {
492
620
  const message = parsed.error ?? parsed.message ?? `HTTP ${resp.status}`;
493
- throw new errors_js_1.CrwApiError(message, resp.status);
621
+ throw new errors_js_1.CrwApiError(message, resp.status, bodyErrorCode(parsed));
494
622
  }
495
623
  return parsed;
496
624
  }
@@ -11,10 +11,13 @@ class CrwError extends Error {
11
11
  exports.CrwError = CrwError;
12
12
  class CrwApiError extends CrwError {
13
13
  statusCode;
14
- constructor(message, statusCode) {
14
+ /** The body's machine-readable `errorCode`, when the server sent one. */
15
+ errorCode;
16
+ constructor(message, statusCode, errorCode) {
15
17
  super(message);
16
18
  this.name = "CrwApiError";
17
19
  this.statusCode = statusCode;
20
+ this.errorCode = errorCode;
18
21
  }
19
22
  }
20
23
  exports.CrwApiError = CrwApiError;
package/dist/cjs/local.js CHANGED
@@ -9,6 +9,7 @@
9
9
  Object.defineProperty(exports, "__esModule", { value: true });
10
10
  exports.LocalTransport = void 0;
11
11
  exports.localChildEnv = localChildEnv;
12
+ exports.proxyChildEnv = proxyChildEnv;
12
13
  const node_child_process_1 = require("node:child_process");
13
14
  const errors_js_1 = require("./errors.js");
14
15
  const BINARY_NAME = process.platform === "win32" ? "crw-mcp.exe" : "crw-mcp";
@@ -38,11 +39,29 @@ function localChildEnv() {
38
39
  const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
39
40
  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
40
41
  }
42
+ /**
43
+ * Environment for a `crw-mcp` subprocess in PROXY mode: it talks to `apiUrl`
44
+ * with the client's own key, whatever the shell says. Only the data tools use
45
+ * it, for a source the API answered with `client_execution_required`.
46
+ */
47
+ function proxyChildEnv(apiUrl, apiKey) {
48
+ const env = { ...process.env, CRW_API_URL: apiUrl };
49
+ if (apiKey)
50
+ env.CRW_API_KEY = apiKey;
51
+ else
52
+ delete env.CRW_API_KEY;
53
+ return env;
54
+ }
41
55
  class LocalTransport {
56
+ env;
42
57
  proc = null;
43
58
  nextId = 0;
44
59
  pending = new Map();
45
60
  buffer = "";
61
+ /** `env` defaults to {@link localChildEnv} (embedded mode). */
62
+ constructor(env) {
63
+ this.env = env;
64
+ }
46
65
  resolveBinary() {
47
66
  const env = process.env.CRW_BINARY;
48
67
  if (env)
@@ -55,10 +74,11 @@ class LocalTransport {
55
74
  if (this.proc && this.proc.exitCode === null)
56
75
  return this.proc;
57
76
  const bin = this.resolveBinary();
58
- const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
77
+ const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: this.env ?? localChildEnv() });
59
78
  proc.on("error", (err) => {
79
+ const where = bin === BINARY_NAME ? "on PATH" : `at CRW_BINARY=${bin}`;
60
80
  const failure = err.code === "ENOENT"
61
- ? new errors_js_1.CrwBinaryNotFoundError(`crw-mcp binary not found on PATH. Install it (e.g. \`npm i -g crw-mcp\` or ` +
81
+ ? new errors_js_1.CrwBinaryNotFoundError(`crw-mcp binary not found ${where}. Install it (e.g. \`npm i -g crw-mcp\` or ` +
62
82
  `\`cargo install crw-mcp\`) or set CRW_BINARY to its path.`)
63
83
  : new errors_js_1.CrwError(`crw-mcp failed to start: ${err.message}`);
64
84
  for (const p of this.pending.values())
@@ -104,6 +124,14 @@ class LocalTransport {
104
124
  }
105
125
  }
106
126
  }
127
+ /** MCP handshake; `crw-mcp` is stateless, so a respawn needs no repeat. */
128
+ initialize() {
129
+ return this.jsonrpc("initialize", {
130
+ protocolVersion: "2025-06-18",
131
+ capabilities: {},
132
+ clientInfo: { name: "crw-sdk", version: "1" },
133
+ });
134
+ }
107
135
  jsonrpc(method, params) {
108
136
  const proc = this.ensureProcess();
109
137
  const id = ++this.nextId;
@@ -117,16 +145,35 @@ class LocalTransport {
117
145
  const content = result.content?.[0];
118
146
  if (!content)
119
147
  throw new errors_js_1.CrwError(`Empty response from ${name}`);
120
- if (result.isError)
121
- throw new errors_js_1.CrwApiError(content.text ?? "Unknown error");
148
+ if (result.isError) {
149
+ // A tool error has no HTTP status; its bracketed code is the errorCode.
150
+ const text = content.text ?? "Unknown error";
151
+ throw new errors_js_1.CrwApiError(text, undefined, /^\[([a-z_]+)\]/.exec(text)?.[1]);
152
+ }
122
153
  return JSON.parse(content.text ?? "{}");
123
154
  }
124
- close() {
125
- if (this.proc && this.proc.exitCode === null) {
126
- this.proc.stdin.end();
127
- this.proc.kill();
128
- }
155
+ /**
156
+ * EOF first so the child can finish what it started (a proxy-mode data call
157
+ * sends a usage ping after answering), then kill it if it is still up after
158
+ * `graceMs`. Resolves once it is gone; not awaiting it is fine. The default
159
+ * is above crw-mcp's own 2 s wait for pings in flight.
160
+ */
161
+ close(graceMs = 3000) {
162
+ const proc = this.proc;
129
163
  this.proc = null;
164
+ if (!proc || proc.exitCode !== null || proc.signalCode !== null)
165
+ return Promise.resolve();
166
+ return new Promise((resolve) => {
167
+ const timer = setTimeout(() => {
168
+ proc.kill();
169
+ resolve();
170
+ }, graceMs);
171
+ proc.once("exit", () => {
172
+ clearTimeout(timer);
173
+ resolve();
174
+ });
175
+ proc.stdin.end();
176
+ });
130
177
  }
131
178
  }
132
179
  exports.LocalTransport = LocalTransport;
@@ -1,5 +1,5 @@
1
1
  /** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
2
- import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, MapResult, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
2
+ import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DataOptions, DataResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, FindDataOptions, FindDataResult, Json, MapOptions, MapResult, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
3
3
  export declare const CLOUD_API_URL = "https://api.fastcrw.com";
4
4
  export declare const DASHBOARD_URL = "https://fastcrw.com/dashboard";
5
5
  export declare const DOCS_URL = "https://docs.fastcrw.com";
@@ -7,6 +7,7 @@ export declare class CrwClient {
7
7
  private apiUrl;
8
8
  private apiKey;
9
9
  private local;
10
+ private dataProxy;
10
11
  /**
11
12
  * CRW is cloud-first. With no arguments the client targets the managed cloud
12
13
  * (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
@@ -58,10 +59,28 @@ export declare class CrwClient {
58
59
  capabilities(): Promise<Capabilities>;
59
60
  /** Diff a page against a prior snapshot (HTTP mode only). */
60
61
  changeTrackingDiff(current: Json, previous?: Json, opts?: ChangeTrackingOptions): Promise<DiffResult>;
61
- /** Shut down the local subprocess if running. */
62
- close(): void;
62
+ /** Find data sources (`GET /v1/data/find`). Page with `offset: result.nextOffset`. */
63
+ findData(opts?: FindDataOptions): Promise<FindDataResult>;
64
+ /**
65
+ * Call one data source capability (`GET /v1/data/<provider>/<capability>`).
66
+ * A source that runs on your machine instead of the API is run through a
67
+ * local `crw-mcp` (CRW_BINARY or PATH) that talks to the same API with your key.
68
+ */
69
+ getData(provider: string, capability: string, options?: DataOptions): Promise<DataResult>;
70
+ /**
71
+ * Shut down any local subprocess (CRW_LOCAL engine or data-source proxy).
72
+ * Each gets up to three seconds to exit on its own (a data proxy may still be
73
+ * sending usage counts); await it to know they are gone.
74
+ */
75
+ close(): Promise<void>;
63
76
  private localTransport;
64
77
  private pollLocalCrawl;
78
+ /**
79
+ * A proxy-mode `crw-mcp`, spawned on first use and kept until close(). The
80
+ * promise is cached so concurrent calls share one process; a failed start is
81
+ * forgotten so the next call retries.
82
+ */
83
+ private dataProxyTransport;
65
84
  private httpRequest;
66
85
  private httpMultipart;
67
86
  /** Parse the JSON body; surface a non-2xx body's `error` as CrwApiError. */
@@ -1,6 +1,6 @@
1
1
  /** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
2
2
  import { CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError } from "./errors.js";
3
- import { LocalTransport } from "./local.js";
3
+ import { LocalTransport, proxyChildEnv } from "./local.js";
4
4
  // CRW is cloud-first: with no explicit apiUrl and no CRW_LOCAL opt-in, the client
5
5
  // talks to the managed cloud. Mirrors the Python SDK + CLI onboarding.
6
6
  export const CLOUD_API_URL = "https://api.fastcrw.com";
@@ -17,7 +17,11 @@ function httpOnlyHint(name, reason) {
17
17
  return (`${name}() requires HTTP mode (${reason}). It is not available with CRW_LOCAL=1. ` +
18
18
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
19
19
  }
20
+ const DATA_LOCAL_HINT = "data sources need the fastCRW API (set apiUrl/apiKey); not available in CRW_LOCAL mode";
21
+ const SLUG = /^[a-z0-9-]+$/;
20
22
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
23
+ /** A body's machine-readable `errorCode`, when it is a string. */
24
+ const bodyErrorCode = (body) => (typeof body.errorCode === "string" ? body.errorCode : undefined);
21
25
  /**
22
26
  * Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
23
27
  * It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
@@ -58,6 +62,7 @@ export class CrwClient {
58
62
  apiUrl;
59
63
  apiKey;
60
64
  local = null;
65
+ dataProxy = null;
61
66
  /**
62
67
  * CRW is cloud-first. With no arguments the client targets the managed cloud
63
68
  * (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
@@ -414,10 +419,114 @@ export class CrwClient {
414
419
  Object.assign(body, rest);
415
420
  return this.httpPost("/v1/change-tracking/diff", body);
416
421
  }
417
- /** Shut down the local subprocess if running. */
418
- close() {
419
- this.local?.close();
422
+ /** Find data sources (`GET /v1/data/find`). Page with `offset: result.nextOffset`. */
423
+ async findData(opts = {}) {
424
+ if (!this.apiUrl)
425
+ throw new CrwError(DATA_LOCAL_HINT);
426
+ const q = new URLSearchParams();
427
+ for (const key of ["query", "category", "provider"]) {
428
+ const value = opts[key]?.trim();
429
+ if (value)
430
+ q.set(key, key === "provider" ? value.toLowerCase() : value);
431
+ }
432
+ if (opts.limit !== undefined)
433
+ q.set("limit", String(opts.limit));
434
+ if (opts.offset !== undefined)
435
+ q.set("offset", String(opts.offset));
436
+ const qs = q.toString();
437
+ const reply = await this.httpRequest("GET", `/v1/data/find${qs ? `?${qs}` : ""}`, undefined, {
438
+ raw: true,
439
+ });
440
+ const tools = Array.isArray(reply.items) ? reply.items : [];
441
+ const list = (v) => (Array.isArray(v) ? v : []);
442
+ const items = tools.map((t) => {
443
+ const options = (t.options ?? {});
444
+ const provenance = (t.provenance ?? {});
445
+ return {
446
+ provider: t.provider,
447
+ capability: t.capability,
448
+ name: t.name ?? null,
449
+ description: t.description ?? null,
450
+ category: t.category ?? null,
451
+ creditsCost: t.creditsCost ?? null,
452
+ requiredOptions: list(options.required),
453
+ optionalOptions: list(options.optional),
454
+ keyRequired: t.keyRequired === true,
455
+ licence: provenance.licence ?? "",
456
+ };
457
+ });
458
+ const result = { items, total: reply.total ?? null };
459
+ if (typeof reply.nextOffset === "number")
460
+ result.nextOffset = reply.nextOffset;
461
+ return result;
462
+ }
463
+ /**
464
+ * Call one data source capability (`GET /v1/data/<provider>/<capability>`).
465
+ * A source that runs on your machine instead of the API is run through a
466
+ * local `crw-mcp` (CRW_BINARY or PATH) that talks to the same API with your key.
467
+ */
468
+ async getData(provider, capability, options = {}) {
469
+ if (!this.apiUrl)
470
+ throw new CrwError(DATA_LOCAL_HINT);
471
+ for (const [field, value] of [["provider", provider], ["capability", capability]]) {
472
+ if (!SLUG.test(value)) {
473
+ throw new CrwError(`${field} must be a lowercase id like the ones findData returns, got ${JSON.stringify(value)}`);
474
+ }
475
+ }
476
+ const q = new URLSearchParams();
477
+ for (const [key, value] of Object.entries(options)) {
478
+ if (value === null || value === undefined)
479
+ continue;
480
+ // Same rule as the Python SDK, both MCPs and the API: scalars only.
481
+ if (!["string", "number", "boolean"].includes(typeof value)) {
482
+ throw new CrwError(`[invalid_options] Option "${key}" must be a string, number or boolean`);
483
+ }
484
+ q.set(key, String(value));
485
+ }
486
+ const qs = q.toString();
487
+ const path = `/v1/data/${provider}/${capability}${qs ? `?${qs}` : ""}`;
488
+ let body;
489
+ try {
490
+ body = await this.httpRequest("GET", path, undefined, { raw: true });
491
+ }
492
+ catch (err) {
493
+ if (err instanceof CrwApiError && err.statusCode === 409 && err.errorCode === "client_execution_required") {
494
+ const proxy = await this.dataProxyTransport();
495
+ return (await proxy.toolCall("crw_get_data", { provider, capability, options }));
496
+ }
497
+ throw err;
498
+ }
499
+ const p = (body.provenance ?? {});
500
+ const field = (v) => v ?? null;
501
+ return {
502
+ provider: body.provider,
503
+ capability: body.capability,
504
+ creditsUsed: body.credits_used ?? null,
505
+ provenance: {
506
+ source: field(p.source),
507
+ upstreamUrl: field(p.upstream_url),
508
+ licence: field(p.licence),
509
+ licenceUrl: field(p.licence_url),
510
+ attributionText: field(p.attribution_text),
511
+ retrievedAt: field(p.retrieved_at),
512
+ },
513
+ data: body.data ?? null,
514
+ };
515
+ }
516
+ /**
517
+ * Shut down any local subprocess (CRW_LOCAL engine or data-source proxy).
518
+ * Each gets up to three seconds to exit on its own (a data proxy may still be
519
+ * sending usage counts); await it to know they are gone.
520
+ */
521
+ async close() {
522
+ const local = this.local;
523
+ const proxy = this.dataProxy;
420
524
  this.local = null;
525
+ this.dataProxy = null;
526
+ await Promise.all([
527
+ local?.close(),
528
+ proxy?.then((t) => t.close(), () => { }),
529
+ ]);
421
530
  }
422
531
  // --- local (subprocess) mode ---
423
532
  localTransport() {
@@ -440,6 +549,24 @@ export class CrwClient {
440
549
  await sleep(pollInterval * 1000);
441
550
  }
442
551
  }
552
+ /**
553
+ * A proxy-mode `crw-mcp`, spawned on first use and kept until close(). The
554
+ * promise is cached so concurrent calls share one process; a failed start is
555
+ * forgotten so the next call retries.
556
+ */
557
+ dataProxyTransport() {
558
+ if (!this.dataProxy) {
559
+ const transport = new LocalTransport(proxyChildEnv(this.apiUrl, this.apiKey));
560
+ const ready = transport.initialize().then(() => transport, (err) => {
561
+ void transport.close();
562
+ if (this.dataProxy === ready)
563
+ this.dataProxy = null;
564
+ throw err;
565
+ });
566
+ this.dataProxy = ready;
567
+ }
568
+ return this.dataProxy;
569
+ }
443
570
  // --- HTTP mode ---
444
571
  async httpRequest(method, path, body, { raw = false, checkSuccess = true, headers: extraHeaders, } = {}) {
445
572
  if (this.apiUrl === null)
@@ -454,7 +581,7 @@ export class CrwClient {
454
581
  const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined });
455
582
  const result = await this.readJson(resp);
456
583
  if (checkSuccess && result.success === false) {
457
- throw new CrwApiError(result.error ?? "API error", resp.status);
584
+ throw new CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
458
585
  }
459
586
  if (raw)
460
587
  return result;
@@ -469,8 +596,9 @@ export class CrwClient {
469
596
  headers.Authorization = `Bearer ${this.apiKey}`;
470
597
  const resp = await fetch(url, { method: "POST", headers, body: form });
471
598
  const result = await this.readJson(resp);
472
- if (result.success === false)
473
- throw new CrwApiError(result.error ?? "API error", resp.status);
599
+ if (result.success === false) {
600
+ throw new CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
601
+ }
474
602
  return result.data ?? result;
475
603
  }
476
604
  /** Parse the JSON body; surface a non-2xx body's `error` as CrwApiError. */
@@ -487,7 +615,7 @@ export class CrwClient {
487
615
  }
488
616
  if (!resp.ok) {
489
617
  const message = parsed.error ?? parsed.message ?? `HTTP ${resp.status}`;
490
- throw new CrwApiError(message, resp.status);
618
+ throw new CrwApiError(message, resp.status, bodyErrorCode(parsed));
491
619
  }
492
620
  return parsed;
493
621
  }
@@ -5,7 +5,9 @@ export declare class CrwError extends Error {
5
5
  }
6
6
  export declare class CrwApiError extends CrwError {
7
7
  statusCode?: number;
8
- constructor(message: string, statusCode?: number);
8
+ /** The body's machine-readable `errorCode`, when the server sent one. */
9
+ errorCode?: string;
10
+ constructor(message: string, statusCode?: number, errorCode?: string);
9
11
  }
10
12
  export declare class CrwTimeoutError extends CrwError {
11
13
  constructor(message: string);
@@ -7,10 +7,13 @@ export class CrwError extends Error {
7
7
  }
8
8
  export class CrwApiError extends CrwError {
9
9
  statusCode;
10
- constructor(message, statusCode) {
10
+ /** The body's machine-readable `errorCode`, when the server sent one. */
11
+ errorCode;
12
+ constructor(message, statusCode, errorCode) {
11
13
  super(message);
12
14
  this.name = "CrwApiError";
13
15
  this.statusCode = statusCode;
16
+ this.errorCode = errorCode;
14
17
  }
15
18
  }
16
19
  export class CrwTimeoutError extends CrwError {
@@ -1,3 +1,3 @@
1
1
  export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
2
2
  export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
3
- export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, MapResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
3
+ export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, MapResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, FindDataOptions, FindDataResult, DataSource, DataOptions, DataResult, DataProvenance, Json, } from "./types.js";
@@ -19,15 +19,32 @@ import type { Json } from "./types.js";
19
19
  * is why `map()` still handles the envelope shape in `mapPayload`.
20
20
  */
21
21
  export declare function localChildEnv(): NodeJS.ProcessEnv;
22
+ /**
23
+ * Environment for a `crw-mcp` subprocess in PROXY mode: it talks to `apiUrl`
24
+ * with the client's own key, whatever the shell says. Only the data tools use
25
+ * it, for a source the API answered with `client_execution_required`.
26
+ */
27
+ export declare function proxyChildEnv(apiUrl: string, apiKey: string | undefined): NodeJS.ProcessEnv;
22
28
  export declare class LocalTransport {
29
+ private readonly env?;
23
30
  private proc;
24
31
  private nextId;
25
32
  private pending;
26
33
  private buffer;
34
+ /** `env` defaults to {@link localChildEnv} (embedded mode). */
35
+ constructor(env?: NodeJS.ProcessEnv | undefined);
27
36
  private resolveBinary;
28
37
  private ensureProcess;
29
38
  private onData;
30
- private jsonrpc;
39
+ /** MCP handshake; `crw-mcp` is stateless, so a respawn needs no repeat. */
40
+ initialize(): Promise<Json>;
41
+ jsonrpc(method: string, params: Json): Promise<Json>;
31
42
  toolCall(name: string, args: Json): Promise<Json>;
32
- close(): void;
43
+ /**
44
+ * EOF first so the child can finish what it started (a proxy-mode data call
45
+ * sends a usage ping after answering), then kill it if it is still up after
46
+ * `graceMs`. Resolves once it is gone; not awaiting it is fine. The default
47
+ * is above crw-mcp's own 2 s wait for pings in flight.
48
+ */
49
+ close(graceMs?: number): Promise<void>;
33
50
  }
package/dist/esm/local.js CHANGED
@@ -34,11 +34,29 @@ export function localChildEnv() {
34
34
  const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
35
35
  return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
36
36
  }
37
+ /**
38
+ * Environment for a `crw-mcp` subprocess in PROXY mode: it talks to `apiUrl`
39
+ * with the client's own key, whatever the shell says. Only the data tools use
40
+ * it, for a source the API answered with `client_execution_required`.
41
+ */
42
+ export function proxyChildEnv(apiUrl, apiKey) {
43
+ const env = { ...process.env, CRW_API_URL: apiUrl };
44
+ if (apiKey)
45
+ env.CRW_API_KEY = apiKey;
46
+ else
47
+ delete env.CRW_API_KEY;
48
+ return env;
49
+ }
37
50
  export class LocalTransport {
51
+ env;
38
52
  proc = null;
39
53
  nextId = 0;
40
54
  pending = new Map();
41
55
  buffer = "";
56
+ /** `env` defaults to {@link localChildEnv} (embedded mode). */
57
+ constructor(env) {
58
+ this.env = env;
59
+ }
42
60
  resolveBinary() {
43
61
  const env = process.env.CRW_BINARY;
44
62
  if (env)
@@ -51,10 +69,11 @@ export class LocalTransport {
51
69
  if (this.proc && this.proc.exitCode === null)
52
70
  return this.proc;
53
71
  const bin = this.resolveBinary();
54
- const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
72
+ const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: this.env ?? localChildEnv() });
55
73
  proc.on("error", (err) => {
74
+ const where = bin === BINARY_NAME ? "on PATH" : `at CRW_BINARY=${bin}`;
56
75
  const failure = err.code === "ENOENT"
57
- ? new CrwBinaryNotFoundError(`crw-mcp binary not found on PATH. Install it (e.g. \`npm i -g crw-mcp\` or ` +
76
+ ? new CrwBinaryNotFoundError(`crw-mcp binary not found ${where}. Install it (e.g. \`npm i -g crw-mcp\` or ` +
58
77
  `\`cargo install crw-mcp\`) or set CRW_BINARY to its path.`)
59
78
  : new CrwError(`crw-mcp failed to start: ${err.message}`);
60
79
  for (const p of this.pending.values())
@@ -100,6 +119,14 @@ export class LocalTransport {
100
119
  }
101
120
  }
102
121
  }
122
+ /** MCP handshake; `crw-mcp` is stateless, so a respawn needs no repeat. */
123
+ initialize() {
124
+ return this.jsonrpc("initialize", {
125
+ protocolVersion: "2025-06-18",
126
+ capabilities: {},
127
+ clientInfo: { name: "crw-sdk", version: "1" },
128
+ });
129
+ }
103
130
  jsonrpc(method, params) {
104
131
  const proc = this.ensureProcess();
105
132
  const id = ++this.nextId;
@@ -113,15 +140,34 @@ export class LocalTransport {
113
140
  const content = result.content?.[0];
114
141
  if (!content)
115
142
  throw new CrwError(`Empty response from ${name}`);
116
- if (result.isError)
117
- throw new CrwApiError(content.text ?? "Unknown error");
143
+ if (result.isError) {
144
+ // A tool error has no HTTP status; its bracketed code is the errorCode.
145
+ const text = content.text ?? "Unknown error";
146
+ throw new CrwApiError(text, undefined, /^\[([a-z_]+)\]/.exec(text)?.[1]);
147
+ }
118
148
  return JSON.parse(content.text ?? "{}");
119
149
  }
120
- close() {
121
- if (this.proc && this.proc.exitCode === null) {
122
- this.proc.stdin.end();
123
- this.proc.kill();
124
- }
150
+ /**
151
+ * EOF first so the child can finish what it started (a proxy-mode data call
152
+ * sends a usage ping after answering), then kill it if it is still up after
153
+ * `graceMs`. Resolves once it is gone; not awaiting it is fine. The default
154
+ * is above crw-mcp's own 2 s wait for pings in flight.
155
+ */
156
+ close(graceMs = 3000) {
157
+ const proc = this.proc;
125
158
  this.proc = null;
159
+ if (!proc || proc.exitCode !== null || proc.signalCode !== null)
160
+ return Promise.resolve();
161
+ return new Promise((resolve) => {
162
+ const timer = setTimeout(() => {
163
+ proc.kill();
164
+ resolve();
165
+ }, graceMs);
166
+ proc.once("exit", () => {
167
+ clearTimeout(timer);
168
+ resolve();
169
+ });
170
+ proc.stdin.end();
171
+ });
126
172
  }
127
173
  }
@@ -539,3 +539,54 @@ export interface ResearchSimilarOptions {
539
539
  rerank?: boolean;
540
540
  anchor?: string[];
541
541
  }
542
+ /** Filters for `findData()` (`GET /v1/data/find`). */
543
+ export interface FindDataOptions {
544
+ /** Words to match against source names and descriptions. */
545
+ query?: string;
546
+ /** Only sources in this category, e.g. `Research`. */
547
+ category?: string;
548
+ /** Only this provider, e.g. `openalex`. */
549
+ provider?: string;
550
+ /** Results per page (server default 10, max 50). */
551
+ limit?: number;
552
+ /** Results to skip; pass the previous page's `nextOffset`. */
553
+ offset?: number;
554
+ }
555
+ /** One data source capability, as `findData()` lists it. */
556
+ export interface DataSource {
557
+ provider: string;
558
+ capability: string;
559
+ name: string | null;
560
+ description: string | null;
561
+ /** The catalogue category, e.g. `Research`; `null` from an older server. */
562
+ category: string | null;
563
+ creditsCost: number | null;
564
+ requiredOptions: string[];
565
+ optionalOptions: string[];
566
+ keyRequired: boolean;
567
+ licence: string | null;
568
+ }
569
+ export interface FindDataResult {
570
+ items: DataSource[];
571
+ total: number | null;
572
+ /** Present when another page exists. */
573
+ nextOffset?: number;
574
+ }
575
+ /** Values for a source's `requiredOptions` and `optionalOptions`. */
576
+ export type DataOptions = Record<string, string | number | boolean | null | undefined>;
577
+ export interface DataProvenance {
578
+ source: string | null;
579
+ upstreamUrl: string | null;
580
+ licence: string | null;
581
+ licenceUrl: string | null;
582
+ attributionText: string | null;
583
+ retrievedAt: string | null;
584
+ }
585
+ /** `getData()` result: the source's record plus where it came from. */
586
+ export interface DataResult {
587
+ provider: string;
588
+ capability: string;
589
+ creditsUsed: number | null;
590
+ provenance: DataProvenance;
591
+ data: unknown;
592
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crw-sdk",
3
- "version": "0.39.1",
3
+ "version": "0.41.0",
4
4
  "description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
5
5
  "license": "MIT",
6
6
  "author": "fastCRW <hello@fastcrw.com>",