crw-sdk 0.40.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/dist/cjs/client.js +135 -7
- package/dist/cjs/errors.js +4 -1
- package/dist/cjs/local.js +56 -9
- package/dist/esm/client.d.ts +22 -3
- package/dist/esm/client.js +136 -8
- package/dist/esm/errors.d.ts +3 -1
- package/dist/esm/errors.js +4 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/local.d.ts +19 -2
- package/dist/esm/local.js +55 -9
- package/dist/esm/types.d.ts +51 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -57,7 +57,8 @@ CRW_LOCAL=1 node app.js
|
|
|
57
57
|
| `batchScrape(urls, opts?)` | Scrape many URLs (async) | HTTP |
|
|
58
58
|
| `capabilities()` | Feature-detect the engine | HTTP |
|
|
59
59
|
| `changeTrackingDiff(cur, prev?)` | Diff vs a prior snapshot | HTTP |
|
|
60
|
-
| `
|
|
60
|
+
| `findData(opts?)` / `getData(provider, capability, options?)` | Structured data from named public APIs | HTTP |
|
|
61
|
+
| `close()` | Shut down local subprocesses (returns a promise) | both |
|
|
61
62
|
|
|
62
63
|
¹ Local search needs a search backend configured on the engine.
|
|
63
64
|
|
|
@@ -83,6 +84,29 @@ import { readFileSync } from "node:fs";
|
|
|
83
84
|
const doc = await crw.parseFile(readFileSync("invoice.pdf"), { formats: ["markdown"] });
|
|
84
85
|
```
|
|
85
86
|
|
|
87
|
+
## Data sources
|
|
88
|
+
|
|
89
|
+
Structured records from named public APIs (papers, packages, registries). Find a
|
|
90
|
+
source, then call it with the options it lists:
|
|
91
|
+
|
|
92
|
+
```ts
|
|
93
|
+
const { items, nextOffset } = await crw.findData({ query: "packages" });
|
|
94
|
+
// items: [{ provider, capability, name, requiredOptions, optionalOptions, creditsCost, licence, ... }]
|
|
95
|
+
|
|
96
|
+
try {
|
|
97
|
+
const result = await crw.getData("depsdev", "package", { system: "npm", name: "react" });
|
|
98
|
+
console.log(result.data, result.provenance.licence);
|
|
99
|
+
} finally {
|
|
100
|
+
await crw.close();
|
|
101
|
+
}
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
A few sources run on your own machine rather than on the API. For those,
|
|
105
|
+
`getData()` starts a local `crw-mcp` (found via `CRW_BINARY` or `PATH`; install with
|
|
106
|
+
`npm i -g crw-mcp`) and calls it with your API URL and key. That `crw-mcp` child
|
|
107
|
+
stays alive for later calls until you call `close()`.
|
|
108
|
+
Data sources are not available with `CRW_LOCAL=1`.
|
|
109
|
+
|
|
86
110
|
## Parity
|
|
87
111
|
|
|
88
112
|
This SDK mirrors the Python [`crw`](https://pypi.org/project/crw/) client method-for-method,
|
package/dist/cjs/client.js
CHANGED
|
@@ -20,7 +20,11 @@ function httpOnlyHint(name, reason) {
|
|
|
20
20
|
return (`${name}() requires HTTP mode (${reason}). It is not available with CRW_LOCAL=1. ` +
|
|
21
21
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
22
22
|
}
|
|
23
|
+
const DATA_LOCAL_HINT = "data sources need the fastCRW API (set apiUrl/apiKey); not available in CRW_LOCAL mode";
|
|
24
|
+
const SLUG = /^[a-z0-9-]+$/;
|
|
23
25
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
26
|
+
/** A body's machine-readable `errorCode`, when it is a string. */
|
|
27
|
+
const bodyErrorCode = (body) => (typeof body.errorCode === "string" ? body.errorCode : undefined);
|
|
24
28
|
/**
|
|
25
29
|
* Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
|
|
26
30
|
* It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
|
|
@@ -61,6 +65,7 @@ class CrwClient {
|
|
|
61
65
|
apiUrl;
|
|
62
66
|
apiKey;
|
|
63
67
|
local = null;
|
|
68
|
+
dataProxy = null;
|
|
64
69
|
/**
|
|
65
70
|
* CRW is cloud-first. With no arguments the client targets the managed cloud
|
|
66
71
|
* (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
|
|
@@ -417,10 +422,114 @@ class CrwClient {
|
|
|
417
422
|
Object.assign(body, rest);
|
|
418
423
|
return this.httpPost("/v1/change-tracking/diff", body);
|
|
419
424
|
}
|
|
420
|
-
/**
|
|
421
|
-
|
|
422
|
-
this.
|
|
425
|
+
/** Find data sources (`GET /v1/data/find`). Page with `offset: result.nextOffset`. */
|
|
426
|
+
async findData(opts = {}) {
|
|
427
|
+
if (!this.apiUrl)
|
|
428
|
+
throw new errors_js_1.CrwError(DATA_LOCAL_HINT);
|
|
429
|
+
const q = new URLSearchParams();
|
|
430
|
+
for (const key of ["query", "category", "provider"]) {
|
|
431
|
+
const value = opts[key]?.trim();
|
|
432
|
+
if (value)
|
|
433
|
+
q.set(key, key === "provider" ? value.toLowerCase() : value);
|
|
434
|
+
}
|
|
435
|
+
if (opts.limit !== undefined)
|
|
436
|
+
q.set("limit", String(opts.limit));
|
|
437
|
+
if (opts.offset !== undefined)
|
|
438
|
+
q.set("offset", String(opts.offset));
|
|
439
|
+
const qs = q.toString();
|
|
440
|
+
const reply = await this.httpRequest("GET", `/v1/data/find${qs ? `?${qs}` : ""}`, undefined, {
|
|
441
|
+
raw: true,
|
|
442
|
+
});
|
|
443
|
+
const tools = Array.isArray(reply.items) ? reply.items : [];
|
|
444
|
+
const list = (v) => (Array.isArray(v) ? v : []);
|
|
445
|
+
const items = tools.map((t) => {
|
|
446
|
+
const options = (t.options ?? {});
|
|
447
|
+
const provenance = (t.provenance ?? {});
|
|
448
|
+
return {
|
|
449
|
+
provider: t.provider,
|
|
450
|
+
capability: t.capability,
|
|
451
|
+
name: t.name ?? null,
|
|
452
|
+
description: t.description ?? null,
|
|
453
|
+
category: t.category ?? null,
|
|
454
|
+
creditsCost: t.creditsCost ?? null,
|
|
455
|
+
requiredOptions: list(options.required),
|
|
456
|
+
optionalOptions: list(options.optional),
|
|
457
|
+
keyRequired: t.keyRequired === true,
|
|
458
|
+
licence: provenance.licence ?? "",
|
|
459
|
+
};
|
|
460
|
+
});
|
|
461
|
+
const result = { items, total: reply.total ?? null };
|
|
462
|
+
if (typeof reply.nextOffset === "number")
|
|
463
|
+
result.nextOffset = reply.nextOffset;
|
|
464
|
+
return result;
|
|
465
|
+
}
|
|
466
|
+
/**
|
|
467
|
+
* Call one data source capability (`GET /v1/data/<provider>/<capability>`).
|
|
468
|
+
* A source that runs on your machine instead of the API is run through a
|
|
469
|
+
* local `crw-mcp` (CRW_BINARY or PATH) that talks to the same API with your key.
|
|
470
|
+
*/
|
|
471
|
+
async getData(provider, capability, options = {}) {
|
|
472
|
+
if (!this.apiUrl)
|
|
473
|
+
throw new errors_js_1.CrwError(DATA_LOCAL_HINT);
|
|
474
|
+
for (const [field, value] of [["provider", provider], ["capability", capability]]) {
|
|
475
|
+
if (!SLUG.test(value)) {
|
|
476
|
+
throw new errors_js_1.CrwError(`${field} must be a lowercase id like the ones findData returns, got ${JSON.stringify(value)}`);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
const q = new URLSearchParams();
|
|
480
|
+
for (const [key, value] of Object.entries(options)) {
|
|
481
|
+
if (value === null || value === undefined)
|
|
482
|
+
continue;
|
|
483
|
+
// Same rule as the Python SDK, both MCPs and the API: scalars only.
|
|
484
|
+
if (!["string", "number", "boolean"].includes(typeof value)) {
|
|
485
|
+
throw new errors_js_1.CrwError(`[invalid_options] Option "${key}" must be a string, number or boolean`);
|
|
486
|
+
}
|
|
487
|
+
q.set(key, String(value));
|
|
488
|
+
}
|
|
489
|
+
const qs = q.toString();
|
|
490
|
+
const path = `/v1/data/${provider}/${capability}${qs ? `?${qs}` : ""}`;
|
|
491
|
+
let body;
|
|
492
|
+
try {
|
|
493
|
+
body = await this.httpRequest("GET", path, undefined, { raw: true });
|
|
494
|
+
}
|
|
495
|
+
catch (err) {
|
|
496
|
+
if (err instanceof errors_js_1.CrwApiError && err.statusCode === 409 && err.errorCode === "client_execution_required") {
|
|
497
|
+
const proxy = await this.dataProxyTransport();
|
|
498
|
+
return (await proxy.toolCall("crw_get_data", { provider, capability, options }));
|
|
499
|
+
}
|
|
500
|
+
throw err;
|
|
501
|
+
}
|
|
502
|
+
const p = (body.provenance ?? {});
|
|
503
|
+
const field = (v) => v ?? null;
|
|
504
|
+
return {
|
|
505
|
+
provider: body.provider,
|
|
506
|
+
capability: body.capability,
|
|
507
|
+
creditsUsed: body.credits_used ?? null,
|
|
508
|
+
provenance: {
|
|
509
|
+
source: field(p.source),
|
|
510
|
+
upstreamUrl: field(p.upstream_url),
|
|
511
|
+
licence: field(p.licence),
|
|
512
|
+
licenceUrl: field(p.licence_url),
|
|
513
|
+
attributionText: field(p.attribution_text),
|
|
514
|
+
retrievedAt: field(p.retrieved_at),
|
|
515
|
+
},
|
|
516
|
+
data: body.data ?? null,
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Shut down any local subprocess (CRW_LOCAL engine or data-source proxy).
|
|
521
|
+
* Each gets up to three seconds to exit on its own (a data proxy may still be
|
|
522
|
+
* sending usage counts); await it to know they are gone.
|
|
523
|
+
*/
|
|
524
|
+
async close() {
|
|
525
|
+
const local = this.local;
|
|
526
|
+
const proxy = this.dataProxy;
|
|
423
527
|
this.local = null;
|
|
528
|
+
this.dataProxy = null;
|
|
529
|
+
await Promise.all([
|
|
530
|
+
local?.close(),
|
|
531
|
+
proxy?.then((t) => t.close(), () => { }),
|
|
532
|
+
]);
|
|
424
533
|
}
|
|
425
534
|
// --- local (subprocess) mode ---
|
|
426
535
|
localTransport() {
|
|
@@ -443,6 +552,24 @@ class CrwClient {
|
|
|
443
552
|
await sleep(pollInterval * 1000);
|
|
444
553
|
}
|
|
445
554
|
}
|
|
555
|
+
/**
|
|
556
|
+
* A proxy-mode `crw-mcp`, spawned on first use and kept until close(). The
|
|
557
|
+
* promise is cached so concurrent calls share one process; a failed start is
|
|
558
|
+
* forgotten so the next call retries.
|
|
559
|
+
*/
|
|
560
|
+
dataProxyTransport() {
|
|
561
|
+
if (!this.dataProxy) {
|
|
562
|
+
const transport = new local_js_1.LocalTransport((0, local_js_1.proxyChildEnv)(this.apiUrl, this.apiKey));
|
|
563
|
+
const ready = transport.initialize().then(() => transport, (err) => {
|
|
564
|
+
void transport.close();
|
|
565
|
+
if (this.dataProxy === ready)
|
|
566
|
+
this.dataProxy = null;
|
|
567
|
+
throw err;
|
|
568
|
+
});
|
|
569
|
+
this.dataProxy = ready;
|
|
570
|
+
}
|
|
571
|
+
return this.dataProxy;
|
|
572
|
+
}
|
|
446
573
|
// --- HTTP mode ---
|
|
447
574
|
async httpRequest(method, path, body, { raw = false, checkSuccess = true, headers: extraHeaders, } = {}) {
|
|
448
575
|
if (this.apiUrl === null)
|
|
@@ -457,7 +584,7 @@ class CrwClient {
|
|
|
457
584
|
const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined });
|
|
458
585
|
const result = await this.readJson(resp);
|
|
459
586
|
if (checkSuccess && result.success === false) {
|
|
460
|
-
throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status);
|
|
587
|
+
throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
|
|
461
588
|
}
|
|
462
589
|
if (raw)
|
|
463
590
|
return result;
|
|
@@ -472,8 +599,9 @@ class CrwClient {
|
|
|
472
599
|
headers.Authorization = `Bearer ${this.apiKey}`;
|
|
473
600
|
const resp = await fetch(url, { method: "POST", headers, body: form });
|
|
474
601
|
const result = await this.readJson(resp);
|
|
475
|
-
if (result.success === false)
|
|
476
|
-
throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status);
|
|
602
|
+
if (result.success === false) {
|
|
603
|
+
throw new errors_js_1.CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
|
|
604
|
+
}
|
|
477
605
|
return result.data ?? result;
|
|
478
606
|
}
|
|
479
607
|
/** Parse the JSON body; surface a non-2xx body's `error` as CrwApiError. */
|
|
@@ -490,7 +618,7 @@ class CrwClient {
|
|
|
490
618
|
}
|
|
491
619
|
if (!resp.ok) {
|
|
492
620
|
const message = parsed.error ?? parsed.message ?? `HTTP ${resp.status}`;
|
|
493
|
-
throw new errors_js_1.CrwApiError(message, resp.status);
|
|
621
|
+
throw new errors_js_1.CrwApiError(message, resp.status, bodyErrorCode(parsed));
|
|
494
622
|
}
|
|
495
623
|
return parsed;
|
|
496
624
|
}
|
package/dist/cjs/errors.js
CHANGED
|
@@ -11,10 +11,13 @@ class CrwError extends Error {
|
|
|
11
11
|
exports.CrwError = CrwError;
|
|
12
12
|
class CrwApiError extends CrwError {
|
|
13
13
|
statusCode;
|
|
14
|
-
|
|
14
|
+
/** The body's machine-readable `errorCode`, when the server sent one. */
|
|
15
|
+
errorCode;
|
|
16
|
+
constructor(message, statusCode, errorCode) {
|
|
15
17
|
super(message);
|
|
16
18
|
this.name = "CrwApiError";
|
|
17
19
|
this.statusCode = statusCode;
|
|
20
|
+
this.errorCode = errorCode;
|
|
18
21
|
}
|
|
19
22
|
}
|
|
20
23
|
exports.CrwApiError = CrwApiError;
|
package/dist/cjs/local.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
10
|
exports.LocalTransport = void 0;
|
|
11
11
|
exports.localChildEnv = localChildEnv;
|
|
12
|
+
exports.proxyChildEnv = proxyChildEnv;
|
|
12
13
|
const node_child_process_1 = require("node:child_process");
|
|
13
14
|
const errors_js_1 = require("./errors.js");
|
|
14
15
|
const BINARY_NAME = process.platform === "win32" ? "crw-mcp.exe" : "crw-mcp";
|
|
@@ -38,11 +39,29 @@ function localChildEnv() {
|
|
|
38
39
|
const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
|
|
39
40
|
return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
|
|
40
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Environment for a `crw-mcp` subprocess in PROXY mode: it talks to `apiUrl`
|
|
44
|
+
* with the client's own key, whatever the shell says. Only the data tools use
|
|
45
|
+
* it, for a source the API answered with `client_execution_required`.
|
|
46
|
+
*/
|
|
47
|
+
function proxyChildEnv(apiUrl, apiKey) {
|
|
48
|
+
const env = { ...process.env, CRW_API_URL: apiUrl };
|
|
49
|
+
if (apiKey)
|
|
50
|
+
env.CRW_API_KEY = apiKey;
|
|
51
|
+
else
|
|
52
|
+
delete env.CRW_API_KEY;
|
|
53
|
+
return env;
|
|
54
|
+
}
|
|
41
55
|
class LocalTransport {
|
|
56
|
+
env;
|
|
42
57
|
proc = null;
|
|
43
58
|
nextId = 0;
|
|
44
59
|
pending = new Map();
|
|
45
60
|
buffer = "";
|
|
61
|
+
/** `env` defaults to {@link localChildEnv} (embedded mode). */
|
|
62
|
+
constructor(env) {
|
|
63
|
+
this.env = env;
|
|
64
|
+
}
|
|
46
65
|
resolveBinary() {
|
|
47
66
|
const env = process.env.CRW_BINARY;
|
|
48
67
|
if (env)
|
|
@@ -55,10 +74,11 @@ class LocalTransport {
|
|
|
55
74
|
if (this.proc && this.proc.exitCode === null)
|
|
56
75
|
return this.proc;
|
|
57
76
|
const bin = this.resolveBinary();
|
|
58
|
-
const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
|
|
77
|
+
const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: this.env ?? localChildEnv() });
|
|
59
78
|
proc.on("error", (err) => {
|
|
79
|
+
const where = bin === BINARY_NAME ? "on PATH" : `at CRW_BINARY=${bin}`;
|
|
60
80
|
const failure = err.code === "ENOENT"
|
|
61
|
-
? new errors_js_1.CrwBinaryNotFoundError(`crw-mcp binary not found
|
|
81
|
+
? new errors_js_1.CrwBinaryNotFoundError(`crw-mcp binary not found ${where}. Install it (e.g. \`npm i -g crw-mcp\` or ` +
|
|
62
82
|
`\`cargo install crw-mcp\`) or set CRW_BINARY to its path.`)
|
|
63
83
|
: new errors_js_1.CrwError(`crw-mcp failed to start: ${err.message}`);
|
|
64
84
|
for (const p of this.pending.values())
|
|
@@ -104,6 +124,14 @@ class LocalTransport {
|
|
|
104
124
|
}
|
|
105
125
|
}
|
|
106
126
|
}
|
|
127
|
+
/** MCP handshake; `crw-mcp` is stateless, so a respawn needs no repeat. */
|
|
128
|
+
initialize() {
|
|
129
|
+
return this.jsonrpc("initialize", {
|
|
130
|
+
protocolVersion: "2025-06-18",
|
|
131
|
+
capabilities: {},
|
|
132
|
+
clientInfo: { name: "crw-sdk", version: "1" },
|
|
133
|
+
});
|
|
134
|
+
}
|
|
107
135
|
jsonrpc(method, params) {
|
|
108
136
|
const proc = this.ensureProcess();
|
|
109
137
|
const id = ++this.nextId;
|
|
@@ -117,16 +145,35 @@ class LocalTransport {
|
|
|
117
145
|
const content = result.content?.[0];
|
|
118
146
|
if (!content)
|
|
119
147
|
throw new errors_js_1.CrwError(`Empty response from ${name}`);
|
|
120
|
-
if (result.isError)
|
|
121
|
-
|
|
148
|
+
if (result.isError) {
|
|
149
|
+
// A tool error has no HTTP status; its bracketed code is the errorCode.
|
|
150
|
+
const text = content.text ?? "Unknown error";
|
|
151
|
+
throw new errors_js_1.CrwApiError(text, undefined, /^\[([a-z_]+)\]/.exec(text)?.[1]);
|
|
152
|
+
}
|
|
122
153
|
return JSON.parse(content.text ?? "{}");
|
|
123
154
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
155
|
+
/**
|
|
156
|
+
* EOF first so the child can finish what it started (a proxy-mode data call
|
|
157
|
+
* sends a usage ping after answering), then kill it if it is still up after
|
|
158
|
+
* `graceMs`. Resolves once it is gone; not awaiting it is fine. The default
|
|
159
|
+
* is above crw-mcp's own 2 s wait for pings in flight.
|
|
160
|
+
*/
|
|
161
|
+
close(graceMs = 3000) {
|
|
162
|
+
const proc = this.proc;
|
|
129
163
|
this.proc = null;
|
|
164
|
+
if (!proc || proc.exitCode !== null || proc.signalCode !== null)
|
|
165
|
+
return Promise.resolve();
|
|
166
|
+
return new Promise((resolve) => {
|
|
167
|
+
const timer = setTimeout(() => {
|
|
168
|
+
proc.kill();
|
|
169
|
+
resolve();
|
|
170
|
+
}, graceMs);
|
|
171
|
+
proc.once("exit", () => {
|
|
172
|
+
clearTimeout(timer);
|
|
173
|
+
resolve();
|
|
174
|
+
});
|
|
175
|
+
proc.stdin.end();
|
|
176
|
+
});
|
|
130
177
|
}
|
|
131
178
|
}
|
|
132
179
|
exports.LocalTransport = LocalTransport;
|
package/dist/esm/client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
|
|
2
|
-
import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, MapResult, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
|
|
2
|
+
import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DataOptions, DataResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, FindDataOptions, FindDataResult, Json, MapOptions, MapResult, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
|
|
3
3
|
export declare const CLOUD_API_URL = "https://api.fastcrw.com";
|
|
4
4
|
export declare const DASHBOARD_URL = "https://fastcrw.com/dashboard";
|
|
5
5
|
export declare const DOCS_URL = "https://docs.fastcrw.com";
|
|
@@ -7,6 +7,7 @@ export declare class CrwClient {
|
|
|
7
7
|
private apiUrl;
|
|
8
8
|
private apiKey;
|
|
9
9
|
private local;
|
|
10
|
+
private dataProxy;
|
|
10
11
|
/**
|
|
11
12
|
* CRW is cloud-first. With no arguments the client targets the managed cloud
|
|
12
13
|
* (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
|
|
@@ -58,10 +59,28 @@ export declare class CrwClient {
|
|
|
58
59
|
capabilities(): Promise<Capabilities>;
|
|
59
60
|
/** Diff a page against a prior snapshot (HTTP mode only). */
|
|
60
61
|
changeTrackingDiff(current: Json, previous?: Json, opts?: ChangeTrackingOptions): Promise<DiffResult>;
|
|
61
|
-
/**
|
|
62
|
-
|
|
62
|
+
/** Find data sources (`GET /v1/data/find`). Page with `offset: result.nextOffset`. */
|
|
63
|
+
findData(opts?: FindDataOptions): Promise<FindDataResult>;
|
|
64
|
+
/**
|
|
65
|
+
* Call one data source capability (`GET /v1/data/<provider>/<capability>`).
|
|
66
|
+
* A source that runs on your machine instead of the API is run through a
|
|
67
|
+
* local `crw-mcp` (CRW_BINARY or PATH) that talks to the same API with your key.
|
|
68
|
+
*/
|
|
69
|
+
getData(provider: string, capability: string, options?: DataOptions): Promise<DataResult>;
|
|
70
|
+
/**
|
|
71
|
+
* Shut down any local subprocess (CRW_LOCAL engine or data-source proxy).
|
|
72
|
+
* Each gets up to three seconds to exit on its own (a data proxy may still be
|
|
73
|
+
* sending usage counts); await it to know they are gone.
|
|
74
|
+
*/
|
|
75
|
+
close(): Promise<void>;
|
|
63
76
|
private localTransport;
|
|
64
77
|
private pollLocalCrawl;
|
|
78
|
+
/**
|
|
79
|
+
* A proxy-mode `crw-mcp`, spawned on first use and kept until close(). The
|
|
80
|
+
* promise is cached so concurrent calls share one process; a failed start is
|
|
81
|
+
* forgotten so the next call retries.
|
|
82
|
+
*/
|
|
83
|
+
private dataProxyTransport;
|
|
65
84
|
private httpRequest;
|
|
66
85
|
private httpMultipart;
|
|
67
86
|
/** Parse the JSON body; surface a non-2xx body's `error` as CrwApiError. */
|
package/dist/esm/client.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
|
|
2
2
|
import { CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError } from "./errors.js";
|
|
3
|
-
import { LocalTransport } from "./local.js";
|
|
3
|
+
import { LocalTransport, proxyChildEnv } from "./local.js";
|
|
4
4
|
// CRW is cloud-first: with no explicit apiUrl and no CRW_LOCAL opt-in, the client
|
|
5
5
|
// talks to the managed cloud. Mirrors the Python SDK + CLI onboarding.
|
|
6
6
|
export const CLOUD_API_URL = "https://api.fastcrw.com";
|
|
@@ -17,7 +17,11 @@ function httpOnlyHint(name, reason) {
|
|
|
17
17
|
return (`${name}() requires HTTP mode (${reason}). It is not available with CRW_LOCAL=1. ` +
|
|
18
18
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
19
19
|
}
|
|
20
|
+
const DATA_LOCAL_HINT = "data sources need the fastCRW API (set apiUrl/apiKey); not available in CRW_LOCAL mode";
|
|
21
|
+
const SLUG = /^[a-z0-9-]+$/;
|
|
20
22
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
23
|
+
/** A body's machine-readable `errorCode`, when it is a string. */
|
|
24
|
+
const bodyErrorCode = (body) => (typeof body.errorCode === "string" ? body.errorCode : undefined);
|
|
21
25
|
/**
|
|
22
26
|
* Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
|
|
23
27
|
* It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
|
|
@@ -58,6 +62,7 @@ export class CrwClient {
|
|
|
58
62
|
apiUrl;
|
|
59
63
|
apiKey;
|
|
60
64
|
local = null;
|
|
65
|
+
dataProxy = null;
|
|
61
66
|
/**
|
|
62
67
|
* CRW is cloud-first. With no arguments the client targets the managed cloud
|
|
63
68
|
* (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
|
|
@@ -414,10 +419,114 @@ export class CrwClient {
|
|
|
414
419
|
Object.assign(body, rest);
|
|
415
420
|
return this.httpPost("/v1/change-tracking/diff", body);
|
|
416
421
|
}
|
|
417
|
-
/**
|
|
418
|
-
|
|
419
|
-
this.
|
|
422
|
+
/** Find data sources (`GET /v1/data/find`). Page with `offset: result.nextOffset`. */
|
|
423
|
+
async findData(opts = {}) {
|
|
424
|
+
if (!this.apiUrl)
|
|
425
|
+
throw new CrwError(DATA_LOCAL_HINT);
|
|
426
|
+
const q = new URLSearchParams();
|
|
427
|
+
for (const key of ["query", "category", "provider"]) {
|
|
428
|
+
const value = opts[key]?.trim();
|
|
429
|
+
if (value)
|
|
430
|
+
q.set(key, key === "provider" ? value.toLowerCase() : value);
|
|
431
|
+
}
|
|
432
|
+
if (opts.limit !== undefined)
|
|
433
|
+
q.set("limit", String(opts.limit));
|
|
434
|
+
if (opts.offset !== undefined)
|
|
435
|
+
q.set("offset", String(opts.offset));
|
|
436
|
+
const qs = q.toString();
|
|
437
|
+
const reply = await this.httpRequest("GET", `/v1/data/find${qs ? `?${qs}` : ""}`, undefined, {
|
|
438
|
+
raw: true,
|
|
439
|
+
});
|
|
440
|
+
const tools = Array.isArray(reply.items) ? reply.items : [];
|
|
441
|
+
const list = (v) => (Array.isArray(v) ? v : []);
|
|
442
|
+
const items = tools.map((t) => {
|
|
443
|
+
const options = (t.options ?? {});
|
|
444
|
+
const provenance = (t.provenance ?? {});
|
|
445
|
+
return {
|
|
446
|
+
provider: t.provider,
|
|
447
|
+
capability: t.capability,
|
|
448
|
+
name: t.name ?? null,
|
|
449
|
+
description: t.description ?? null,
|
|
450
|
+
category: t.category ?? null,
|
|
451
|
+
creditsCost: t.creditsCost ?? null,
|
|
452
|
+
requiredOptions: list(options.required),
|
|
453
|
+
optionalOptions: list(options.optional),
|
|
454
|
+
keyRequired: t.keyRequired === true,
|
|
455
|
+
licence: provenance.licence ?? "",
|
|
456
|
+
};
|
|
457
|
+
});
|
|
458
|
+
const result = { items, total: reply.total ?? null };
|
|
459
|
+
if (typeof reply.nextOffset === "number")
|
|
460
|
+
result.nextOffset = reply.nextOffset;
|
|
461
|
+
return result;
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Call one data source capability (`GET /v1/data/<provider>/<capability>`).
|
|
465
|
+
* A source that runs on your machine instead of the API is run through a
|
|
466
|
+
* local `crw-mcp` (CRW_BINARY or PATH) that talks to the same API with your key.
|
|
467
|
+
*/
|
|
468
|
+
async getData(provider, capability, options = {}) {
|
|
469
|
+
if (!this.apiUrl)
|
|
470
|
+
throw new CrwError(DATA_LOCAL_HINT);
|
|
471
|
+
for (const [field, value] of [["provider", provider], ["capability", capability]]) {
|
|
472
|
+
if (!SLUG.test(value)) {
|
|
473
|
+
throw new CrwError(`${field} must be a lowercase id like the ones findData returns, got ${JSON.stringify(value)}`);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
const q = new URLSearchParams();
|
|
477
|
+
for (const [key, value] of Object.entries(options)) {
|
|
478
|
+
if (value === null || value === undefined)
|
|
479
|
+
continue;
|
|
480
|
+
// Same rule as the Python SDK, both MCPs and the API: scalars only.
|
|
481
|
+
if (!["string", "number", "boolean"].includes(typeof value)) {
|
|
482
|
+
throw new CrwError(`[invalid_options] Option "${key}" must be a string, number or boolean`);
|
|
483
|
+
}
|
|
484
|
+
q.set(key, String(value));
|
|
485
|
+
}
|
|
486
|
+
const qs = q.toString();
|
|
487
|
+
const path = `/v1/data/${provider}/${capability}${qs ? `?${qs}` : ""}`;
|
|
488
|
+
let body;
|
|
489
|
+
try {
|
|
490
|
+
body = await this.httpRequest("GET", path, undefined, { raw: true });
|
|
491
|
+
}
|
|
492
|
+
catch (err) {
|
|
493
|
+
if (err instanceof CrwApiError && err.statusCode === 409 && err.errorCode === "client_execution_required") {
|
|
494
|
+
const proxy = await this.dataProxyTransport();
|
|
495
|
+
return (await proxy.toolCall("crw_get_data", { provider, capability, options }));
|
|
496
|
+
}
|
|
497
|
+
throw err;
|
|
498
|
+
}
|
|
499
|
+
const p = (body.provenance ?? {});
|
|
500
|
+
const field = (v) => v ?? null;
|
|
501
|
+
return {
|
|
502
|
+
provider: body.provider,
|
|
503
|
+
capability: body.capability,
|
|
504
|
+
creditsUsed: body.credits_used ?? null,
|
|
505
|
+
provenance: {
|
|
506
|
+
source: field(p.source),
|
|
507
|
+
upstreamUrl: field(p.upstream_url),
|
|
508
|
+
licence: field(p.licence),
|
|
509
|
+
licenceUrl: field(p.licence_url),
|
|
510
|
+
attributionText: field(p.attribution_text),
|
|
511
|
+
retrievedAt: field(p.retrieved_at),
|
|
512
|
+
},
|
|
513
|
+
data: body.data ?? null,
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Shut down any local subprocess (CRW_LOCAL engine or data-source proxy).
|
|
518
|
+
* Each gets up to three seconds to exit on its own (a data proxy may still be
|
|
519
|
+
* sending usage counts); await it to know they are gone.
|
|
520
|
+
*/
|
|
521
|
+
async close() {
|
|
522
|
+
const local = this.local;
|
|
523
|
+
const proxy = this.dataProxy;
|
|
420
524
|
this.local = null;
|
|
525
|
+
this.dataProxy = null;
|
|
526
|
+
await Promise.all([
|
|
527
|
+
local?.close(),
|
|
528
|
+
proxy?.then((t) => t.close(), () => { }),
|
|
529
|
+
]);
|
|
421
530
|
}
|
|
422
531
|
// --- local (subprocess) mode ---
|
|
423
532
|
localTransport() {
|
|
@@ -440,6 +549,24 @@ export class CrwClient {
|
|
|
440
549
|
await sleep(pollInterval * 1000);
|
|
441
550
|
}
|
|
442
551
|
}
|
|
552
|
+
/**
|
|
553
|
+
* A proxy-mode `crw-mcp`, spawned on first use and kept until close(). The
|
|
554
|
+
* promise is cached so concurrent calls share one process; a failed start is
|
|
555
|
+
* forgotten so the next call retries.
|
|
556
|
+
*/
|
|
557
|
+
dataProxyTransport() {
|
|
558
|
+
if (!this.dataProxy) {
|
|
559
|
+
const transport = new LocalTransport(proxyChildEnv(this.apiUrl, this.apiKey));
|
|
560
|
+
const ready = transport.initialize().then(() => transport, (err) => {
|
|
561
|
+
void transport.close();
|
|
562
|
+
if (this.dataProxy === ready)
|
|
563
|
+
this.dataProxy = null;
|
|
564
|
+
throw err;
|
|
565
|
+
});
|
|
566
|
+
this.dataProxy = ready;
|
|
567
|
+
}
|
|
568
|
+
return this.dataProxy;
|
|
569
|
+
}
|
|
443
570
|
// --- HTTP mode ---
|
|
444
571
|
async httpRequest(method, path, body, { raw = false, checkSuccess = true, headers: extraHeaders, } = {}) {
|
|
445
572
|
if (this.apiUrl === null)
|
|
@@ -454,7 +581,7 @@ export class CrwClient {
|
|
|
454
581
|
const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined });
|
|
455
582
|
const result = await this.readJson(resp);
|
|
456
583
|
if (checkSuccess && result.success === false) {
|
|
457
|
-
throw new CrwApiError(result.error ?? "API error", resp.status);
|
|
584
|
+
throw new CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
|
|
458
585
|
}
|
|
459
586
|
if (raw)
|
|
460
587
|
return result;
|
|
@@ -469,8 +596,9 @@ export class CrwClient {
|
|
|
469
596
|
headers.Authorization = `Bearer ${this.apiKey}`;
|
|
470
597
|
const resp = await fetch(url, { method: "POST", headers, body: form });
|
|
471
598
|
const result = await this.readJson(resp);
|
|
472
|
-
if (result.success === false)
|
|
473
|
-
throw new CrwApiError(result.error ?? "API error", resp.status);
|
|
599
|
+
if (result.success === false) {
|
|
600
|
+
throw new CrwApiError(result.error ?? "API error", resp.status, bodyErrorCode(result));
|
|
601
|
+
}
|
|
474
602
|
return result.data ?? result;
|
|
475
603
|
}
|
|
476
604
|
/** Parse the JSON body; surface a non-2xx body's `error` as CrwApiError. */
|
|
@@ -487,7 +615,7 @@ export class CrwClient {
|
|
|
487
615
|
}
|
|
488
616
|
if (!resp.ok) {
|
|
489
617
|
const message = parsed.error ?? parsed.message ?? `HTTP ${resp.status}`;
|
|
490
|
-
throw new CrwApiError(message, resp.status);
|
|
618
|
+
throw new CrwApiError(message, resp.status, bodyErrorCode(parsed));
|
|
491
619
|
}
|
|
492
620
|
return parsed;
|
|
493
621
|
}
|
package/dist/esm/errors.d.ts
CHANGED
|
@@ -5,7 +5,9 @@ export declare class CrwError extends Error {
|
|
|
5
5
|
}
|
|
6
6
|
export declare class CrwApiError extends CrwError {
|
|
7
7
|
statusCode?: number;
|
|
8
|
-
|
|
8
|
+
/** The body's machine-readable `errorCode`, when the server sent one. */
|
|
9
|
+
errorCode?: string;
|
|
10
|
+
constructor(message: string, statusCode?: number, errorCode?: string);
|
|
9
11
|
}
|
|
10
12
|
export declare class CrwTimeoutError extends CrwError {
|
|
11
13
|
constructor(message: string);
|
package/dist/esm/errors.js
CHANGED
|
@@ -7,10 +7,13 @@ export class CrwError extends Error {
|
|
|
7
7
|
}
|
|
8
8
|
export class CrwApiError extends CrwError {
|
|
9
9
|
statusCode;
|
|
10
|
-
|
|
10
|
+
/** The body's machine-readable `errorCode`, when the server sent one. */
|
|
11
|
+
errorCode;
|
|
12
|
+
constructor(message, statusCode, errorCode) {
|
|
11
13
|
super(message);
|
|
12
14
|
this.name = "CrwApiError";
|
|
13
15
|
this.statusCode = statusCode;
|
|
16
|
+
this.errorCode = errorCode;
|
|
14
17
|
}
|
|
15
18
|
}
|
|
16
19
|
export class CrwTimeoutError extends CrwError {
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
|
|
2
2
|
export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
|
|
3
|
-
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, MapResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
|
|
3
|
+
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, MapResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, FindDataOptions, FindDataResult, DataSource, DataOptions, DataResult, DataProvenance, Json, } from "./types.js";
|
package/dist/esm/local.d.ts
CHANGED
|
@@ -19,15 +19,32 @@ import type { Json } from "./types.js";
|
|
|
19
19
|
* is why `map()` still handles the envelope shape in `mapPayload`.
|
|
20
20
|
*/
|
|
21
21
|
export declare function localChildEnv(): NodeJS.ProcessEnv;
|
|
22
|
+
/**
|
|
23
|
+
* Environment for a `crw-mcp` subprocess in PROXY mode: it talks to `apiUrl`
|
|
24
|
+
* with the client's own key, whatever the shell says. Only the data tools use
|
|
25
|
+
* it, for a source the API answered with `client_execution_required`.
|
|
26
|
+
*/
|
|
27
|
+
export declare function proxyChildEnv(apiUrl: string, apiKey: string | undefined): NodeJS.ProcessEnv;
|
|
22
28
|
export declare class LocalTransport {
|
|
29
|
+
private readonly env?;
|
|
23
30
|
private proc;
|
|
24
31
|
private nextId;
|
|
25
32
|
private pending;
|
|
26
33
|
private buffer;
|
|
34
|
+
/** `env` defaults to {@link localChildEnv} (embedded mode). */
|
|
35
|
+
constructor(env?: NodeJS.ProcessEnv | undefined);
|
|
27
36
|
private resolveBinary;
|
|
28
37
|
private ensureProcess;
|
|
29
38
|
private onData;
|
|
30
|
-
|
|
39
|
+
/** MCP handshake; `crw-mcp` is stateless, so a respawn needs no repeat. */
|
|
40
|
+
initialize(): Promise<Json>;
|
|
41
|
+
jsonrpc(method: string, params: Json): Promise<Json>;
|
|
31
42
|
toolCall(name: string, args: Json): Promise<Json>;
|
|
32
|
-
|
|
43
|
+
/**
|
|
44
|
+
* EOF first so the child can finish what it started (a proxy-mode data call
|
|
45
|
+
* sends a usage ping after answering), then kill it if it is still up after
|
|
46
|
+
* `graceMs`. Resolves once it is gone; not awaiting it is fine. The default
|
|
47
|
+
* is above crw-mcp's own 2 s wait for pings in flight.
|
|
48
|
+
*/
|
|
49
|
+
close(graceMs?: number): Promise<void>;
|
|
33
50
|
}
|
package/dist/esm/local.js
CHANGED
|
@@ -34,11 +34,29 @@ export function localChildEnv() {
|
|
|
34
34
|
const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
|
|
35
35
|
return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
|
|
36
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Environment for a `crw-mcp` subprocess in PROXY mode: it talks to `apiUrl`
|
|
39
|
+
* with the client's own key, whatever the shell says. Only the data tools use
|
|
40
|
+
* it, for a source the API answered with `client_execution_required`.
|
|
41
|
+
*/
|
|
42
|
+
export function proxyChildEnv(apiUrl, apiKey) {
|
|
43
|
+
const env = { ...process.env, CRW_API_URL: apiUrl };
|
|
44
|
+
if (apiKey)
|
|
45
|
+
env.CRW_API_KEY = apiKey;
|
|
46
|
+
else
|
|
47
|
+
delete env.CRW_API_KEY;
|
|
48
|
+
return env;
|
|
49
|
+
}
|
|
37
50
|
export class LocalTransport {
|
|
51
|
+
env;
|
|
38
52
|
proc = null;
|
|
39
53
|
nextId = 0;
|
|
40
54
|
pending = new Map();
|
|
41
55
|
buffer = "";
|
|
56
|
+
/** `env` defaults to {@link localChildEnv} (embedded mode). */
|
|
57
|
+
constructor(env) {
|
|
58
|
+
this.env = env;
|
|
59
|
+
}
|
|
42
60
|
resolveBinary() {
|
|
43
61
|
const env = process.env.CRW_BINARY;
|
|
44
62
|
if (env)
|
|
@@ -51,10 +69,11 @@ export class LocalTransport {
|
|
|
51
69
|
if (this.proc && this.proc.exitCode === null)
|
|
52
70
|
return this.proc;
|
|
53
71
|
const bin = this.resolveBinary();
|
|
54
|
-
const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
|
|
72
|
+
const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: this.env ?? localChildEnv() });
|
|
55
73
|
proc.on("error", (err) => {
|
|
74
|
+
const where = bin === BINARY_NAME ? "on PATH" : `at CRW_BINARY=${bin}`;
|
|
56
75
|
const failure = err.code === "ENOENT"
|
|
57
|
-
? new CrwBinaryNotFoundError(`crw-mcp binary not found
|
|
76
|
+
? new CrwBinaryNotFoundError(`crw-mcp binary not found ${where}. Install it (e.g. \`npm i -g crw-mcp\` or ` +
|
|
58
77
|
`\`cargo install crw-mcp\`) or set CRW_BINARY to its path.`)
|
|
59
78
|
: new CrwError(`crw-mcp failed to start: ${err.message}`);
|
|
60
79
|
for (const p of this.pending.values())
|
|
@@ -100,6 +119,14 @@ export class LocalTransport {
|
|
|
100
119
|
}
|
|
101
120
|
}
|
|
102
121
|
}
|
|
122
|
+
/** MCP handshake; `crw-mcp` is stateless, so a respawn needs no repeat. */
|
|
123
|
+
initialize() {
|
|
124
|
+
return this.jsonrpc("initialize", {
|
|
125
|
+
protocolVersion: "2025-06-18",
|
|
126
|
+
capabilities: {},
|
|
127
|
+
clientInfo: { name: "crw-sdk", version: "1" },
|
|
128
|
+
});
|
|
129
|
+
}
|
|
103
130
|
jsonrpc(method, params) {
|
|
104
131
|
const proc = this.ensureProcess();
|
|
105
132
|
const id = ++this.nextId;
|
|
@@ -113,15 +140,34 @@ export class LocalTransport {
|
|
|
113
140
|
const content = result.content?.[0];
|
|
114
141
|
if (!content)
|
|
115
142
|
throw new CrwError(`Empty response from ${name}`);
|
|
116
|
-
if (result.isError)
|
|
117
|
-
|
|
143
|
+
if (result.isError) {
|
|
144
|
+
// A tool error has no HTTP status; its bracketed code is the errorCode.
|
|
145
|
+
const text = content.text ?? "Unknown error";
|
|
146
|
+
throw new CrwApiError(text, undefined, /^\[([a-z_]+)\]/.exec(text)?.[1]);
|
|
147
|
+
}
|
|
118
148
|
return JSON.parse(content.text ?? "{}");
|
|
119
149
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
150
|
+
/**
|
|
151
|
+
* EOF first so the child can finish what it started (a proxy-mode data call
|
|
152
|
+
* sends a usage ping after answering), then kill it if it is still up after
|
|
153
|
+
* `graceMs`. Resolves once it is gone; not awaiting it is fine. The default
|
|
154
|
+
* is above crw-mcp's own 2 s wait for pings in flight.
|
|
155
|
+
*/
|
|
156
|
+
close(graceMs = 3000) {
|
|
157
|
+
const proc = this.proc;
|
|
125
158
|
this.proc = null;
|
|
159
|
+
if (!proc || proc.exitCode !== null || proc.signalCode !== null)
|
|
160
|
+
return Promise.resolve();
|
|
161
|
+
return new Promise((resolve) => {
|
|
162
|
+
const timer = setTimeout(() => {
|
|
163
|
+
proc.kill();
|
|
164
|
+
resolve();
|
|
165
|
+
}, graceMs);
|
|
166
|
+
proc.once("exit", () => {
|
|
167
|
+
clearTimeout(timer);
|
|
168
|
+
resolve();
|
|
169
|
+
});
|
|
170
|
+
proc.stdin.end();
|
|
171
|
+
});
|
|
126
172
|
}
|
|
127
173
|
}
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -539,3 +539,54 @@ export interface ResearchSimilarOptions {
|
|
|
539
539
|
rerank?: boolean;
|
|
540
540
|
anchor?: string[];
|
|
541
541
|
}
|
|
542
|
+
/** Filters for `findData()` (`GET /v1/data/find`). */
|
|
543
|
+
export interface FindDataOptions {
|
|
544
|
+
/** Words to match against source names and descriptions. */
|
|
545
|
+
query?: string;
|
|
546
|
+
/** Only sources in this category, e.g. `Research`. */
|
|
547
|
+
category?: string;
|
|
548
|
+
/** Only this provider, e.g. `openalex`. */
|
|
549
|
+
provider?: string;
|
|
550
|
+
/** Results per page (server default 10, max 50). */
|
|
551
|
+
limit?: number;
|
|
552
|
+
/** Results to skip; pass the previous page's `nextOffset`. */
|
|
553
|
+
offset?: number;
|
|
554
|
+
}
|
|
555
|
+
/** One data source capability, as `findData()` lists it. */
|
|
556
|
+
export interface DataSource {
|
|
557
|
+
provider: string;
|
|
558
|
+
capability: string;
|
|
559
|
+
name: string | null;
|
|
560
|
+
description: string | null;
|
|
561
|
+
/** The catalogue category, e.g. `Research`; `null` from an older server. */
|
|
562
|
+
category: string | null;
|
|
563
|
+
creditsCost: number | null;
|
|
564
|
+
requiredOptions: string[];
|
|
565
|
+
optionalOptions: string[];
|
|
566
|
+
keyRequired: boolean;
|
|
567
|
+
licence: string | null;
|
|
568
|
+
}
|
|
569
|
+
export interface FindDataResult {
|
|
570
|
+
items: DataSource[];
|
|
571
|
+
total: number | null;
|
|
572
|
+
/** Present when another page exists. */
|
|
573
|
+
nextOffset?: number;
|
|
574
|
+
}
|
|
575
|
+
/** Values for a source's `requiredOptions` and `optionalOptions`. */
|
|
576
|
+
export type DataOptions = Record<string, string | number | boolean | null | undefined>;
|
|
577
|
+
export interface DataProvenance {
|
|
578
|
+
source: string | null;
|
|
579
|
+
upstreamUrl: string | null;
|
|
580
|
+
licence: string | null;
|
|
581
|
+
licenceUrl: string | null;
|
|
582
|
+
attributionText: string | null;
|
|
583
|
+
retrievedAt: string | null;
|
|
584
|
+
}
|
|
585
|
+
/** `getData()` result: the source's record plus where it came from. */
|
|
586
|
+
export interface DataResult {
|
|
587
|
+
provider: string;
|
|
588
|
+
capability: string;
|
|
589
|
+
creditsUsed: number | null;
|
|
590
|
+
provenance: DataProvenance;
|
|
591
|
+
data: unknown;
|
|
592
|
+
}
|
package/package.json
CHANGED