crw-sdk 0.35.1 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/client.js +36 -6
- package/dist/cjs/local.js +28 -1
- package/dist/esm/client.d.ts +2 -2
- package/dist/esm/client.js +36 -6
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/local.d.ts +13 -0
- package/dist/esm/local.js +27 -1
- package/dist/esm/types.d.ts +23 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -49,7 +49,7 @@ CRW_LOCAL=1 node app.js
|
|
|
49
49
|
|---|---|---|
|
|
50
50
|
| `scrape(url, opts?)` | Scrape one URL | both |
|
|
51
51
|
| `crawl(url, opts?)` | Crawl a site (async, polled) | both |
|
|
52
|
-
| `map(url, opts?)` | Discover URLs | both |
|
|
52
|
+
| `map(url, opts?)` | Discover URLs (+ the site's `sitemaps`) | both |
|
|
53
53
|
| `search(query, opts?)` | Web search (+ optional scrape) | both¹ |
|
|
54
54
|
| `parseFile(bytes, opts?)` | PDF → markdown / structured JSON | both |
|
|
55
55
|
| `extract({urls, schema?})` | Structured LLM extraction | HTTP |
|
package/dist/cjs/client.js
CHANGED
|
@@ -21,6 +21,22 @@ function httpOnlyHint(name, reason) {
|
|
|
21
21
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
22
22
|
}
|
|
23
23
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
24
|
+
/**
|
|
25
|
+
* Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
|
|
26
|
+
* It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
|
|
27
|
+
* the REST envelope (`{success, data: {…}}`) when it proxies. It proxies
|
|
28
|
+
* whenever `CRW_API_URL` is set, which the subprocess inherits from us even
|
|
29
|
+
* though CRW_LOCAL mode never reads that variable itself. Without this, `links`
|
|
30
|
+
* came back empty in that state.
|
|
31
|
+
*
|
|
32
|
+
* Probe `data.links` rather than a bare `data`, so a flat response that grows
|
|
33
|
+
* some unrelated top-level `data` later cannot make us unwrap into it. Same
|
|
34
|
+
* discriminator the engine's own MCP bounds use.
|
|
35
|
+
*/
|
|
36
|
+
function mapPayload(result) {
|
|
37
|
+
const envelope = result.data;
|
|
38
|
+
return envelope && typeof envelope === "object" && "links" in envelope ? envelope : result;
|
|
39
|
+
}
|
|
24
40
|
/**
|
|
25
41
|
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
|
|
26
42
|
* server's `next` is an absolute URL from its own public origin/prefix; reusing
|
|
@@ -107,12 +123,26 @@ class CrwClient {
|
|
|
107
123
|
async map(url, opts = {}) {
|
|
108
124
|
const { maxDepth = 2, useSitemap = true, ...rest } = opts;
|
|
109
125
|
const args = { url, maxDepth, useSitemap, ...rest };
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
126
|
+
const data = this.apiUrl
|
|
127
|
+
? await this.httpPost("/v1/map", args)
|
|
128
|
+
: mapPayload(await this.localTransport().toolCall("crw_map", args));
|
|
129
|
+
// Guard the shapes, as search() does before its own attach: a self-hosted
|
|
130
|
+
// server or gateway answering `{"links": "none"}` would otherwise make
|
|
131
|
+
// Object.defineProperty throw, or smuggle a non-array out typed as string[].
|
|
132
|
+
const links = (Array.isArray(data.links) ? data.links : []);
|
|
133
|
+
// `sitemaps` rides BESIDE `links` in the engine's response and the old
|
|
134
|
+
// unwrap dropped it: a caller could not see which sitemaps a site exposes
|
|
135
|
+
// even though they paid for the discovery. Attach it the way search()
|
|
136
|
+
// attaches its answer siblings: non-enumerably, so `links.sitemaps` works
|
|
137
|
+
// while `for…of`, spread and JSON.stringify keep seeing exactly the array
|
|
138
|
+
// they saw before.
|
|
139
|
+
Object.defineProperty(links, "sitemaps", {
|
|
140
|
+
value: Array.isArray(data.sitemaps) ? data.sitemaps : [],
|
|
141
|
+
enumerable: false,
|
|
142
|
+
configurable: true,
|
|
143
|
+
writable: true,
|
|
144
|
+
});
|
|
145
|
+
return links;
|
|
116
146
|
}
|
|
117
147
|
/**
|
|
118
148
|
* Works in both modes; local mode needs a search backend configured on the engine.
|
package/dist/cjs/local.js
CHANGED
|
@@ -8,9 +8,36 @@
|
|
|
8
8
|
*/
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
10
|
exports.LocalTransport = void 0;
|
|
11
|
+
exports.localChildEnv = localChildEnv;
|
|
11
12
|
const node_child_process_1 = require("node:child_process");
|
|
12
13
|
const errors_js_1 = require("./errors.js");
|
|
13
14
|
const BINARY_NAME = process.platform === "win32" ? "crw-mcp.exe" : "crw-mcp";
|
|
15
|
+
/**
|
|
16
|
+
* Variables that switch `crw-mcp` from embedded to proxy mode. Both are
|
|
17
|
+
* verified against the shipped binary: `CRW_API_URL` is bound by its CLI, and
|
|
18
|
+
* `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
|
|
19
|
+
*/
|
|
20
|
+
const PROXY_MODE_ENV_VARS = ["CRW_API_URL", "CRW_CLIENT__API_URL"];
|
|
21
|
+
/**
|
|
22
|
+
* Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
|
|
23
|
+
*
|
|
24
|
+
* The child inherits our environment, so either variable above left in the
|
|
25
|
+
* shell silently turned "run the local engine" into "call the cloud": every
|
|
26
|
+
* tool then answered with the REST envelope instead of the flat payload, and
|
|
27
|
+
* the call was billed. CRW_LOCAL means local, so the child does not get them.
|
|
28
|
+
*
|
|
29
|
+
* This cannot close every route: `crw-mcp` also reads `client.api_url` from
|
|
30
|
+
* `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
|
|
31
|
+
* is why `map()` still handles the envelope shape in `mapPayload`.
|
|
32
|
+
*/
|
|
33
|
+
function localChildEnv() {
|
|
34
|
+
// Windows environment names are case-insensitive, so the child would read a
|
|
35
|
+
// differently-spelled key that an exact-match filter left behind. POSIX names
|
|
36
|
+
// are case-sensitive, so there only the exact spellings matter.
|
|
37
|
+
const fold = (k) => (process.platform === "win32" ? k.toUpperCase() : k);
|
|
38
|
+
const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
|
|
39
|
+
return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
|
|
40
|
+
}
|
|
14
41
|
class LocalTransport {
|
|
15
42
|
proc = null;
|
|
16
43
|
nextId = 0;
|
|
@@ -28,7 +55,7 @@ class LocalTransport {
|
|
|
28
55
|
if (this.proc && this.proc.exitCode === null)
|
|
29
56
|
return this.proc;
|
|
30
57
|
const bin = this.resolveBinary();
|
|
31
|
-
const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"] });
|
|
58
|
+
const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
|
|
32
59
|
proc.on("error", (err) => {
|
|
33
60
|
const failure = err.code === "ENOENT"
|
|
34
61
|
? new errors_js_1.CrwBinaryNotFoundError(`crw-mcp binary not found on PATH. Install it (e.g. \`npm i -g crw-mcp\` or ` +
|
package/dist/esm/client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
|
|
2
|
-
import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
|
|
2
|
+
import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, MapResult, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
|
|
3
3
|
export declare const CLOUD_API_URL = "https://api.fastcrw.com";
|
|
4
4
|
export declare const DASHBOARD_URL = "https://fastcrw.com/dashboard";
|
|
5
5
|
export declare const DOCS_URL = "https://us.github.io/crw";
|
|
@@ -15,7 +15,7 @@ export declare class CrwClient {
|
|
|
15
15
|
constructor(opts?: ClientOptions);
|
|
16
16
|
scrape(url: string, opts?: ScrapeOptions): Promise<ScrapeResult>;
|
|
17
17
|
crawl(url: string, opts?: CrawlOptions): Promise<CrawlResult>;
|
|
18
|
-
map(url: string, opts?: MapOptions): Promise<
|
|
18
|
+
map(url: string, opts?: MapOptions): Promise<MapResult>;
|
|
19
19
|
/**
|
|
20
20
|
* Works in both modes; local mode needs a search backend configured on the engine.
|
|
21
21
|
*/
|
package/dist/esm/client.js
CHANGED
|
@@ -18,6 +18,22 @@ function httpOnlyHint(name, reason) {
|
|
|
18
18
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
19
19
|
}
|
|
20
20
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
21
|
+
/**
|
|
22
|
+
* Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
|
|
23
|
+
* It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
|
|
24
|
+
* the REST envelope (`{success, data: {…}}`) when it proxies. It proxies
|
|
25
|
+
* whenever `CRW_API_URL` is set, which the subprocess inherits from us even
|
|
26
|
+
* though CRW_LOCAL mode never reads that variable itself. Without this, `links`
|
|
27
|
+
* came back empty in that state.
|
|
28
|
+
*
|
|
29
|
+
* Probe `data.links` rather than a bare `data`, so a flat response that grows
|
|
30
|
+
* some unrelated top-level `data` later cannot make us unwrap into it. Same
|
|
31
|
+
* discriminator the engine's own MCP bounds use.
|
|
32
|
+
*/
|
|
33
|
+
function mapPayload(result) {
|
|
34
|
+
const envelope = result.data;
|
|
35
|
+
return envelope && typeof envelope === "object" && "links" in envelope ? envelope : result;
|
|
36
|
+
}
|
|
21
37
|
/**
|
|
22
38
|
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
|
|
23
39
|
* server's `next` is an absolute URL from its own public origin/prefix; reusing
|
|
@@ -104,12 +120,26 @@ export class CrwClient {
|
|
|
104
120
|
async map(url, opts = {}) {
|
|
105
121
|
const { maxDepth = 2, useSitemap = true, ...rest } = opts;
|
|
106
122
|
const args = { url, maxDepth, useSitemap, ...rest };
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
123
|
+
const data = this.apiUrl
|
|
124
|
+
? await this.httpPost("/v1/map", args)
|
|
125
|
+
: mapPayload(await this.localTransport().toolCall("crw_map", args));
|
|
126
|
+
// Guard the shapes, as search() does before its own attach: a self-hosted
|
|
127
|
+
// server or gateway answering `{"links": "none"}` would otherwise make
|
|
128
|
+
// Object.defineProperty throw, or smuggle a non-array out typed as string[].
|
|
129
|
+
const links = (Array.isArray(data.links) ? data.links : []);
|
|
130
|
+
// `sitemaps` rides BESIDE `links` in the engine's response and the old
|
|
131
|
+
// unwrap dropped it: a caller could not see which sitemaps a site exposes
|
|
132
|
+
// even though they paid for the discovery. Attach it the way search()
|
|
133
|
+
// attaches its answer siblings: non-enumerably, so `links.sitemaps` works
|
|
134
|
+
// while `for…of`, spread and JSON.stringify keep seeing exactly the array
|
|
135
|
+
// they saw before.
|
|
136
|
+
Object.defineProperty(links, "sitemaps", {
|
|
137
|
+
value: Array.isArray(data.sitemaps) ? data.sitemaps : [],
|
|
138
|
+
enumerable: false,
|
|
139
|
+
configurable: true,
|
|
140
|
+
writable: true,
|
|
141
|
+
});
|
|
142
|
+
return links;
|
|
113
143
|
}
|
|
114
144
|
/**
|
|
115
145
|
* Works in both modes; local mode needs a search backend configured on the engine.
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
|
|
2
2
|
export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
|
|
3
|
-
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
|
|
3
|
+
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, MapResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
|
package/dist/esm/local.d.ts
CHANGED
|
@@ -6,6 +6,19 @@
|
|
|
6
6
|
* (as the Python SDK does) is a fast-follow.
|
|
7
7
|
*/
|
|
8
8
|
import type { Json } from "./types.js";
|
|
9
|
+
/**
|
|
10
|
+
* Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
|
|
11
|
+
*
|
|
12
|
+
* The child inherits our environment, so either variable above left in the
|
|
13
|
+
* shell silently turned "run the local engine" into "call the cloud": every
|
|
14
|
+
* tool then answered with the REST envelope instead of the flat payload, and
|
|
15
|
+
* the call was billed. CRW_LOCAL means local, so the child does not get them.
|
|
16
|
+
*
|
|
17
|
+
* This cannot close every route: `crw-mcp` also reads `client.api_url` from
|
|
18
|
+
* `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
|
|
19
|
+
* is why `map()` still handles the envelope shape in `mapPayload`.
|
|
20
|
+
*/
|
|
21
|
+
export declare function localChildEnv(): NodeJS.ProcessEnv;
|
|
9
22
|
export declare class LocalTransport {
|
|
10
23
|
private proc;
|
|
11
24
|
private nextId;
|
package/dist/esm/local.js
CHANGED
|
@@ -8,6 +8,32 @@
|
|
|
8
8
|
import { spawn } from "node:child_process";
|
|
9
9
|
import { CrwApiError, CrwBinaryNotFoundError, CrwError } from "./errors.js";
|
|
10
10
|
const BINARY_NAME = process.platform === "win32" ? "crw-mcp.exe" : "crw-mcp";
|
|
11
|
+
/**
|
|
12
|
+
* Variables that switch `crw-mcp` from embedded to proxy mode. Both are
|
|
13
|
+
* verified against the shipped binary: `CRW_API_URL` is bound by its CLI, and
|
|
14
|
+
* `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
|
|
15
|
+
*/
|
|
16
|
+
const PROXY_MODE_ENV_VARS = ["CRW_API_URL", "CRW_CLIENT__API_URL"];
|
|
17
|
+
/**
|
|
18
|
+
* Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
|
|
19
|
+
*
|
|
20
|
+
* The child inherits our environment, so either variable above left in the
|
|
21
|
+
* shell silently turned "run the local engine" into "call the cloud": every
|
|
22
|
+
* tool then answered with the REST envelope instead of the flat payload, and
|
|
23
|
+
* the call was billed. CRW_LOCAL means local, so the child does not get them.
|
|
24
|
+
*
|
|
25
|
+
* This cannot close every route: `crw-mcp` also reads `client.api_url` from
|
|
26
|
+
* `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
|
|
27
|
+
* is why `map()` still handles the envelope shape in `mapPayload`.
|
|
28
|
+
*/
|
|
29
|
+
export function localChildEnv() {
|
|
30
|
+
// Windows environment names are case-insensitive, so the child would read a
|
|
31
|
+
// differently-spelled key that an exact-match filter left behind. POSIX names
|
|
32
|
+
// are case-sensitive, so there only the exact spellings matter.
|
|
33
|
+
const fold = (k) => (process.platform === "win32" ? k.toUpperCase() : k);
|
|
34
|
+
const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
|
|
35
|
+
return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
|
|
36
|
+
}
|
|
11
37
|
export class LocalTransport {
|
|
12
38
|
proc = null;
|
|
13
39
|
nextId = 0;
|
|
@@ -25,7 +51,7 @@ export class LocalTransport {
|
|
|
25
51
|
if (this.proc && this.proc.exitCode === null)
|
|
26
52
|
return this.proc;
|
|
27
53
|
const bin = this.resolveBinary();
|
|
28
|
-
const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"] });
|
|
54
|
+
const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
|
|
29
55
|
proc.on("error", (err) => {
|
|
30
56
|
const failure = err.code === "ENOENT"
|
|
31
57
|
? new CrwBinaryNotFoundError(`crw-mcp binary not found on PATH. Install it (e.g. \`npm i -g crw-mcp\` or ` +
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -478,6 +478,29 @@ export interface V2Document {
|
|
|
478
478
|
export type ScrapeResult = ScrapeDocument;
|
|
479
479
|
/** Crawl (`/v1/crawl`) returns the scraped document for each page, in crawl order. */
|
|
480
480
|
export type CrawlResult = ScrapeDocument[];
|
|
481
|
+
/**
|
|
482
|
+
* Map (`/v1/map`) returns the discovered URLs, with the sitemap URLs the engine
|
|
483
|
+
* read attached as `sitemaps`, the same way {@link SearchAnswer} rides
|
|
484
|
+
* alongside search results. Assignable to `string[]`, so
|
|
485
|
+
* `const urls: string[] = await crw.map(u)` keeps compiling.
|
|
486
|
+
*
|
|
487
|
+
* `sitemaps` is always an array at runtime: the client normalizes a missing
|
|
488
|
+
* field to `[]`. It is typed optional only so a plain `string[]` still assigns
|
|
489
|
+
* to this type, which keeps a consumer's own test stub
|
|
490
|
+
* (`mockResolvedValue(["https://a"])`) compiling. Same shape as
|
|
491
|
+
* {@link SearchAnswer}. It is non-enumerable, so it does not survive
|
|
492
|
+
* `structuredClone` (and therefore an RSC, worker or `postMessage` boundary) or
|
|
493
|
+
* any transform that builds a new array (`slice`, `filter`, spread).
|
|
494
|
+
* `JSON.stringify` is unaffected, since an array serializes its indices either
|
|
495
|
+
* way. Read it off the returned array directly.
|
|
496
|
+
*
|
|
497
|
+
* In local (subprocess) mode the MCP layer caps both lists at its map limit, so
|
|
498
|
+
* a site with a very deep sitemap index reports fewer entries there than over
|
|
499
|
+
* HTTP.
|
|
500
|
+
*/
|
|
501
|
+
export type MapResult = string[] & {
|
|
502
|
+
sitemaps?: string[];
|
|
503
|
+
};
|
|
481
504
|
/** Parse (`/v2/parse`) returns a v2 document (markdown/json/metadata). */
|
|
482
505
|
export type ParseResult = V2Document;
|
|
483
506
|
/** Batch scrape (`/v2/batch/scrape`) returns one v2 document per URL. */
|
package/package.json
CHANGED