crw-sdk 0.35.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -49,7 +49,7 @@ CRW_LOCAL=1 node app.js
49
49
  |---|---|---|
50
50
  | `scrape(url, opts?)` | Scrape one URL | both |
51
51
  | `crawl(url, opts?)` | Crawl a site (async, polled) | both |
52
- | `map(url, opts?)` | Discover URLs | both |
52
+ | `map(url, opts?)` | Discover URLs (+ the site's `sitemaps`) | both |
53
53
  | `search(query, opts?)` | Web search (+ optional scrape) | both¹ |
54
54
  | `parseFile(bytes, opts?)` | PDF → markdown / structured JSON | both |
55
55
  | `extract({urls, schema?})` | Structured LLM extraction | HTTP |
@@ -21,6 +21,22 @@ function httpOnlyHint(name, reason) {
21
21
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
22
22
  }
23
23
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
24
+ /**
25
+ * Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
26
+ * It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
27
+ * the REST envelope (`{success, data: {…}}`) when it proxies. It proxies
28
+ * whenever `CRW_API_URL` is set, which the subprocess inherits from us even
29
+ * though CRW_LOCAL mode never reads that variable itself. Without this, `links`
30
+ * came back empty in that state.
31
+ *
32
+ * Probe `data.links` rather than a bare `data`, so a flat response that grows
33
+ * some unrelated top-level `data` later cannot make us unwrap into it. Same
34
+ * discriminator the engine's own MCP bounds use.
35
+ */
36
+ function mapPayload(result) {
37
+ const envelope = result.data;
38
+ return envelope && typeof envelope === "object" && "links" in envelope ? envelope : result;
39
+ }
24
40
  /**
25
41
  * Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
26
42
  * server's `next` is an absolute URL from its own public origin/prefix; reusing
@@ -107,12 +123,26 @@ class CrwClient {
107
123
  async map(url, opts = {}) {
108
124
  const { maxDepth = 2, useSitemap = true, ...rest } = opts;
109
125
  const args = { url, maxDepth, useSitemap, ...rest };
110
- if (this.apiUrl) {
111
- const data = await this.httpPost("/v1/map", args);
112
- return data.links ?? [];
113
- }
114
- const result = await this.localTransport().toolCall("crw_map", args);
115
- return result.links ?? [];
126
+ const data = this.apiUrl
127
+ ? await this.httpPost("/v1/map", args)
128
+ : mapPayload(await this.localTransport().toolCall("crw_map", args));
129
+ // Guard the shapes, as search() does before its own attach: a self-hosted
130
+ // server or gateway answering `{"links": "none"}` would otherwise make
131
+ // Object.defineProperty throw, or smuggle a non-array out typed as string[].
132
+ const links = (Array.isArray(data.links) ? data.links : []);
133
+ // `sitemaps` rides BESIDE `links` in the engine's response and the old
134
+ // unwrap dropped it: a caller could not see which sitemaps a site exposes
135
+ // even though they paid for the discovery. Attach it the way search()
136
+ // attaches its answer siblings: non-enumerably, so `links.sitemaps` works
137
+ // while `for…of`, spread and JSON.stringify keep seeing exactly the array
138
+ // they saw before.
139
+ Object.defineProperty(links, "sitemaps", {
140
+ value: Array.isArray(data.sitemaps) ? data.sitemaps : [],
141
+ enumerable: false,
142
+ configurable: true,
143
+ writable: true,
144
+ });
145
+ return links;
116
146
  }
117
147
  /**
118
148
  * Works in both modes; local mode needs a search backend configured on the engine.
package/dist/cjs/local.js CHANGED
@@ -8,9 +8,36 @@
8
8
  */
9
9
  Object.defineProperty(exports, "__esModule", { value: true });
10
10
  exports.LocalTransport = void 0;
11
+ exports.localChildEnv = localChildEnv;
11
12
  const node_child_process_1 = require("node:child_process");
12
13
  const errors_js_1 = require("./errors.js");
13
14
  const BINARY_NAME = process.platform === "win32" ? "crw-mcp.exe" : "crw-mcp";
15
+ /**
16
+ * Variables that switch `crw-mcp` from embedded to proxy mode. Both are
17
+ * verified against the shipped binary: `CRW_API_URL` is bound by its CLI, and
18
+ * `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
19
+ */
20
+ const PROXY_MODE_ENV_VARS = ["CRW_API_URL", "CRW_CLIENT__API_URL"];
21
+ /**
22
+ * Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
23
+ *
24
+ * The child inherits our environment, so either variable above left in the
25
+ * shell silently turned "run the local engine" into "call the cloud": every
26
+ * tool then answered with the REST envelope instead of the flat payload, and
27
+ * the call was billed. CRW_LOCAL means local, so the child does not get them.
28
+ *
29
+ * This cannot close every route: `crw-mcp` also reads `client.api_url` from
30
+ * `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
31
+ * is why `map()` still handles the envelope shape in `mapPayload`.
32
+ */
33
+ function localChildEnv() {
34
+ // Windows environment names are case-insensitive, so the child would read a
35
+ // differently-spelled key that an exact-match filter left behind. POSIX names
36
+ // are case-sensitive, so there only the exact spellings matter.
37
+ const fold = (k) => (process.platform === "win32" ? k.toUpperCase() : k);
38
+ const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
39
+ return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
40
+ }
14
41
  class LocalTransport {
15
42
  proc = null;
16
43
  nextId = 0;
@@ -28,7 +55,7 @@ class LocalTransport {
28
55
  if (this.proc && this.proc.exitCode === null)
29
56
  return this.proc;
30
57
  const bin = this.resolveBinary();
31
- const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"] });
58
+ const proc = (0, node_child_process_1.spawn)(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
32
59
  proc.on("error", (err) => {
33
60
  const failure = err.code === "ENOENT"
34
61
  ? new errors_js_1.CrwBinaryNotFoundError(`crw-mcp binary not found on PATH. Install it (e.g. \`npm i -g crw-mcp\` or ` +
@@ -1,5 +1,5 @@
1
1
  /** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
2
- import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
2
+ import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, MapResult, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
3
3
  export declare const CLOUD_API_URL = "https://api.fastcrw.com";
4
4
  export declare const DASHBOARD_URL = "https://fastcrw.com/dashboard";
5
5
  export declare const DOCS_URL = "https://us.github.io/crw";
@@ -15,7 +15,7 @@ export declare class CrwClient {
15
15
  constructor(opts?: ClientOptions);
16
16
  scrape(url: string, opts?: ScrapeOptions): Promise<ScrapeResult>;
17
17
  crawl(url: string, opts?: CrawlOptions): Promise<CrawlResult>;
18
- map(url: string, opts?: MapOptions): Promise<string[]>;
18
+ map(url: string, opts?: MapOptions): Promise<MapResult>;
19
19
  /**
20
20
  * Works in both modes; local mode needs a search backend configured on the engine.
21
21
  */
@@ -18,6 +18,22 @@ function httpOnlyHint(name, reason) {
18
18
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
19
19
  }
20
20
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
21
+ /**
22
+ * Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
23
+ * It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
24
+ * the REST envelope (`{success, data: {…}}`) when it proxies. It proxies
25
+ * whenever `CRW_API_URL` is set, which the subprocess inherits from us even
26
+ * though CRW_LOCAL mode never reads that variable itself. Without this, `links`
27
+ * came back empty in that state.
28
+ *
29
+ * Probe `data.links` rather than a bare `data`, so a flat response that grows
30
+ * some unrelated top-level `data` later cannot make us unwrap into it. Same
31
+ * discriminator the engine's own MCP bounds use.
32
+ */
33
+ function mapPayload(result) {
34
+ const envelope = result.data;
35
+ return envelope && typeof envelope === "object" && "links" in envelope ? envelope : result;
36
+ }
21
37
  /**
22
38
  * Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
23
39
  * server's `next` is an absolute URL from its own public origin/prefix; reusing
@@ -104,12 +120,26 @@ export class CrwClient {
104
120
  async map(url, opts = {}) {
105
121
  const { maxDepth = 2, useSitemap = true, ...rest } = opts;
106
122
  const args = { url, maxDepth, useSitemap, ...rest };
107
- if (this.apiUrl) {
108
- const data = await this.httpPost("/v1/map", args);
109
- return data.links ?? [];
110
- }
111
- const result = await this.localTransport().toolCall("crw_map", args);
112
- return result.links ?? [];
123
+ const data = this.apiUrl
124
+ ? await this.httpPost("/v1/map", args)
125
+ : mapPayload(await this.localTransport().toolCall("crw_map", args));
126
+ // Guard the shapes, as search() does before its own attach: a self-hosted
127
+ // server or gateway answering `{"links": "none"}` would otherwise make
128
+ // Object.defineProperty throw, or smuggle a non-array out typed as string[].
129
+ const links = (Array.isArray(data.links) ? data.links : []);
130
+ // `sitemaps` rides BESIDE `links` in the engine's response and the old
131
+ // unwrap dropped it: a caller could not see which sitemaps a site exposes
132
+ // even though they paid for the discovery. Attach it the way search()
133
+ // attaches its answer siblings: non-enumerably, so `links.sitemaps` works
134
+ // while `for…of`, spread and JSON.stringify keep seeing exactly the array
135
+ // they saw before.
136
+ Object.defineProperty(links, "sitemaps", {
137
+ value: Array.isArray(data.sitemaps) ? data.sitemaps : [],
138
+ enumerable: false,
139
+ configurable: true,
140
+ writable: true,
141
+ });
142
+ return links;
113
143
  }
114
144
  /**
115
145
  * Works in both modes; local mode needs a search backend configured on the engine.
@@ -1,3 +1,3 @@
1
1
  export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
2
2
  export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
3
- export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
3
+ export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, MapResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
@@ -6,6 +6,19 @@
6
6
  * (as the Python SDK does) is a fast-follow.
7
7
  */
8
8
  import type { Json } from "./types.js";
9
+ /**
10
+ * Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
11
+ *
12
+ * The child inherits our environment, so either variable above left in the
13
+ * shell silently turned "run the local engine" into "call the cloud": every
14
+ * tool then answered with the REST envelope instead of the flat payload, and
15
+ * the call was billed. CRW_LOCAL means local, so the child does not get them.
16
+ *
17
+ * This cannot close every route: `crw-mcp` also reads `client.api_url` from
18
+ * `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
19
+ * is why `map()` still handles the envelope shape in `mapPayload`.
20
+ */
21
+ export declare function localChildEnv(): NodeJS.ProcessEnv;
9
22
  export declare class LocalTransport {
10
23
  private proc;
11
24
  private nextId;
package/dist/esm/local.js CHANGED
@@ -8,6 +8,32 @@
8
8
  import { spawn } from "node:child_process";
9
9
  import { CrwApiError, CrwBinaryNotFoundError, CrwError } from "./errors.js";
10
10
  const BINARY_NAME = process.platform === "win32" ? "crw-mcp.exe" : "crw-mcp";
11
+ /**
12
+ * Variables that switch `crw-mcp` from embedded to proxy mode. Both are
13
+ * verified against the shipped binary: `CRW_API_URL` is bound by its CLI, and
14
+ * `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
15
+ */
16
+ const PROXY_MODE_ENV_VARS = ["CRW_API_URL", "CRW_CLIENT__API_URL"];
17
+ /**
18
+ * Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
19
+ *
20
+ * The child inherits our environment, so either variable above left in the
21
+ * shell silently turned "run the local engine" into "call the cloud": every
22
+ * tool then answered with the REST envelope instead of the flat payload, and
23
+ * the call was billed. CRW_LOCAL means local, so the child does not get them.
24
+ *
25
+ * This cannot close every route: `crw-mcp` also reads `client.api_url` from
26
+ * `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
27
+ * is why `map()` still handles the envelope shape in `mapPayload`.
28
+ */
29
+ export function localChildEnv() {
30
+ // Windows environment names are case-insensitive, so the child would read a
31
+ // differently-spelled key that an exact-match filter left behind. POSIX names
32
+ // are case-sensitive, so there only the exact spellings matter.
33
+ const fold = (k) => (process.platform === "win32" ? k.toUpperCase() : k);
34
+ const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
35
+ return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
36
+ }
11
37
  export class LocalTransport {
12
38
  proc = null;
13
39
  nextId = 0;
@@ -25,7 +51,7 @@ export class LocalTransport {
25
51
  if (this.proc && this.proc.exitCode === null)
26
52
  return this.proc;
27
53
  const bin = this.resolveBinary();
28
- const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"] });
54
+ const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
29
55
  proc.on("error", (err) => {
30
56
  const failure = err.code === "ENOENT"
31
57
  ? new CrwBinaryNotFoundError(`crw-mcp binary not found on PATH. Install it (e.g. \`npm i -g crw-mcp\` or ` +
@@ -478,6 +478,29 @@ export interface V2Document {
478
478
  export type ScrapeResult = ScrapeDocument;
479
479
  /** Crawl (`/v1/crawl`) returns the scraped document for each page, in crawl order. */
480
480
  export type CrawlResult = ScrapeDocument[];
481
+ /**
482
+ * Map (`/v1/map`) returns the discovered URLs, with the sitemap URLs the engine
483
+ * read attached as `sitemaps`, the same way {@link SearchAnswer} rides
484
+ * alongside search results. Assignable to `string[]`, so
485
+ * `const urls: string[] = await crw.map(u)` keeps compiling.
486
+ *
487
+ * `sitemaps` is always an array at runtime: the client normalizes a missing
488
+ * field to `[]`. It is typed optional only so a plain `string[]` still assigns
489
+ * to this type, which keeps a consumer's own test stub
490
+ * (`mockResolvedValue(["https://a"])`) compiling. Same shape as
491
+ * {@link SearchAnswer}. It is non-enumerable, so it does not survive
492
+ * `structuredClone` (and therefore an RSC, worker or `postMessage` boundary) or
493
+ * any transform that builds a new array (`slice`, `filter`, spread).
494
+ * `JSON.stringify` is unaffected, since an array serializes its indices either
495
+ * way. Read it off the returned array directly.
496
+ *
497
+ * In local (subprocess) mode the MCP layer caps both lists at its map limit, so
498
+ * a site with a very deep sitemap index reports fewer entries there than over
499
+ * HTTP.
500
+ */
501
+ export type MapResult = string[] & {
502
+ sitemaps?: string[];
503
+ };
481
504
  /** Parse (`/v2/parse`) returns a v2 document (markdown/json/metadata). */
482
505
  export type ParseResult = V2Document;
483
506
  /** Batch scrape (`/v2/batch/scrape`) returns one v2 document per URL. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crw-sdk",
3
- "version": "0.35.0",
3
+ "version": "0.36.0",
4
4
  "description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
5
5
  "license": "MIT",
6
6
  "author": "fastCRW <hello@fastcrw.com>",