firecrawl 4.4.1 → 4.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,7 +8,7 @@ var require_package = __commonJS({
8
8
  "package.json"(exports, module) {
9
9
  module.exports = {
10
10
  name: "@mendable/firecrawl-js",
11
- version: "4.4.1",
11
+ version: "4.5.1",
12
12
  description: "JavaScript SDK for Firecrawl API",
13
13
  main: "dist/index.js",
14
14
  types: "dist/index.d.ts",
@@ -36,6 +36,7 @@ var require_package = __commonJS({
36
36
  dependencies: {
37
37
  axios: "^1.12.2",
38
38
  "typescript-event-target": "^1.1.1",
39
+ ws: "^8.18.3",
39
40
  zod: "^3.23.8",
40
41
  "zod-to-json-schema": "^3.23.0"
41
42
  },
@@ -50,6 +51,7 @@ var require_package = __commonJS({
50
51
  "@types/mocha": "^10.0.6",
51
52
  "@types/node": "^20.12.12",
52
53
  "@types/uuid": "^9.0.8",
54
+ "@types/ws": "^8.18.1",
53
55
  dotenv: "^16.4.5",
54
56
  jest: "^30.0.5",
55
57
  "ts-jest": "^29.4.0",
package/dist/index.cjs CHANGED
@@ -35,7 +35,7 @@ var require_package = __commonJS({
35
35
  "package.json"(exports2, module2) {
36
36
  module2.exports = {
37
37
  name: "@mendable/firecrawl-js",
38
- version: "4.4.1",
38
+ version: "4.5.1",
39
39
  description: "JavaScript SDK for Firecrawl API",
40
40
  main: "dist/index.js",
41
41
  types: "dist/index.d.ts",
@@ -63,6 +63,7 @@ var require_package = __commonJS({
63
63
  dependencies: {
64
64
  axios: "^1.12.2",
65
65
  "typescript-event-target": "^1.1.1",
66
+ ws: "^8.18.3",
66
67
  zod: "^3.23.8",
67
68
  "zod-to-json-schema": "^3.23.0"
68
69
  },
@@ -77,6 +78,7 @@ var require_package = __commonJS({
77
78
  "@types/mocha": "^10.0.6",
78
79
  "@types/node": "^20.12.12",
79
80
  "@types/uuid": "^9.0.8",
81
+ "@types/ws": "^8.18.1",
80
82
  dotenv: "^16.4.5",
81
83
  jest: "^30.0.5",
82
84
  "ts-jest": "^29.4.0",
@@ -823,6 +825,7 @@ async function getTokenUsageHistorical(http, byApiKey) {
823
825
 
824
826
  // src/v2/watcher.ts
825
827
  var import_events = require("events");
828
+ var import_ws = require("ws");
826
829
  var Watcher = class extends import_events.EventEmitter {
827
830
  http;
828
831
  jobId;
@@ -848,9 +851,15 @@ var Watcher = class extends import_events.EventEmitter {
848
851
  async start() {
849
852
  try {
850
853
  const url = this.buildWsUrl();
851
- this.ws = new WebSocket(url, this.http.getApiKey());
852
- this.attachWsHandlers(this.ws);
853
- } catch {
854
+ if (typeof WebSocket !== "undefined") {
855
+ this.ws = new WebSocket(url, this.http.getApiKey());
856
+ } else {
857
+ this.ws = new import_ws.WebSocket(url, this.http.getApiKey());
858
+ }
859
+ if (this.ws) {
860
+ this.attachWsHandlers(this.ws);
861
+ }
862
+ } catch (err) {
854
863
  this.pollLoop();
855
864
  }
856
865
  }
@@ -878,7 +887,9 @@ var Watcher = class extends import_events.EventEmitter {
878
887
  return;
879
888
  }
880
889
  if (type === "done") {
881
- this.emit("done", { status: "completed", data: [], id: this.jobId });
890
+ const payload2 = body.data || body;
891
+ const data = payload2.data || [];
892
+ this.emit("done", { status: "completed", data, id: this.jobId });
882
893
  this.close();
883
894
  return;
884
895
  }
package/dist/index.d.cts CHANGED
@@ -4,7 +4,7 @@ import { AxiosResponse, AxiosRequestHeaders } from 'axios';
4
4
  import { EventEmitter } from 'events';
5
5
  import { TypedEventTarget } from 'typescript-event-target';
6
6
 
7
- type FormatString = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes';
7
+ type FormatString = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes' | 'branding';
8
8
  interface Viewport {
9
9
  width: number;
10
10
  height: number;
@@ -121,6 +121,130 @@ interface WebhookConfig {
121
121
  metadata?: Record<string, string>;
122
122
  events?: Array<'completed' | 'failed' | 'page' | 'started'>;
123
123
  }
124
+ interface BrandingProfile {
125
+ colorScheme?: 'light' | 'dark';
126
+ logo?: string | null;
127
+ fonts?: Array<{
128
+ family: string;
129
+ [key: string]: unknown;
130
+ }>;
131
+ colors?: {
132
+ primary?: string;
133
+ secondary?: string;
134
+ accent?: string;
135
+ background?: string;
136
+ textPrimary?: string;
137
+ textSecondary?: string;
138
+ link?: string;
139
+ success?: string;
140
+ warning?: string;
141
+ error?: string;
142
+ [key: string]: string | undefined;
143
+ };
144
+ typography?: {
145
+ fontFamilies?: {
146
+ primary?: string;
147
+ heading?: string;
148
+ code?: string;
149
+ [key: string]: string | undefined;
150
+ };
151
+ fontStacks?: {
152
+ primary?: string[];
153
+ heading?: string[];
154
+ body?: string[];
155
+ paragraph?: string[];
156
+ [key: string]: string[] | undefined;
157
+ };
158
+ fontSizes?: {
159
+ h1?: string;
160
+ h2?: string;
161
+ h3?: string;
162
+ body?: string;
163
+ small?: string;
164
+ [key: string]: string | undefined;
165
+ };
166
+ lineHeights?: {
167
+ heading?: number;
168
+ body?: number;
169
+ [key: string]: number | undefined;
170
+ };
171
+ fontWeights?: {
172
+ light?: number;
173
+ regular?: number;
174
+ medium?: number;
175
+ bold?: number;
176
+ [key: string]: number | undefined;
177
+ };
178
+ };
179
+ spacing?: {
180
+ baseUnit?: number;
181
+ padding?: Record<string, number>;
182
+ margins?: Record<string, number>;
183
+ gridGutter?: number;
184
+ borderRadius?: string;
185
+ [key: string]: number | string | Record<string, number> | undefined;
186
+ };
187
+ components?: {
188
+ buttonPrimary?: {
189
+ background?: string;
190
+ textColor?: string;
191
+ borderColor?: string;
192
+ borderRadius?: string;
193
+ [key: string]: string | undefined;
194
+ };
195
+ buttonSecondary?: {
196
+ background?: string;
197
+ textColor?: string;
198
+ borderColor?: string;
199
+ borderRadius?: string;
200
+ [key: string]: string | undefined;
201
+ };
202
+ input?: {
203
+ borderColor?: string;
204
+ focusBorderColor?: string;
205
+ borderRadius?: string;
206
+ [key: string]: string | undefined;
207
+ };
208
+ [key: string]: unknown;
209
+ };
210
+ icons?: {
211
+ style?: string;
212
+ primaryColor?: string;
213
+ [key: string]: string | undefined;
214
+ };
215
+ images?: {
216
+ logo?: string | null;
217
+ favicon?: string | null;
218
+ ogImage?: string | null;
219
+ [key: string]: string | null | undefined;
220
+ };
221
+ animations?: {
222
+ transitionDuration?: string;
223
+ easing?: string;
224
+ [key: string]: string | undefined;
225
+ };
226
+ layout?: {
227
+ grid?: {
228
+ columns?: number;
229
+ maxWidth?: string;
230
+ [key: string]: number | string | undefined;
231
+ };
232
+ headerHeight?: string;
233
+ footerHeight?: string;
234
+ [key: string]: number | string | Record<string, number | string | undefined> | undefined;
235
+ };
236
+ tone?: {
237
+ voice?: string;
238
+ emojiUsage?: string;
239
+ [key: string]: string | undefined;
240
+ };
241
+ personality?: {
242
+ tone: 'professional' | 'playful' | 'modern' | 'traditional' | 'minimalist' | 'bold';
243
+ energy: 'low' | 'medium' | 'high';
244
+ targetAudience: string;
245
+ };
246
+ [key: string]: unknown;
247
+ }
124
248
  interface DocumentMetadata {
125
249
  title?: string;
126
250
  description?: string;
@@ -183,6 +307,7 @@ interface Document {
183
307
  actions?: Record<string, unknown>;
184
308
  warning?: string;
185
309
  changeTracking?: Record<string, unknown>;
310
+ branding?: BrandingProfile;
186
311
  }
187
312
  interface PaginationConfig {
188
313
  /** When true (default), automatically follow `next` links and aggregate all documents. */
@@ -1532,4 +1657,4 @@ declare class Firecrawl extends FirecrawlClient {
1532
1657
  get v1(): FirecrawlApp;
1533
1658
  }
1534
1659
 
1535
- export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
1660
+ export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
package/dist/index.d.ts CHANGED
@@ -4,7 +4,7 @@ import { AxiosResponse, AxiosRequestHeaders } from 'axios';
4
4
  import { EventEmitter } from 'events';
5
5
  import { TypedEventTarget } from 'typescript-event-target';
6
6
 
7
- type FormatString = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes';
7
+ type FormatString = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes' | 'branding';
8
8
  interface Viewport {
9
9
  width: number;
10
10
  height: number;
@@ -121,6 +121,130 @@ interface WebhookConfig {
121
121
  metadata?: Record<string, string>;
122
122
  events?: Array<'completed' | 'failed' | 'page' | 'started'>;
123
123
  }
124
+ interface BrandingProfile {
125
+ colorScheme?: 'light' | 'dark';
126
+ logo?: string | null;
127
+ fonts?: Array<{
128
+ family: string;
129
+ [key: string]: unknown;
130
+ }>;
131
+ colors?: {
132
+ primary?: string;
133
+ secondary?: string;
134
+ accent?: string;
135
+ background?: string;
136
+ textPrimary?: string;
137
+ textSecondary?: string;
138
+ link?: string;
139
+ success?: string;
140
+ warning?: string;
141
+ error?: string;
142
+ [key: string]: string | undefined;
143
+ };
144
+ typography?: {
145
+ fontFamilies?: {
146
+ primary?: string;
147
+ heading?: string;
148
+ code?: string;
149
+ [key: string]: string | undefined;
150
+ };
151
+ fontStacks?: {
152
+ primary?: string[];
153
+ heading?: string[];
154
+ body?: string[];
155
+ paragraph?: string[];
156
+ [key: string]: string[] | undefined;
157
+ };
158
+ fontSizes?: {
159
+ h1?: string;
160
+ h2?: string;
161
+ h3?: string;
162
+ body?: string;
163
+ small?: string;
164
+ [key: string]: string | undefined;
165
+ };
166
+ lineHeights?: {
167
+ heading?: number;
168
+ body?: number;
169
+ [key: string]: number | undefined;
170
+ };
171
+ fontWeights?: {
172
+ light?: number;
173
+ regular?: number;
174
+ medium?: number;
175
+ bold?: number;
176
+ [key: string]: number | undefined;
177
+ };
178
+ };
179
+ spacing?: {
180
+ baseUnit?: number;
181
+ padding?: Record<string, number>;
182
+ margins?: Record<string, number>;
183
+ gridGutter?: number;
184
+ borderRadius?: string;
185
+ [key: string]: number | string | Record<string, number> | undefined;
186
+ };
187
+ components?: {
188
+ buttonPrimary?: {
189
+ background?: string;
190
+ textColor?: string;
191
+ borderColor?: string;
192
+ borderRadius?: string;
193
+ [key: string]: string | undefined;
194
+ };
195
+ buttonSecondary?: {
196
+ background?: string;
197
+ textColor?: string;
198
+ borderColor?: string;
199
+ borderRadius?: string;
200
+ [key: string]: string | undefined;
201
+ };
202
+ input?: {
203
+ borderColor?: string;
204
+ focusBorderColor?: string;
205
+ borderRadius?: string;
206
+ [key: string]: string | undefined;
207
+ };
208
+ [key: string]: unknown;
209
+ };
210
+ icons?: {
211
+ style?: string;
212
+ primaryColor?: string;
213
+ [key: string]: string | undefined;
214
+ };
215
+ images?: {
216
+ logo?: string | null;
217
+ favicon?: string | null;
218
+ ogImage?: string | null;
219
+ [key: string]: string | null | undefined;
220
+ };
221
+ animations?: {
222
+ transitionDuration?: string;
223
+ easing?: string;
224
+ [key: string]: string | undefined;
225
+ };
226
+ layout?: {
227
+ grid?: {
228
+ columns?: number;
229
+ maxWidth?: string;
230
+ [key: string]: number | string | undefined;
231
+ };
232
+ headerHeight?: string;
233
+ footerHeight?: string;
234
+ [key: string]: number | string | Record<string, number | string | undefined> | undefined;
235
+ };
236
+ tone?: {
237
+ voice?: string;
238
+ emojiUsage?: string;
239
+ [key: string]: string | undefined;
240
+ };
241
+ personality?: {
242
+ tone: 'professional' | 'playful' | 'modern' | 'traditional' | 'minimalist' | 'bold';
243
+ energy: 'low' | 'medium' | 'high';
244
+ targetAudience: string;
245
+ };
246
+ [key: string]: unknown;
247
+ }
124
248
  interface DocumentMetadata {
125
249
  title?: string;
126
250
  description?: string;
@@ -183,6 +307,7 @@ interface Document {
183
307
  actions?: Record<string, unknown>;
184
308
  warning?: string;
185
309
  changeTracking?: Record<string, unknown>;
310
+ branding?: BrandingProfile;
186
311
  }
187
312
  interface PaginationConfig {
188
313
  /** When true (default), automatically follow `next` links and aggregate all documents. */
@@ -1532,4 +1657,4 @@ declare class Firecrawl extends FirecrawlClient {
1532
1657
  get v1(): FirecrawlApp;
1533
1658
  }
1534
1659
 
1535
- export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
1660
+ export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type JsonFormat, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type PDFAction, type PaginationConfig, type PressAction, type QueueStatusResponse$1 as QueueStatusResponse, type ScrapeAction, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type Viewport, type WaitAction, type WebhookConfig, type WriteAction, Firecrawl as default };
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-J3BEBX3Y.js";
3
+ } from "./chunk-TNGVHCPH.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -707,6 +707,7 @@ async function getTokenUsageHistorical(http, byApiKey) {
707
707
 
708
708
  // src/v2/watcher.ts
709
709
  import { EventEmitter } from "events";
710
+ import { WebSocket as WS } from "ws";
710
711
  var Watcher = class extends EventEmitter {
711
712
  http;
712
713
  jobId;
@@ -732,9 +733,15 @@ var Watcher = class extends EventEmitter {
732
733
  async start() {
733
734
  try {
734
735
  const url = this.buildWsUrl();
735
- this.ws = new WebSocket(url, this.http.getApiKey());
736
- this.attachWsHandlers(this.ws);
737
- } catch {
736
+ if (typeof WebSocket !== "undefined") {
737
+ this.ws = new WebSocket(url, this.http.getApiKey());
738
+ } else {
739
+ this.ws = new WS(url, this.http.getApiKey());
740
+ }
741
+ if (this.ws) {
742
+ this.attachWsHandlers(this.ws);
743
+ }
744
+ } catch (err) {
738
745
  this.pollLoop();
739
746
  }
740
747
  }
@@ -762,7 +769,9 @@ var Watcher = class extends EventEmitter {
762
769
  return;
763
770
  }
764
771
  if (type === "done") {
765
- this.emit("done", { status: "completed", data: [], id: this.jobId });
772
+ const payload2 = body.data || body;
773
+ const data = payload2.data || [];
774
+ this.emit("done", { status: "completed", data, id: this.jobId });
766
775
  this.close();
767
776
  return;
768
777
  }
@@ -1071,7 +1080,7 @@ var FirecrawlApp = class {
1071
1080
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
1072
1081
  return process.env.npm_package_version;
1073
1082
  }
1074
- const packageJson = await import("./package-SYKCM5JT.js");
1083
+ const packageJson = await import("./package-SV2UYR6U.js");
1075
1084
  return packageJson.default.version;
1076
1085
  } catch (error) {
1077
1086
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-J3BEBX3Y.js";
3
+ } from "./chunk-TNGVHCPH.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "firecrawl",
3
- "version": "4.4.1",
3
+ "version": "4.5.1",
4
4
  "description": "JavaScript SDK for Firecrawl API",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -28,6 +28,7 @@
28
28
  "dependencies": {
29
29
  "axios": "^1.12.2",
30
30
  "typescript-event-target": "^1.1.1",
31
+ "ws": "^8.18.3",
31
32
  "zod": "^3.23.8",
32
33
  "zod-to-json-schema": "^3.23.0"
33
34
  },
@@ -42,6 +43,7 @@
42
43
  "@types/mocha": "^10.0.6",
43
44
  "@types/node": "^20.12.12",
44
45
  "@types/uuid": "^9.0.8",
46
+ "@types/ws": "^8.18.1",
45
47
  "dotenv": "^16.4.5",
46
48
  "jest": "^30.0.5",
47
49
  "ts-jest": "^29.4.0",
@@ -0,0 +1,186 @@
1
+ import { describe, test, expect, jest } from "@jest/globals";
2
+ import { scrape } from "../../../v2/methods/scrape";
3
+
4
+ describe("JS SDK v2 branding format", () => {
5
+ function makeHttp(postImpl: (url: string, data: any) => any) {
6
+ return { post: jest.fn(async (u: string, d: any) => postImpl(u, d)) } as any;
7
+ }
8
+
9
+ test("scrape with branding format returns branding data", async () => {
10
+ const mockResponse = {
11
+ status: 200,
12
+ data: {
13
+ success: true,
14
+ data: {
15
+ markdown: "# Example",
16
+ branding: {
17
+ colorScheme: "light",
18
+ colors: {
19
+ primary: "#E11D48",
20
+ secondary: "#3B82F6",
21
+ accent: "#F59E0B"
22
+ },
23
+ typography: {
24
+ fontFamilies: {
25
+ primary: "Inter",
26
+ heading: "Poppins"
27
+ },
28
+ fontSizes: {
29
+ h1: "2.5rem",
30
+ body: "1rem"
31
+ }
32
+ },
33
+ spacing: {
34
+ baseUnit: 8
35
+ },
36
+ components: {
37
+ buttonPrimary: {
38
+ background: "#E11D48",
39
+ textColor: "#FFFFFF",
40
+ borderRadius: "0.5rem"
41
+ }
42
+ }
43
+ }
44
+ }
45
+ }
46
+ };
47
+
48
+ const http = makeHttp(() => mockResponse);
49
+ const result = await scrape(http, "https://example.com", { formats: ["branding"] });
50
+
51
+ expect(result.branding).toBeDefined();
52
+ expect(result.branding?.colorScheme).toBe("light");
53
+ expect(result.branding?.colors?.primary).toBe("#E11D48");
54
+ expect(result.branding?.typography?.fontFamilies?.primary).toBe("Inter");
55
+ expect(result.branding?.spacing?.baseUnit).toBe(8);
56
+ expect(result.branding?.components?.buttonPrimary?.background).toBe("#E11D48");
57
+ });
58
+
59
+ test("scrape with branding and markdown formats returns both", async () => {
60
+ const mockResponse = {
61
+ status: 200,
62
+ data: {
63
+ success: true,
64
+ data: {
65
+ markdown: "# Example Content",
66
+ branding: {
67
+ colorScheme: "dark",
68
+ colors: {
69
+ primary: "#10B981"
70
+ },
71
+ typography: {
72
+ fontFamilies: {
73
+ primary: "Roboto"
74
+ }
75
+ }
76
+ }
77
+ }
78
+ }
79
+ };
80
+
81
+ const http = makeHttp(() => mockResponse);
82
+ const result = await scrape(http, "https://example.com", { formats: ["markdown", "branding"] });
83
+
84
+ expect(result.markdown).toBe("# Example Content");
85
+ expect(result.branding).toBeDefined();
86
+ expect(result.branding?.colorScheme).toBe("dark");
87
+ expect(result.branding?.colors?.primary).toBe("#10B981");
88
+ });
89
+
90
+ test("scrape without branding format does not return branding", async () => {
91
+ const mockResponse = {
92
+ status: 200,
93
+ data: {
94
+ success: true,
95
+ data: {
96
+ markdown: "# Example"
97
+ }
98
+ }
99
+ };
100
+
101
+ const http = makeHttp(() => mockResponse);
102
+ const result = await scrape(http, "https://example.com", { formats: ["markdown"] });
103
+
104
+ expect(result.markdown).toBe("# Example");
105
+ expect(result.branding).toBeUndefined();
106
+ });
107
+
108
+ test("branding format with all nested fields", async () => {
109
+ const mockResponse = {
110
+ status: 200,
111
+ data: {
112
+ success: true,
113
+ data: {
114
+ branding: {
115
+ colorScheme: "light",
116
+ logo: "https://example.com/logo.png",
117
+ fonts: [
118
+ { family: "Inter", weight: 400 },
119
+ { family: "Poppins", weight: 700 }
120
+ ],
121
+ colors: {
122
+ primary: "#E11D48",
123
+ background: "#FFFFFF"
124
+ },
125
+ typography: {
126
+ fontFamilies: { primary: "Inter" },
127
+ fontStacks: { body: ["Inter", "sans-serif"] },
128
+ fontSizes: { h1: "2.5rem" },
129
+ lineHeights: { body: 1.5 },
130
+ fontWeights: { regular: 400 }
131
+ },
132
+ spacing: {
133
+ baseUnit: 8,
134
+ padding: { sm: 8, md: 16 }
135
+ },
136
+ components: {
137
+ buttonPrimary: {
138
+ background: "#E11D48",
139
+ textColor: "#FFFFFF"
140
+ }
141
+ },
142
+ icons: {
143
+ style: "outline",
144
+ primaryColor: "#E11D48"
145
+ },
146
+ images: {
147
+ logo: "https://example.com/logo.png",
148
+ favicon: "https://example.com/favicon.ico"
149
+ },
150
+ animations: {
151
+ transitionDuration: "200ms",
152
+ easing: "ease-in-out"
153
+ },
154
+ layout: {
155
+ grid: { columns: 12, maxWidth: "1200px" },
156
+ headerHeight: "64px"
157
+ },
158
+ tone: {
159
+ voice: "professional",
160
+ emojiUsage: "minimal"
161
+ },
162
+ personality: {
163
+ tone: "professional",
164
+ energy: "medium",
165
+ targetAudience: "developers"
166
+ }
167
+ }
168
+ }
169
+ }
170
+ };
171
+
172
+ const http = makeHttp(() => mockResponse);
173
+ const result = await scrape(http, "https://example.com", { formats: ["branding"] });
174
+
175
+ expect(result.branding).toBeDefined();
176
+ expect(result.branding?.logo).toBe("https://example.com/logo.png");
177
+ expect(result.branding?.fonts).toHaveLength(2);
178
+ expect(result.branding?.typography?.fontStacks?.body).toEqual(["Inter", "sans-serif"]);
179
+ expect(result.branding?.spacing?.padding).toEqual({ sm: 8, md: 16 });
180
+ expect(result.branding?.icons?.style).toBe("outline");
181
+ expect(result.branding?.images?.favicon).toBe("https://example.com/favicon.ico");
182
+ expect(result.branding?.animations?.easing).toBe("ease-in-out");
183
+ expect(result.branding?.layout?.grid?.columns).toBe(12);
184
+ expect(result.branding?.personality?.tone).toBe("professional");
185
+ });
186
+ });
package/src/v2/types.ts CHANGED
@@ -11,7 +11,8 @@ export type FormatString =
11
11
  | 'summary'
12
12
  | 'changeTracking'
13
13
  | 'json'
14
- | 'attributes';
14
+ | 'attributes'
15
+ | 'branding';
15
16
 
16
17
  export interface Viewport {
17
18
  width: number;
@@ -165,6 +166,141 @@ export interface WebhookConfig {
165
166
  events?: Array<'completed' | 'failed' | 'page' | 'started'>;
166
167
  }
167
168
 
169
+ export interface BrandingProfile {
170
+ colorScheme?: 'light' | 'dark';
171
+ logo?: string | null;
172
+ fonts?: Array<{
173
+ family: string;
174
+ [key: string]: unknown;
175
+ }>;
176
+ colors?: {
177
+ primary?: string;
178
+ secondary?: string;
179
+ accent?: string;
180
+ background?: string;
181
+ textPrimary?: string;
182
+ textSecondary?: string;
183
+ link?: string;
184
+ success?: string;
185
+ warning?: string;
186
+ error?: string;
187
+ [key: string]: string | undefined;
188
+ };
189
+ typography?: {
190
+ fontFamilies?: {
191
+ primary?: string;
192
+ heading?: string;
193
+ code?: string;
194
+ [key: string]: string | undefined;
195
+ };
196
+ fontStacks?: {
197
+ primary?: string[];
198
+ heading?: string[];
199
+ body?: string[];
200
+ paragraph?: string[];
201
+ [key: string]: string[] | undefined;
202
+ };
203
+ fontSizes?: {
204
+ h1?: string;
205
+ h2?: string;
206
+ h3?: string;
207
+ body?: string;
208
+ small?: string;
209
+ [key: string]: string | undefined;
210
+ };
211
+ lineHeights?: {
212
+ heading?: number;
213
+ body?: number;
214
+ [key: string]: number | undefined;
215
+ };
216
+ fontWeights?: {
217
+ light?: number;
218
+ regular?: number;
219
+ medium?: number;
220
+ bold?: number;
221
+ [key: string]: number | undefined;
222
+ };
223
+ };
224
+ spacing?: {
225
+ baseUnit?: number;
226
+ padding?: Record<string, number>;
227
+ margins?: Record<string, number>;
228
+ gridGutter?: number;
229
+ borderRadius?: string;
230
+ [key: string]: number | string | Record<string, number> | undefined;
231
+ };
232
+ components?: {
233
+ buttonPrimary?: {
234
+ background?: string;
235
+ textColor?: string;
236
+ borderColor?: string;
237
+ borderRadius?: string;
238
+ [key: string]: string | undefined;
239
+ };
240
+ buttonSecondary?: {
241
+ background?: string;
242
+ textColor?: string;
243
+ borderColor?: string;
244
+ borderRadius?: string;
245
+ [key: string]: string | undefined;
246
+ };
247
+ input?: {
248
+ borderColor?: string;
249
+ focusBorderColor?: string;
250
+ borderRadius?: string;
251
+ [key: string]: string | undefined;
252
+ };
253
+ [key: string]: unknown;
254
+ };
255
+ icons?: {
256
+ style?: string;
257
+ primaryColor?: string;
258
+ [key: string]: string | undefined;
259
+ };
260
+ images?: {
261
+ logo?: string | null;
262
+ favicon?: string | null;
263
+ ogImage?: string | null;
264
+ [key: string]: string | null | undefined;
265
+ };
266
+ animations?: {
267
+ transitionDuration?: string;
268
+ easing?: string;
269
+ [key: string]: string | undefined;
270
+ };
271
+ layout?: {
272
+ grid?: {
273
+ columns?: number;
274
+ maxWidth?: string;
275
+ [key: string]: number | string | undefined;
276
+ };
277
+ headerHeight?: string;
278
+ footerHeight?: string;
279
+ [key: string]:
280
+ | number
281
+ | string
282
+ | Record<string, number | string | undefined>
283
+ | undefined;
284
+ };
285
+ tone?: {
286
+ voice?: string;
287
+ emojiUsage?: string;
288
+ [key: string]: string | undefined;
289
+ };
290
+ personality?: {
291
+ tone:
292
+ | 'professional'
293
+ | 'playful'
294
+ | 'modern'
295
+ | 'traditional'
296
+ | 'minimalist'
297
+ | 'bold';
298
+ energy: 'low' | 'medium' | 'high';
299
+ targetAudience: string;
300
+ };
301
+ [key: string]: unknown;
302
+ }
303
+
168
304
  export interface DocumentMetadata {
169
305
  // Common metadata fields
170
306
  title?: string;
@@ -239,6 +375,7 @@ export interface Document {
239
375
  actions?: Record<string, unknown>;
240
376
  warning?: string;
241
377
  changeTracking?: Record<string, unknown>;
378
+ branding?: BrandingProfile;
242
379
  }
243
380
 
244
381
  // Pagination configuration for auto-fetching pages from v2 endpoints that return a `next` URL
package/src/v2/watcher.ts CHANGED
@@ -3,6 +3,7 @@ import type { BatchScrapeJob, CrawlJob, Document } from "./types";
3
3
  import type { HttpClient } from "./utils/httpClient";
4
4
  import { getBatchScrapeStatus } from "./methods/batch";
5
5
  import { getCrawlStatus } from "./methods/crawl";
6
+ import { WebSocket as WS } from "ws";
6
7
 
7
8
  type JobKind = "crawl" | "batch";
8
9
 
@@ -43,11 +44,17 @@ export class Watcher extends EventEmitter {
43
44
  async start(): Promise<void> {
44
45
  try {
45
46
  const url = this.buildWsUrl();
46
- // Pass API key as subprotocol for browser compatibility
47
- this.ws = new WebSocket(url, this.http.getApiKey());
48
- this.attachWsHandlers(this.ws);
49
- } catch {
50
- // Fallback to polling immediately
47
+
48
+ if (typeof WebSocket !== 'undefined') {
49
+ this.ws = new WebSocket(url, this.http.getApiKey()) as any;
50
+ } else {
51
+ this.ws = new WS(url, this.http.getApiKey()) as any;
52
+ }
53
+
54
+ if (this.ws) {
55
+ this.attachWsHandlers(this.ws);
56
+ }
57
+ } catch (err) {
51
58
  this.pollLoop();
52
59
  }
53
60
  }
@@ -76,7 +83,9 @@ export class Watcher extends EventEmitter {
76
83
  return;
77
84
  }
78
85
  if (type === "done") {
79
- this.emit("done", { status: "completed", data: [], id: this.jobId });
86
+ const payload = body.data || body;
87
+ const data = (payload.data || []) as Document[];
88
+ this.emit("done", { status: "completed", data, id: this.jobId });
80
89
  this.close();
81
90
  return;
82
91
  }
package/tsup.config.ts CHANGED
@@ -8,6 +8,7 @@ export default defineConfig({
8
8
  clean: true,
9
9
  platform: "node",
10
10
  target: "node22",
11
+ external: ["ws"],
11
12
  noExternal: ["typescript-event-target"],
12
13
  esbuildOptions(options) {
13
14
  options.define = {