firecrawl-cli 1.15.2 → 1.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/__tests__/commands/scrape.test.js +36 -0
- package/dist/__tests__/commands/scrape.test.js.map +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +54 -13
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/crawl.d.ts.map +1 -1
- package/dist/commands/crawl.js +31 -1
- package/dist/commands/crawl.js.map +1 -1
- package/dist/commands/parse.d.ts +18 -0
- package/dist/commands/parse.d.ts.map +1 -0
- package/dist/commands/parse.js +219 -0
- package/dist/commands/parse.js.map +1 -0
- package/dist/commands/scrape.d.ts.map +1 -1
- package/dist/commands/scrape.js +17 -2
- package/dist/commands/scrape.js.map +1 -1
- package/dist/index.js +286 -4
- package/dist/index.js.map +1 -1
- package/dist/types/agent.d.ts +6 -1
- package/dist/types/agent.d.ts.map +1 -1
- package/dist/types/crawl.d.ts +6 -0
- package/dist/types/crawl.d.ts.map +1 -1
- package/dist/types/parse.d.ts +40 -0
- package/dist/types/parse.d.ts.map +1 -0
- package/dist/types/parse.js +6 -0
- package/dist/types/parse.js.map +1 -0
- package/dist/types/scrape.d.ts +8 -0
- package/dist/types/scrape.d.ts.map +1 -1
- package/dist/utils/options.d.ts.map +1 -1
- package/dist/utils/options.js +1 -0
- package/dist/utils/options.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types and interfaces for the parse command
|
|
3
|
+
*/
|
|
4
|
+
import type { ScrapeFormat, ScrapeLocation } from './scrape';
|
|
5
|
+
export interface ParseOptions {
|
|
6
|
+
/** Local file path to parse */
|
|
7
|
+
file: string;
|
|
8
|
+
/** Output format(s) */
|
|
9
|
+
formats?: ScrapeFormat[];
|
|
10
|
+
/** Include only main content */
|
|
11
|
+
onlyMainContent?: boolean;
|
|
12
|
+
/** Include tags */
|
|
13
|
+
includeTags?: string[];
|
|
14
|
+
/** Exclude tags */
|
|
15
|
+
excludeTags?: string[];
|
|
16
|
+
/** Timeout in milliseconds for the parse job */
|
|
17
|
+
timeout?: number;
|
|
18
|
+
/** API key for Firecrawl */
|
|
19
|
+
apiKey?: string;
|
|
20
|
+
/** API URL for Firecrawl */
|
|
21
|
+
apiUrl?: string;
|
|
22
|
+
/** Output file path */
|
|
23
|
+
output?: string;
|
|
24
|
+
/** Pretty print JSON output */
|
|
25
|
+
pretty?: boolean;
|
|
26
|
+
/** Force JSON output */
|
|
27
|
+
json?: boolean;
|
|
28
|
+
/** Show request timing */
|
|
29
|
+
timing?: boolean;
|
|
30
|
+
/** Location for geo-targeted parsing (typically unused for local files) */
|
|
31
|
+
location?: ScrapeLocation;
|
|
32
|
+
/** Ask a question about the parsed content (query format) */
|
|
33
|
+
query?: string;
|
|
34
|
+
}
|
|
35
|
+
export interface ParseResult {
|
|
36
|
+
success: boolean;
|
|
37
|
+
data?: any;
|
|
38
|
+
error?: string;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=parse.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse.d.ts","sourceRoot":"","sources":["../../src/types/parse.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE7D,MAAM,WAAW,YAAY;IAC3B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,gCAAgC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wBAAwB;IACxB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,0BAA0B;IAC1B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse.js","sourceRoot":"","sources":["../../src/types/parse.ts"],"names":[],"mappings":";AAAA;;GAEG"}
|
package/dist/types/scrape.d.ts
CHANGED
|
@@ -43,11 +43,19 @@ export interface ScrapeOptions {
|
|
|
43
43
|
location?: ScrapeLocation;
|
|
44
44
|
/** Question to ask about the page content (query format) */
|
|
45
45
|
query?: string;
|
|
46
|
+
/** JSON schema for json format */
|
|
47
|
+
schema?: Record<string, unknown>;
|
|
48
|
+
/** Actions to run during scrape */
|
|
49
|
+
actions?: Record<string, unknown>[];
|
|
50
|
+
/** Proxy mode */
|
|
51
|
+
proxy?: string;
|
|
46
52
|
/** Persistent browser profile for maintaining state across scrapes */
|
|
47
53
|
profile?: {
|
|
48
54
|
name: string;
|
|
49
55
|
saveChanges?: boolean;
|
|
50
56
|
};
|
|
57
|
+
/** Enable lockdown mode for the scrape */
|
|
58
|
+
lockdown?: boolean;
|
|
51
59
|
}
|
|
52
60
|
export interface ScrapeResult {
|
|
53
61
|
success: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,MAAM,GACN,SAAS,GACT,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,SAAS,GACT,gBAAgB,GAChB,MAAM,GACN,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,cAAc;IAC7B,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,2DAA2D;IAC3D,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,gCAAgC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gCAAgC;IAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wBAAwB;IACxB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,MAAM,GACN,SAAS,GACT,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,SAAS,GACT,gBAAgB,GAChB,MAAM,GACN,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,cAAc;IAC7B,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,2DAA2D;IAC3D,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,gCAAgC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gCAAgC;IAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wBAAwB;IACxB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,uDAAuD;IACvD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,wEAAwE;IACxE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,mCAAmC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;IACpC,iBAAiB;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IACF,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EAEb,MAAM,iBAAiB,CAAC;AA0BzB;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY,EAAE,CA2BjE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EAEb,MAAM,iBAAiB,CAAC;AA0BzB;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY,EAAE,CA2BjE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,CAsD9D"}
|
package/dist/utils/options.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAqCH,oCA2BC;AAKD,
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAqCH,oCA2BC;AAKD,gDAsDC;AAnHD;;GAEG;AACH,MAAM,aAAa,GAAmB;IACpC,UAAU;IACV,MAAM;IACN,SAAS;IACT,OAAO;IACP,QAAQ;IACR,YAAY;IACZ,SAAS;IACT,gBAAgB;IAChB,MAAM;IACN,YAAY;IACZ,UAAU;CACX,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,GAAiC,MAAM,CAAC,WAAW,CACjE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CACf,CAAC;AAElC;;;;GAIG;AACH,SAAgB,YAAY,CAAC,YAAoB;IAC/C,MAAM,YAAY,GAAG,YAAY;SAC9B,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,qCAAqC;IACrC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,YAAY,GAAmB,EAAE,CAAC;IAExC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,MAAM,EAAE,CAAC;YACX,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,sBAAsB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;IAED,2CAA2C;IAC3C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAAC,OAAY;IAC7C,4CAA4C;IAC5C,IAAI,OAAmC,CAAC;IACxC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAED,6DAA6D;IAC7D,IAAI,QAAoC,CAAC;IACzC,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACzC,QAAQ,GAAG,EAAE,CAAC;QACd,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QACrC,CAAC;QACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS;iBACnC,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,IAAI,OAA4D,CAAC;IACjE,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG;YACR,IAAI,EAAE,OAAO,CAAC,OAAO;YACrB,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;QAC9C,WAAW,EAAE,OAAO,CAAC,WAAW;YAC9B,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,SAAS;QACb,WAAW,EAAE,OAAO,CAAC,WAAW;YAC9B,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,SAAS;QACb,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ;QACR,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO;QACP,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED