firecrawl-cli 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/commands/scrape.test.js +21 -6
- package/dist/__tests__/commands/scrape.test.js.map +1 -1
- package/dist/commands/scrape.d.ts.map +1 -1
- package/dist/commands/scrape.js +15 -8
- package/dist/commands/scrape.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types/scrape.d.ts +2 -2
- package/dist/types/scrape.d.ts.map +1 -1
- package/dist/utils/options.d.ts +7 -1
- package/dist/utils/options.d.ts.map +1 -1
- package/dist/utils/options.js +55 -1
- package/dist/utils/options.js.map +1 -1
- package/dist/utils/output.d.ts +7 -5
- package/dist/utils/output.d.ts.map +1 -1
- package/dist/utils/output.js +83 -48
- package/dist/utils/output.js.map +1 -1
- package/package.json +1 -1
|
@@ -53,7 +53,7 @@ vitest_1.vi.mock('../../utils/client', async () => {
|
|
|
53
53
|
mockClient.scrape.mockResolvedValue(mockResponse);
|
|
54
54
|
await (0, scrape_1.executeScrape)({
|
|
55
55
|
url: 'https://example.com',
|
|
56
|
-
|
|
56
|
+
formats: ['html'],
|
|
57
57
|
});
|
|
58
58
|
(0, vitest_1.expect)(mockClient.scrape).toHaveBeenCalledWith('https://example.com', {
|
|
59
59
|
formats: ['html'],
|
|
@@ -81,7 +81,7 @@ vitest_1.vi.mock('../../utils/client', async () => {
|
|
|
81
81
|
mockClient.scrape.mockResolvedValue(mockResponse);
|
|
82
82
|
await (0, scrape_1.executeScrape)({
|
|
83
83
|
url: 'https://example.com',
|
|
84
|
-
|
|
84
|
+
formats: ['markdown'],
|
|
85
85
|
screenshot: true,
|
|
86
86
|
});
|
|
87
87
|
(0, vitest_1.expect)(mockClient.scrape).toHaveBeenCalledWith('https://example.com', {
|
|
@@ -141,7 +141,7 @@ vitest_1.vi.mock('../../utils/client', async () => {
|
|
|
141
141
|
mockClient.scrape.mockResolvedValue(mockResponse);
|
|
142
142
|
await (0, scrape_1.executeScrape)({
|
|
143
143
|
url: 'https://example.com',
|
|
144
|
-
|
|
144
|
+
formats: ['markdown'],
|
|
145
145
|
screenshot: true,
|
|
146
146
|
onlyMainContent: true,
|
|
147
147
|
waitFor: 3000,
|
|
@@ -211,21 +211,36 @@ vitest_1.vi.mock('../../utils/client', async () => {
|
|
|
211
211
|
});
|
|
212
212
|
(0, vitest_1.describe)('Type safety', () => {
|
|
213
213
|
(0, vitest_1.it)('should accept valid ScrapeFormat types', async () => {
|
|
214
|
-
const
|
|
214
|
+
const formatList = [
|
|
215
215
|
'markdown',
|
|
216
216
|
'html',
|
|
217
217
|
'rawHtml',
|
|
218
218
|
'links',
|
|
219
219
|
];
|
|
220
|
-
for (const format of
|
|
220
|
+
for (const format of formatList) {
|
|
221
221
|
mockClient.scrape.mockResolvedValue({ [format]: 'test' });
|
|
222
222
|
const result = await (0, scrape_1.executeScrape)({
|
|
223
223
|
url: 'https://example.com',
|
|
224
|
-
format,
|
|
224
|
+
formats: [format],
|
|
225
225
|
});
|
|
226
226
|
(0, vitest_1.expect)(result.success).toBe(true);
|
|
227
227
|
}
|
|
228
228
|
});
|
|
229
|
+
(0, vitest_1.it)('should accept multiple formats', async () => {
|
|
230
|
+
mockClient.scrape.mockResolvedValue({
|
|
231
|
+
markdown: '# Test',
|
|
232
|
+
links: ['http://a.com'],
|
|
233
|
+
images: ['http://img.com/a.png'],
|
|
234
|
+
});
|
|
235
|
+
const result = await (0, scrape_1.executeScrape)({
|
|
236
|
+
url: 'https://example.com',
|
|
237
|
+
formats: ['markdown', 'links', 'images'],
|
|
238
|
+
});
|
|
239
|
+
(0, vitest_1.expect)(result.success).toBe(true);
|
|
240
|
+
(0, vitest_1.expect)(mockClient.scrape).toHaveBeenCalledWith('https://example.com', {
|
|
241
|
+
formats: ['markdown', 'links', 'images'],
|
|
242
|
+
});
|
|
243
|
+
});
|
|
229
244
|
});
|
|
230
245
|
});
|
|
231
246
|
//# sourceMappingURL=scrape.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.test.js","sourceRoot":"","sources":["../../../src/__tests__/commands/scrape.test.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAEH,mCAAyE;AACzE,kDAAsD;AACtD,+CAA+C;AAC/C,+CAAsD;AACtD,sDAA+D;AAE/D,mCAAmC;AACnC,WAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;IACvC,MAAM,MAAM,GAAG,MAAM,WAAE,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG,MAAM;QACT,SAAS,EAAE,WAAE,CAAC,EAAE,EAAE;KACnB,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,UAAe,CAAC;IAEpB,IAAA,mBAAU,EAAC,GAAG,EAAE;QACd,IAAA,uBAAS,GAAE,CAAC;QACZ,sCAAsC;QACtC,IAAA,yBAAgB,EAAC;YACf,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,2BAA2B;SACpC,CAAC,CAAC;QAEH,qBAAqB;QACrB,UAAU,GAAG;YACX,MAAM,EAAE,WAAE,CAAC,EAAE,EAAE;SAChB,CAAC;QAEF,oCAAoC;QACpC,WAAE,CAAC,MAAM,CAAC,kBAAS,CAAC,CAAC,eAAe,CAAC,UAAiB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,GAAG,EAAE;QACb,IAAA,0BAAY,GAAE,CAAC;QACf,WAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,IAAA,iBAAQ,EAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,IAAA,WAAE,EAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;YACpD,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;aAC3B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACnD,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,CAAC;aACtB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,YAAY,GAAG,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;YAClD,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,
|
|
1
|
+
{"version":3,"file":"scrape.test.js","sourceRoot":"","sources":["../../../src/__tests__/commands/scrape.test.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAEH,mCAAyE;AACzE,kDAAsD;AACtD,+CAA+C;AAC/C,+CAAsD;AACtD,sDAA+D;AAE/D,mCAAmC;AACnC,WAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;IACvC,MAAM,MAAM,GAAG,MAAM,WAAE,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG,MAAM;QACT,SAAS,EAAE,WAAE,CAAC,EAAE,EAAE;KACnB,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAA,iBAAQ,EAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,IAAI,UAAe,CAAC;IAEpB,IAAA,mBAAU,EAAC,GAAG,EAAE;QACd,IAAA,uBAAS,GAAE,CAAC;QACZ,sCAAsC;QACtC,IAAA,yBAAgB,EAAC;YACf,MAAM,EAAE,cAAc;YACtB,MAAM,EAAE,2BAA2B;SACpC,CAAC,CAAC;QAEH,qBAAqB;QACrB,UAAU,GAAG;YACX,MAAM,EAAE,WAAE,CAAC,EAAE,EAAE;SAChB,CAAC;QAEF,oCAAoC;QACpC,WAAE,CAAC,MAAM,CAAC,kBAAS,CAAC,CAAC,eAAe,CAAC,UAAiB,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,IAAA,kBAAS,EAAC,GAAG,EAAE;QACb,IAAA,0BAAY,GAAE,CAAC;QACf,WAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,IAAA,iBAAQ,EAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,IAAA,WAAE,EAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;YACpD,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;aAC3B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACnD,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,CAAC;aACtB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,YAAY,GAAG,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;YAClD,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,OAAO,EAAE,CAAC,MAAM,CAAC;aAClB,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,MAAM,CAAC;aAClB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,YAAY,GAAG;gBACnB,QAAQ,EAAE,QAAQ;gBAClB,UAAU,EAAE,gBAAgB;aAC7B,CAAC;YACF,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,YAAY,CAAC;aACxB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,YAAY,GAAG;gBACnB,QAAQ,EAAE,QAAQ;gBAClB,UAAU,EAAE,gBAAgB;aAC7B,CAAC;YACF,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,UAAU,EAAE,IAAI;aACjB,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC;aACpC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;YAC5C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,eAAe,EAAE,IAAI;aACtB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;YAC5C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;YAC5C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,WAAW,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC;aACjC,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,WAAW,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC;aACjC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;YAC5C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,WAAW,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;aAC/B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,WAAW,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;aAC/B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;YACrE,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,IAAA,sBAAa,EAAC;gBAClB,GAAG,EAAE,qBAAqB;gBAC1B,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,IAAI;gBACrB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,CAAC,SAAS,CAAC;gBACxB,WAAW,EAAE,CAAC,KAAK,CAAC;aACrB,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC;gBACnC,eAAe,EAAE,IAAI;gBACrB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,CAAC,SAAS,CAAC;gBACxB,WAAW,EAAE,CAAC,KAAK,CAAC;aACrB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAA,iBAAQ,EAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,IAAA,WAAE,EAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,MAAM,YAAY,GAAG;gBACnB,QAAQ,EAAE,gBAAgB;gBAC1B,GAAG,EAAE,qBAAqB;aAC3B,CAAC;YACF,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC;gBACjC,GAAG,EAAE,qBAAqB;aAC3B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,OAAO,EAAE,IAAI;gBACb,IAAI,EAAE,YAAY;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,GAAG;gBACnB,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE;oBACR,KAAK,EAAE,WAAW;oBAClB,WAAW,EAAE,kBAAkB;iBAChC;aACF,CAAC;YACF,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAElD,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC;gBACjC,GAAG,EAAE,qBAAqB;aAC3B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,YAAY,GAAG,wBAAwB,CAAC;YAC9C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;YAE7D,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC;gBACjC,GAAG,EAAE,qBAAqB;aAC3B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC;gBACrB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC;YAEpD,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC;gBACjC,GAAG,EAAE,qBAAqB;aAC3B,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAA,iBAAQ,EAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,IAAA,WAAE,EAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,UAAU,GAAqD;gBACnE,UAAU;gBACV,MAAM;gBACN,SAAS;gBACT,OAAO;aACR,CAAC;YAEF,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC1D,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC;oBACjC,GAAG,EAAE,qBAAqB;oBAC1B,OAAO,EAAE,CAAC,MAAM,CAAC;iBAClB,CAAC,CAAC;gBACH,IAAA,eAAM,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAA,WAAE,EAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC;gBAClC,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,CAAC,cAAc,CAAC;gBACvB,MAAM,EAAE,CAAC,sBAAsB,CAAC;aACjC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAa,EAAC;gBACjC,GAAG,EAAE,qBAAqB;gBAC1B,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;aACzC,CAAC,CAAC;YAEH,IAAA,eAAM,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,IAAA,eAAM,EAAC,UAAU,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,qBAAqB,EAAE;gBACpE,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;aACzC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EAEb,MAAM,iBAAiB,CAAC;AAoCzB;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAqEvB;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,IAAI,CAAC,CAef"}
|
package/dist/commands/scrape.js
CHANGED
|
@@ -33,14 +33,13 @@ async function executeScrape(options) {
|
|
|
33
33
|
const app = (0, client_1.getClient)({ apiKey: options.apiKey });
|
|
34
34
|
// Build scrape options
|
|
35
35
|
const formats = [];
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
// Add requested formats
|
|
37
|
+
if (options.formats && options.formats.length > 0) {
|
|
38
|
+
formats.push(...options.formats);
|
|
38
39
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
formats.push('screenshot');
|
|
43
|
-
}
|
|
40
|
+
// Add screenshot format if requested and not already included
|
|
41
|
+
if (options.screenshot && !formats.includes('screenshot')) {
|
|
42
|
+
formats.push('screenshot');
|
|
44
43
|
}
|
|
45
44
|
// If no formats specified, default to markdown
|
|
46
45
|
if (formats.length === 0) {
|
|
@@ -86,6 +85,14 @@ async function executeScrape(options) {
|
|
|
86
85
|
*/
|
|
87
86
|
async function handleScrapeCommand(options) {
|
|
88
87
|
const result = await executeScrape(options);
|
|
89
|
-
|
|
88
|
+
// Determine effective formats for output handling
|
|
89
|
+
const effectiveFormats = options.formats && options.formats.length > 0
|
|
90
|
+
? [...options.formats]
|
|
91
|
+
: ['markdown'];
|
|
92
|
+
// Add screenshot to effective formats if it was requested separately
|
|
93
|
+
if (options.screenshot && !effectiveFormats.includes('screenshot')) {
|
|
94
|
+
effectiveFormats.push('screenshot');
|
|
95
|
+
}
|
|
96
|
+
(0, output_1.handleScrapeOutput)(result, effectiveFormats, options.output, options.pretty);
|
|
90
97
|
}
|
|
91
98
|
//# sourceMappingURL=scrape.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG;;
|
|
1
|
+
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/commands/scrape.ts"],"names":[],"mappings":";AAAA;;GAEG;;AA8CH,sCAuEC;AAKD,kDAiBC;AAnID,4CAA4C;AAC5C,4CAAqD;AAErD;;GAEG;AACH,SAAS,YAAY,CACnB,OAAsB,EACtB,gBAAwB,EACxB,cAAsB,EACtB,KAAuB;IAEvB,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO;IAE5B,MAAM,eAAe,GAAG,cAAc,GAAG,gBAAgB,CAAC;IAC1D,MAAM,UAAU,GAMZ;QACF,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE;QACrD,QAAQ,EAAE,GAAG,eAAe,IAAI;QAChC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;KACpC,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,UAAU,CAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IAC9E,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,aAAa,CACjC,OAAsB;IAEtB,iEAAiE;IACjE,MAAM,GAAG,GAAG,IAAA,kBAAS,EAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAElD,uBAAuB;IACvB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,wBAAwB;IACxB,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;IAED,8DAA8D;IAC9D,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7B,CAAC;IAED,+CAA+C;IAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,YAAY,GAMd;QACF,OAAO;KACR,CAAC;IAEF,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC1C,YAAY,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;IACzD,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,YAAY,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IACjD,CAAC;IAED,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC;QAExD,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,MAAM;SACb,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;QAE/D,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,mBAAmB,CACvC,OAAsB;IAEtB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC;IAE5C,kDAAkD;IAClD,MAAM,gBAAgB,GACpB,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QAC3C,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;QACtB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnB,qEAAqE;IACrE,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QACnE,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACtC,CAAC;IAED,IAAA,2BAAkB,EAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC/E,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -46,7 +46,7 @@ function createScrapeCommand() {
|
|
|
46
46
|
.argument('[url]', 'URL to scrape')
|
|
47
47
|
.option('-u, --url <url>', 'URL to scrape (alternative to positional argument)')
|
|
48
48
|
.option('-H, --html', 'Output raw HTML (shortcut for --format html)')
|
|
49
|
-
.option('-f, --format <
|
|
49
|
+
.option('-f, --format <formats>', 'Output format(s). Multiple formats can be specified with commas (e.g., "markdown,links,images"). Available: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding. Single format outputs raw content; multiple formats output JSON.', 'markdown')
|
|
50
50
|
.option('--only-main-content', 'Include only main content', false)
|
|
51
51
|
.option('--wait-for <ms>', 'Wait time before scraping in milliseconds', parseInt)
|
|
52
52
|
.option('--screenshot', 'Take a screenshot', false)
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA;;;GAGG;;;;;AAEH,yCAAoC;AACpC,8CAAwD;AACxD,2CAAkD;AAClD,2CAA2C;AAC3C,8CAA8C;AAC9C,0DAAmE;AACnE,4CAAsD;AACtD,wCAAkD;AAClD,gDAA0D;AAC1D,qCAAkD;AAClD,6CAAqD;AACrD,qCAAsC;AACtC,mEAA0C;AAE1C,6DAA6D;AAC7D,IAAA,yBAAgB,GAAE,CAAC;AAEnB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,qCAAqC,CAAC;KAClD,OAAO,CAAC,sBAAW,CAAC,OAAO,CAAC;KAC5B,MAAM,CACL,qBAAqB,EACrB,iFAAiF,CAClF;KACA,kBAAkB,EAAE,CAAC,oDAAoD;KACzE,IAAI,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE;IAChD,gEAAgE;IAChE,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,IAAA,kBAAS,EAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,SAAS,mBAAmB;IAC1B,MAAM,SAAS,GAAG,IAAI,mBAAO,CAAC,QAAQ,CAAC;SACpC,WAAW,CAAC,8BAA8B,CAAC;SAC3C,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;SAClC,MAAM,CACL,iBAAiB,EACjB,oDAAoD,CACrD;SACA,MAAM,CAAC,YAAY,EAAE,8CAA8C,CAAC;SACpE,MAAM,CACL,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAEA;;;GAGG;;;;;AAEH,yCAAoC;AACpC,8CAAwD;AACxD,2CAAkD;AAClD,2CAA2C;AAC3C,8CAA8C;AAC9C,0DAAmE;AACnE,4CAAsD;AACtD,wCAAkD;AAClD,gDAA0D;AAC1D,qCAAkD;AAClD,6CAAqD;AACrD,qCAAsC;AACtC,mEAA0C;AAE1C,6DAA6D;AAC7D,IAAA,yBAAgB,GAAE,CAAC;AAEnB,MAAM,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,qCAAqC,CAAC;KAClD,OAAO,CAAC,sBAAW,CAAC,OAAO,CAAC;KAC5B,MAAM,CACL,qBAAqB,EACrB,iFAAiF,CAClF;KACA,kBAAkB,EAAE,CAAC,oDAAoD;KACzE,IAAI,CAAC,WAAW,EAAE,CAAC,WAAW,EAAE,aAAa,EAAE,EAAE;IAChD,gEAAgE;IAChE,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,IAAA,kBAAS,EAAC,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;AACH,CAAC,CAAC,CAAC;AAEL;;GAEG;AACH,SAAS,mBAAmB;IAC1B,MAAM,SAAS,GAAG,IAAI,mBAAO,CAAC,QAAQ,CAAC;SACpC,WAAW,CAAC,8BAA8B,CAAC;SAC3C,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;SAClC,MAAM,CACL,iBAAiB,EACjB,oDAAoD,CACrD;SACA,MAAM,CAAC,YAAY,EAAE,8CAA8C,CAAC;SACpE,MAAM,CACL,wBAAwB,EACxB,uRAAuR,EACvR,UAAU,CACX;SACA,MAAM,CAAC,qBAAqB,EAAE,2BAA2B,EAAE,KAAK,CAAC;SACjE,MAAM,CACL,iBAAiB,EACjB,2CAA2C,EAC3C,QAAQ,CACT;SACA,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,KAAK,CAAC;SAClD,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CAAC,uBAAuB,EAAE,yCAAyC,CAAC;SAC1E,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;SACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CACL,UAAU,EACV,kDAAkD,EAClD,KAAK,CACN;SACA,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE;QACvC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;QACzC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,8BAA8B;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtD,MAAM,aAAa,GAAG,IAAA,4BAAkB,EAAC,EAAE,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QACtE,MAAM,IAAA,4BAAmB,EAAC,aAAa,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEL,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qCAAqC;AACrC,OAAO,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAE1C;;GAEG;AACH,SAAS,kBAAkB;IACzB,MAAM,QAAQ,GAAG,IAAI,mBAAO,CAAC,OAAO,CAAC;SAClC,WAAW,CAAC,iCAAiC,CAAC;SAC9C,QAAQ,CAAC,iBAAiB,EAAE,wCAAwC,CAAC;SACrE,MAAM,CACL,iBAAiB,EACjB,mDAAmD,CACpD;SACA,MAAM,CAAC,UAAU,EAAE,oCAAoC,EAAE,KAAK,CAAC;SAC/D,MAAM,CACL,QAAQ,EACR,qDAAqD,EACrD,KAAK,CACN;SACA,MAAM,CACL,2BAA2B,EAC3B,uDAAuD,EACvD,UAAU,CACX;SACA,MAAM,CACL,qBAAqB,EACrB,uDAAuD,EACvD,UAAU,CACX;SACA,MAAM,CAAC,YAAY,EAAE,kCAAkC,EAAE,KAAK,CAAC;SAC/D,MAAM,CAAC,kBAAkB,EAAE,kCAAkC,EAAE,QAAQ,CAAC;SACxE,MAAM,CAAC,sBAAsB,EAAE,qBAAqB,EAAE,QAAQ,CAAC;SAC/D,MAAM,CACL,yBAAyB,EACzB,0CAA0C,CAC3C;SACA,MAAM,CACL,yBAAyB,EACzB,0CAA0C,CAC3C;SACA,MAAM,CAAC,kBAAkB,EAAE,iCAAiC,EAAE,SAAS,CAAC;SACxE,MAAM,CACL,2BAA2B,EAC3B,uCAAuC,EACvC,KAAK,CACN;SACA,MAAM,CAAC,uBAAuB,EAAE,qBAAqB,EAAE,KAAK,CAAC;SAC7D,MAAM,CAAC,wBAAwB,EAAE,sBAAsB,EAAE,KAAK,CAAC;SAC/D,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,KAAK,CAAC;SACvD,MAAM,CAAC,cAAc,EAAE,wCAAwC,EAAE,QAAQ,CAAC;SAC1E,MAAM,CACL,4BAA4B,EAC5B,6BAA6B,EAC7B,QAAQ,CACT;SACA,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;SACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,EAAE;QAC9C,kEAAkE;QAClE,MAAM,UAAU,GAAG,oBAAoB,IAAI,OAAO,CAAC,GAAG,CAAC;QACvD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CACX,+EAA+E,CAChF,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,6CAA6C;QAC7C,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,IAAI,IAAA,aAAO,EAAC,UAAU,CAAC,CAAC;QAE5D,MAAM,YAAY,GAAG;YACnB,UAAU;YACV,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;gBAChC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9D,CAAC,CAAC,SAAS;YACb,YAAY,EAAE,OAAO,CAAC,YAAY;gBAChC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9D,CAAC,CAAC,SAAS;YACb,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,qBAAqB,EAAE,OAAO,CAAC,qBAAqB;YACpD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;YAC9C,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC;QAEF,MAAM,IAAA,0BAAkB,EAAC,YAAY,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEL,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB;IACvB,MAAM,MAAM,GAAG,IAAI,mBAAO,CAAC,KAAK,CAAC;SAC9B,WAAW,CAAC,uCAAuC,CAAC;SACpD,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;SAC/B,MAAM,CACL,iBAAiB,EACjB,iDAAiD,CAClD;SACA,MAAM,CAAC,QAAQ,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACnD,MAAM,CAAC,kBAAkB,EAAE,0BAA0B,EAAE,QAAQ,CAAC;SAChE,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;SACzD,MAAM,CACL,kBAAkB,EAClB,uCAAuC,EACvC,SAAS,CACV;SACA,MAAM,CAAC,sBAAsB,EAAE,oBAAoB,EAAE,KAAK,CAAC;SAC3D,MAAM,CAAC,2BAA2B,EAAE,yBAAyB,EAAE,KAAK,CAAC;SACrE,MAAM,CAAC,qBAAqB,EAAE,oBAAoB,EAAE,UAAU,CAAC;SAC/D,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;SACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;SACnE,MAAM,CAAC,QAAQ,EAAE,uBAAuB,EAAE,KAAK,CAAC;SAChD,MAAM,CAAC,UAAU,EAAE,0BAA0B,EAAE,KAAK,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE;QACvC,6DAA6D;QAC7D,MAAM,GAAG,GAAG,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC;QACzC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,UAAU,GAAG;YACjB,UAAU,EAAE,GAAG;YACf,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,qBAAqB,EAAE,OAAO,CAAC,qBAAqB;YACpD,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC;QAEF,MAAM,IAAA,sBAAgB,EAAC,UAAU,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEL,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,6CAA6C;AAC7C,OAAO,CAAC,UAAU,CAAC,kBAAkB,EAAE,CAAC,CAAC;AACzC,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC;AAEvC,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,6CAA6C,CAAC;KAC1D,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,MAAM,IAAA,kBAAS,GAAE,CAAC;AACpB,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,cAAc,CAAC;KACvB,WAAW,CAAC,mCAAmC,CAAC;KAChD,MAAM,CACL,qBAAqB,EACrB,gDAAgD,CACjD;KACA,MAAM,CAAC,qBAAqB,EAAE,oCAAoC,CAAC;KACnE,MAAM,CAAC,QAAQ,EAAE,uBAAuB,EAAE,KAAK,CAAC;KAChD,MAAM,CACL,UAAU,EACV,qDAAqD,EACrD,KAAK,CACN;KACA,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,MAAM,IAAA,uCAAwB,EAAC,OAAO,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,6BAA6B,CAAC;KAC1C,MAAM,CAAC,GAAG,EAAE;IACX,IAAA,8BAAoB,GAAE,CAAC;AACzB,CAAC,CAAC,CAAC;AAEL,kBAAkB;AAClB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,uDAAuD;AACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAA,WAAK,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,+DAA+D;IAC/D,MAAM,GAAG,GAAG,IAAA,kBAAY,EAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,YAAY,GAAG;QACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACf,QAAQ;QACR,GAAG;QACH,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;KACjB,CAAC;IAEF,yEAAyE;IACzE,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/C,OAAO,CAAC,KAAK,CACX,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CACzD,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;KAAM,CAAC;IACN,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAE,CAAC;AAClB,CAAC"}
|
package/dist/types/scrape.d.ts
CHANGED
|
@@ -5,8 +5,8 @@ export type ScrapeFormat = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images'
|
|
|
5
5
|
export interface ScrapeOptions {
|
|
6
6
|
/** URL to scrape */
|
|
7
7
|
url: string;
|
|
8
|
-
/** Output format
|
|
9
|
-
|
|
8
|
+
/** Output format(s) - single format or array of formats */
|
|
9
|
+
formats?: ScrapeFormat[];
|
|
10
10
|
/** Include only main content */
|
|
11
11
|
onlyMainContent?: boolean;
|
|
12
12
|
/** Wait time before scraping (ms) */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,MAAM,GACN,SAAS,GACT,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,SAAS,GACT,gBAAgB,GAChB,MAAM,GACN,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/types/scrape.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,MAAM,YAAY,GACpB,UAAU,GACV,MAAM,GACN,SAAS,GACT,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,SAAS,GACT,gBAAgB,GAChB,MAAM,GACN,YAAY,GACZ,UAAU,CAAC;AAEf,MAAM,WAAW,aAAa;IAC5B,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,2DAA2D;IAC3D,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;IACzB,gCAAgC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB;IACtB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,uDAAuD;IACvD,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB"}
|
package/dist/utils/options.d.ts
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Option parsing utilities
|
|
3
3
|
*/
|
|
4
|
-
import type { ScrapeOptions } from '../types/scrape';
|
|
4
|
+
import type { ScrapeOptions, ScrapeFormat } from '../types/scrape';
|
|
5
|
+
/**
|
|
6
|
+
* Parse format string into array of ScrapeFormat
|
|
7
|
+
* Handles comma-separated values: "markdown,links,images"
|
|
8
|
+
* Case-insensitive input, returns correct camelCase for API
|
|
9
|
+
*/
|
|
10
|
+
export declare function parseFormats(formatString: string): ScrapeFormat[];
|
|
5
11
|
/**
|
|
6
12
|
* Convert commander options to ScrapeOptions
|
|
7
13
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AA0BnE;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,YAAY,EAAE,CA2BjE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,GAAG,GAAG,aAAa,CAwB9D"}
|
package/dist/utils/options.js
CHANGED
|
@@ -3,14 +3,68 @@
|
|
|
3
3
|
* Option parsing utilities
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.parseFormats = parseFormats;
|
|
6
7
|
exports.parseScrapeOptions = parseScrapeOptions;
|
|
8
|
+
/**
|
|
9
|
+
* Valid scrape format values
|
|
10
|
+
*/
|
|
11
|
+
const VALID_FORMATS = [
|
|
12
|
+
'markdown',
|
|
13
|
+
'html',
|
|
14
|
+
'rawHtml',
|
|
15
|
+
'links',
|
|
16
|
+
'images',
|
|
17
|
+
'screenshot',
|
|
18
|
+
'summary',
|
|
19
|
+
'changeTracking',
|
|
20
|
+
'json',
|
|
21
|
+
'attributes',
|
|
22
|
+
'branding',
|
|
23
|
+
];
|
|
24
|
+
/**
|
|
25
|
+
* Map from lowercase to correct camelCase format
|
|
26
|
+
*/
|
|
27
|
+
const FORMAT_MAP = Object.fromEntries(VALID_FORMATS.map((f) => [f.toLowerCase(), f]));
|
|
28
|
+
/**
|
|
29
|
+
* Parse format string into array of ScrapeFormat
|
|
30
|
+
* Handles comma-separated values: "markdown,links,images"
|
|
31
|
+
* Case-insensitive input, returns correct camelCase for API
|
|
32
|
+
*/
|
|
33
|
+
function parseFormats(formatString) {
|
|
34
|
+
const inputFormats = formatString
|
|
35
|
+
.split(',')
|
|
36
|
+
.map((f) => f.trim().toLowerCase())
|
|
37
|
+
.filter((f) => f.length > 0);
|
|
38
|
+
// Validate and map to correct casing
|
|
39
|
+
const invalidFormats = [];
|
|
40
|
+
const validFormats = [];
|
|
41
|
+
for (const input of inputFormats) {
|
|
42
|
+
const mapped = FORMAT_MAP[input];
|
|
43
|
+
if (mapped) {
|
|
44
|
+
validFormats.push(mapped);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
invalidFormats.push(input);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (invalidFormats.length > 0) {
|
|
51
|
+
throw new Error(`Invalid format(s): ${invalidFormats.join(', ')}. Valid formats are: ${VALID_FORMATS.join(', ')}`);
|
|
52
|
+
}
|
|
53
|
+
// Remove duplicates while preserving order
|
|
54
|
+
return [...new Set(validFormats)];
|
|
55
|
+
}
|
|
7
56
|
/**
|
|
8
57
|
* Convert commander options to ScrapeOptions
|
|
9
58
|
*/
|
|
10
59
|
function parseScrapeOptions(options) {
|
|
60
|
+
// Parse formats from comma-separated string
|
|
61
|
+
let formats;
|
|
62
|
+
if (options.format) {
|
|
63
|
+
formats = parseFormats(options.format);
|
|
64
|
+
}
|
|
11
65
|
return {
|
|
12
66
|
url: options.url,
|
|
13
|
-
|
|
67
|
+
formats,
|
|
14
68
|
onlyMainContent: options.onlyMainContent,
|
|
15
69
|
waitFor: options.waitFor,
|
|
16
70
|
screenshot: options.screenshot,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../../src/utils/options.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAiCH,oCA2BC;AAKD,gDAwBC;AArFD;;GAEG;AACH,MAAM,aAAa,GAAmB;IACpC,UAAU;IACV,MAAM;IACN,SAAS;IACT,OAAO;IACP,QAAQ;IACR,YAAY;IACZ,SAAS;IACT,gBAAgB;IAChB,MAAM;IACN,YAAY;IACZ,UAAU;CACX,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,GAAiC,MAAM,CAAC,WAAW,CACjE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CACf,CAAC;AAElC;;;;GAIG;AACH,SAAgB,YAAY,CAAC,YAAoB;IAC/C,MAAM,YAAY,GAAG,YAAY;SAC9B,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,qCAAqC;IACrC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,YAAY,GAAmB,EAAE,CAAC;IAExC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,MAAM,EAAE,CAAC;YACX,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,sBAAsB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;IAED,2CAA2C;IAC3C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAAC,OAAY;IAC7C,4CAA4C;IAC5C,IAAI,OAAmC,CAAC;IACxC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAED,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;QACP,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAC9B,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,SAAS;QACb,WAAW,EAAE,OAAO,CAAC,WAAW;YAC9B,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,CAAC,CAAC,SAAS;QACb,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC"}
|
package/dist/utils/output.d.ts
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Output utilities for CLI
|
|
3
3
|
*/
|
|
4
|
-
import type { ScrapeResult } from '../types/scrape';
|
|
5
|
-
import type { ScrapeFormat } from '../types/scrape';
|
|
4
|
+
import type { ScrapeResult, ScrapeFormat } from '../types/scrape';
|
|
6
5
|
/**
|
|
7
6
|
* Write output to file or stdout
|
|
8
7
|
*/
|
|
9
8
|
export declare function writeOutput(content: string, outputPath?: string, silent?: boolean): void;
|
|
10
9
|
/**
|
|
11
10
|
* Handle scrape result output
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
*
|
|
12
|
+
* Output behavior:
|
|
13
|
+
* - Single text format (html, markdown, links, images, summary, rawHtml): raw content
|
|
14
|
+
* - Single complex format (screenshot, json, branding, etc.): JSON output
|
|
15
|
+
* - Multiple formats: JSON with all requested data
|
|
14
16
|
*/
|
|
15
|
-
export declare function handleScrapeOutput(result: ScrapeResult,
|
|
17
|
+
export declare function handleScrapeOutput(result: ScrapeResult, formats: ScrapeFormat[], outputPath?: string, pretty?: boolean): void;
|
|
16
18
|
//# sourceMappingURL=output.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAuFlE;;GAEG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAmBN;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,YAAY,EAAE,EACvB,UAAU,CAAC,EAAE,MAAM,EACnB,MAAM,GAAE,OAAe,GACtB,IAAI,CAkDN"}
|
package/dist/utils/output.js
CHANGED
|
@@ -40,44 +40,77 @@ exports.writeOutput = writeOutput;
|
|
|
40
40
|
exports.handleScrapeOutput = handleScrapeOutput;
|
|
41
41
|
const fs = __importStar(require("fs"));
|
|
42
42
|
const path = __importStar(require("path"));
|
|
43
|
+
/**
|
|
44
|
+
* Text formats that can be output as raw content (curl-like)
|
|
45
|
+
*/
|
|
46
|
+
const RAW_TEXT_FORMATS = [
|
|
47
|
+
'html',
|
|
48
|
+
'rawHtml',
|
|
49
|
+
'markdown',
|
|
50
|
+
'links',
|
|
51
|
+
'images',
|
|
52
|
+
'summary',
|
|
53
|
+
];
|
|
43
54
|
/**
|
|
44
55
|
* Extract content from Firecrawl Document based on format
|
|
45
56
|
*/
|
|
46
57
|
function extractContent(data, format) {
|
|
47
58
|
if (!data)
|
|
48
59
|
return null;
|
|
49
|
-
//
|
|
50
|
-
if (format) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
60
|
+
// Handle html/rawHtml formats - extract HTML content directly
|
|
61
|
+
if (format === 'html' || format === 'rawHtml') {
|
|
62
|
+
return data.html || data.rawHtml || data[format] || null;
|
|
63
|
+
}
|
|
64
|
+
// Handle markdown format
|
|
65
|
+
if (format === 'markdown') {
|
|
66
|
+
return data.markdown || data[format] || null;
|
|
67
|
+
}
|
|
68
|
+
// Handle links format (array of URLs -> newline-separated string)
|
|
69
|
+
if (format === 'links') {
|
|
70
|
+
const links = data.links || data[format];
|
|
71
|
+
if (Array.isArray(links)) {
|
|
72
|
+
return links.join('\n');
|
|
54
73
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
74
|
+
return links || null;
|
|
75
|
+
}
|
|
76
|
+
// Handle images format (array of URLs -> newline-separated string)
|
|
77
|
+
if (format === 'images') {
|
|
78
|
+
const images = data.images || data[format];
|
|
79
|
+
if (Array.isArray(images)) {
|
|
80
|
+
return images.join('\n');
|
|
58
81
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
82
|
+
return images || null;
|
|
83
|
+
}
|
|
84
|
+
// Handle summary format
|
|
85
|
+
if (format === 'summary') {
|
|
86
|
+
return data.summary || data[format] || null;
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Extract multiple format contents from response data
|
|
92
|
+
*/
|
|
93
|
+
function extractMultipleFormats(data, formats) {
|
|
94
|
+
const result = {};
|
|
95
|
+
for (const format of formats) {
|
|
96
|
+
const key = format;
|
|
97
|
+
if (data[key] !== undefined) {
|
|
98
|
+
result[key] = data[key];
|
|
62
99
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
100
|
+
else if (format === 'html' && data.rawHtml !== undefined) {
|
|
101
|
+
// Fallback for html -> rawHtml
|
|
102
|
+
result[key] = data.rawHtml;
|
|
66
103
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
104
|
+
else if (format === 'rawHtml' && data.html !== undefined) {
|
|
105
|
+
// Fallback for rawHtml -> html
|
|
106
|
+
result[key] = data.html;
|
|
70
107
|
}
|
|
71
108
|
}
|
|
72
|
-
//
|
|
73
|
-
if (
|
|
74
|
-
|
|
75
|
-
}
|
|
76
|
-
// If it's an object, try to find string content
|
|
77
|
-
if (typeof data === 'object') {
|
|
78
|
-
return data.html || data.markdown || data.rawHtml || data.content || null;
|
|
109
|
+
// Always include metadata if present
|
|
110
|
+
if (data.metadata) {
|
|
111
|
+
result.metadata = data.metadata;
|
|
79
112
|
}
|
|
80
|
-
return
|
|
113
|
+
return result;
|
|
81
114
|
}
|
|
82
115
|
/**
|
|
83
116
|
* Write output to file or stdout
|
|
@@ -105,10 +138,13 @@ function writeOutput(content, outputPath, silent = false) {
|
|
|
105
138
|
}
|
|
106
139
|
/**
|
|
107
140
|
* Handle scrape result output
|
|
108
|
-
*
|
|
109
|
-
*
|
|
141
|
+
*
|
|
142
|
+
* Output behavior:
|
|
143
|
+
* - Single text format (html, markdown, links, images, summary, rawHtml): raw content
|
|
144
|
+
* - Single complex format (screenshot, json, branding, etc.): JSON output
|
|
145
|
+
* - Multiple formats: JSON with all requested data
|
|
110
146
|
*/
|
|
111
|
-
function handleScrapeOutput(result,
|
|
147
|
+
function handleScrapeOutput(result, formats, outputPath, pretty = false) {
|
|
112
148
|
if (!result.success) {
|
|
113
149
|
// Always use stderr for errors to allow piping
|
|
114
150
|
console.error('Error:', result.error);
|
|
@@ -117,41 +153,40 @@ function handleScrapeOutput(result, format, outputPath, pretty = false) {
|
|
|
117
153
|
if (!result.data) {
|
|
118
154
|
return;
|
|
119
155
|
}
|
|
120
|
-
//
|
|
121
|
-
const
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
'summary',
|
|
128
|
-
];
|
|
129
|
-
const shouldOutputRaw = format && rawTextFormats.includes(format);
|
|
130
|
-
if (shouldOutputRaw) {
|
|
131
|
-
// Extract and output raw content
|
|
132
|
-
const content = extractContent(result.data, format);
|
|
156
|
+
// Determine output mode based on number of formats
|
|
157
|
+
const isSingleFormat = formats.length === 1;
|
|
158
|
+
const singleFormat = isSingleFormat ? formats[0] : null;
|
|
159
|
+
const isRawTextFormat = singleFormat && RAW_TEXT_FORMATS.includes(singleFormat);
|
|
160
|
+
// Single raw text format: output raw content (current behavior)
|
|
161
|
+
if (isSingleFormat && isRawTextFormat && singleFormat) {
|
|
162
|
+
const content = extractContent(result.data, singleFormat);
|
|
133
163
|
if (content !== null) {
|
|
134
164
|
writeOutput(content, outputPath, !!outputPath);
|
|
135
165
|
return;
|
|
136
166
|
}
|
|
137
167
|
}
|
|
138
|
-
//
|
|
139
|
-
|
|
168
|
+
// Multiple formats or complex format: output JSON
|
|
169
|
+
let outputData;
|
|
170
|
+
if (isSingleFormat) {
|
|
171
|
+
// Single complex format - output entire data object
|
|
172
|
+
outputData = result.data;
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
// Multiple formats - extract only requested formats
|
|
176
|
+
outputData = extractMultipleFormats(result.data, formats);
|
|
177
|
+
}
|
|
140
178
|
let jsonContent;
|
|
141
179
|
try {
|
|
142
180
|
jsonContent = pretty
|
|
143
|
-
? JSON.stringify(
|
|
144
|
-
: JSON.stringify(
|
|
181
|
+
? JSON.stringify(outputData, null, 2)
|
|
182
|
+
: JSON.stringify(outputData);
|
|
145
183
|
}
|
|
146
184
|
catch (error) {
|
|
147
|
-
// If stringification fails, try to create a minimal error response
|
|
148
185
|
jsonContent = JSON.stringify({
|
|
149
186
|
error: 'Failed to serialize response',
|
|
150
187
|
message: error instanceof Error ? error.message : 'Unknown error',
|
|
151
188
|
});
|
|
152
189
|
}
|
|
153
|
-
// Ensure clean JSON output (no extra newlines or text before JSON)
|
|
154
|
-
// Output directly to stdout without any prefix
|
|
155
190
|
writeOutput(jsonContent, outputPath, !!outputPath);
|
|
156
191
|
}
|
|
157
192
|
//# sourceMappingURL=output.js.map
|
package/dist/utils/output.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/utils/output.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8FH,kCAuBC;AAUD,gDAuDC;AApLD,uCAAyB;AACzB,2CAA6B;AAG7B;;GAEG;AACH,MAAM,gBAAgB,GAAmB;IACvC,MAAM;IACN,SAAS;IACT,UAAU;IACV,OAAO;IACP,QAAQ;IACR,SAAS;CACV,CAAC;AAEF;;GAEG;AACH,SAAS,cAAc,CAAC,IAAS,EAAE,MAAoB;IACrD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,8DAA8D;IAC9D,IAAI,MAAM,KAAK,MAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED,yBAAyB;IACzB,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IAC/C,CAAC;IAED,kEAAkE;IAClE,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QACD,OAAO,KAAK,IAAI,IAAI,CAAC;IACvB,CAAC;IAED,mEAAmE;IACnE,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,MAAM,IAAI,IAAI,CAAC;IACxB,CAAC;IAED,wBAAwB;IACxB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IAC9C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAC7B,IAAS,EACT,OAAuB;IAEvB,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,CAAC;QAEnB,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;aAAM,IAAI,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,CAAC;aAAM,IAAI,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAClC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAgB,WAAW,CACzB,OAAe,EACf,UAAmB,EACnB,SAAkB,KAAK;IAEvB,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,mDAAmD;YACnD,OAAO,CAAC,KAAK,CAAC,sBAAsB,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sDAAsD;QACtD,qDAAqD;QACrD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,kBAAkB,CAChC,MAAoB,EACpB,OAAuB,EACvB,UAAmB,EACnB,SAAkB,KAAK;IAEvB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,+CAA+C;QAC/C,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,mDAAmD;IACnD,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,MAAM,eAAe,GACnB,YAAY,IAAI,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAE1D,gEAAgE;IAChE,IAAI,cAAc,IAAI,eAAe,IAAI,YAAY,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAC1D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,UAAe,CAAC;IAEpB,IAAI,cAAc,EAAE,CAAC;QACnB,oDAAoD;QACpD,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,oDAAoD;QACpD,UAAU,GAAG,sBAAsB,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,WAAmB,CAAC;IACxB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM;YAClB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YACrC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC;YAC3B,KAAK,EAAE,8BAA8B;YACrC,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAClE,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;AACrD,CAAC"}
|
package/package.json
CHANGED