@monostate/node-scraper 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/index.js +12 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
# @
|
|
1
|
+
# @monostate/node-scraper
|
|
2
2
|
|
|
3
3
|
> **Lightning-fast web scraping with intelligent fallback system - 11.35x faster than Firecrawl**
|
|
4
4
|
|
|
5
|
-
[](https://badge.fury.io/js/%40monostate%2Fnode-scraper)
|
|
6
6
|
[](../../test-results/)
|
|
7
7
|
[](../../LICENSE)
|
|
8
8
|
[](https://nodejs.org/)
|
|
@@ -12,17 +12,17 @@
|
|
|
12
12
|
### Installation
|
|
13
13
|
|
|
14
14
|
```bash
|
|
15
|
-
npm install @
|
|
15
|
+
npm install @monostate/node-scraper
|
|
16
16
|
# or
|
|
17
|
-
yarn add @
|
|
17
|
+
yarn add @monostate/node-scraper
|
|
18
18
|
# or
|
|
19
|
-
pnpm add @
|
|
19
|
+
pnpm add @monostate/node-scraper
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
### Basic Usage
|
|
23
23
|
|
|
24
24
|
```javascript
|
|
25
|
-
import { smartScrape, BNCASmartScraper } from '@
|
|
25
|
+
import { smartScrape, BNCASmartScraper } from '@monostate/node-scraper';
|
|
26
26
|
|
|
27
27
|
// Simple one-line scraping
|
|
28
28
|
const result = await smartScrape('https://example.com');
|
|
@@ -34,7 +34,7 @@ console.log(result.performance.totalTime); // Time taken in ms
|
|
|
34
34
|
### Advanced Usage
|
|
35
35
|
|
|
36
36
|
```javascript
|
|
37
|
-
import { BNCASmartScraper } from '@
|
|
37
|
+
import { BNCASmartScraper } from '@monostate/node-scraper';
|
|
38
38
|
|
|
39
39
|
const scraper = new BNCASmartScraper({
|
|
40
40
|
timeout: 10000,
|
|
@@ -147,7 +147,7 @@ await scraper.cleanup();
|
|
|
147
147
|
|
|
148
148
|
```javascript
|
|
149
149
|
// pages/api/scrape.js or app/api/scrape/route.js
|
|
150
|
-
import { smartScrape } from '@
|
|
150
|
+
import { smartScrape } from '@monostate/node-scraper';
|
|
151
151
|
|
|
152
152
|
export async function POST(request) {
|
|
153
153
|
try {
|
|
@@ -297,7 +297,7 @@ chmod +x lightpanda
|
|
|
297
297
|
Full TypeScript definitions included:
|
|
298
298
|
|
|
299
299
|
```typescript
|
|
300
|
-
import { BNCASmartScraper, ScrapingResult, ScrapingOptions } from '@
|
|
300
|
+
import { BNCASmartScraper, ScrapingResult, ScrapingOptions } from '@monostate/node-scraper';
|
|
301
301
|
|
|
302
302
|
const scraper: BNCASmartScraper = new BNCASmartScraper({
|
|
303
303
|
timeout: 5000,
|
package/index.js
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
import fetch from 'node-fetch';
|
|
2
2
|
import { spawn } from 'child_process';
|
|
3
|
-
import puppeteer from 'puppeteer';
|
|
4
3
|
import fs from 'fs/promises';
|
|
5
4
|
import path from 'path';
|
|
6
5
|
import { fileURLToPath } from 'url';
|
|
7
6
|
|
|
7
|
+
let puppeteer = null;
|
|
8
|
+
try {
|
|
9
|
+
puppeteer = await import('puppeteer');
|
|
10
|
+
puppeteer = puppeteer.default || puppeteer;
|
|
11
|
+
} catch (e) {
|
|
12
|
+
// Puppeteer is optional
|
|
13
|
+
}
|
|
14
|
+
|
|
8
15
|
const __filename = fileURLToPath(import.meta.url);
|
|
9
16
|
const __dirname = path.dirname(__filename);
|
|
10
17
|
|
|
@@ -254,6 +261,10 @@ export class BNCASmartScraper {
|
|
|
254
261
|
async tryPuppeteer(url, config) {
|
|
255
262
|
this.stats.puppeteer.attempts++;
|
|
256
263
|
|
|
264
|
+
if (!puppeteer) {
|
|
265
|
+
throw new Error('Puppeteer is not available');
|
|
266
|
+
}
|
|
267
|
+
|
|
257
268
|
try {
|
|
258
269
|
if (!this.browser) {
|
|
259
270
|
this.browser = await puppeteer.launch({
|