@jambudipa/spider 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +310 -332
- package/dist/index.js.map +1 -1
- package/dist/lib/Logging/SpiderLogger.service.d.ts.map +1 -1
- package/dist/lib/Middleware/SpiderMiddleware.d.ts +1 -1
- package/dist/lib/Middleware/SpiderMiddleware.d.ts.map +1 -1
- package/dist/lib/Resumability/backends/PostgresStorageBackend.d.ts.map +1 -1
- package/dist/lib/Resumability/types.d.ts +3 -3
- package/dist/lib/Scheduler/SpiderScheduler.service.d.ts +7 -7
- package/dist/lib/Scheduler/SpiderScheduler.service.d.ts.map +1 -1
- package/dist/lib/Scraper/Scraper.service.d.ts +1 -1
- package/dist/lib/Scraper/Scraper.service.d.ts.map +1 -1
- package/dist/lib/Spider/Spider.defaults.d.ts +24 -0
- package/dist/lib/Spider/Spider.defaults.d.ts.map +1 -0
- package/dist/lib/Spider/Spider.service.d.ts +0 -10
- package/dist/lib/Spider/Spider.service.d.ts.map +1 -1
- package/dist/lib/UrlDeduplicator/UrlDeduplicator.service.d.ts.map +1 -1
- package/dist/lib/WebScrapingEngine/WebScrapingEngine.service.d.ts +1 -2
- package/dist/lib/WebScrapingEngine/WebScrapingEngine.service.d.ts.map +1 -1
- package/dist/lib/api-facades.d.ts +1 -1
- package/dist/lib/api-facades.d.ts.map +1 -1
- package/dist/lib/errors/effect-errors.d.ts +167 -34
- package/dist/lib/errors/effect-errors.d.ts.map +1 -1
- package/dist/lib/utils/JsonUtils.d.ts.map +1 -1
- package/dist/lib/utils/UrlUtils.d.ts.map +1 -1
- package/dist/lib/utils/url-deduplication.d.ts.map +1 -1
- package/package.json +7 -10
- package/dist/browser/BrowserManager.d.ts +0 -63
- package/dist/browser/BrowserManager.d.ts.map +0 -1
- package/dist/browser/PlaywrightAdapter.d.ts +0 -166
- package/dist/browser/PlaywrightAdapter.d.ts.map +0 -1
- package/dist/examples/01-basic-crawl-working.d.ts +0 -13
- package/dist/examples/01-basic-crawl-working.d.ts.map +0 -1
- package/dist/examples/02-multiple-urls-working.d.ts +0 -13
- package/dist/examples/02-multiple-urls-working.d.ts.map +0 -1
- package/dist/examples/03-url-filtering.d.ts +0 -13
- package/dist/examples/03-url-filtering.d.ts.map +0 -1
- package/dist/examples/04-robots-compliance.d.ts +0 -14
- package/dist/examples/04-robots-compliance.d.ts.map +0 -1
- package/dist/examples/05-link-extraction-selectors.d.ts +0 -14
- package/dist/examples/05-link-extraction-selectors.d.ts.map +0 -1
- package/dist/examples/06-custom-middleware.d.ts +0 -18
- package/dist/examples/06-custom-middleware.d.ts.map +0 -1
- package/dist/examples/07-resumability-demo.d.ts +0 -14
- package/dist/examples/07-resumability-demo.d.ts.map +0 -1
- package/dist/examples/08-worker-monitoring.d.ts +0 -15
- package/dist/examples/08-worker-monitoring.d.ts.map +0 -1
- package/dist/examples/09-error-handling-recovery.d.ts +0 -15
- package/dist/examples/09-error-handling-recovery.d.ts.map +0 -1
- package/dist/lib/errors.d.ts +0 -172
- package/dist/lib/errors.d.ts.map +0 -1
- package/dist/lib/utils/url-deduplication.test.d.ts +0 -5
- package/dist/lib/utils/url-deduplication.test.d.ts.map +0 -1
- package/dist/test/infrastructure/EffectTestUtils.d.ts +0 -167
- package/dist/test/infrastructure/EffectTestUtils.d.ts.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jambudipa/spider",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "A comprehensive web scraping library with resumable operations, middleware support, and built-in rate limiting",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -42,9 +42,8 @@
|
|
|
42
42
|
"version": "changeset version",
|
|
43
43
|
"release": "npm run build && changeset publish",
|
|
44
44
|
"docs:api": "typedoc",
|
|
45
|
-
"docs:build": "npm run docs:api
|
|
45
|
+
"docs:build": "npm run docs:api",
|
|
46
46
|
"examples:run": "tsx",
|
|
47
|
-
"examples:validate": "tsx scripts/analyse-exports.ts",
|
|
48
47
|
"examples:run-all": "tsx examples/run-all.ts",
|
|
49
48
|
"test:examples": "vitest run tests/examples.test.ts",
|
|
50
49
|
"test:core": "vitest run tests/core",
|
|
@@ -75,7 +74,6 @@
|
|
|
75
74
|
},
|
|
76
75
|
"homepage": "https://github.com/jambudipa/spider#readme",
|
|
77
76
|
"peerDependencies": {
|
|
78
|
-
"@types/tough-cookie": "^4.0.5",
|
|
79
77
|
"cheerio": "^1.1.2",
|
|
80
78
|
"domhandler": "^5.0.3",
|
|
81
79
|
"effect": "^3.16.16",
|
|
@@ -83,8 +81,10 @@
|
|
|
83
81
|
"tslib": "^2.3.0"
|
|
84
82
|
},
|
|
85
83
|
"devDependencies": {
|
|
84
|
+
"@anthropic-ai/claude-agent-sdk": "^0.1.76",
|
|
86
85
|
"@changesets/cli": "^2.29.5",
|
|
87
86
|
"@playwright/test": "^1.54.2",
|
|
87
|
+
"@types/cli-progress": "^3.11.6",
|
|
88
88
|
"@types/node": "^20.19.11",
|
|
89
89
|
"@types/tough-cookie": "^4.0.5",
|
|
90
90
|
"@typescript-eslint/eslint-plugin": "^8.51.0",
|
|
@@ -94,6 +94,8 @@
|
|
|
94
94
|
"@vitest/eslint-plugin": "^1.6.4",
|
|
95
95
|
"@vitest/ui": "^1.6.0",
|
|
96
96
|
"cheerio": "^1.1.2",
|
|
97
|
+
"cli-progress": "^3.12.0",
|
|
98
|
+
"commander": "^14.0.2",
|
|
97
99
|
"domhandler": "^5.0.3",
|
|
98
100
|
"effect": "^3.16.16",
|
|
99
101
|
"eslint": "^9.39.2",
|
|
@@ -116,10 +118,5 @@
|
|
|
116
118
|
"publishConfig": {
|
|
117
119
|
"access": "public"
|
|
118
120
|
},
|
|
119
|
-
"dependencies": {
|
|
120
|
-
"@anthropic-ai/claude-agent-sdk": "^0.1.76",
|
|
121
|
-
"@types/cli-progress": "^3.11.6",
|
|
122
|
-
"cli-progress": "^3.12.0",
|
|
123
|
-
"commander": "^14.0.2"
|
|
124
|
-
}
|
|
121
|
+
"dependencies": {}
|
|
125
122
|
}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Browser Manager for Playwright Integration
|
|
3
|
-
* Handles browser lifecycle, pooling, and resource management
|
|
4
|
-
*/
|
|
5
|
-
import { Effect } from 'effect';
|
|
6
|
-
import { BrowserContext, Page, BrowserContextOptions } from 'playwright';
|
|
7
|
-
import { BrowserError } from '../lib/errors';
|
|
8
|
-
export interface BrowserConfig {
|
|
9
|
-
headless?: boolean;
|
|
10
|
-
timeout?: number;
|
|
11
|
-
poolSize?: number;
|
|
12
|
-
viewport?: {
|
|
13
|
-
width: number;
|
|
14
|
-
height: number;
|
|
15
|
-
};
|
|
16
|
-
userAgent?: string;
|
|
17
|
-
locale?: string;
|
|
18
|
-
extraHTTPHeaders?: Record<string, string>;
|
|
19
|
-
}
|
|
20
|
-
export declare class BrowserManager {
|
|
21
|
-
private browsers;
|
|
22
|
-
private contexts;
|
|
23
|
-
private config;
|
|
24
|
-
private isInitialised;
|
|
25
|
-
constructor(config?: BrowserConfig);
|
|
26
|
-
/**
|
|
27
|
-
* Initialise browser pool
|
|
28
|
-
*/
|
|
29
|
-
initialise(): Effect.Effect<void, BrowserError>;
|
|
30
|
-
/**
|
|
31
|
-
* Launch a new browser instance
|
|
32
|
-
*/
|
|
33
|
-
private launchBrowser;
|
|
34
|
-
/**
|
|
35
|
-
* Get or create a browser context
|
|
36
|
-
*/
|
|
37
|
-
getContext(id: string, options?: BrowserContextOptions): Effect.Effect<BrowserContext, BrowserError>;
|
|
38
|
-
/**
|
|
39
|
-
* Create a new page in a context
|
|
40
|
-
*/
|
|
41
|
-
createPage(contextId: string): Effect.Effect<Page, BrowserError>;
|
|
42
|
-
/**
|
|
43
|
-
* Get the browser with least contexts
|
|
44
|
-
*/
|
|
45
|
-
private getLeastLoadedBrowser;
|
|
46
|
-
/**
|
|
47
|
-
* Close a specific context
|
|
48
|
-
*/
|
|
49
|
-
closeContext(id: string): Effect.Effect<void, BrowserError>;
|
|
50
|
-
/**
|
|
51
|
-
* Close all resources
|
|
52
|
-
*/
|
|
53
|
-
close(): Effect.Effect<void>;
|
|
54
|
-
/**
|
|
55
|
-
* Get statistics about browser pool
|
|
56
|
-
*/
|
|
57
|
-
getStats(): {
|
|
58
|
-
browsers: number;
|
|
59
|
-
contexts: number;
|
|
60
|
-
pages: number;
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
//# sourceMappingURL=BrowserManager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"BrowserManager.d.ts","sourceRoot":"","sources":["../../src/browser/BrowserManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAkC,MAAM,QAAQ,CAAC;AAChE,OAAO,EAAW,cAAc,EAAE,IAAI,EAAY,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAuB,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAiF;IACjG,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,aAAa,CAAS;gBAElB,MAAM,GAAE,aAAkB;IAYtC;;OAEG;IACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;IAc/C;;OAEG;IACH,OAAO,CAAC,aAAa;IAmBrB;;OAEG;IACH,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,YAAY,CAAC;IA+BpG;;OAEG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;IAahE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAe7B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;IAc3D;;OAEG;IACH,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAiD5B;;OAEG;IACH,QAAQ,IAAI;QACV,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;KACf;CAYF"}
|
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Playwright Adapter for Spider Integration
|
|
3
|
-
* Provides high-level browser automation capabilities
|
|
4
|
-
*/
|
|
5
|
-
import { Effect, Option } from 'effect';
|
|
6
|
-
import { Page, Response as PlaywrightResponse, Route, Cookie, Request as PlaywrightRequest } from 'playwright';
|
|
7
|
-
import { BrowserManager } from './BrowserManager';
|
|
8
|
-
import { AdapterNotInitialisedError } from '../lib/errors';
|
|
9
|
-
export type RequestHandler = (request: PlaywrightRequest) => void;
|
|
10
|
-
export type ResponseHandler = (response: PlaywrightResponse) => void;
|
|
11
|
-
export interface WaitOptions {
|
|
12
|
-
timeout?: number;
|
|
13
|
-
state?: 'load' | 'domcontentloaded' | 'networkidle';
|
|
14
|
-
}
|
|
15
|
-
export interface ScrollOptions {
|
|
16
|
-
delay?: number;
|
|
17
|
-
maxScrolls?: number;
|
|
18
|
-
scrollDistance?: number;
|
|
19
|
-
}
|
|
20
|
-
declare const PageClosedError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
21
|
-
readonly _tag: "PageClosedError";
|
|
22
|
-
} & Readonly<A>;
|
|
23
|
-
/**
|
|
24
|
-
* Error thrown when page is closed during an operation
|
|
25
|
-
*/
|
|
26
|
-
export declare class PageClosedError extends PageClosedError_base<{
|
|
27
|
-
readonly operation: string;
|
|
28
|
-
readonly message: string;
|
|
29
|
-
}> {
|
|
30
|
-
static create(operation: string): PageClosedError;
|
|
31
|
-
}
|
|
32
|
-
declare const StreamReadError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
33
|
-
readonly _tag: "StreamReadError";
|
|
34
|
-
} & Readonly<A>;
|
|
35
|
-
/**
|
|
36
|
-
* Error thrown when stream reading fails
|
|
37
|
-
*/
|
|
38
|
-
export declare class StreamReadError extends StreamReadError_base<{
|
|
39
|
-
readonly cause: unknown;
|
|
40
|
-
readonly message: string;
|
|
41
|
-
}> {
|
|
42
|
-
static fromCause(cause: unknown): StreamReadError;
|
|
43
|
-
}
|
|
44
|
-
export declare class PlaywrightAdapter {
|
|
45
|
-
private browserManager;
|
|
46
|
-
private page;
|
|
47
|
-
private contextId;
|
|
48
|
-
private requestHandlers;
|
|
49
|
-
private responseHandlers;
|
|
50
|
-
constructor(browserManager: BrowserManager, contextId: string);
|
|
51
|
-
/**
|
|
52
|
-
* Initialise the adapter with a new page
|
|
53
|
-
*/
|
|
54
|
-
initialise(): Effect.Effect<Page, AdapterNotInitialisedError>;
|
|
55
|
-
/**
|
|
56
|
-
* Get the current page instance (Effect)
|
|
57
|
-
*/
|
|
58
|
-
getPageEffect(): Effect.Effect<Page, AdapterNotInitialisedError>;
|
|
59
|
-
/**
|
|
60
|
-
* Get the current page instance (direct)
|
|
61
|
-
* Returns Option for type-safe handling
|
|
62
|
-
*/
|
|
63
|
-
getPage(): Option.Option<Page>;
|
|
64
|
-
/**
|
|
65
|
-
* Internal helper to get page or fail
|
|
66
|
-
*/
|
|
67
|
-
private requirePage;
|
|
68
|
-
/**
|
|
69
|
-
* Navigate to a URL
|
|
70
|
-
*/
|
|
71
|
-
goto(url: string, options?: WaitOptions): Effect.Effect<Option.Option<PlaywrightResponse>, AdapterNotInitialisedError>;
|
|
72
|
-
/**
|
|
73
|
-
* Wait for dynamic content to load
|
|
74
|
-
*/
|
|
75
|
-
waitForDynamicContent(selector: string, options?: WaitOptions): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
76
|
-
/**
|
|
77
|
-
* Scroll to bottom progressively
|
|
78
|
-
*/
|
|
79
|
-
scrollToBottom(options?: ScrollOptions): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
80
|
-
/**
|
|
81
|
-
* Click an element and wait for navigation or content
|
|
82
|
-
*/
|
|
83
|
-
clickAndWait(selector: string, waitFor?: string | WaitOptions): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
84
|
-
/**
|
|
85
|
-
* Intercept requests
|
|
86
|
-
*/
|
|
87
|
-
interceptRequests(handler: RequestHandler): Effect.Effect<void>;
|
|
88
|
-
/**
|
|
89
|
-
* Intercept responses
|
|
90
|
-
*/
|
|
91
|
-
interceptResponses(handler: ResponseHandler): Effect.Effect<void>;
|
|
92
|
-
/**
|
|
93
|
-
* Route specific URLs
|
|
94
|
-
*/
|
|
95
|
-
route(pattern: string | RegExp, handler: (route: Route) => void): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
96
|
-
/**
|
|
97
|
-
* Execute JavaScript in page context
|
|
98
|
-
*/
|
|
99
|
-
evaluate<T>(fn: () => T): Effect.Effect<T, AdapterNotInitialisedError>;
|
|
100
|
-
/**
|
|
101
|
-
* Take a screenshot
|
|
102
|
-
*/
|
|
103
|
-
screenshot(path: string): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
104
|
-
/**
|
|
105
|
-
* Get page content
|
|
106
|
-
*/
|
|
107
|
-
content(): Effect.Effect<string, AdapterNotInitialisedError>;
|
|
108
|
-
/**
|
|
109
|
-
* Fill a form field
|
|
110
|
-
*/
|
|
111
|
-
fill(selector: string, value: string): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
112
|
-
/**
|
|
113
|
-
* Select an option
|
|
114
|
-
*/
|
|
115
|
-
select(selector: string, value: string): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
116
|
-
/**
|
|
117
|
-
* Check if element exists
|
|
118
|
-
*/
|
|
119
|
-
exists(selector: string): Effect.Effect<boolean, AdapterNotInitialisedError>;
|
|
120
|
-
/**
|
|
121
|
-
* Wait for network idle
|
|
122
|
-
*/
|
|
123
|
-
waitForNetworkIdle(options?: WaitOptions): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
124
|
-
/**
|
|
125
|
-
* Handle new tabs/windows
|
|
126
|
-
*/
|
|
127
|
-
handleNewTab(callback: (page: Page) => Effect.Effect<void>): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
128
|
-
/**
|
|
129
|
-
* Get cookies
|
|
130
|
-
*/
|
|
131
|
-
getCookies(): Effect.Effect<readonly Cookie[], AdapterNotInitialisedError>;
|
|
132
|
-
/**
|
|
133
|
-
* Set cookies
|
|
134
|
-
*/
|
|
135
|
-
setCookies(cookies: readonly Cookie[]): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
136
|
-
/**
|
|
137
|
-
* Clear cookies
|
|
138
|
-
*/
|
|
139
|
-
clearCookies(): Effect.Effect<void, AdapterNotInitialisedError>;
|
|
140
|
-
/**
|
|
141
|
-
* Helper to read a Node stream as a Buffer using Effect Stream
|
|
142
|
-
*/
|
|
143
|
-
private readStreamAsBuffer;
|
|
144
|
-
/**
|
|
145
|
-
* Download file from URL
|
|
146
|
-
*/
|
|
147
|
-
downloadFile(url: string, filename?: string): Effect.Effect<{
|
|
148
|
-
buffer: Buffer;
|
|
149
|
-
filename: string;
|
|
150
|
-
mimeType: string;
|
|
151
|
-
}, AdapterNotInitialisedError | PageClosedError | StreamReadError>;
|
|
152
|
-
/**
|
|
153
|
-
* Trigger download by clicking element
|
|
154
|
-
*/
|
|
155
|
-
downloadFromClick(selector: string): Effect.Effect<{
|
|
156
|
-
buffer: Buffer;
|
|
157
|
-
filename: string;
|
|
158
|
-
mimeType: string;
|
|
159
|
-
}, AdapterNotInitialisedError | PageClosedError | StreamReadError>;
|
|
160
|
-
/**
|
|
161
|
-
* Close the page
|
|
162
|
-
*/
|
|
163
|
-
close(): Effect.Effect<void>;
|
|
164
|
-
}
|
|
165
|
-
export {};
|
|
166
|
-
//# sourceMappingURL=PlaywrightAdapter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"PlaywrightAdapter.d.ts","sourceRoot":"","sources":["../../src/browser/PlaywrightAdapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAe,MAAM,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AAE7D,OAAO,EAAE,IAAI,EAAE,QAAQ,IAAI,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/G,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,0BAA0B,EAAE,MAAM,eAAe,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,CAAC;AAClE,MAAM,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,kBAAkB,KAAK,IAAI,CAAC;AAErE,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;CACrD;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;;;;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe;CAMlD;;;;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe;CAMlD;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,IAAI,CAAsC;IAClD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAA8C;IACrE,OAAO,CAAC,gBAAgB,CAA+C;gBAE3D,cAAc,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM;IAK7D;;OAEG;IACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAyB7D;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAShE;;;OAGG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAI9B;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;IAetH;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAc/G;;OAEG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IA+CxF;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IA0C/G;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAM/D;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAMjE;;OAEG;IACH,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWjH;;OAEG;IACH,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,0BAA0B,CAAC;IAWtE;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWzE;;OAEG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC;IAW5D;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWtF;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWxF;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,0BAA0B,CAAC;IAY5E;;OAEG;IACH,kBAAkB,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAa1F;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAmB5G;;OAEG;IACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,MAAM,EAAE,EAAE,0BAA0B,CAAC;IAW1E;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWvF;;OAEG;IACH,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAW/D;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAY1B;;OAEG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,EAAE,0BAA0B,GAAG,eAAe,GAAG,eAAe,CAAC;IAiElE;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACjD,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,EAAE,0BAA0B,GAAG,eAAe,GAAG,eAAe,CAAC;IA2DlE;;OAEG;IACH,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;CAsB7B"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 01: Basic Web Crawling
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Basic Spider setup and configuration
|
|
6
|
-
* - Single URL crawling with depth control
|
|
7
|
-
* - Result collection and processing
|
|
8
|
-
* - Error handling
|
|
9
|
-
*
|
|
10
|
-
* Tests against: web-scraping.dev (static content)
|
|
11
|
-
*/
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=01-basic-crawl-working.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"01-basic-crawl-working.d.ts","sourceRoot":"","sources":["../../src/examples/01-basic-crawl-working.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 02: Multiple Starting URLs and Domain Isolation
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Crawling multiple domains concurrently
|
|
6
|
-
* - Domain isolation and deduplication
|
|
7
|
-
* - Metadata passing through crawl
|
|
8
|
-
* - Concurrent request handling
|
|
9
|
-
*
|
|
10
|
-
* Tests against: Multiple sections of web-scraping.dev
|
|
11
|
-
*/
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=02-multiple-urls-working.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"02-multiple-urls-working.d.ts","sourceRoot":"","sources":["../../src/examples/02-multiple-urls-working.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 03: URL Filtering and Domain Restrictions
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Custom URL filtering with regex patterns
|
|
6
|
-
* - File extension filtering
|
|
7
|
-
* - Domain restrictions
|
|
8
|
-
* - Technical URL filtering (malformed, long URLs, etc.)
|
|
9
|
-
*
|
|
10
|
-
* Tests against: web-scraping.dev with various filters
|
|
11
|
-
*/
|
|
12
|
-
export {};
|
|
13
|
-
//# sourceMappingURL=03-url-filtering.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"03-url-filtering.d.ts","sourceRoot":"","sources":["../../src/examples/03-url-filtering.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 04: Robots.txt Compliance and Rate Limiting
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Robots.txt compliance checking
|
|
6
|
-
* - Crawl delay respect from robots.txt
|
|
7
|
-
* - Rate limiting per domain
|
|
8
|
-
* - Request delay configuration
|
|
9
|
-
* - Respectful crawling practices
|
|
10
|
-
*
|
|
11
|
-
* Tests against: web-scraping.dev robots.txt rules
|
|
12
|
-
*/
|
|
13
|
-
export {};
|
|
14
|
-
//# sourceMappingURL=04-robots-compliance.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"04-robots-compliance.d.ts","sourceRoot":"","sources":["../../src/examples/04-robots-compliance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 05: Advanced Link Extraction with CSS Selectors
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Custom CSS selectors for targeted link extraction
|
|
6
|
-
* - Link extraction from different HTML elements
|
|
7
|
-
* - Form action URL extraction
|
|
8
|
-
* - Custom attribute extraction
|
|
9
|
-
* - Link extraction statistics and analysis
|
|
10
|
-
*
|
|
11
|
-
* Tests against: web-scraping.dev with targeted selectors
|
|
12
|
-
*/
|
|
13
|
-
export {};
|
|
14
|
-
//# sourceMappingURL=05-link-extraction-selectors.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"05-link-extraction-selectors.d.ts","sourceRoot":"","sources":["../../src/examples/05-link-extraction-selectors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 06: Custom Middleware Usage
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Creating custom middleware for request/response processing
|
|
6
|
-
* - Using built-in middleware (RateLimit, Logging, UserAgent, Stats)
|
|
7
|
-
* - Middleware error handling
|
|
8
|
-
* - Request preprocessing and response postprocessing
|
|
9
|
-
* - Statistics collection through middleware
|
|
10
|
-
*
|
|
11
|
-
* Tests against: web-scraping.dev with custom middleware pipeline
|
|
12
|
-
*/
|
|
13
|
-
import { SpiderMiddleware } from '../index.js';
|
|
14
|
-
declare const TimingMiddleware: SpiderMiddleware;
|
|
15
|
-
declare const ContentAnalysisMiddleware: SpiderMiddleware;
|
|
16
|
-
declare const PatternDetectionMiddleware: SpiderMiddleware;
|
|
17
|
-
export { TimingMiddleware, ContentAnalysisMiddleware, PatternDetectionMiddleware };
|
|
18
|
-
//# sourceMappingURL=06-custom-middleware.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"06-custom-middleware.d.ts","sourceRoot":"","sources":["../../src/examples/06-custom-middleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,EAKL,gBAAgB,EAIjB,MAAM,aAAa,CAAC;AAGrB,QAAA,MAAM,gBAAgB,EAAE,gBAyCvB,CAAC;AAGF,QAAA,MAAM,yBAAyB,EAAE,gBAiBhC,CAAC;AAGF,QAAA,MAAM,0BAA0B,EAAE,gBAiBjC,CAAC;AAGF,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,CAAC"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 07: Resumability and State Persistence
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Resumable crawling with state persistence
|
|
6
|
-
* - Multiple storage backends (File, Redis, Postgres)
|
|
7
|
-
* - Session management and restoration
|
|
8
|
-
* - State persistence strategies (Full, Delta, Hybrid)
|
|
9
|
-
* - Graceful recovery from interruptions
|
|
10
|
-
*
|
|
11
|
-
* Tests against: web-scraping.dev with resumable sessions
|
|
12
|
-
*/
|
|
13
|
-
export {};
|
|
14
|
-
//# sourceMappingURL=07-resumability-demo.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"07-resumability-demo.d.ts","sourceRoot":"","sources":["../../src/examples/07-resumability-demo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 08: Worker Health Monitoring and Performance Analysis
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Worker health monitoring and stuck worker detection
|
|
6
|
-
* - Memory usage monitoring and limits
|
|
7
|
-
* - Performance metrics collection
|
|
8
|
-
* - Concurrent worker management
|
|
9
|
-
* - Worker lifecycle tracking
|
|
10
|
-
* - Domain failure detection and recovery
|
|
11
|
-
*
|
|
12
|
-
* Tests against: web-scraping.dev with worker monitoring
|
|
13
|
-
*/
|
|
14
|
-
export {};
|
|
15
|
-
//# sourceMappingURL=08-worker-monitoring.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"08-worker-monitoring.d.ts","sourceRoot":"","sources":["../../src/examples/08-worker-monitoring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Example 09: Error Handling and Recovery Strategies
|
|
3
|
-
*
|
|
4
|
-
* This example demonstrates:
|
|
5
|
-
* - Comprehensive error handling for network failures
|
|
6
|
-
* - Recovery strategies for failed requests
|
|
7
|
-
* - Timeout handling and request cancellation
|
|
8
|
-
* - Domain failure detection and recovery
|
|
9
|
-
* - Graceful degradation under adverse conditions
|
|
10
|
-
* - Error classification and reporting
|
|
11
|
-
*
|
|
12
|
-
* Tests against: web-scraping.dev with simulated failures
|
|
13
|
-
*/
|
|
14
|
-
export {};
|
|
15
|
-
//# sourceMappingURL=09-error-handling-recovery.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"09-error-handling-recovery.d.ts","sourceRoot":"","sources":["../../src/examples/09-error-handling-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
package/dist/lib/errors.d.ts
DELETED
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
declare const NetworkError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
2
|
-
readonly _tag: "NetworkError";
|
|
3
|
-
} & Readonly<A>;
|
|
4
|
-
/**
|
|
5
|
-
* Network-related errors (fetch failures, timeouts, etc.)
|
|
6
|
-
*/
|
|
7
|
-
export declare class NetworkError extends NetworkError_base<{
|
|
8
|
-
readonly url: string;
|
|
9
|
-
readonly cause?: unknown;
|
|
10
|
-
readonly message: string;
|
|
11
|
-
}> {
|
|
12
|
-
static fromCause(url: string, cause: unknown): NetworkError;
|
|
13
|
-
}
|
|
14
|
-
declare const ResponseError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
15
|
-
readonly _tag: "ResponseError";
|
|
16
|
-
} & Readonly<A>;
|
|
17
|
-
/**
|
|
18
|
-
* Response processing errors (invalid content, parsing failures)
|
|
19
|
-
*/
|
|
20
|
-
export declare class ResponseError extends ResponseError_base<{
|
|
21
|
-
readonly url: string;
|
|
22
|
-
readonly cause?: unknown;
|
|
23
|
-
readonly message: string;
|
|
24
|
-
}> {
|
|
25
|
-
static fromCause(url: string, cause: unknown): ResponseError;
|
|
26
|
-
}
|
|
27
|
-
declare const RobotsTxtError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
28
|
-
readonly _tag: "RobotsTxtError";
|
|
29
|
-
} & Readonly<A>;
|
|
30
|
-
/**
|
|
31
|
-
* Robots.txt fetching errors
|
|
32
|
-
*/
|
|
33
|
-
export declare class RobotsTxtError extends RobotsTxtError_base<{
|
|
34
|
-
readonly url: string;
|
|
35
|
-
readonly cause?: unknown;
|
|
36
|
-
readonly message: string;
|
|
37
|
-
}> {
|
|
38
|
-
static fromCause(url: string, cause: unknown): RobotsTxtError;
|
|
39
|
-
}
|
|
40
|
-
declare const ConfigurationError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
41
|
-
readonly _tag: "ConfigurationError";
|
|
42
|
-
} & Readonly<A>;
|
|
43
|
-
/**
|
|
44
|
-
* Configuration errors
|
|
45
|
-
*/
|
|
46
|
-
export declare class ConfigurationError extends ConfigurationError_base<{
|
|
47
|
-
readonly message: string;
|
|
48
|
-
readonly details?: unknown;
|
|
49
|
-
}> {
|
|
50
|
-
}
|
|
51
|
-
declare const MiddlewareError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
52
|
-
readonly _tag: "MiddlewareError";
|
|
53
|
-
} & Readonly<A>;
|
|
54
|
-
/**
|
|
55
|
-
* Middleware processing errors
|
|
56
|
-
*/
|
|
57
|
-
export declare class MiddlewareError extends MiddlewareError_base<{
|
|
58
|
-
readonly phase: 'transform' | 'error';
|
|
59
|
-
readonly middlewareName: string;
|
|
60
|
-
readonly cause?: unknown;
|
|
61
|
-
readonly message: string;
|
|
62
|
-
}> {
|
|
63
|
-
static transform(middlewareName: string, cause: unknown): MiddlewareError;
|
|
64
|
-
static error(middlewareName: string, cause: unknown): MiddlewareError;
|
|
65
|
-
}
|
|
66
|
-
declare const FileSystemError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
67
|
-
readonly _tag: "FileSystemError";
|
|
68
|
-
} & Readonly<A>;
|
|
69
|
-
/**
|
|
70
|
-
* File system errors
|
|
71
|
-
*/
|
|
72
|
-
export declare class FileSystemError extends FileSystemError_base<{
|
|
73
|
-
readonly operation: 'read' | 'write' | 'create' | 'delete';
|
|
74
|
-
readonly path: string;
|
|
75
|
-
readonly cause?: unknown;
|
|
76
|
-
readonly message: string;
|
|
77
|
-
}> {
|
|
78
|
-
static write(path: string, cause: unknown): FileSystemError;
|
|
79
|
-
static create(path: string, cause: unknown): FileSystemError;
|
|
80
|
-
}
|
|
81
|
-
declare const PersistenceError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
82
|
-
readonly _tag: "PersistenceError";
|
|
83
|
-
} & Readonly<A>;
|
|
84
|
-
/**
|
|
85
|
-
* Persistence layer errors
|
|
86
|
-
*/
|
|
87
|
-
export declare class PersistenceError extends PersistenceError_base<{
|
|
88
|
-
readonly operation: 'save' | 'load' | 'delete';
|
|
89
|
-
readonly key?: string;
|
|
90
|
-
readonly cause?: unknown;
|
|
91
|
-
readonly message: string;
|
|
92
|
-
}> {
|
|
93
|
-
static save(cause: unknown, key?: string): PersistenceError;
|
|
94
|
-
static load(cause: unknown, key?: string): PersistenceError;
|
|
95
|
-
static delete(cause: unknown, key?: string): PersistenceError;
|
|
96
|
-
}
|
|
97
|
-
declare const ContentTypeError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
98
|
-
readonly _tag: "ContentTypeError";
|
|
99
|
-
} & Readonly<A>;
|
|
100
|
-
/**
|
|
101
|
-
* Content type validation errors
|
|
102
|
-
*/
|
|
103
|
-
export declare class ContentTypeError extends ContentTypeError_base<{
|
|
104
|
-
readonly url: string;
|
|
105
|
-
readonly contentType: string;
|
|
106
|
-
readonly expectedTypes: readonly string[];
|
|
107
|
-
readonly message: string;
|
|
108
|
-
}> {
|
|
109
|
-
static create(url: string, contentType: string, expectedTypes: readonly string[]): ContentTypeError;
|
|
110
|
-
}
|
|
111
|
-
declare const RequestAbortError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
112
|
-
readonly _tag: "RequestAbortError";
|
|
113
|
-
} & Readonly<A>;
|
|
114
|
-
/**
|
|
115
|
-
* Request abort errors
|
|
116
|
-
*/
|
|
117
|
-
export declare class RequestAbortError extends RequestAbortError_base<{
|
|
118
|
-
readonly url: string;
|
|
119
|
-
readonly duration: number;
|
|
120
|
-
readonly reason: 'timeout' | 'cancelled';
|
|
121
|
-
readonly message: string;
|
|
122
|
-
}> {
|
|
123
|
-
static timeout(url: string, duration: number): RequestAbortError;
|
|
124
|
-
static cancelled(url: string, duration: number): RequestAbortError;
|
|
125
|
-
}
|
|
126
|
-
declare const AdapterNotInitialisedError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
127
|
-
readonly _tag: "AdapterNotInitialisedError";
|
|
128
|
-
} & Readonly<A>;
|
|
129
|
-
/**
|
|
130
|
-
* Adapter initialisation errors
|
|
131
|
-
*/
|
|
132
|
-
export declare class AdapterNotInitialisedError extends AdapterNotInitialisedError_base<{
|
|
133
|
-
readonly adapterId: string;
|
|
134
|
-
readonly operation: string;
|
|
135
|
-
readonly message: string;
|
|
136
|
-
}> {
|
|
137
|
-
static create(adapterId: string, operation: string): AdapterNotInitialisedError;
|
|
138
|
-
}
|
|
139
|
-
declare const BrowserError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
140
|
-
readonly _tag: "BrowserError";
|
|
141
|
-
} & Readonly<A>;
|
|
142
|
-
/**
|
|
143
|
-
* Browser operation errors
|
|
144
|
-
*/
|
|
145
|
-
export declare class BrowserError extends BrowserError_base<{
|
|
146
|
-
readonly operation: 'launch' | 'createContext' | 'createPage' | 'closeContext';
|
|
147
|
-
readonly cause: unknown;
|
|
148
|
-
readonly message: string;
|
|
149
|
-
}> {
|
|
150
|
-
static launch(cause: unknown): BrowserError;
|
|
151
|
-
static createContext(cause: unknown): BrowserError;
|
|
152
|
-
static createPage(cause: unknown): BrowserError;
|
|
153
|
-
static closeContext(cause: unknown): BrowserError;
|
|
154
|
-
}
|
|
155
|
-
declare const BrowserCleanupError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
|
|
156
|
-
readonly _tag: "BrowserCleanupError";
|
|
157
|
-
} & Readonly<A>;
|
|
158
|
-
/**
|
|
159
|
-
* Browser cleanup errors
|
|
160
|
-
*/
|
|
161
|
-
export declare class BrowserCleanupError extends BrowserCleanupError_base<{
|
|
162
|
-
readonly resourceType: 'context' | 'browser';
|
|
163
|
-
readonly resourceId: string;
|
|
164
|
-
readonly cause: unknown;
|
|
165
|
-
readonly message: string;
|
|
166
|
-
}> {
|
|
167
|
-
static context(id: string, cause: unknown): BrowserCleanupError;
|
|
168
|
-
static browser(id: string, cause: unknown): BrowserCleanupError;
|
|
169
|
-
}
|
|
170
|
-
export type SpiderError = NetworkError | ResponseError | RobotsTxtError | ConfigurationError | MiddlewareError | FileSystemError | PersistenceError | ContentTypeError | RequestAbortError | AdapterNotInitialisedError | BrowserError | BrowserCleanupError;
|
|
171
|
-
export {};
|
|
172
|
-
//# sourceMappingURL=errors.d.ts.map
|
package/dist/lib/errors.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/lib/errors.ts"],"names":[],"mappings":";;;AAEA;;GAEG;AACH,qBAAa,YAAa,SAAQ,kBAAiC;IACjE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,YAAY;CAO5D;;;;AAED;;GAEG;AACH,qBAAa,aAAc,SAAQ,mBAAkC;IACnE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,aAAa;CAO7D;;;;AAED;;GAEG;AACH,qBAAa,cAAe,SAAQ,oBAAmC;IACrE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,cAAc;CAO9D;;;;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,wBAAuC;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;CAAG;;;;AAEL;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC;IACtC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;IASzE,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;CAQtE;;;;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC3D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;IAS3D,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;CAQ7D;;;;AAED;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC/C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,gBAAgB;IAW3D,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,gBAAgB;IAW3D,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,gBAAgB;CAU9D;;;;AAED;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,MAAM,CACX,GAAG,EAAE,MAAM,EACX,WAAW,EAAE,MAAM,EACnB,aAAa,EAAE,SAAS,MAAM,EAAE,GAC/B,gBAAgB;CAQpB;;;;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,uBAAsC;IAC3E,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,iBAAiB;IAShE,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,iBAAiB;CAQnE;;;;AAED;;GAEG;AACH,qBAAa,0BAA2B,SAAQ,gCAA+C;IAC7F,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,0BAA0B;CAOhF;;;;AAED;;GAEG;AACH,qBAAa,YAAa,SAAQ,kBAAiC;IACjE,QAAQ,CAAC,SAAS,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,GAAG,cAAc,CAAC;IAC/E,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY;IAQ3C,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY;IAQlD,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY;IAQ/C,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY;CAOlD;;;;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,yBAAwC;IAC/E,QAAQ,CAAC,YAAY,EAAE,SAAS,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,mBAAmB;IAS/D,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,mBAAmB;CAQhE;AAGD,MAAM,MAAM,WAAW,GACnB,YAAY,GACZ,aAAa,GACb,cAAc,GACd,kBAAkB,GAClB,eAAe,GACf,eAAe,GACf,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,0BAA0B,GAC1B,YAAY,GACZ,mBAAmB,CAAC"}
|