@crawlee/types 4.0.0-beta.14 → 4.0.0-beta.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/browser.d.ts +104 -1
- package/browser.js +0 -1
- package/cookies.d.ts +82 -0
- package/cookies.js +1 -0
- package/http-client.d.ts +83 -0
- package/http-client.js +1 -0
- package/index.d.ts +8 -4
- package/index.js +1 -4
- package/logger.d.ts +66 -0
- package/logger.js +1 -0
- package/package.json +3 -3
- package/session.d.ts +152 -0
- package/session.js +1 -0
- package/status-message.d.ts +13 -0
- package/status-message.js +1 -0
- package/storages.d.ts +297 -169
- package/storages.js +0 -1
- package/utility-types.d.ts +0 -2
- package/utility-types.js +0 -1
- package/browser.d.ts.map +0 -1
- package/browser.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/storages.d.ts.map +0 -1
- package/storages.js.map +0 -1
- package/utility-types.d.ts.map +0 -1
- package/utility-types.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/browser.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ISession } from './session.js';
|
|
1
2
|
import type { Dictionary } from './utility-types.js';
|
|
2
3
|
export interface Cookie {
|
|
3
4
|
/**
|
|
@@ -60,4 +61,106 @@ export interface BrowserLikeResponse {
|
|
|
60
61
|
url(): string;
|
|
61
62
|
headers(): Dictionary<string | string[]>;
|
|
62
63
|
}
|
|
63
|
-
|
|
64
|
+
/**
|
|
65
|
+
* A snapshot of the relevant state of a page, as extracted by
|
|
66
|
+
* {@link IBrowserPool.extractPageState}.
|
|
67
|
+
*/
|
|
68
|
+
export interface PageState {
|
|
69
|
+
/**
|
|
70
|
+
* Cookies currently set in the page's browsing context.
|
|
71
|
+
*/
|
|
72
|
+
cookies: Cookie[];
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Options accepted by {@link IBrowserPool.newPage}.
|
|
76
|
+
*/
|
|
77
|
+
export interface NewPageOptions {
|
|
78
|
+
/**
|
|
79
|
+
* Assign a custom ID to the page. If you don't provide one, a random string
|
|
80
|
+
* ID is generated.
|
|
81
|
+
*/
|
|
82
|
+
id?: string;
|
|
83
|
+
/**
|
|
84
|
+
* The crawling session that will use the returned page.
|
|
85
|
+
*
|
|
86
|
+
* The pool derives proxy configuration from the session's {@link
|
|
87
|
+
* ProxyInfo|proxy} (including TLS-error handling) — there are intentionally
|
|
88
|
+
* no standalone `proxyUrl` / `ignoreTlsErrors` options; configure them
|
|
89
|
+
* through the session instead.
|
|
90
|
+
*
|
|
91
|
+
* Session injection is **best-effort**: the pool may use the session's
|
|
92
|
+
* {@link ProxyInfo|proxy}, cookies, or fingerprint data to configure the
|
|
93
|
+
* page or the underlying browser, but none of this is guaranteed. Different
|
|
94
|
+
* pool implementations (or pool configurations such as `useIncognitoPages`)
|
|
95
|
+
* may support different subsets of session properties — or ignore them
|
|
96
|
+
* entirely.
|
|
97
|
+
*
|
|
98
|
+
* The crawler is still responsible for deterministic session setup (e.g.
|
|
99
|
+
* injecting cookies into the page before navigation) that must happen
|
|
100
|
+
* regardless of pool implementation.
|
|
101
|
+
*/
|
|
102
|
+
session?: ISession;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Minimal contract that any object passed to a browser crawler as its `browserPool`
|
|
106
|
+
* option must satisfy.
|
|
107
|
+
*
|
|
108
|
+
* Lifecycle (`destroy`) is the responsibility of whoever owns the pool — since a
|
|
109
|
+
* user-supplied pool is never owned by the crawler, the crawler never tears it
|
|
110
|
+
* down.
|
|
111
|
+
*
|
|
112
|
+
* Implement this interface to plug a custom page-provisioning strategy into any
|
|
113
|
+
* Crawlee browser crawler — for example a remote browser farm, a session-aware
|
|
114
|
+
* pool that pins pages to fingerprints differently, or a thin wrapper around the
|
|
115
|
+
* built-in `BrowserPool`.
|
|
116
|
+
*
|
|
117
|
+
* @category Browser management
|
|
118
|
+
*/
|
|
119
|
+
export interface IBrowserPool<Page = unknown> {
|
|
120
|
+
/**
|
|
121
|
+
* Opens a new page. The pool decides which browser to use, launching a new
|
|
122
|
+
* one if needed.
|
|
123
|
+
*/
|
|
124
|
+
newPage(options?: NewPageOptions): Promise<Page>;
|
|
125
|
+
/**
|
|
126
|
+
* Signals the pool that the caller is done with the page. The pool is
|
|
127
|
+
* responsible for closing the page and performing any necessary cleanup
|
|
128
|
+
* (e.g. retiring the underlying browser when a session has gone bad).
|
|
129
|
+
*
|
|
130
|
+
* @param page The page to release back to the pool.
|
|
131
|
+
* @param options.error If the page is being released because of an error,
|
|
132
|
+
* pass the error here. In particular, if the error is a
|
|
133
|
+
* {@link SessionError}, implementations should treat it as a signal
|
|
134
|
+
* to purge all state associated with the session (e.g. discard any
|
|
135
|
+
* browser that served the page).
|
|
136
|
+
*/
|
|
137
|
+
closePage(page: Page, options?: {
|
|
138
|
+
error?: Error;
|
|
139
|
+
}): Promise<void>;
|
|
140
|
+
/**
|
|
141
|
+
* Extracts the relevant state from a page so the caller can persist it —
|
|
142
|
+
* for example, back-propagating cookies into the crawling {@link
|
|
143
|
+
* ISession|session}.
|
|
144
|
+
*
|
|
145
|
+
* @param page The page to read state from.
|
|
146
|
+
*/
|
|
147
|
+
extractPageState(page: Page): Promise<PageState>;
|
|
148
|
+
/**
|
|
149
|
+
* Injects state (currently just cookies) into a page. This is the
|
|
150
|
+
* counterpart to {@link IBrowserPool.extractPageState} and lets the
|
|
151
|
+
* caller set up a page — for example, seeding it with the crawling
|
|
152
|
+
* {@link ISession|session}'s cookies before navigation.
|
|
153
|
+
*
|
|
154
|
+
* As with {@link IBrowserPool.newPage}, the caller decides *what* state
|
|
155
|
+
* to inject, while the pool decides *how*.
|
|
156
|
+
*
|
|
157
|
+
* Isolation between pages is **best-effort**: depending on the pool
|
|
158
|
+
* implementation and its configuration, multiple pages may share a browsing
|
|
159
|
+
* context, so injected state (such as cookies) can bleed across pages
|
|
160
|
+
* served by the same underlying browser.
|
|
161
|
+
*
|
|
162
|
+
* @param page The page to inject state into.
|
|
163
|
+
* @param state The state to inject.
|
|
164
|
+
*/
|
|
165
|
+
injectPageState(page: Page, state: PageState): Promise<void>;
|
|
166
|
+
}
|
package/browser.js
CHANGED
package/cookies.d.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cookie shape used by {@link CookieJar}. Structurally compatible with `tough-cookie`'s
|
|
3
|
+
* `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending
|
|
4
|
+
* on `tough-cookie` itself.
|
|
5
|
+
*/
|
|
6
|
+
export interface SessionCookie {
|
|
7
|
+
key: string;
|
|
8
|
+
value: string;
|
|
9
|
+
expires: Date | 'Infinity' | null;
|
|
10
|
+
maxAge: number | 'Infinity' | '-Infinity' | null;
|
|
11
|
+
domain: string | null;
|
|
12
|
+
path: string | null;
|
|
13
|
+
secure: boolean;
|
|
14
|
+
httpOnly: boolean;
|
|
15
|
+
extensions: string[] | null;
|
|
16
|
+
creation: Date | 'Infinity' | null;
|
|
17
|
+
creationIndex: number;
|
|
18
|
+
hostOnly: boolean | null;
|
|
19
|
+
pathIsDefault: boolean | null;
|
|
20
|
+
lastAccessed: Date | 'Infinity' | null;
|
|
21
|
+
sameSite: string | undefined;
|
|
22
|
+
toJSON(): Record<string, unknown>;
|
|
23
|
+
clone(): SessionCookie | undefined;
|
|
24
|
+
validate(): boolean;
|
|
25
|
+
setExpires(exp: string | Date): void;
|
|
26
|
+
setMaxAge(age: number): void;
|
|
27
|
+
cookieString(): string;
|
|
28
|
+
toString(): string;
|
|
29
|
+
TTL(now?: number): number;
|
|
30
|
+
expiryTime(now?: Date): number | undefined;
|
|
31
|
+
expiryDate(now?: Date): Date | undefined;
|
|
32
|
+
isPersistent(): boolean;
|
|
33
|
+
canonicalizedDomain(): string | undefined;
|
|
34
|
+
cdomain(): string | undefined;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Options for {@link CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s
|
|
38
|
+
* `SetCookieOptions`.
|
|
39
|
+
*/
|
|
40
|
+
export interface CookieJarSetCookieOptions {
|
|
41
|
+
loose?: boolean;
|
|
42
|
+
sameSiteContext?: 'strict' | 'lax' | 'none';
|
|
43
|
+
ignoreError?: boolean;
|
|
44
|
+
http?: boolean;
|
|
45
|
+
now?: Date;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Options for {@link CookieJar}'s cookie-reading methods. Structurally compatible with
|
|
49
|
+
* `tough-cookie`'s `GetCookiesOptions`.
|
|
50
|
+
*/
|
|
51
|
+
export interface CookieJarGetCookiesOptions {
|
|
52
|
+
http?: boolean;
|
|
53
|
+
expire?: boolean;
|
|
54
|
+
allPaths?: boolean;
|
|
55
|
+
sameSiteContext?: 'none' | 'lax' | 'strict';
|
|
56
|
+
sort?: boolean;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is
|
|
60
|
+
* what backs {@link Session} by default — so `@crawlee/types` does not have to depend on
|
|
61
|
+
* `tough-cookie` for it.
|
|
62
|
+
*/
|
|
63
|
+
export interface CookieJar {
|
|
64
|
+
setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
|
|
65
|
+
getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
|
|
66
|
+
getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
|
|
67
|
+
getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
|
|
68
|
+
serialize(): Promise<SerializedCookieJar>;
|
|
69
|
+
toJSON(): SerializedCookieJar | undefined;
|
|
70
|
+
clone(): Promise<CookieJar>;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* JSON representation of a {@link CookieJar}. Structurally compatible with `tough-cookie`'s
|
|
74
|
+
* `SerializedCookieJar`.
|
|
75
|
+
*/
|
|
76
|
+
export interface SerializedCookieJar {
|
|
77
|
+
version: string;
|
|
78
|
+
storeType: string | null;
|
|
79
|
+
rejectPublicSuffixes: boolean;
|
|
80
|
+
cookies: Record<string, unknown>[];
|
|
81
|
+
[key: string]: unknown;
|
|
82
|
+
}
|
package/cookies.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/http-client.d.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { Readable } from 'node:stream';
|
|
2
|
+
import type { CookieJar } from './cookies.js';
|
|
3
|
+
import type { ISession } from './session.js';
|
|
4
|
+
import type { AllowedHttpMethods } from './utility-types.js';
|
|
5
|
+
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
|
|
6
|
+
/**
|
|
7
|
+
* HTTP Request as accepted by {@link BaseHttpClient} methods.
|
|
8
|
+
*/
|
|
9
|
+
export interface HttpRequest {
|
|
10
|
+
url: string | URL;
|
|
11
|
+
method?: AllowedHttpMethods;
|
|
12
|
+
headers?: Headers;
|
|
13
|
+
body?: Readable;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
timeout?: number;
|
|
16
|
+
cookieJar?: CookieJar;
|
|
17
|
+
followRedirect?: boolean | ((response: any) => boolean);
|
|
18
|
+
maxRedirects?: number;
|
|
19
|
+
encoding?: BufferEncoding;
|
|
20
|
+
throwHttpErrors?: boolean;
|
|
21
|
+
proxyUrl?: string;
|
|
22
|
+
headerGeneratorOptions?: Record<string, unknown>;
|
|
23
|
+
useHeaderGenerator?: boolean;
|
|
24
|
+
headerGenerator?: {
|
|
25
|
+
getHeaders: (options: Record<string, unknown>) => Record<string, string>;
|
|
26
|
+
};
|
|
27
|
+
insecureHTTPParser?: boolean;
|
|
28
|
+
sessionToken?: object;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Additional options for HTTP requests that need to be handled separately before passing to {@link BaseHttpClient}.
|
|
32
|
+
*/
|
|
33
|
+
export interface HttpRequestOptions extends HttpRequest {
|
|
34
|
+
/** Search (query string) parameters to be appended to the request URL */
|
|
35
|
+
searchParams?: SearchParams;
|
|
36
|
+
/** A form to be sent in the HTTP request body (URL encoding will be used) */
|
|
37
|
+
form?: Record<string, string>;
|
|
38
|
+
/** Arbitrary object to be JSON-serialized and sent as the HTTP request body */
|
|
39
|
+
json?: unknown;
|
|
40
|
+
/** Basic HTTP Auth username */
|
|
41
|
+
username?: string;
|
|
42
|
+
/** Basic HTTP Auth password */
|
|
43
|
+
password?: string;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Type of a function called when an HTTP redirect takes place. It is allowed to mutate the `updatedRequest` argument.
|
|
47
|
+
*/
|
|
48
|
+
export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
|
|
49
|
+
url?: string | URL;
|
|
50
|
+
headers: Headers;
|
|
51
|
+
}) => void;
|
|
52
|
+
export interface SendRequestOptions {
|
|
53
|
+
session?: ISession;
|
|
54
|
+
cookieJar?: CookieJar;
|
|
55
|
+
/** Timeout for the HTTP request in milliseconds. */
|
|
56
|
+
timeoutMillis?: number;
|
|
57
|
+
/** An AbortSignal to cancel the HTTP request. */
|
|
58
|
+
signal?: AbortSignal;
|
|
59
|
+
/**
|
|
60
|
+
* Overrides the proxy URL set in the `session` for this request.
|
|
61
|
+
*
|
|
62
|
+
* Note that setting this manually can interfere with session proxy rotation.
|
|
63
|
+
*/
|
|
64
|
+
proxyUrl?: string;
|
|
65
|
+
/**
|
|
66
|
+
* When `true`, TLS certificate errors are ignored for this request. Also enabled by
|
|
67
|
+
* `session.proxyInfo.ignoreTlsErrors` (MITM proxies). Best-effort: clients that cannot
|
|
68
|
+
* disable TLS verification (e.g. the native fetch fallback) ignore it.
|
|
69
|
+
*/
|
|
70
|
+
ignoreTlsErrors?: boolean;
|
|
71
|
+
}
|
|
72
|
+
export interface StreamOptions extends SendRequestOptions {
|
|
73
|
+
onRedirect?: RedirectHandler;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Interface for user-defined HTTP clients to be used for plain HTTP crawling and for sending additional requests during a crawl.
|
|
77
|
+
*/
|
|
78
|
+
export interface BaseHttpClient {
|
|
79
|
+
/**
|
|
80
|
+
* Perform an HTTP Request and return the complete response.
|
|
81
|
+
*/
|
|
82
|
+
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
|
83
|
+
}
|
package/http-client.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/index.d.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
|
|
1
|
+
export type * from './status-message.js';
|
|
2
|
+
export type * from './storages.js';
|
|
3
|
+
export type * from './utility-types.js';
|
|
4
|
+
export type * from './browser.js';
|
|
5
|
+
export type * from './cookies.js';
|
|
6
|
+
export type * from './http-client.js';
|
|
7
|
+
export type * from './session.js';
|
|
8
|
+
export type * from './logger.js';
|
package/index.js
CHANGED
package/logger.d.ts
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration options for Crawlee logger implementations.
|
|
3
|
+
*/
|
|
4
|
+
export interface CrawleeLoggerOptions {
|
|
5
|
+
/** Prefix to be prepended to each logged line. */
|
|
6
|
+
prefix?: string | null;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Interface for Crawlee logger implementations.
|
|
10
|
+
* This allows users to inject custom loggers (e.g., Winston, Pino) while maintaining
|
|
11
|
+
* compatibility with the default `@apify/log` implementation.
|
|
12
|
+
*/
|
|
13
|
+
export interface CrawleeLogger {
|
|
14
|
+
/**
|
|
15
|
+
* Returns the logger configuration.
|
|
16
|
+
*/
|
|
17
|
+
getOptions(): CrawleeLoggerOptions;
|
|
18
|
+
/**
|
|
19
|
+
* Configures logger options.
|
|
20
|
+
*/
|
|
21
|
+
setOptions(options: Partial<CrawleeLoggerOptions>): void;
|
|
22
|
+
/**
|
|
23
|
+
* Creates a new instance of logger that inherits settings from a parent logger.
|
|
24
|
+
*/
|
|
25
|
+
child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
|
|
26
|
+
/**
|
|
27
|
+
* Logs an `ERROR` message.
|
|
28
|
+
*/
|
|
29
|
+
error(message: string, data?: Record<string, unknown>): void;
|
|
30
|
+
/**
|
|
31
|
+
* Logs an `ERROR` level message with a nicely formatted exception.
|
|
32
|
+
*/
|
|
33
|
+
exception(exception: Error, message: string, data?: Record<string, unknown>): void;
|
|
34
|
+
/**
|
|
35
|
+
* Logs a `SOFT_FAIL` level message.
|
|
36
|
+
*/
|
|
37
|
+
softFail(message: string, data?: Record<string, unknown>): void;
|
|
38
|
+
/**
|
|
39
|
+
* Logs a `WARNING` level message.
|
|
40
|
+
*/
|
|
41
|
+
warning(message: string, data?: Record<string, unknown>): void;
|
|
42
|
+
/**
|
|
43
|
+
* Logs a `WARNING` level message only once.
|
|
44
|
+
*/
|
|
45
|
+
warningOnce(message: string): void;
|
|
46
|
+
/**
|
|
47
|
+
* Logs an `INFO` message.
|
|
48
|
+
*/
|
|
49
|
+
info(message: string, data?: Record<string, unknown>): void;
|
|
50
|
+
/**
|
|
51
|
+
* Logs a `DEBUG` message.
|
|
52
|
+
*/
|
|
53
|
+
debug(message: string, data?: Record<string, unknown>): void;
|
|
54
|
+
/**
|
|
55
|
+
* Logs a `PERF` level message for performance tracking.
|
|
56
|
+
*/
|
|
57
|
+
perf(message: string, data?: Record<string, unknown>): void;
|
|
58
|
+
/**
|
|
59
|
+
* Logs given message only once as WARNING for deprecated features.
|
|
60
|
+
*/
|
|
61
|
+
deprecated(message: string): void;
|
|
62
|
+
/**
|
|
63
|
+
* Logs a message at the given level. Useful when the log level is determined dynamically.
|
|
64
|
+
*/
|
|
65
|
+
logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
|
|
66
|
+
}
|
package/logger.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/types",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.140",
|
|
4
4
|
"description": "Shared types for the crawlee projects",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
},
|
|
35
35
|
"homepage": "https://crawlee.dev",
|
|
36
36
|
"scripts": {
|
|
37
|
-
"build": "
|
|
37
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
38
38
|
"clean": "rimraf ./dist",
|
|
39
39
|
"compile": "tsc -p tsconfig.build.json",
|
|
40
40
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -52,5 +52,5 @@
|
|
|
52
52
|
}
|
|
53
53
|
}
|
|
54
54
|
},
|
|
55
|
-
"gitHead": "
|
|
55
|
+
"gitHead": "70f846b4f05d89b34ccfded676e766fdbbdc5e66"
|
|
56
56
|
}
|
package/session.d.ts
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import type { CookieJar, SerializedCookieJar } from './cookies.js';
|
|
2
|
+
/**
|
|
3
|
+
* The main purpose of the ProxyInfo object is to provide information
|
|
4
|
+
* about the current proxy connection used by the crawler for the request.
|
|
5
|
+
* Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
|
|
6
|
+
*
|
|
7
|
+
* **Example usage:**
|
|
8
|
+
*
|
|
9
|
+
* ```javascript
|
|
10
|
+
* const proxyConfiguration = new ProxyConfiguration({
|
|
11
|
+
* proxyUrls: ['...', '...'] // List of Proxy URLs to rotate
|
|
12
|
+
* });
|
|
13
|
+
*
|
|
14
|
+
* // Getting proxyInfo object by calling class method directly
|
|
15
|
+
* const proxyInfo = await proxyConfiguration.newProxyInfo();
|
|
16
|
+
*
|
|
17
|
+
* // In crawler
|
|
18
|
+
* const crawler = new CheerioCrawler({
|
|
19
|
+
* // ...
|
|
20
|
+
* proxyConfiguration,
|
|
21
|
+
* requestHandler({ proxyInfo }) {
|
|
22
|
+
* // Getting used proxy URL
|
|
23
|
+
* const proxyUrl = proxyInfo.url;
|
|
24
|
+
* }
|
|
25
|
+
* })
|
|
26
|
+
*
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
export interface ProxyInfo {
|
|
30
|
+
/**
|
|
31
|
+
* The URL of the proxy.
|
|
32
|
+
*/
|
|
33
|
+
url: string;
|
|
34
|
+
/**
|
|
35
|
+
* Username for the proxy.
|
|
36
|
+
*/
|
|
37
|
+
username?: string;
|
|
38
|
+
/**
|
|
39
|
+
* User's password for the proxy.
|
|
40
|
+
*/
|
|
41
|
+
password: string;
|
|
42
|
+
/**
|
|
43
|
+
* Hostname of your proxy.
|
|
44
|
+
*/
|
|
45
|
+
hostname: string;
|
|
46
|
+
/**
|
|
47
|
+
* Proxy port.
|
|
48
|
+
*/
|
|
49
|
+
port: number | string;
|
|
50
|
+
/**
|
|
51
|
+
* When `true`, the proxy is likely intercepting HTTPS traffic and is able to view and modify its content.
|
|
52
|
+
* The built-in HTTP clients and the browser pool disable TLS certificate verification for the session's
|
|
53
|
+
* requests when this is set.
|
|
54
|
+
*
|
|
55
|
+
* @default false
|
|
56
|
+
*/
|
|
57
|
+
ignoreTlsErrors?: boolean;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Identifies the browser-like profile a {@link Session} is impersonating, so
|
|
61
|
+
* repeated requests with the same session look consistent to the target server.
|
|
62
|
+
*
|
|
63
|
+
* These fields are *hints* — `browser`, `platform`, `device`. Consumers
|
|
64
|
+
* (`@crawlee/browser-pool`, `@crawlee/impit-client`, …) derive their own rich
|
|
65
|
+
* state from them (e.g. a full browser fingerprint, a TLS impersonation profile)
|
|
66
|
+
* and cache it on their own; the session itself is read-only intent.
|
|
67
|
+
*/
|
|
68
|
+
export interface SessionFingerprint {
|
|
69
|
+
/** Browser family — consumed by HTTP clients that impersonate (e.g. `impit`). */
|
|
70
|
+
browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
|
|
71
|
+
/** Platform hint — used by header generators and as a virtual session key. */
|
|
72
|
+
platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
|
|
73
|
+
/** Device class — drives header generation and viewport defaults. */
|
|
74
|
+
device?: 'desktop' | 'mobile';
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Persistable {@link Session} state.
|
|
78
|
+
*/
|
|
79
|
+
export interface SessionState {
|
|
80
|
+
id: string;
|
|
81
|
+
cookieJar: SerializedCookieJar;
|
|
82
|
+
proxyInfo?: ProxyInfo;
|
|
83
|
+
userData: object;
|
|
84
|
+
fingerprint?: SessionFingerprint;
|
|
85
|
+
errorScore: number;
|
|
86
|
+
maxErrorScore: number;
|
|
87
|
+
errorScoreDecrement: number;
|
|
88
|
+
usageCount: number;
|
|
89
|
+
maxUsageCount: number;
|
|
90
|
+
expiresAt: string;
|
|
91
|
+
createdAt: string;
|
|
92
|
+
retired: boolean;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
96
|
+
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
97
|
+
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
98
|
+
* @category Scaling
|
|
99
|
+
*/
|
|
100
|
+
export interface ISession {
|
|
101
|
+
readonly id: string;
|
|
102
|
+
cookieJar: CookieJar;
|
|
103
|
+
proxyInfo?: ProxyInfo;
|
|
104
|
+
fingerprint?: SessionFingerprint;
|
|
105
|
+
/**
|
|
106
|
+
* Indicates whether the session can be used for next requests.
|
|
107
|
+
* Session is usable when it is not expired, not blocked and the maximum usage count has not been reached.
|
|
108
|
+
*/
|
|
109
|
+
isUsable(): boolean;
|
|
110
|
+
/**
|
|
111
|
+
* This method should be called after a successful session usage.
|
|
112
|
+
*/
|
|
113
|
+
markGood(): void;
|
|
114
|
+
/**
|
|
115
|
+
* Marks session as blocked.
|
|
116
|
+
* This method should be used if the session usage was unsuccessful
|
|
117
|
+
* and you are sure that it is because of the session configuration and not any external matters.
|
|
118
|
+
* For example when server returns 403 status code.
|
|
119
|
+
* If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
|
|
120
|
+
*/
|
|
121
|
+
retire(): void;
|
|
122
|
+
/**
|
|
123
|
+
* Increases usage and error count.
|
|
124
|
+
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
125
|
+
*/
|
|
126
|
+
markBad(): void;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Minimal contract that any object passed to a crawler as its `sessionPool` option must satisfy.
|
|
130
|
+
*
|
|
131
|
+
* Crawlers only depend on a single method of the built-in `SessionPool`: `getSession()` /
|
|
132
|
+
* `getSession(id)` to hand out an {@link ISession} for a request. Lifecycle (reset, teardown)
|
|
133
|
+
* is the responsibility of whoever owns the pool — since a user-supplied pool is never owned by
|
|
134
|
+
* the crawler, the crawler never tears it down.
|
|
135
|
+
*
|
|
136
|
+
* Implement this interface to plug a custom session-management strategy into any Crawlee crawler —
|
|
137
|
+
* for example a remote, multi-process pool, a database-backed pool, or a thin wrapper around the
|
|
138
|
+
* built-in `SessionPool` with different rotation rules.
|
|
139
|
+
*
|
|
140
|
+
* @category Scaling
|
|
141
|
+
*/
|
|
142
|
+
export interface ISessionPool {
|
|
143
|
+
/**
|
|
144
|
+
* Returns a usable {@link ISession}. Without an id, the pool decides which session to return
|
|
145
|
+
* (creating a new one when appropriate). With an id, the pool returns the matching session if
|
|
146
|
+
* it is still usable.
|
|
147
|
+
*
|
|
148
|
+
* In case the `SessionPool` cannot provide a usable session given the configuration,
|
|
149
|
+
* this method may return `undefined`.
|
|
150
|
+
*/
|
|
151
|
+
getSession(sessionId?: string): Promise<ISession | undefined>;
|
|
152
|
+
}
|
package/session.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Options for setting a crawler run's status message via {@link BasicCrawler.setStatusMessage}.
|
|
3
|
+
*
|
|
4
|
+
* Setting a status message is not a storage concern — the crawler broadcasts it through the event
|
|
5
|
+
* system (`EventType.STATUS_MESSAGE`), and integrations such as the Apify SDK forward it to their
|
|
6
|
+
* status-reporting backend.
|
|
7
|
+
*/
|
|
8
|
+
export interface SetStatusMessageOptions {
|
|
9
|
+
/** Whether this is the final status message of the run. */
|
|
10
|
+
isStatusMessageTerminal?: boolean;
|
|
11
|
+
/** The log level to log the message with. Defaults to `'DEBUG'`. */
|
|
12
|
+
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
|
13
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/storages.d.ts
CHANGED
|
@@ -11,16 +11,13 @@ export interface QueueOperationInfo {
|
|
|
11
11
|
/** The ID of the added request */
|
|
12
12
|
requestId: string;
|
|
13
13
|
}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
modifiedAt: Date;
|
|
22
|
-
accessedAt: Date;
|
|
23
|
-
}
|
|
14
|
+
/**
|
|
15
|
+
* A single page of items returned by {@link DatasetBackend.getData}.
|
|
16
|
+
*
|
|
17
|
+
* Datasets paginate by offset, so a page is self-describing via `total` / `offset` / `limit`: a
|
|
18
|
+
* frontend assembling all pages knows it has reached the end once `offset + items.length >= total`.
|
|
19
|
+
* The cursor-based counterpart for key-value stores is {@link KeyValueStoreListKeysResult}.
|
|
20
|
+
*/
|
|
24
21
|
export interface PaginatedList<Data> {
|
|
25
22
|
/** Total count of entries in the dataset. */
|
|
26
23
|
total: number;
|
|
@@ -35,20 +32,7 @@ export interface PaginatedList<Data> {
|
|
|
35
32
|
/** Dataset entries based on chosen format parameter. */
|
|
36
33
|
items: Data[];
|
|
37
34
|
}
|
|
38
|
-
export interface
|
|
39
|
-
itemCount: number;
|
|
40
|
-
}
|
|
41
|
-
/**
|
|
42
|
-
* Dataset collection client.
|
|
43
|
-
*/
|
|
44
|
-
export interface DatasetCollectionClient {
|
|
45
|
-
list(): Promise<PaginatedList<Dataset>>;
|
|
46
|
-
getOrCreate(name?: string): Promise<DatasetCollectionData>;
|
|
47
|
-
}
|
|
48
|
-
export interface DatasetClientUpdateOptions {
|
|
49
|
-
name?: string;
|
|
50
|
-
}
|
|
51
|
-
export interface DatasetClientListOptions {
|
|
35
|
+
export interface DatasetBackendListOptions {
|
|
52
36
|
desc?: boolean;
|
|
53
37
|
limit?: number;
|
|
54
38
|
offset?: number;
|
|
@@ -60,164 +44,158 @@ export interface DatasetInfo {
|
|
|
60
44
|
modifiedAt: Date;
|
|
61
45
|
accessedAt: Date;
|
|
62
46
|
itemCount: number;
|
|
63
|
-
actId?: string;
|
|
64
|
-
actRunId?: string;
|
|
65
47
|
}
|
|
66
|
-
export interface
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
deleteCount?: number;
|
|
84
|
-
listCount?: number;
|
|
85
|
-
storageBytes?: number;
|
|
48
|
+
export interface DatasetBackend<Data extends Dictionary = Dictionary> {
|
|
49
|
+
/**
|
|
50
|
+
* Returns metadata about the dataset (id, name, timestamps, item count, etc.).
|
|
51
|
+
*
|
|
52
|
+
* Implementations should throw if the underlying storage no longer exists
|
|
53
|
+
* (e.g. it was deleted externally). This method should never return stale data
|
|
54
|
+
* for a storage that has been removed.
|
|
55
|
+
*/
|
|
56
|
+
getMetadata(): Promise<DatasetInfo>;
|
|
57
|
+
/** Remove the dataset and all its data. */
|
|
58
|
+
drop(): Promise<void>;
|
|
59
|
+
/** Remove all items from the dataset but keep the dataset itself. */
|
|
60
|
+
purge(): Promise<void>;
|
|
61
|
+
/** Add items to the dataset. */
|
|
62
|
+
pushData(items: Data[]): Promise<void>;
|
|
63
|
+
/** Fetch a page of items from the dataset. */
|
|
64
|
+
getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Data>>;
|
|
86
65
|
}
|
|
87
66
|
export interface KeyValueStoreInfo {
|
|
88
67
|
id: string;
|
|
89
68
|
name?: string;
|
|
90
|
-
userId?: string;
|
|
91
69
|
createdAt: Date;
|
|
92
70
|
modifiedAt: Date;
|
|
93
71
|
accessedAt: Date;
|
|
94
|
-
actId?: string;
|
|
95
|
-
actRunId?: string;
|
|
96
|
-
stats?: KeyValueStoreStats;
|
|
97
72
|
}
|
|
98
73
|
/**
|
|
99
|
-
*
|
|
74
|
+
* The value a serialized record carries on the way *into* a storage backend (`setValue`).
|
|
75
|
+
*
|
|
76
|
+
* The `KeyValueStore` frontend has already serialized by this point, so it is a pre-serialized
|
|
77
|
+
* string, raw bytes (`Buffer` / `ArrayBuffer` / typed array), or a stream the client drains.
|
|
78
|
+
*/
|
|
79
|
+
export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
|
|
80
|
+
/**
|
|
81
|
+
* A record as returned by a storage backend (`getValue`).
|
|
82
|
+
*
|
|
83
|
+
* Storage backends are byte transports: they persist and return bytes verbatim and never serialize
|
|
84
|
+
* or parse. Interpretation is the `KeyValueStore` frontend's job (it parses according to the content
|
|
85
|
+
* type via `parseValue`). The value is therefore always raw bytes — a `Buffer` (Node) or an
|
|
86
|
+
* `ArrayBuffer` (browser backends).
|
|
100
87
|
*/
|
|
101
|
-
export interface KeyValueStoreCollectionClient {
|
|
102
|
-
list(): Promise<PaginatedList<KeyValueStoreInfo>>;
|
|
103
|
-
getOrCreate(name?: string): Promise<KeyValueStoreInfo>;
|
|
104
|
-
}
|
|
105
88
|
export interface KeyValueStoreRecord {
|
|
106
89
|
key: string;
|
|
107
|
-
value:
|
|
90
|
+
value: Buffer | ArrayBuffer;
|
|
108
91
|
contentType?: string;
|
|
109
92
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
export interface
|
|
115
|
-
|
|
93
|
+
/**
|
|
94
|
+
* A record passed to a storage backend for writing (`setValue`). Like {@link KeyValueStoreRecord} but
|
|
95
|
+
* with the lenient, pre-serialized {@link KeyValueStoreRecordInputValue} for the value.
|
|
96
|
+
*/
|
|
97
|
+
export interface KeyValueStoreInputRecord {
|
|
98
|
+
key: string;
|
|
99
|
+
value: KeyValueStoreRecordInputValue;
|
|
100
|
+
contentType?: string;
|
|
116
101
|
}
|
|
117
|
-
export interface
|
|
118
|
-
|
|
119
|
-
exclusiveStartKey?: string;
|
|
120
|
-
collection?: string;
|
|
102
|
+
export interface KeyValueStoreListKeysOptions {
|
|
103
|
+
/** If set, only keys that start with this prefix are returned. */
|
|
121
104
|
prefix?: string;
|
|
105
|
+
/** All keys up to this one are skipped from the result. */
|
|
106
|
+
exclusiveStartKey?: string;
|
|
107
|
+
/** Maximum number of keys to return. */
|
|
108
|
+
limit?: number;
|
|
122
109
|
}
|
|
123
110
|
export interface KeyValueStoreItemData {
|
|
124
111
|
key: string;
|
|
125
112
|
size: number;
|
|
113
|
+
/** The MIME content type the record was stored with, if known. */
|
|
114
|
+
contentType: string;
|
|
126
115
|
}
|
|
127
|
-
|
|
116
|
+
/**
|
|
117
|
+
* A single page of keys returned by {@link KeyValueStoreBackend.listKeys}.
|
|
118
|
+
*
|
|
119
|
+
* This mirrors {@link PaginatedList} (the shape returned by {@link DatasetBackend.getData}) so that
|
|
120
|
+
* both listing operations on the storage backend layer return a self-describing page. The difference
|
|
121
|
+
* is the pagination model: datasets are offset-based (`total` / `offset`), whereas key-value stores
|
|
122
|
+
* are cursor-based. A frontend assembling all pages should therefore not guess "is this the last
|
|
123
|
+
* page?" from `items.length < limit` — it should rely on {@link isTruncated} and resume from
|
|
124
|
+
* {@link nextExclusiveStartKey}.
|
|
125
|
+
*/
|
|
126
|
+
export interface KeyValueStoreListKeysResult {
|
|
127
|
+
/** Keys returned on this page. */
|
|
128
|
+
items: KeyValueStoreItemData[];
|
|
129
|
+
/** Number of keys returned on this page (`items.length`). */
|
|
128
130
|
count: number;
|
|
131
|
+
/** Maximum number of keys requested for this page. */
|
|
129
132
|
limit: number;
|
|
133
|
+
/** The `exclusiveStartKey` that produced this page, if any. */
|
|
130
134
|
exclusiveStartKey?: string;
|
|
135
|
+
/** `true` if there are more keys beyond this page. When `true`, {@link nextExclusiveStartKey} is set. */
|
|
131
136
|
isTruncated: boolean;
|
|
137
|
+
/** Cursor to pass as the next call's `exclusiveStartKey`, or `undefined` when {@link isTruncated} is `false`. */
|
|
132
138
|
nextExclusiveStartKey?: string;
|
|
133
|
-
items: KeyValueStoreItemData[];
|
|
134
|
-
}
|
|
135
|
-
export interface KeyValueStoreClientGetRecordOptions {
|
|
136
|
-
buffer?: boolean;
|
|
137
|
-
stream?: boolean;
|
|
138
139
|
}
|
|
139
140
|
/**
|
|
140
141
|
* Key-value Store client.
|
|
141
142
|
*/
|
|
142
|
-
export interface
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
143
|
+
export interface KeyValueStoreBackend {
|
|
144
|
+
/**
|
|
145
|
+
* Returns metadata about the key-value store (id, name, timestamps, etc.).
|
|
146
|
+
*
|
|
147
|
+
* Implementations should throw if the underlying storage no longer exists
|
|
148
|
+
* (e.g. it was deleted externally). This method should never return stale data
|
|
149
|
+
* for a storage that has been removed.
|
|
150
|
+
*/
|
|
151
|
+
getMetadata(): Promise<KeyValueStoreInfo>;
|
|
152
|
+
/** Remove the key-value store and all its data. */
|
|
153
|
+
drop(): Promise<void>;
|
|
154
|
+
/** Remove all records from the store but keep the store itself. */
|
|
155
|
+
purge(): Promise<void>;
|
|
156
|
+
/**
|
|
157
|
+
* Get a record by key. Returns the raw bytes plus content type, or `undefined` if not found.
|
|
158
|
+
*
|
|
159
|
+
* Clients are byte transports and must not parse the body — the `KeyValueStore` frontend
|
|
160
|
+
* interprets it according to the content type.
|
|
161
|
+
*/
|
|
162
|
+
getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
|
|
163
|
+
/** Set a record value. */
|
|
164
|
+
setValue(record: KeyValueStoreInputRecord): Promise<void>;
|
|
165
|
+
/** Delete a record by key. */
|
|
166
|
+
deleteValue(key: string): Promise<void>;
|
|
167
|
+
/**
|
|
168
|
+
* List a single page of keys in the store. Returns at most `limit` keys starting after
|
|
169
|
+
* `exclusiveStartKey`, wrapped in a self-describing page.
|
|
170
|
+
*
|
|
171
|
+
* Like {@link DatasetBackend.getData}, this returns one page rather than the whole collection;
|
|
172
|
+
* assembling all pages (e.g. for `KeyValueStore.keys()`) is the frontend's job. The result carries
|
|
173
|
+
* a cursor (`isTruncated` / `nextExclusiveStartKey`) so the frontend can paginate deterministically
|
|
174
|
+
* instead of inferring the end from `items.length < limit`.
|
|
175
|
+
*/
|
|
176
|
+
listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
|
|
177
|
+
/** Get the public URL for a record, or `undefined` if unavailable. */
|
|
178
|
+
getPublicUrl(key: string): Promise<string | undefined>;
|
|
179
|
+
/** Check whether a record with the given key exists. */
|
|
147
180
|
recordExists(key: string): Promise<boolean>;
|
|
148
|
-
getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
|
|
149
|
-
setRecord(record: KeyValueStoreRecord, options?: KeyValueStoreRecordOptions): Promise<void>;
|
|
150
|
-
deleteRecord(key: string): Promise<void>;
|
|
151
|
-
}
|
|
152
|
-
export interface RequestQueueStats {
|
|
153
|
-
readCount?: number;
|
|
154
|
-
writeCount?: number;
|
|
155
|
-
deleteCount?: number;
|
|
156
|
-
headItemReadCount?: number;
|
|
157
|
-
storageBytes?: number;
|
|
158
181
|
}
|
|
159
182
|
export interface RequestQueueInfo {
|
|
160
183
|
id: string;
|
|
161
184
|
name?: string;
|
|
162
|
-
userId?: string;
|
|
163
185
|
createdAt: Date;
|
|
164
186
|
modifiedAt: Date;
|
|
165
187
|
accessedAt: Date;
|
|
166
|
-
expireAt?: string;
|
|
167
188
|
totalRequestCount: number;
|
|
168
189
|
handledRequestCount: number;
|
|
169
190
|
pendingRequestCount: number;
|
|
170
|
-
actId?: string;
|
|
171
|
-
actRunId?: string;
|
|
172
|
-
hadMultipleClients?: boolean;
|
|
173
|
-
stats?: RequestQueueStats;
|
|
174
191
|
}
|
|
175
192
|
/**
|
|
176
|
-
*
|
|
193
|
+
* Options for request-queue operations that add or return requests to the queue
|
|
194
|
+
* ({@link RequestQueueBackend.addBatchOfRequests}, {@link RequestQueueBackend.reclaimRequest}).
|
|
177
195
|
*/
|
|
178
|
-
export interface
|
|
179
|
-
|
|
180
|
-
getOrCreate(name: string): Promise<RequestQueueInfo>;
|
|
181
|
-
}
|
|
182
|
-
export interface RequestQueueHeadItem {
|
|
183
|
-
id: string;
|
|
184
|
-
retryCount: number;
|
|
185
|
-
uniqueKey: string;
|
|
186
|
-
url: string;
|
|
187
|
-
method: AllowedHttpMethods;
|
|
188
|
-
}
|
|
189
|
-
export interface QueueHead {
|
|
190
|
-
limit: number;
|
|
191
|
-
queueModifiedAt: Date;
|
|
192
|
-
hadMultipleClients?: boolean;
|
|
193
|
-
items: RequestQueueHeadItem[];
|
|
194
|
-
}
|
|
195
|
-
export interface ListOptions {
|
|
196
|
-
/**
|
|
197
|
-
* @default 100
|
|
198
|
-
*/
|
|
199
|
-
limit?: number;
|
|
200
|
-
}
|
|
201
|
-
export interface ListAndLockOptions extends ListOptions {
|
|
202
|
-
lockSecs: number;
|
|
203
|
-
}
|
|
204
|
-
export interface ListAndLockHeadResult extends QueueHead {
|
|
205
|
-
lockSecs: number;
|
|
206
|
-
queueHasLockedRequests?: boolean;
|
|
207
|
-
}
|
|
208
|
-
export interface ProlongRequestLockOptions {
|
|
209
|
-
lockSecs: number;
|
|
210
|
-
forefront?: boolean;
|
|
211
|
-
}
|
|
212
|
-
export interface ProlongRequestLockResult {
|
|
213
|
-
lockExpiresAt: Date;
|
|
214
|
-
}
|
|
215
|
-
export interface DeleteRequestLockOptions {
|
|
216
|
-
forefront?: boolean;
|
|
217
|
-
}
|
|
218
|
-
export interface RequestOptions {
|
|
196
|
+
export interface RequestQueueOperationOptions {
|
|
197
|
+
/** Place the affected request(s) at the beginning of the queue so they are processed sooner. */
|
|
219
198
|
forefront?: boolean;
|
|
220
|
-
[k: string]: unknown;
|
|
221
199
|
}
|
|
222
200
|
export interface RequestSchema {
|
|
223
201
|
id?: string;
|
|
@@ -251,45 +229,195 @@ export interface BatchAddRequestsResult {
|
|
|
251
229
|
processedRequests: ProcessedRequest[];
|
|
252
230
|
unprocessedRequests: UnprocessedRequest[];
|
|
253
231
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
232
|
+
/**
|
|
233
|
+
* Operations on a single request queue.
|
|
234
|
+
*
|
|
235
|
+
* A backend implementation owns all request bookkeeping (pending, in-progress, handled). Any
|
|
236
|
+
* coordination required between multiple distributed clients accessing the same queue (e.g. request
|
|
237
|
+
* locking on the Apify platform) is an internal concern of the implementation and is not exposed on
|
|
238
|
+
* this interface.
|
|
239
|
+
*/
|
|
240
|
+
export interface RequestQueueBackend {
|
|
241
|
+
/**
|
|
242
|
+
* Returns metadata about the request queue (id, name, timestamps, request counts, etc.).
|
|
243
|
+
*
|
|
244
|
+
* Implementations should throw if the underlying storage no longer exists
|
|
245
|
+
* (e.g. it was deleted externally). This method should never return stale data
|
|
246
|
+
* for a storage that has been removed.
|
|
247
|
+
*/
|
|
248
|
+
getMetadata(): Promise<RequestQueueInfo>;
|
|
249
|
+
/** Remove the request queue and all its data. */
|
|
250
|
+
drop(): Promise<void>;
|
|
251
|
+
/** Remove all requests from the queue but keep the queue itself. */
|
|
252
|
+
purge(): Promise<void>;
|
|
253
|
+
/**
|
|
254
|
+
* Add a batch of requests to the queue.
|
|
255
|
+
*
|
|
256
|
+
* Each request is deduplicated by its `uniqueKey`. Duplicates are reported in the result
|
|
257
|
+
* but not re-added. With `forefront`, requests are placed at the beginning of the queue so
|
|
258
|
+
* they are processed sooner.
|
|
259
|
+
*/
|
|
260
|
+
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
261
|
+
/**
|
|
262
|
+
* Retrieve a request from the queue by its `uniqueKey`, or `undefined` if it does not exist.
|
|
263
|
+
*/
|
|
264
|
+
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
|
|
265
|
+
/**
|
|
266
|
+
* Return the next request in the queue to be processed, or `undefined` if there are currently no
|
|
267
|
+
* pending requests.
|
|
268
|
+
*
|
|
269
|
+
* The returned request is marked as in-progress; it will not be returned again until it is
|
|
270
|
+
* either reclaimed via {@link reclaimRequest} or marked as handled via {@link markRequestAsHandled}.
|
|
271
|
+
*
|
|
272
|
+
* An `undefined` return value does not mean processing is finished — only that there are no pending
|
|
273
|
+
* requests right now. Use {@link isEmpty} (together with the frontend's knowledge of pending
|
|
274
|
+
* add operations) to determine whether the queue is truly finished.
|
|
275
|
+
*/
|
|
276
|
+
fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
|
|
277
|
+
/**
|
|
278
|
+
* Mark a request previously returned by {@link fetchNextRequest} as handled.
|
|
279
|
+
*
|
|
280
|
+
* Handled requests are never returned again by {@link fetchNextRequest}. Returns information
|
|
281
|
+
* about the operation, or `undefined` if the request was not in progress.
|
|
282
|
+
*
|
|
283
|
+
* An `undefined` result is a no-op, not an error: the request is simply not something this client is
|
|
284
|
+
* currently processing, so nothing is changed and the request is never added to the queue as a side
|
|
285
|
+
* effect. (Marking an already-handled request is idempotent and still returns operation info with
|
|
286
|
+
* `wasAlreadyHandled: true` rather than `undefined`.)
|
|
287
|
+
*/
|
|
288
|
+
markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
|
|
289
|
+
/**
|
|
290
|
+
* Reclaim a failed request back to the queue so it can be processed again by a later call to
|
|
291
|
+
* {@link fetchNextRequest}. With `forefront`, the request is returned to the beginning of the
|
|
292
|
+
* queue. Returns information about the operation, or `undefined` if the request was not in progress.
|
|
293
|
+
*
|
|
294
|
+
* The request is expected to already be present in the queue (it should have been obtained via
|
|
295
|
+
* {@link fetchNextRequest}); reclaiming releases its lock rather than inserting it. An `undefined` result
|
|
296
|
+
* is a no-op, not an error: the request is simply not something this client is currently processing,
|
|
297
|
+
* so nothing is changed and the request is never added to the queue as a side effect. Use
|
|
298
|
+
* {@link addBatchOfRequests} to insert a new request.
|
|
299
|
+
*/
|
|
300
|
+
reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
|
|
301
|
+
/**
|
|
302
|
+
* Resolves to `true` if the next call to {@link fetchNextRequest} would return `undefined` — i.e. there
|
|
303
|
+
* are no pending requests to fetch right now.
|
|
304
|
+
*
|
|
305
|
+
* Requests that are currently in progress (fetched but not yet handled or reclaimed, including
|
|
306
|
+
* requests locked by other clients sharing the same queue) are **not** counted. An empty queue
|
|
307
|
+
* therefore does not mean crawling is finished — those in-progress requests may still be reclaimed,
|
|
308
|
+
* and background tasks may still add more requests. Use {@link isFinished} to detect completion.
|
|
309
|
+
*/
|
|
310
|
+
isEmpty(): Promise<boolean>;
|
|
311
|
+
/**
|
|
312
|
+
* Resolves to `true` only when there is no outstanding work left in the queue at all — i.e. there
|
|
313
|
+
* are no pending requests to fetch **and** no requests currently in progress (fetched but not yet
|
|
314
|
+
* handled or reclaimed, including requests locked by other clients sharing the same queue).
|
|
315
|
+
*
|
|
316
|
+
* This is the strong counterpart of {@link isEmpty}: a queue whose only remaining requests are in
|
|
317
|
+
* progress is empty (nothing to fetch) but not finished (that work might still be reclaimed). It is
|
|
318
|
+
* the building block for determining whether crawling is done — though a frontend may still need to
|
|
319
|
+
* account for its own pending background add operations on top of this.
|
|
320
|
+
*/
|
|
321
|
+
isFinished(): Promise<boolean>;
|
|
322
|
+
/**
|
|
323
|
+
* Tells the client how long (in seconds) a consumer expects to hold a request fetched via
|
|
324
|
+
* {@link fetchNextRequest} before marking it handled or reclaiming it — typically the consumer's
|
|
325
|
+
* request-processing timeout plus some padding.
|
|
326
|
+
*
|
|
327
|
+
* A client that coordinates consumers via locking uses this to keep the request reserved for at least
|
|
328
|
+
* this long, so that a long-running consumer does not have its request handed out again while it is
|
|
329
|
+
* still being processed. Clients that do not lock may ignore it.
|
|
330
|
+
*/
|
|
331
|
+
setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
|
|
277
332
|
}
|
|
278
333
|
/**
|
|
279
|
-
*
|
|
334
|
+
* Identifies a storage by its ID, name, or alias. At most one may be provided.
|
|
335
|
+
*
|
|
336
|
+
* - `{ id }` — open a pre-existing storage by its unique ID.
|
|
337
|
+
* - `{ name }` — open or create a globally named storage (persists across runs). The name `default`
|
|
338
|
+
* is reserved: it resolves to the default storage, and is emptied on start along with it.
|
|
339
|
+
* - `{ alias }` — open or create a run-scoped unnamed storage identified by this alias.
|
|
340
|
+
* The alias is used locally (e.g. as a directory name or cache key) but the storage
|
|
341
|
+
* itself has no persistent name. Use this for non-default unnamed storages. Like the
|
|
342
|
+
* default storage, an aliased one is emptied on start unless `purgeOnStart` is disabled.
|
|
343
|
+
* - `{}` / omitted — open the default storage.
|
|
344
|
+
*/
|
|
345
|
+
export type StorageIdentifier = {
|
|
346
|
+
id: string;
|
|
347
|
+
name?: never;
|
|
348
|
+
alias?: never;
|
|
349
|
+
} | {
|
|
350
|
+
id?: never;
|
|
351
|
+
name: string;
|
|
352
|
+
alias?: never;
|
|
353
|
+
} | {
|
|
354
|
+
id?: never;
|
|
355
|
+
name?: never;
|
|
356
|
+
alias: string;
|
|
357
|
+
} | {
|
|
358
|
+
id?: never;
|
|
359
|
+
name?: never;
|
|
360
|
+
alias?: never;
|
|
361
|
+
};
|
|
362
|
+
/**
|
|
363
|
+
* Represents a storage backend capable of working with datasets, key-value stores and request queues.
|
|
364
|
+
*
|
|
365
|
+
* A new storage backend needs to implement 4 classes:
|
|
366
|
+
* - `StorageBackend` - the factory that creates sub-backends
|
|
367
|
+
* - `DatasetBackend` - operations on a single dataset
|
|
368
|
+
* - `KeyValueStoreBackend` - operations on a single key-value store
|
|
369
|
+
* - `RequestQueueBackend` - operations on a single request queue
|
|
370
|
+
*
|
|
371
|
+
* The `StorageBackend` acts as an async factory: each `create*` method either opens an existing
|
|
372
|
+
* storage or creates a new one, returning a sub-backend bound to that storage instance.
|
|
280
373
|
*/
|
|
281
|
-
export interface
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
374
|
+
export interface StorageBackend {
|
|
375
|
+
/**
|
|
376
|
+
* Create (or open) a dataset backend.
|
|
377
|
+
* If `id` is provided, opens the dataset with that ID.
|
|
378
|
+
* If `name` is provided, opens an existing dataset with that name or creates a new one.
|
|
379
|
+
* If neither is provided, opens or creates the default dataset.
|
|
380
|
+
*/
|
|
381
|
+
createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
|
|
382
|
+
/**
|
|
383
|
+
* Create (or open) a key-value store backend.
|
|
384
|
+
* If `id` is provided, opens the key-value store with that ID.
|
|
385
|
+
* If `name` is provided, opens an existing store with that name or creates a new one.
|
|
386
|
+
* If neither is provided, opens or creates the default key-value store.
|
|
387
|
+
*/
|
|
388
|
+
createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
|
|
389
|
+
/**
|
|
390
|
+
* Create (or open) a request queue backend.
|
|
391
|
+
* If `id` is provided, opens the request queue with that ID.
|
|
392
|
+
* If `name` is provided, opens an existing queue with that name or creates a new one.
|
|
393
|
+
* If neither is provided, opens or creates the default request queue.
|
|
394
|
+
*/
|
|
395
|
+
createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
|
|
396
|
+
/**
|
|
397
|
+
* Check whether a storage with the given ID exists.
|
|
398
|
+
*
|
|
399
|
+
* Used internally to resolve ambiguous `idOrName` strings passed to `Dataset.open()`,
|
|
400
|
+
* `KeyValueStore.open()`, and `RequestQueue.open()`.
|
|
401
|
+
*/
|
|
402
|
+
storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
|
|
403
|
+
/**
|
|
404
|
+
* Return an opaque key that uniquely identifies this storage backend instance.
|
|
405
|
+
*
|
|
406
|
+
* The key is used by `StorageInstanceManager` to partition the storage cache per-backend,
|
|
407
|
+
* so that two storages with the same name but backed by different clients
|
|
408
|
+
* (e.g. a local `MemoryStorageBackend` and a cloud `ApifyClient`) are cached as separate instances.
|
|
409
|
+
*
|
|
410
|
+
* When not provided, the fallback uses the client's constructor name, so different
|
|
411
|
+
* `StorageBackend` implementations automatically get separate cache partitions.
|
|
412
|
+
*/
|
|
413
|
+
getStorageBackendCacheKey?(): string;
|
|
414
|
+
/**
|
|
415
|
+
* Empty the run-scoped storages — the default one and every alias-keyed one, including any left
|
|
416
|
+
* behind by a previous run. Named storages persist across runs, as does the default store's `INPUT`.
|
|
417
|
+
*/
|
|
288
418
|
purge?(): Promise<void>;
|
|
289
419
|
teardown?(): Promise<void>;
|
|
290
|
-
setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;
|
|
291
420
|
stats?: {
|
|
292
421
|
rateLimitErrors: number[];
|
|
293
422
|
};
|
|
294
423
|
}
|
|
295
|
-
//# sourceMappingURL=storages.d.ts.map
|
package/storages.js
CHANGED
package/utility-types.d.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
/** @ignore */
|
|
2
1
|
export type Dictionary<T = any> = Record<PropertyKey, T>;
|
|
3
2
|
/** @ignore */
|
|
4
3
|
export type Constructor<T = unknown> = new (...args: any[]) => T;
|
|
5
4
|
/** @ignore */
|
|
6
5
|
export type Awaitable<T> = T | PromiseLike<T>;
|
|
7
6
|
export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';
|
|
8
|
-
//# sourceMappingURL=utility-types.d.ts.map
|
package/utility-types.js
CHANGED
package/browser.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,WAAW,MAAM;IACnB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;OAEG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IACrC;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IACrC;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC;IAChD;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAChC,GAAG,IAAI,MAAM,CAAC;IACd,OAAO,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;CAC5C"}
|
package/browser.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":""}
|
package/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC"}
|
package/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC"}
|
package/storages.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"storages.d.ts","sourceRoot":"","sources":["../src/storages.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEzE;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,8BAA8B;IAC3C,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;CACpB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI;IAC/B,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,OAAQ,SAAQ,qBAAqB;IAClD,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACpC,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IACxC,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;CAC9D;AAED,MAAM,WAAW,0BAA0B;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACrC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AACD,MAAM,WAAW,YAAY;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAC/D,GAAG,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC;IACxC,MAAM,CAAC,SAAS,EAAE,0BAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7E,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,aAAa,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,SAAS,CAAC,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5E,SAAS,CAAC,KAAK,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC1C,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAClD,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,mBAAmB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,GAAG,CAAC;IACX,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,0BAA0B;IACvC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,gCAAgC;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,2BAA2B;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC;IACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,KAAK,EAAE,qBAAqB,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,mCAAmC;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,GAAG,IAAI,OAAO,CAAC,iBAAiB,GAAG,SAAS,CAAC,CAAC;IAC9C,MAAM,CAAC,SAAS,EAAE,gCAAgC,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,8BAA8B,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACzF,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mCAAmC,GAAG,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC;IAChH,SAAS,CAAC,MAAM,EAAE,mBAAmB,EAAE,OAAO,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5F,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,iBAAiB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,IAAI,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,KAAK,CAAC,EAAE,iBAAiB,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IACzC,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACjD,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,IAAI,CAAC;IACtB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,KAAK,EAAE,oBAAoB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,WAAW;IACxB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAmB,SAAQ,WAAW;IACnD,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,qBAAsB,SAAQ,SAAS;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,OAAO,CAAC;CACpC;AAED,MAAM,WAAW,yBAAyB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACrC,aAAa,EAAE,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACrC,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC3B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC7B,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,mBAAoB,SAAQ,aAAa;IACtD,EAAE,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC/B;AAED,MAAM,WAAW,sBAAsB;IACnC,iBAAiB,EAAE,gBAAgB,EAAE,CAAC;IACtC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;CAC7C;AAED,MAAM,WAAW,kBAAkB;IAC/B,GAAG,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IAC7C,MAAM,CAAC,SAAS,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC,CAAC;IACrF,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpD,UAAU,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1F,gBAAgB,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACvG,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IAC5D,aAAa,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACnG,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,eAAe,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC7E,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;IACtG,iBAAiB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpF;AAED,MAAM,WAAW,mBAAmB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACpC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,KAAK,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;CAClD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,QAAQ,IAAI,uBAAuB,CAAC;IACpC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa,CAAC;IACnC,cAAc,IAAI,6BAA6B,CAAC;IAChD,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,mBAAmB,CAAC;IAC/C,aAAa,IAAI,4BAA4B,CAAC;IAC9C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,CAAC;IAC5E,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,gBAAgB,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrF,KAAK,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACzC"}
|
package/storages.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"storages.js","sourceRoot":"","sources":["../src/storages.ts"],"names":[],"mappings":""}
|
package/utility-types.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utility-types.d.ts","sourceRoot":"","sources":["../src/utility-types.ts"],"names":[],"mappings":"AAAA,cAAc;AACd,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAEzD,cAAc;AACd,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI,KAAK,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAEjE,cAAc;AACd,MAAM,MAAM,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;AAE9C,MAAM,MAAM,kBAAkB,GACxB,KAAK,GACL,MAAM,GACN,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,OAAO,GACP,KAAK,GACL,MAAM,GACN,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,SAAS,GACT,SAAS,GACT,OAAO,CAAC"}
|
package/utility-types.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utility-types.js","sourceRoot":"","sources":["../src/utility-types.ts"],"names":[],"mappings":""}
|