@crawlee/core 4.0.0-beta.7 → 4.0.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/autoscaling/autoscaled_pool.d.ts +3 -5
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +3 -9
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -13
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +18 -29
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +0 -3
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +2 -3
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +88 -227
- package/configuration.d.ts.map +1 -1
- package/configuration.js +162 -223
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +4 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +18 -12
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +71 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +123 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +20 -29
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +12 -20
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +2 -2
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -1
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/crawlers/statistics.d.ts +16 -16
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +22 -25
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +36 -21
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +47 -26
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +25 -8
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +69 -37
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +33 -3
- package/errors.d.ts.map +1 -1
- package/errors.js +48 -4
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +34 -7
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +8 -9
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts +14 -4
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +33 -39
- package/events/local_event_manager.js.map +1 -1
- package/index.d.ts +4 -2
- package/index.d.ts.map +1 -1
- package/index.js +3 -1
- package/index.js.map +1 -1
- package/log.d.ts +82 -2
- package/log.d.ts.map +1 -1
- package/log.js +102 -0
- package/log.js.map +1 -1
- package/memory-storage/consts.d.ts +5 -0
- package/memory-storage/consts.d.ts.map +1 -0
- package/memory-storage/consts.js +5 -0
- package/memory-storage/consts.js.map +1 -0
- package/memory-storage/index.d.ts +2 -0
- package/memory-storage/index.d.ts.map +1 -0
- package/memory-storage/index.js +2 -0
- package/memory-storage/index.js.map +1 -0
- package/memory-storage/memory-storage.d.ts +47 -0
- package/memory-storage/memory-storage.d.ts.map +1 -0
- package/memory-storage/memory-storage.js +137 -0
- package/memory-storage/memory-storage.js.map +1 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +5 -0
- package/memory-storage/resource-clients/common/base-client.d.ts.map +1 -0
- package/memory-storage/resource-clients/common/base-client.js +7 -0
- package/memory-storage/resource-clients/common/base-client.js.map +1 -0
- package/memory-storage/resource-clients/dataset.d.ts +41 -0
- package/memory-storage/resource-clients/dataset.d.ts.map +1 -0
- package/memory-storage/resource-clients/dataset.js +114 -0
- package/memory-storage/resource-clients/dataset.js.map +1 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +64 -0
- package/memory-storage/resource-clients/key-value-store.d.ts.map +1 -0
- package/memory-storage/resource-clients/key-value-store.js +204 -0
- package/memory-storage/resource-clients/key-value-store.js.map +1 -0
- package/memory-storage/resource-clients/request-queue.d.ts +97 -0
- package/memory-storage/resource-clients/request-queue.d.ts.map +1 -0
- package/memory-storage/resource-clients/request-queue.js +422 -0
- package/memory-storage/resource-clients/request-queue.js.map +1 -0
- package/memory-storage/utils.d.ts +12 -0
- package/memory-storage/utils.d.ts.map +1 -0
- package/memory-storage/utils.js +28 -0
- package/memory-storage/utils.js.map +1 -0
- package/package.json +12 -10
- package/proxy_configuration.d.ts +14 -148
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +19 -167
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +121 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +142 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +84 -12
- package/request.d.ts.map +1 -1
- package/request.js +107 -27
- package/request.js.map +1 -1
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/serialization.js +1 -1
- package/serialization.js.map +1 -1
- package/service_locator.d.ts +157 -0
- package/service_locator.d.ts.map +1 -0
- package/service_locator.js +239 -0
- package/service_locator.js.map +1 -0
- package/session_pool/consts.d.ts +1 -1
- package/session_pool/consts.d.ts.map +1 -1
- package/session_pool/consts.js +1 -1
- package/session_pool/consts.js.map +1 -1
- package/session_pool/fingerprint.d.ts +10 -0
- package/session_pool/fingerprint.d.ts.map +1 -0
- package/session_pool/fingerprint.js +31 -0
- package/session_pool/fingerprint.js.map +1 -0
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +35 -72
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +46 -98
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +71 -71
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +111 -99
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +5 -1
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +98 -47
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +166 -123
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +8 -6
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +6 -6
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +163 -33
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +267 -119
- package/storages/key_value_store.js.map +1 -1
- package/storages/key_value_store_codec.d.ts +33 -0
- package/storages/key_value_store_codec.d.ts.map +1 -0
- package/storages/key_value_store_codec.js +114 -0
- package/storages/key_value_store_codec.js.map +1 -0
- package/storages/request_list.d.ts +32 -81
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +49 -39
- package/storages/request_list.js.map +1 -1
- package/storages/request_loader.d.ts +97 -0
- package/storages/request_loader.d.ts.map +1 -0
- package/storages/request_loader.js +2 -0
- package/storages/request_loader.js.map +1 -0
- package/storages/request_manager.d.ts +34 -0
- package/storages/request_manager.d.ts.map +1 -0
- package/storages/request_manager.js +2 -0
- package/storages/request_manager.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +107 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +198 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_queue.d.ts +282 -45
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +592 -213
- package/storages/request_queue.js.map +1 -1
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +29 -24
- package/storages/sitemap_request_loader.d.ts.map +1 -0
- package/storages/{sitemap_request_list.js → sitemap_request_loader.js} +50 -46
- package/storages/sitemap_request_loader.js.map +1 -0
- package/storages/storage_instance_manager.d.ts +92 -0
- package/storages/storage_instance_manager.d.ts.map +1 -0
- package/storages/storage_instance_manager.js +261 -0
- package/storages/storage_instance_manager.js.map +1 -0
- package/storages/storage_stats.d.ts +49 -0
- package/storages/storage_stats.d.ts.map +1 -0
- package/storages/storage_stats.js +30 -0
- package/storages/storage_stats.js.map +1 -0
- package/storages/utils.d.ts +51 -5
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +60 -8
- package/storages/utils.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/validators.d.ts +8 -0
- package/validators.d.ts.map +1 -1
- package/validators.js +8 -0
- package/validators.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -134
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -33
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -15
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -69
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/storages/request_provider.d.ts +0 -307
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -555
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -1,23 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { Cookie, SerializedCookieJar } from 'tough-cookie';
|
|
1
|
+
import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
|
|
3
2
|
import { CookieJar } from 'tough-cookie';
|
|
4
|
-
import type {
|
|
5
|
-
import type { ResponseLike } from '../cookie_utils.js';
|
|
6
|
-
/**
|
|
7
|
-
* Persistable {@link Session} state.
|
|
8
|
-
*/
|
|
9
|
-
export interface SessionState {
|
|
10
|
-
id: string;
|
|
11
|
-
cookieJar: SerializedCookieJar;
|
|
12
|
-
userData: object;
|
|
13
|
-
errorScore: number;
|
|
14
|
-
maxErrorScore: number;
|
|
15
|
-
errorScoreDecrement: number;
|
|
16
|
-
usageCount: number;
|
|
17
|
-
maxUsageCount: number;
|
|
18
|
-
expiresAt: string;
|
|
19
|
-
createdAt: string;
|
|
20
|
-
}
|
|
3
|
+
import type { CrawleeLogger } from '../log.js';
|
|
21
4
|
export interface SessionOptions {
|
|
22
5
|
/** Id of session used for generating fingerprints. It is used as proxy session name. */
|
|
23
6
|
id?: string;
|
|
@@ -58,11 +41,22 @@ export interface SessionOptions {
|
|
|
58
41
|
* @default 50
|
|
59
42
|
*/
|
|
60
43
|
maxUsageCount?: number;
|
|
61
|
-
/**
|
|
62
|
-
|
|
63
|
-
|
|
44
|
+
/**
|
|
45
|
+
* Marks the session as already retired. Used when restoring a previously persisted session
|
|
46
|
+
* so that `isUsable()` reflects the terminal state regardless of error score or usage count.
|
|
47
|
+
* @default false
|
|
48
|
+
*/
|
|
49
|
+
retired?: boolean;
|
|
50
|
+
log?: CrawleeLogger;
|
|
64
51
|
errorScore?: number;
|
|
65
52
|
cookieJar?: CookieJar;
|
|
53
|
+
proxyInfo?: ProxyInfo;
|
|
54
|
+
/**
|
|
55
|
+
* Browser / HTTP client fingerprint tied to this session. Backends use this to make
|
|
56
|
+
* repeated requests with the same session look consistent (same user-agent, headers,
|
|
57
|
+
* TLS profile). See {@link SessionFingerprint}.
|
|
58
|
+
*/
|
|
59
|
+
fingerprint?: SessionFingerprint;
|
|
66
60
|
}
|
|
67
61
|
/**
|
|
68
62
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
@@ -70,9 +64,8 @@ export interface SessionOptions {
|
|
|
70
64
|
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
71
65
|
* @category Scaling
|
|
72
66
|
*/
|
|
73
|
-
export declare class Session {
|
|
67
|
+
export declare class Session implements ISession {
|
|
74
68
|
readonly id: string;
|
|
75
|
-
private maxAgeSecs;
|
|
76
69
|
userData: Dictionary;
|
|
77
70
|
private _maxErrorScore;
|
|
78
71
|
private _errorScoreDecrement;
|
|
@@ -80,9 +73,11 @@ export declare class Session {
|
|
|
80
73
|
private _expiresAt;
|
|
81
74
|
private _usageCount;
|
|
82
75
|
private _maxUsageCount;
|
|
83
|
-
private sessionPool;
|
|
84
76
|
private _errorScore;
|
|
77
|
+
private _retired;
|
|
78
|
+
private _proxyInfo?;
|
|
85
79
|
private _cookieJar;
|
|
80
|
+
private _fingerprint?;
|
|
86
81
|
private log;
|
|
87
82
|
get errorScore(): number;
|
|
88
83
|
get usageCount(): number;
|
|
@@ -92,10 +87,18 @@ export declare class Session {
|
|
|
92
87
|
get createdAt(): Date;
|
|
93
88
|
get maxUsageCount(): number;
|
|
94
89
|
get cookieJar(): CookieJar;
|
|
90
|
+
get proxyInfo(): ProxyInfo | undefined;
|
|
91
|
+
get fingerprint(): SessionFingerprint | undefined;
|
|
92
|
+
set fingerprint(fingerprint: SessionFingerprint | undefined);
|
|
93
|
+
/**
|
|
94
|
+
* `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
|
|
95
|
+
* a retired session is never picked by the pool and cannot be revived via `markGood()`.
|
|
96
|
+
*/
|
|
97
|
+
get retired(): boolean;
|
|
95
98
|
/**
|
|
96
99
|
* Session configuration.
|
|
97
100
|
*/
|
|
98
|
-
constructor(options
|
|
101
|
+
constructor(options?: SessionOptions);
|
|
99
102
|
/**
|
|
100
103
|
* Indicates whether the session is blocked.
|
|
101
104
|
* Session is blocked once it reaches the `maxErrorScore`.
|
|
@@ -114,7 +117,7 @@ export declare class Session {
|
|
|
114
117
|
isMaxUsageCountReached(): boolean;
|
|
115
118
|
/**
|
|
116
119
|
* Indicates whether the session can be used for next requests.
|
|
117
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
|
|
120
|
+
* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
|
|
118
121
|
*/
|
|
119
122
|
isUsable(): boolean;
|
|
120
123
|
/**
|
|
@@ -128,11 +131,11 @@ export declare class Session {
|
|
|
128
131
|
*/
|
|
129
132
|
getState(): SessionState;
|
|
130
133
|
/**
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
134
|
+
* Permanently retires the session — `isUsable()` will return `false` from here on,
|
|
135
|
+
* and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
|
|
136
|
+
*
|
|
137
|
+
* Use this when you're confident the session itself is the problem (e.g. a `403` response).
|
|
138
|
+
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
|
|
136
139
|
*/
|
|
137
140
|
retire(): void;
|
|
138
141
|
/**
|
|
@@ -140,42 +143,6 @@ export declare class Session {
|
|
|
140
143
|
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
141
144
|
*/
|
|
142
145
|
markBad(): void;
|
|
143
|
-
/**
|
|
144
|
-
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
145
|
-
* that the target website is blocking us. This function helps to do this conveniently
|
|
146
|
-
* by retiring the session when such code is received. Optionally, the default status
|
|
147
|
-
* codes can be extended in the second parameter.
|
|
148
|
-
* @param statusCode HTTP status code.
|
|
149
|
-
* @returns Whether the session was retired.
|
|
150
|
-
*/
|
|
151
|
-
retireOnBlockedStatusCodes(statusCode: number): boolean;
|
|
152
|
-
/**
|
|
153
|
-
* Saves cookies from an HTTP response to be used with the session.
|
|
154
|
-
* It expects an object with a `headers` property that's either an `Object`
|
|
155
|
-
* (typical Node.js responses) or a `Function` (Puppeteer Response).
|
|
156
|
-
*
|
|
157
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
158
|
-
*/
|
|
159
|
-
setCookiesFromResponse(response: ResponseLike): void;
|
|
160
|
-
/**
|
|
161
|
-
* Saves an array with cookie objects to be used with the session.
|
|
162
|
-
* The objects should be in the format that
|
|
163
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
164
|
-
* but you can also use this function to set cookies manually:
|
|
165
|
-
*
|
|
166
|
-
* ```
|
|
167
|
-
* [
|
|
168
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
169
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
170
|
-
* ]
|
|
171
|
-
* ```
|
|
172
|
-
*/
|
|
173
|
-
setCookies(cookies: CookieObject[], url: string): void;
|
|
174
|
-
/**
|
|
175
|
-
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
176
|
-
* @param url website url. Only cookies stored for this url will be returned
|
|
177
|
-
*/
|
|
178
|
-
getCookies(url: string): CookieObject[];
|
|
179
146
|
/**
|
|
180
147
|
* Returns cookies saved with the session in the typical
|
|
181
148
|
* key1=value1; key2=value2 format, ready to be used in
|
|
@@ -187,10 +154,6 @@ export declare class Session {
|
|
|
187
154
|
* Sets a cookie within this session for the specific URL.
|
|
188
155
|
*/
|
|
189
156
|
setCookie(rawCookie: string, url: string): void;
|
|
190
|
-
/**
|
|
191
|
-
* Sets cookies.
|
|
192
|
-
*/
|
|
193
|
-
protected _setCookies(cookies: Cookie[], url: string): void;
|
|
194
157
|
/**
|
|
195
158
|
* Checks if session is not usable. if it is not retires the session.
|
|
196
159
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAExG,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAKzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAG/C,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,GAAG,CAAC,EAAE,aAAa,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;;;OAIG;IACH,WAAW,CAAC,EAAE,kBAAkB,CAAC;CACpC;AAED;;;;;GAKG;AACH,qBAAa,OAAQ,YAAW,QAAQ;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAC,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,YAAY,CAAC,CAAqB;IAC1C,OAAO,CAAC,GAAG,CAAgB;IAE3B,IAAI,UAAU,WAEb;IAED,IAAI,UAAU,WAEb;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,mBAAmB,WAEtB;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,SAAS,0BAEZ;IAED,IAAI,WAAW,IAAI,kBAAkB,GAAG,SAAS,CAEhD;IAED,IAAI,WAAW,CAAC,WAAW,EAAE,kBAAkB,GAAG,SAAS,EAE1D;IAED;;;OAGG;IACH,IAAI,OAAO,YAEV;IAED;;OAEG;gBACS,OAAO,GAAE,cAAmB;IA4DxC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAkBxB;;;;;;OAMG;IACH,MAAM;IAON;;;OAGG;IACH,OAAO;IAOP;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAQ/C;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import { EventEmitter } from 'node:events';
|
|
2
1
|
import ow from 'ow';
|
|
3
2
|
import { CookieJar } from 'tough-cookie';
|
|
4
3
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { EVENT_SESSION_RETIRED } from './events.js';
|
|
4
|
+
import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
|
|
5
|
+
import { serviceLocator } from '../service_locator.js';
|
|
8
6
|
/**
|
|
9
7
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
10
8
|
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
@@ -13,7 +11,6 @@ import { EVENT_SESSION_RETIRED } from './events.js';
|
|
|
13
11
|
*/
|
|
14
12
|
export class Session {
|
|
15
13
|
id;
|
|
16
|
-
maxAgeSecs;
|
|
17
14
|
userData;
|
|
18
15
|
_maxErrorScore;
|
|
19
16
|
_errorScoreDecrement;
|
|
@@ -21,9 +18,11 @@ export class Session {
|
|
|
21
18
|
_expiresAt;
|
|
22
19
|
_usageCount;
|
|
23
20
|
_maxUsageCount;
|
|
24
|
-
sessionPool;
|
|
25
21
|
_errorScore;
|
|
22
|
+
_retired = false;
|
|
23
|
+
_proxyInfo;
|
|
26
24
|
_cookieJar;
|
|
25
|
+
_fingerprint;
|
|
27
26
|
log;
|
|
28
27
|
get errorScore() {
|
|
29
28
|
return this._errorScore;
|
|
@@ -49,14 +48,30 @@ export class Session {
|
|
|
49
48
|
get cookieJar() {
|
|
50
49
|
return this._cookieJar;
|
|
51
50
|
}
|
|
51
|
+
get proxyInfo() {
|
|
52
|
+
return this._proxyInfo;
|
|
53
|
+
}
|
|
54
|
+
get fingerprint() {
|
|
55
|
+
return this._fingerprint;
|
|
56
|
+
}
|
|
57
|
+
set fingerprint(fingerprint) {
|
|
58
|
+
this._fingerprint = fingerprint;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
|
|
62
|
+
* a retired session is never picked by the pool and cannot be revived via `markGood()`.
|
|
63
|
+
*/
|
|
64
|
+
get retired() {
|
|
65
|
+
return this._retired;
|
|
66
|
+
}
|
|
52
67
|
/**
|
|
53
68
|
* Session configuration.
|
|
54
69
|
*/
|
|
55
|
-
constructor(options) {
|
|
70
|
+
constructor(options = {}) {
|
|
56
71
|
ow(options, ow.object.exactShape({
|
|
57
|
-
sessionPool: ow.object.instanceOf(EventEmitter),
|
|
58
72
|
id: ow.optional.string,
|
|
59
73
|
cookieJar: ow.optional.object,
|
|
74
|
+
proxyInfo: ow.optional.object,
|
|
60
75
|
maxAgeSecs: ow.optional.number,
|
|
61
76
|
userData: ow.optional.object,
|
|
62
77
|
maxErrorScore: ow.optional.number,
|
|
@@ -66,14 +81,17 @@ export class Session {
|
|
|
66
81
|
usageCount: ow.optional.number,
|
|
67
82
|
errorScore: ow.optional.number,
|
|
68
83
|
maxUsageCount: ow.optional.number,
|
|
84
|
+
retired: ow.optional.boolean,
|
|
69
85
|
log: ow.optional.object,
|
|
86
|
+
fingerprint: ow.optional.object,
|
|
70
87
|
}));
|
|
71
|
-
const {
|
|
88
|
+
const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
|
|
72
89
|
const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
|
|
73
90
|
this.log = log.child({ prefix: 'Session' });
|
|
74
91
|
this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
92
|
+
this._proxyInfo = proxyInfo;
|
|
93
|
+
this._fingerprint = fingerprint;
|
|
75
94
|
this.id = id;
|
|
76
|
-
this.maxAgeSecs = maxAgeSecs;
|
|
77
95
|
this.userData = userData;
|
|
78
96
|
this._maxErrorScore = maxErrorScore;
|
|
79
97
|
this._errorScoreDecrement = errorScoreDecrement;
|
|
@@ -83,7 +101,7 @@ export class Session {
|
|
|
83
101
|
this._usageCount = usageCount; // indicates how many times the session has been used
|
|
84
102
|
this._errorScore = errorScore; // indicates number of markBaded request with the session
|
|
85
103
|
this._maxUsageCount = maxUsageCount;
|
|
86
|
-
this.
|
|
104
|
+
this._retired = retired;
|
|
87
105
|
}
|
|
88
106
|
/**
|
|
89
107
|
* Indicates whether the session is blocked.
|
|
@@ -109,10 +127,10 @@ export class Session {
|
|
|
109
127
|
}
|
|
110
128
|
/**
|
|
111
129
|
* Indicates whether the session can be used for next requests.
|
|
112
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
|
|
130
|
+
* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
|
|
113
131
|
*/
|
|
114
132
|
isUsable() {
|
|
115
|
-
return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
|
|
133
|
+
return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
|
|
116
134
|
}
|
|
117
135
|
/**
|
|
118
136
|
* This method should be called after a successful session usage.
|
|
@@ -133,7 +151,9 @@ export class Session {
|
|
|
133
151
|
return {
|
|
134
152
|
id: this.id,
|
|
135
153
|
cookieJar: this.cookieJar.toJSON(),
|
|
154
|
+
proxyInfo: this._proxyInfo,
|
|
136
155
|
userData: this.userData,
|
|
156
|
+
fingerprint: this._fingerprint,
|
|
137
157
|
maxErrorScore: this.maxErrorScore,
|
|
138
158
|
errorScoreDecrement: this.errorScoreDecrement,
|
|
139
159
|
expiresAt: this.expiresAt.toISOString(),
|
|
@@ -141,21 +161,22 @@ export class Session {
|
|
|
141
161
|
usageCount: this.usageCount,
|
|
142
162
|
maxUsageCount: this.maxUsageCount,
|
|
143
163
|
errorScore: this.errorScore,
|
|
164
|
+
retired: this._retired,
|
|
144
165
|
};
|
|
145
166
|
}
|
|
146
167
|
/**
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
168
|
+
* Permanently retires the session — `isUsable()` will return `false` from here on,
|
|
169
|
+
* and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
|
|
170
|
+
*
|
|
171
|
+
* Use this when you're confident the session itself is the problem (e.g. a `403` response).
|
|
172
|
+
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
|
|
152
173
|
*/
|
|
153
174
|
retire() {
|
|
154
|
-
|
|
175
|
+
if (this._retired)
|
|
176
|
+
return;
|
|
155
177
|
this._errorScore += this._maxErrorScore;
|
|
156
178
|
this._usageCount += 1;
|
|
157
|
-
|
|
158
|
-
this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
|
|
179
|
+
this._retired = true;
|
|
159
180
|
}
|
|
160
181
|
/**
|
|
161
182
|
* Increases usage and error count.
|
|
@@ -166,65 +187,6 @@ export class Session {
|
|
|
166
187
|
this._usageCount += 1;
|
|
167
188
|
this._maybeSelfRetire();
|
|
168
189
|
}
|
|
169
|
-
/**
|
|
170
|
-
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
171
|
-
* that the target website is blocking us. This function helps to do this conveniently
|
|
172
|
-
* by retiring the session when such code is received. Optionally, the default status
|
|
173
|
-
* codes can be extended in the second parameter.
|
|
174
|
-
* @param statusCode HTTP status code.
|
|
175
|
-
* @returns Whether the session was retired.
|
|
176
|
-
*/
|
|
177
|
-
retireOnBlockedStatusCodes(statusCode) {
|
|
178
|
-
// eslint-disable-next-line dot-notation -- accessing private property
|
|
179
|
-
const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
|
|
180
|
-
if (isBlocked) {
|
|
181
|
-
this.retire();
|
|
182
|
-
}
|
|
183
|
-
return isBlocked;
|
|
184
|
-
}
|
|
185
|
-
/**
|
|
186
|
-
* Saves cookies from an HTTP response to be used with the session.
|
|
187
|
-
* It expects an object with a `headers` property that's either an `Object`
|
|
188
|
-
* (typical Node.js responses) or a `Function` (Puppeteer Response).
|
|
189
|
-
*
|
|
190
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
191
|
-
*/
|
|
192
|
-
setCookiesFromResponse(response) {
|
|
193
|
-
try {
|
|
194
|
-
const cookies = getCookiesFromResponse(response).filter((c) => c);
|
|
195
|
-
this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
|
|
196
|
-
}
|
|
197
|
-
catch (e) {
|
|
198
|
-
const err = e;
|
|
199
|
-
// if invalid Cookie header is provided just log the exception.
|
|
200
|
-
this.log.exception(err, 'Could not get cookies from response');
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Saves an array with cookie objects to be used with the session.
|
|
205
|
-
* The objects should be in the format that
|
|
206
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
207
|
-
* but you can also use this function to set cookies manually:
|
|
208
|
-
*
|
|
209
|
-
* ```
|
|
210
|
-
* [
|
|
211
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
212
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
213
|
-
* ]
|
|
214
|
-
* ```
|
|
215
|
-
*/
|
|
216
|
-
setCookies(cookies, url) {
|
|
217
|
-
const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
|
|
218
|
-
this._setCookies(normalizedCookies, url);
|
|
219
|
-
}
|
|
220
|
-
/**
|
|
221
|
-
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
222
|
-
* @param url website url. Only cookies stored for this url will be returned
|
|
223
|
-
*/
|
|
224
|
-
getCookies(url) {
|
|
225
|
-
const cookies = this.cookieJar.getCookiesSync(url);
|
|
226
|
-
return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
|
|
227
|
-
}
|
|
228
190
|
/**
|
|
229
191
|
* Returns cookies saved with the session in the typical
|
|
230
192
|
* key1=value1; key2=value2 format, ready to be used in
|
|
@@ -238,25 +200,11 @@ export class Session {
|
|
|
238
200
|
* Sets a cookie within this session for the specific URL.
|
|
239
201
|
*/
|
|
240
202
|
setCookie(rawCookie, url) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
/**
|
|
244
|
-
* Sets cookies.
|
|
245
|
-
*/
|
|
246
|
-
_setCookies(cookies, url) {
|
|
247
|
-
const errorMessages = [];
|
|
248
|
-
for (const cookie of cookies) {
|
|
249
|
-
try {
|
|
250
|
-
this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
|
|
251
|
-
}
|
|
252
|
-
catch (e) {
|
|
253
|
-
const err = e;
|
|
254
|
-
errorMessages.push(err.message);
|
|
255
|
-
}
|
|
203
|
+
try {
|
|
204
|
+
this.cookieJar.setCookieSync(rawCookie, url);
|
|
256
205
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
this.log.debug('Could not set cookies.', { errorMessages });
|
|
206
|
+
catch (e) {
|
|
207
|
+
this.log.warning('Could not set cookie.', { url, error: e.message });
|
|
260
208
|
}
|
|
261
209
|
}
|
|
262
210
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EAAE,8BAA8B,EAAE,MAAM,oBAAoB,CAAC;AAEpE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAuEvD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACpB,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAAS;IACpB,QAAQ,GAAG,KAAK,CAAC;IACjB,UAAU,CAAa;IACvB,UAAU,CAAY;IACtB,YAAY,CAAsB;IAClC,GAAG,CAAgB;IAE3B,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,WAAW;QACX,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,IAAI,WAAW,CAAC,WAA2C;QACvD,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,YAAY,UAA0B,EAAE;QACpC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;YAC5B,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAClC,CAAC,CACL,CAAC;QAEF,MAAM,EACF,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,SAAS,GAAG,SAAS,EACrB,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,KAAK,EACf,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,EAChC,WAAW,GACd,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACtG,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,QAAQ;SACzB,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAG,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACpF,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}
|