@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -6
- package/cookie_utils.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -5
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +22 -15
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +33 -14
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +8 -2
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +40 -18
- package/enqueue_links/shared.js.map +1 -1
- package/http_clients/base-http-client.d.ts +11 -5
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +7 -0
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +8 -3
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +48 -32
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/log.d.ts +4 -2
- package/log.d.ts.map +1 -1
- package/log.js.map +1 -1
- package/package.json +6 -6
- package/proxy_configuration.d.ts +7 -9
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +2 -5
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +123 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +140 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +31 -3
- package/request.d.ts.map +1 -1
- package/request.js +35 -4
- package/request.js.map +1 -1
- package/session_pool/session.d.ts +1 -2
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +1 -1
- package/session_pool/session.js.map +1 -1
- package/storages/dataset.d.ts +7 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +10 -1
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +8 -0
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +5 -3
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +0 -6
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +67 -4
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +74 -44
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue_v2.d.ts +2 -2
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/tsconfig.build.tsbuildinfo +0 -1
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { Configuration, KeyValueStore } from '@crawlee/core';
|
|
2
|
+
import log from '@apify/log';
|
|
3
|
+
/**
|
|
4
|
+
* A class for managing persistent recoverable state using a plain JavaScript object.
|
|
5
|
+
*
|
|
6
|
+
* This class facilitates state persistence to a `KeyValueStore`, allowing data to be saved and retrieved
|
|
7
|
+
* across migrations or restarts. It manages the loading, saving, and resetting of state data,
|
|
8
|
+
* with optional persistence capabilities.
|
|
9
|
+
*
|
|
10
|
+
* The state is represented by a plain JavaScript object that can be serialized to and deserialized from JSON.
|
|
11
|
+
* The class automatically hooks into the event system to persist state when needed.
|
|
12
|
+
*/
|
|
13
|
+
export class RecoverableState {
|
|
14
|
+
defaultState;
|
|
15
|
+
state = null;
|
|
16
|
+
persistenceEnabled;
|
|
17
|
+
persistStateKey;
|
|
18
|
+
persistStateKvsName;
|
|
19
|
+
persistStateKvsId;
|
|
20
|
+
keyValueStore = null;
|
|
21
|
+
log;
|
|
22
|
+
config;
|
|
23
|
+
serialize;
|
|
24
|
+
deserialize;
|
|
25
|
+
/**
|
|
26
|
+
* Initialize a new recoverable state object.
|
|
27
|
+
*
|
|
28
|
+
* @param options Configuration options for the recoverable state
|
|
29
|
+
*/
|
|
30
|
+
constructor(options) {
|
|
31
|
+
this.defaultState = options.defaultState;
|
|
32
|
+
this.persistStateKey = options.persistStateKey;
|
|
33
|
+
this.persistenceEnabled = options.persistenceEnabled ?? false;
|
|
34
|
+
this.persistStateKvsName = options.persistStateKvsName;
|
|
35
|
+
this.persistStateKvsId = options.persistStateKvsId;
|
|
36
|
+
this.log = options.logger ?? log.child({ prefix: 'RecoverableState' });
|
|
37
|
+
this.config = options.config ?? Configuration.getGlobalConfig();
|
|
38
|
+
this.serialize = options.serialize ?? JSON.stringify;
|
|
39
|
+
this.deserialize = options.deserialize ?? JSON.parse;
|
|
40
|
+
this.persistState = this.persistState.bind(this);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Initialize the recoverable state.
|
|
44
|
+
*
|
|
45
|
+
* This method must be called before using the recoverable state. It loads the saved state
|
|
46
|
+
* if persistence is enabled and registers the object to listen for PERSIST_STATE events.
|
|
47
|
+
*
|
|
48
|
+
* @returns The loaded state object
|
|
49
|
+
*/
|
|
50
|
+
async initialize() {
|
|
51
|
+
if (this.state !== null && this.state !== undefined) {
|
|
52
|
+
return this.currentValue;
|
|
53
|
+
}
|
|
54
|
+
if (!this.persistenceEnabled) {
|
|
55
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
56
|
+
return this.currentValue;
|
|
57
|
+
}
|
|
58
|
+
this.keyValueStore = await KeyValueStore.open(this.persistStateKvsName ?? this.persistStateKvsId, {
|
|
59
|
+
config: this.config,
|
|
60
|
+
});
|
|
61
|
+
await this.loadSavedState();
|
|
62
|
+
// Register for persist state events
|
|
63
|
+
const eventManager = this.config.getEventManager();
|
|
64
|
+
eventManager.on("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
65
|
+
return this.currentValue;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Clean up resources used by the recoverable state.
|
|
69
|
+
*
|
|
70
|
+
* If persistence is enabled, this method deregisters the object from PERSIST_STATE events
|
|
71
|
+
* and persists the current state one last time.
|
|
72
|
+
*/
|
|
73
|
+
async teardown() {
|
|
74
|
+
if (!this.persistenceEnabled || !this.persistState) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const eventManager = this.config.getEventManager();
|
|
78
|
+
eventManager.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
|
|
79
|
+
await this.persistState();
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Get the current state.
|
|
83
|
+
*/
|
|
84
|
+
get currentValue() {
|
|
85
|
+
if (this.state === null) {
|
|
86
|
+
throw new Error('Recoverable state has not yet been loaded');
|
|
87
|
+
}
|
|
88
|
+
return this.state;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Reset the state to the default values and clear any persisted state.
|
|
92
|
+
*
|
|
93
|
+
* Resets the current state to the default state and, if persistence is enabled,
|
|
94
|
+
* clears the persisted state from the KeyValueStore.
|
|
95
|
+
*/
|
|
96
|
+
async reset() {
|
|
97
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
98
|
+
if (this.persistenceEnabled) {
|
|
99
|
+
if (this.keyValueStore === null) {
|
|
100
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
101
|
+
}
|
|
102
|
+
await this.keyValueStore.setValue(this.persistStateKey, null);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Persist the current state to the KeyValueStore.
|
|
107
|
+
*
|
|
108
|
+
* This method is typically called in response to a PERSIST_STATE event, but can also be called
|
|
109
|
+
* directly when needed.
|
|
110
|
+
*
|
|
111
|
+
* @param eventData Optional data associated with a PERSIST_STATE event
|
|
112
|
+
*/
|
|
113
|
+
async persistState(eventData) {
|
|
114
|
+
this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
|
|
115
|
+
if (this.keyValueStore === null || this.state === null) {
|
|
116
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
117
|
+
}
|
|
118
|
+
if (this.persistenceEnabled) {
|
|
119
|
+
await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
|
|
120
|
+
contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Load the saved state from the KeyValueStore
|
|
126
|
+
*/
|
|
127
|
+
async loadSavedState() {
|
|
128
|
+
if (this.keyValueStore === null) {
|
|
129
|
+
throw new Error('Recoverable state has not yet been initialized');
|
|
130
|
+
}
|
|
131
|
+
const storedState = await this.keyValueStore.getValue(this.persistStateKey);
|
|
132
|
+
if (storedState === null || storedState === undefined) {
|
|
133
|
+
this.state = this.deserialize(this.serialize(this.defaultState));
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
this.state = this.deserialize(storedState);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=recoverable_state.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recoverable_state.js","sourceRoot":"","sources":["../src/recoverable_state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAa,aAAa,EAAE,MAAM,eAAe,CAAC;AAGxE,OAAO,GAAG,MAAM,YAAY,CAAC;AA6D7B;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAgB;IACR,YAAY,CAAc;IACnC,KAAK,GAAuB,IAAI,CAAC;IACxB,kBAAkB,CAAU;IAC5B,eAAe,CAAS;IACxB,mBAAmB,CAAU;IAC7B,iBAAiB,CAAU;IACpC,aAAa,GAAyB,IAAI,CAAC;IAClC,GAAG,CAAM;IACT,MAAM,CAAgB;IACtB,SAAS,CAAiC;IAC1C,WAAW,CAA2C;IAEvE;;;;OAIG;IACH,YAAY,OAA6C;QACrD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,IAAI,KAAK,CAAC;QAC9D,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,CAAC;QAChE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC;QAErD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrD,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC9F,MAAM,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,oCAAoC;QACpC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACnD,YAAY,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO;QACX,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACnD,YAAY,CAAC,GAAG,+CAA0B,IAAI,CAAC,YAAY,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACtE,CAAC;YAED,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY,CAAC,SAAoC;QACnD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBAChF,WAAW,EAAE,YAAY,EAAE,wHAAwH;aACtJ,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QACxB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC5E,IAAI,WAAW,KAAK,IAAI,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YACpD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,WAAqB,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;CACJ"}
|
package/request.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
|
|
4
|
+
import type { SkippedRequestReason } from './enqueue_links/shared.js';
|
|
4
5
|
import type { AllowedHttpMethods } from './typedefs.js';
|
|
5
6
|
export declare enum RequestState {
|
|
6
7
|
UNPROCESSED = 0,
|
|
@@ -43,7 +44,7 @@ export declare enum RequestState {
|
|
|
43
44
|
* ```
|
|
44
45
|
* @category Sources
|
|
45
46
|
*/
|
|
46
|
-
|
|
47
|
+
declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
|
|
47
48
|
/** Request ID */
|
|
48
49
|
id?: string;
|
|
49
50
|
/** URL of the web page to crawl. */
|
|
@@ -87,10 +88,25 @@ export declare class Request<UserData extends Dictionary = Dictionary> {
|
|
|
87
88
|
* `Request` parameters including the URL, HTTP method and headers, and others.
|
|
88
89
|
*/
|
|
89
90
|
constructor(options: RequestOptions<UserData>);
|
|
91
|
+
/**
|
|
92
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
93
|
+
* @returns The native `fetch` API Request object.
|
|
94
|
+
*/
|
|
95
|
+
intoFetchAPIRequest(): Request;
|
|
90
96
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
91
97
|
get skipNavigation(): boolean;
|
|
92
98
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
93
99
|
set skipNavigation(value: boolean);
|
|
100
|
+
/**
|
|
101
|
+
* Depth of the request in the current crawl tree.
|
|
102
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
103
|
+
*/
|
|
104
|
+
get crawlDepth(): number;
|
|
105
|
+
/**
|
|
106
|
+
* Depth of the request in the current crawl tree.
|
|
107
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
108
|
+
*/
|
|
109
|
+
set crawlDepth(value: number);
|
|
94
110
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
95
111
|
get sessionRotationCount(): number;
|
|
96
112
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
@@ -201,6 +217,18 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
|
|
|
201
217
|
* @default false
|
|
202
218
|
*/
|
|
203
219
|
skipNavigation?: boolean;
|
|
220
|
+
/**
|
|
221
|
+
* Depth of the request in the current crawl tree.
|
|
222
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
223
|
+
* @default 0
|
|
224
|
+
*/
|
|
225
|
+
crawlDepth?: number;
|
|
226
|
+
/**
|
|
227
|
+
* Reason for skipping this request.
|
|
228
|
+
* This is used to provide more information about why the request was skipped.
|
|
229
|
+
* @internal
|
|
230
|
+
*/
|
|
231
|
+
skippedReason?: SkippedRequestReason;
|
|
204
232
|
/**
|
|
205
233
|
* Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
|
|
206
234
|
*/
|
|
@@ -231,11 +259,11 @@ interface ComputeUniqueKeyOptions {
|
|
|
231
259
|
export type Source = (Partial<RequestOptions> & {
|
|
232
260
|
requestsFromUrl?: string;
|
|
233
261
|
regex?: RegExp;
|
|
234
|
-
}) |
|
|
262
|
+
}) | CrawleeRequest;
|
|
235
263
|
/** @internal */
|
|
236
264
|
export interface InternalSource {
|
|
237
265
|
requestsFromUrl: string;
|
|
238
266
|
regex?: RegExp;
|
|
239
267
|
}
|
|
240
|
-
export {};
|
|
268
|
+
export { CrawleeRequest as Request };
|
|
241
269
|
//# sourceMappingURL=request.d.ts.map
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAI9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAMjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AA4BxD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,cAAM,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IA2H7C;;;OAGG;IACI,mBAAmB,IAAI,OAAO;IAQrC,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED;;;OAGG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,EAG3B;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAMrC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAgCtF,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAiB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,oBAAoB,CAAC;IAErC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,cAAc,CAAC;AAE/G,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/request.js
CHANGED
|
@@ -24,6 +24,7 @@ const requestOptionalPredicates = {
|
|
|
24
24
|
keepUrlFragment: ow.optional.boolean,
|
|
25
25
|
useExtendedUniqueKey: ow.optional.boolean,
|
|
26
26
|
skipNavigation: ow.optional.boolean,
|
|
27
|
+
crawlDepth: ow.optional.number.greaterThanOrEqual(0),
|
|
27
28
|
state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
|
|
28
29
|
};
|
|
29
30
|
export var RequestState;
|
|
@@ -68,7 +69,7 @@ export var RequestState;
|
|
|
68
69
|
* ```
|
|
69
70
|
* @category Sources
|
|
70
71
|
*/
|
|
71
|
-
|
|
72
|
+
class CrawleeRequest {
|
|
72
73
|
/** Request ID */
|
|
73
74
|
id;
|
|
74
75
|
/** URL of the web page to crawl. */
|
|
@@ -130,7 +131,7 @@ export class Request {
|
|
|
130
131
|
ow(value, `RequestOptions.${prop}`, predicate);
|
|
131
132
|
}
|
|
132
133
|
});
|
|
133
|
-
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, } = options;
|
|
134
|
+
const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
|
|
134
135
|
let { method = 'GET' } = options;
|
|
135
136
|
method = method.toUpperCase();
|
|
136
137
|
if (method === 'GET' && payload)
|
|
@@ -139,7 +140,8 @@ export class Request {
|
|
|
139
140
|
this.url = url;
|
|
140
141
|
this.loadedUrl = loadedUrl;
|
|
141
142
|
this.uniqueKey =
|
|
142
|
-
uniqueKey ||
|
|
143
|
+
uniqueKey ||
|
|
144
|
+
CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
|
|
143
145
|
this.method = method;
|
|
144
146
|
this.payload = payload;
|
|
145
147
|
this.noRetry = noRetry;
|
|
@@ -191,11 +193,24 @@ export class Request {
|
|
|
191
193
|
this.skipNavigation = skipNavigation;
|
|
192
194
|
if (maxRetries != null)
|
|
193
195
|
this.maxRetries = maxRetries;
|
|
196
|
+
if (crawlDepth != null)
|
|
197
|
+
this.userData.__crawlee.crawlDepth ??= crawlDepth;
|
|
194
198
|
// If it's already set, don't override it (for instance when fetching from storage)
|
|
195
199
|
if (enqueueStrategy) {
|
|
196
200
|
this.enqueueStrategy ??= enqueueStrategy;
|
|
197
201
|
}
|
|
198
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* Converts the Crawlee Request object to a `fetch` API Request object.
|
|
205
|
+
* @returns The native `fetch` API Request object.
|
|
206
|
+
*/
|
|
207
|
+
intoFetchAPIRequest() {
|
|
208
|
+
return new Request(this.url, {
|
|
209
|
+
method: this.method,
|
|
210
|
+
headers: this.headers,
|
|
211
|
+
body: this.payload,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
199
214
|
/** Tells the crawler processing this request to skip the navigation and process the request directly. */
|
|
200
215
|
get skipNavigation() {
|
|
201
216
|
return this.userData.__crawlee?.skipNavigation ?? false;
|
|
@@ -209,6 +224,21 @@ export class Request {
|
|
|
209
224
|
this.userData.__crawlee.skipNavigation = value;
|
|
210
225
|
}
|
|
211
226
|
}
|
|
227
|
+
/**
|
|
228
|
+
* Depth of the request in the current crawl tree.
|
|
229
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
230
|
+
*/
|
|
231
|
+
get crawlDepth() {
|
|
232
|
+
return this.userData.__crawlee?.crawlDepth ?? 0;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Depth of the request in the current crawl tree.
|
|
236
|
+
* Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
|
|
237
|
+
*/
|
|
238
|
+
set crawlDepth(value) {
|
|
239
|
+
this.userData.__crawlee ??= {};
|
|
240
|
+
this.userData.__crawlee.crawlDepth = value;
|
|
241
|
+
}
|
|
212
242
|
/** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
|
|
213
243
|
get sessionRotationCount() {
|
|
214
244
|
return this.userData.__crawlee?.sessionRotationCount ?? 0;
|
|
@@ -329,7 +359,7 @@ export class Request {
|
|
|
329
359
|
}
|
|
330
360
|
return normalizedUrl;
|
|
331
361
|
}
|
|
332
|
-
const payloadHash = payload ?
|
|
362
|
+
const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
|
|
333
363
|
return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
|
|
334
364
|
}
|
|
335
365
|
/** @internal */
|
|
@@ -337,4 +367,5 @@ export class Request {
|
|
|
337
367
|
return crypto.createHash('sha256').update(payload).digest('base64').replace(/[+/=]/g, '').substring(0, 8);
|
|
338
368
|
}
|
|
339
369
|
}
|
|
370
|
+
export { CrawleeRequest as Request };
|
|
340
371
|
//# sourceMappingURL=request.js.map
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,UAAU,CAAC;AAE7C,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAErC,4DAA4D;AAC5D,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAEpD,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACxC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpD,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE,CAAC;AAEF,MAAM,CAAN,IAAY,YASX;AATD,WAAY,YAAY;IACpB,6DAAW,CAAA;IACX,2DAAU,CAAA;IACV,yDAAS,CAAA;IACT,qEAAe,CAAA;IACf,+CAAI,CAAA;IACJ,iEAAa,CAAA;IACb,iDAAK,CAAA;IACL,qDAAO,CAAA;AACX,CAAC,EATW,YAAY,KAAZ,YAAY,QASvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,cAAc;IAChB,iBAAiB;IACjB,EAAE,CAAU;IAEZ,oCAAoC;IACpC,GAAG,CAAS;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,SAAS,CAAS;IAElB,yCAAyC;IACzC,MAAM,CAAqB;IAE3B,oDAAoD;IACpD,OAAO,CAAU;IAEjB,8FAA8F;IAC9F,OAAO,CAAU;IAEjB,2FAA2F;IAC3F,UAAU,CAAS;IAEnB,0DAA0D;IAC1D,aAAa,CAAW;IAExB,wEAAwE;IACxE,OAAO,CAA0B;IAEjC,sEAAsE;IAC9D,SAAS,GAAwB,EAAE,CAAC;IAE5C,gDAAgD;IAChD,QAAQ,GAAa,EAAc,CAAC;IAEpC;;;OAGG;IACH,SAAS,CAAU;IAEnB;;OAEG;IACH,YAAY,OAAiC;QACzC,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,0CAA0C;YAC1C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,oBAAoB,GAAG,CAAC,EACxB,UAAU,EACV,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,EACd,eAAe,EACf,UAAU,GACb,GAAG,OAMH,CAAC;QAEF,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS;YACV,SAAS;gBACT,cAAc,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAI,SAAqB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEzG,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACnD,OAAO;wCACH,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC;gCACN,CAAC;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACjE,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QACrD,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,CAAC;QAE1E,mFAAmF;QACnF,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACP,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAc;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAa;QACvB,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACzD,CAAC;IACL,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAC/C,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAyB;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,IAAY,eAAe;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpD,CAAC;IAED,IAAY,eAAe,CAAC,KAAsC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,OAAO,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE,CAAC;gBACzC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,CAAC,CAAC,8BAA8B;wBAC9B,cAAc,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE,CAAC;gBACrE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,GAAG,yDAAyD,CAAC;gBACxE,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,GAAG,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAM,GAAG,KAAK,EACd,OAAO,EACP,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GACN;QACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACxB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE,CAAC;gBACxC,GAAG,CAAC,WAAW,CACX,uBAAuB,gBAAgB,2BAA2B;oBAC9D,kFAAkF;oBAClF,iHAAiH,CACxH,CAAC;YACN,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAmB;QAClC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;CACJ;AA8ID,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
|
@@ -2,7 +2,6 @@ import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';
|
|
|
2
2
|
import type { Cookie, SerializedCookieJar } from 'tough-cookie';
|
|
3
3
|
import { CookieJar } from 'tough-cookie';
|
|
4
4
|
import type { Log } from '@apify/log';
|
|
5
|
-
import type { ResponseLike } from '../cookie_utils.js';
|
|
6
5
|
import type { ProxyInfo } from '../proxy_configuration.js';
|
|
7
6
|
/**
|
|
8
7
|
* Persistable {@link Session} state.
|
|
@@ -161,7 +160,7 @@ export declare class Session {
|
|
|
161
160
|
*
|
|
162
161
|
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
163
162
|
*/
|
|
164
|
-
setCookiesFromResponse(response:
|
|
163
|
+
setCookiesFromResponse(response: Response): void;
|
|
165
164
|
/**
|
|
166
165
|
* Saves an array with cookie objects to be used with the session.
|
|
167
166
|
* The objects should be in the format that
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAUtC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAG3D;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,mBAAmB,CAAC;IAC/B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,CAAC;IAEtD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAC,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,GAAG,CAAM;IAEjB,IAAI,UAAU,WAEb;IAED,IAAI,UAAU,WAEb;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,mBAAmB,WAEtB;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,SAAS,0BAEZ;IAED;;OAEG;gBACS,OAAO,EAAE,cAAc;IA0DnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAgBxB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IASvD;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,QAAQ;IAWzC;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -199,7 +199,7 @@ export class Session {
|
|
|
199
199
|
setCookiesFromResponse(response) {
|
|
200
200
|
try {
|
|
201
201
|
const cookies = getCookiesFromResponse(response).filter((c) => c);
|
|
202
|
-
this._setCookies(cookies,
|
|
202
|
+
this._setCookies(cookies, response.url);
|
|
203
203
|
}
|
|
204
204
|
catch (e) {
|
|
205
205
|
const err = e;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EACH,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,8BAA8B,GACjC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA6EpD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACZ,UAAU,CAAS;IAC3B,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAA0C;IACrD,WAAW,CAAS;IACpB,UAAU,CAAa;IACvB,UAAU,CAAY;IACtB,GAAG,CAAM;IAEjB,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,OAAuB;QAC/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC;YAC/C,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,SAAS,GAAG,SAAS,EACrB,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,UAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAkB;QACzC,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9E,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAkB;QACrC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -67,6 +67,11 @@ export interface DatasetDataOptions {
|
|
|
67
67
|
skipEmpty?: boolean;
|
|
68
68
|
}
|
|
69
69
|
export interface DatasetExportOptions extends Omit<DatasetDataOptions, 'offset' | 'limit'> {
|
|
70
|
+
/**
|
|
71
|
+
* If true, includes all unique keys from all dataset items in the CSV export header.
|
|
72
|
+
* If omitted or false, only keys from the first item are used.
|
|
73
|
+
*/
|
|
74
|
+
collectAllKeys?: boolean;
|
|
70
75
|
}
|
|
71
76
|
export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset' | 'limit' | 'clean' | 'skipHidden' | 'skipEmpty'> {
|
|
72
77
|
/** @internal */
|
|
@@ -145,6 +150,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
145
150
|
id: string;
|
|
146
151
|
name?: string;
|
|
147
152
|
client: DatasetClient<Data>;
|
|
153
|
+
readonly storageObject?: Record<string, unknown>;
|
|
148
154
|
log: Log;
|
|
149
155
|
/**
|
|
150
156
|
* @internal
|
|
@@ -412,6 +418,7 @@ export interface DatasetOptions {
|
|
|
412
418
|
id: string;
|
|
413
419
|
name?: string;
|
|
414
420
|
client: StorageClient;
|
|
421
|
+
storageObject?: Record<string, unknown>;
|
|
415
422
|
}
|
|
416
423
|
export interface DatasetContent<Data> {
|
|
417
424
|
/** Total count of entries in the dataset. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,KAAK,GAAG,EAAO,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,KAAK,GAAG,EAAO,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtF;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,sBACb,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC3F,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAYjD,QAAQ,CAAC,MAAM;IAXnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,GAAG,EAAE,GAAG,CAAoC;IAE5C;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IAQrD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAgB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAyBjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAmCpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAMjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBzG;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAC/B,OAAO,GAAE,qBAA0B,GACpC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAsBzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
package/storages/dataset.js
CHANGED
|
@@ -132,6 +132,7 @@ export class Dataset {
|
|
|
132
132
|
id;
|
|
133
133
|
name;
|
|
134
134
|
client;
|
|
135
|
+
storageObject;
|
|
135
136
|
log = log.child({ prefix: 'Dataset' });
|
|
136
137
|
/**
|
|
137
138
|
* @internal
|
|
@@ -141,6 +142,7 @@ export class Dataset {
|
|
|
141
142
|
this.id = options.id;
|
|
142
143
|
this.name = options.name;
|
|
143
144
|
this.client = options.client.dataset(this.id);
|
|
145
|
+
this.storageObject = options.storageObject;
|
|
144
146
|
}
|
|
145
147
|
/**
|
|
146
148
|
* Stores an object or an array of objects to the dataset.
|
|
@@ -233,7 +235,14 @@ export class Dataset {
|
|
|
233
235
|
const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { config: this.config });
|
|
234
236
|
const items = await this.export(options);
|
|
235
237
|
if (contentType === 'text/csv') {
|
|
236
|
-
|
|
238
|
+
// To handle empty dataset exports gracefully.
|
|
239
|
+
if (items.length === 0) {
|
|
240
|
+
await kvStore.setValue(key, '', { contentType });
|
|
241
|
+
return items;
|
|
242
|
+
}
|
|
243
|
+
const keys = options?.collectAllKeys
|
|
244
|
+
? Array.from(new Set(items.flatMap(Object.keys)))
|
|
245
|
+
: Object.keys(items[0]);
|
|
237
246
|
const value = stringify([
|
|
238
247
|
keys,
|
|
239
248
|
...items.map((item) => {
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAY,GAAG,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAY,GAAG,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,gBAAgB;AAChB,MAAM,CAAC,MAAM,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;IAChG,CAAC;IAED,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;IACpG,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,CAAC;YAChD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;QAC3B,CAAC;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;QACjE,CAAC;aAAM,CAAC;YACJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;QAC9D,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA2FD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,MAAM,OAAO,OAAO;IAYH;IAXb,EAAE,CAAS;IACX,IAAI,CAAU;IACd,MAAM,CAAsB;IACnB,aAAa,CAA2B;IACjD,GAAG,GAAQ,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAE5C;;OAEG;IACH,YACI,OAAuB,EACd,SAAS,aAAa,CAAC,eAAe,EAAE;QAAxC,WAAM,GAAN,MAAM,CAAkC;QAEjD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;QACrE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAG,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO;QACX,CAAC;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,UAAgC,EAAE;QAC3C,kBAAkB,EAAE,CAAC;QAErB,MAAM,KAAK,GAAW,EAAE,CAAC;QAEzB,MAAM,cAAc,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAiB,EAAE;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;YAEzE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBACrC,MAAM,cAAc,CAAC,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,cAAc,EAAE,CAAC;QAEvB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,OAAgC,EAAE,WAAoB;QAC9E,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC7B,8CAA8C;YAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;gBACjD,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,EAAE,cAAc;gBAChC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBACjD,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,MAAM,KAAK,GAAG,SAAS,CAAC;gBACpB,IAAI;gBACJ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC;aACL,CAAC,CAAC;YACH,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;QAE5D,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAqD;QACjF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAqD;QAChF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,OAAgC;QACnE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,GAAW,EAAE,OAAgC;QAClE,kBAAkB,EAAE,CAAC;QAErB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACtD,MAAM,OAAO,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,KAAK,CAAC,OAAO;QACT,kBAAkB,EAAE,CAAC;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,kBAAkB,EAAE,CAAC;QAErB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAC3C,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAClF,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,+BAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,kBAAkB,EAAE,CAAC;QAErB,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IA6DD,KAAK,CAAC,MAAM,CACR,QAAiC,EACjC,IAAQ,EACR,UAAkC,EAAE;QAEpC,kBAAkB,EAAE,CAAC;QAErB,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,MAAM,WAAW,GAA0B,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YAC7D,IAAI,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACJ,iFAAiF;gBACjF,4DAA4D;gBAC5D,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAgB,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAChE,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,kBAAkB,EAAE,CAAC;QAErB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,eAA+B,EAC/B,UAAiC,EAAE;QAEnC,kBAAkB,EAAE,CAAC;QAErB,EAAE,CAAC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC;YACpD,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACpC,CAAC,CACL,CAAC;QAEF,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,eAAe,EAAE,CAAC;QACnD,OAAO,CAAC,aAAa,KAAK,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAE5D,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3G,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAgB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/E,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,CAChB,UAA8B,EAAE;QAEhC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;CACJ"}
|
package/storages/index.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
3
|
export * from './request_list.js';
|
|
4
|
+
export * from './request_list_adapter.js';
|
|
4
5
|
export * from './request_provider.js';
|
|
5
6
|
export { RequestQueueV1 } from './request_queue.js';
|
|
6
7
|
export { RequestQueue } from './request_queue_v2.js';
|
|
@@ -9,4 +10,5 @@ export * from './storage_manager.js';
|
|
|
9
10
|
export * from './utils.js';
|
|
10
11
|
export * from './access_checking.js';
|
|
11
12
|
export * from './sitemap_request_list.js';
|
|
13
|
+
export * from './request_manager_tandem.js';
|
|
12
14
|
//# sourceMappingURL=index.d.ts.map
|
package/storages/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storages/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,sBAAsB,CAAC;AACrC,cAAc,mBAAmB,CAAC;AAClC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,YAAY,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvE,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,6BAA6B,CAAC"}
|
package/storages/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
2
|
export * from './key_value_store.js';
|
|
3
3
|
export * from './request_list.js';
|
|
4
|
+
export * from './request_list_adapter.js';
|
|
4
5
|
export * from './request_provider.js';
|
|
5
6
|
export { RequestQueueV1 } from './request_queue.js';
|
|
6
7
|
export { RequestQueue } from './request_queue_v2.js';
|
|
@@ -9,4 +10,5 @@ export * from './storage_manager.js';
|
|
|
9
10
|
export * from './utils.js';
|
|
10
11
|
export * from './access_checking.js';
|
|
11
12
|
export * from './sitemap_request_list.js';
|
|
13
|
+
export * from './request_manager_tandem.js';
|
|
12
14
|
//# sourceMappingURL=index.js.map
|