crawlee 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.js +1 -2
- package/cli.js.map +1 -1
- package/index.js +24 -28
- package/index.js.map +1 -1
- package/package.json +20 -26
- package/tsconfig.build.tsbuildinfo +1 -1
- package/index.mjs +0 -175
package/index.mjs
DELETED
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
import mod from "./index.js";
|
|
2
|
-
|
|
3
|
-
export default mod;
|
|
4
|
-
export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
|
|
5
|
-
export const AdaptivePlaywrightCrawler = mod.AdaptivePlaywrightCrawler;
|
|
6
|
-
export const AutoscaledPool = mod.AutoscaledPool;
|
|
7
|
-
export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = mod.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS;
|
|
8
|
-
export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
|
|
9
|
-
export const BROWSER_CONTROLLER_EVENTS = mod.BROWSER_CONTROLLER_EVENTS;
|
|
10
|
-
export const BROWSER_POOL_EVENTS = mod.BROWSER_POOL_EVENTS;
|
|
11
|
-
export const BasicCrawler = mod.BasicCrawler;
|
|
12
|
-
export const BrowserController = mod.BrowserController;
|
|
13
|
-
export const BrowserCrawler = mod.BrowserCrawler;
|
|
14
|
-
export const BrowserLaunchError = mod.BrowserLaunchError;
|
|
15
|
-
export const BrowserLauncher = mod.BrowserLauncher;
|
|
16
|
-
export const BrowserName = mod.BrowserName;
|
|
17
|
-
export const BrowserPlugin = mod.BrowserPlugin;
|
|
18
|
-
export const BrowserPool = mod.BrowserPool;
|
|
19
|
-
export const ByteCounterStream = mod.ByteCounterStream;
|
|
20
|
-
export const CLOUDFLARE_RETRY_CSS_SELECTORS = mod.CLOUDFLARE_RETRY_CSS_SELECTORS;
|
|
21
|
-
export const CheerioCrawler = mod.CheerioCrawler;
|
|
22
|
-
export const Configuration = mod.Configuration;
|
|
23
|
-
export const CookieParseError = mod.CookieParseError;
|
|
24
|
-
export const CrawlerExtension = mod.CrawlerExtension;
|
|
25
|
-
export const CriticalError = mod.CriticalError;
|
|
26
|
-
export const DATASET_ITERATORS_DEFAULT_LIMIT = mod.DATASET_ITERATORS_DEFAULT_LIMIT;
|
|
27
|
-
export const DEFAULT_USER_AGENT = mod.DEFAULT_USER_AGENT;
|
|
28
|
-
export const Dataset = mod.Dataset;
|
|
29
|
-
export const DeviceCategory = mod.DeviceCategory;
|
|
30
|
-
export const EVENT_SESSION_RETIRED = mod.EVENT_SESSION_RETIRED;
|
|
31
|
-
export const EnqueueStrategy = mod.EnqueueStrategy;
|
|
32
|
-
export const ErrorSnapshotter = mod.ErrorSnapshotter;
|
|
33
|
-
export const ErrorTracker = mod.ErrorTracker;
|
|
34
|
-
export const EventManager = mod.EventManager;
|
|
35
|
-
export const EventType = mod.EventType;
|
|
36
|
-
export const FileDownload = mod.FileDownload;
|
|
37
|
-
export const GotScrapingHttpClient = mod.GotScrapingHttpClient;
|
|
38
|
-
export const HttpCrawler = mod.HttpCrawler;
|
|
39
|
-
export const JSDOMCrawler = mod.JSDOMCrawler;
|
|
40
|
-
export const KeyValueStore = mod.KeyValueStore;
|
|
41
|
-
export const LaunchContext = mod.LaunchContext;
|
|
42
|
-
export const LinkeDOMCrawler = mod.LinkeDOMCrawler;
|
|
43
|
-
export const LocalEventManager = mod.LocalEventManager;
|
|
44
|
-
export const Log = mod.Log;
|
|
45
|
-
export const LogLevel = mod.LogLevel;
|
|
46
|
-
export const Logger = mod.Logger;
|
|
47
|
-
export const LoggerJson = mod.LoggerJson;
|
|
48
|
-
export const LoggerText = mod.LoggerText;
|
|
49
|
-
export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
|
|
50
|
-
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
51
|
-
export const MinimumSpeedStream = mod.MinimumSpeedStream;
|
|
52
|
-
export const MissingRouteError = mod.MissingRouteError;
|
|
53
|
-
export const NonRetryableError = mod.NonRetryableError;
|
|
54
|
-
export const OperatingSystemsName = mod.OperatingSystemsName;
|
|
55
|
-
export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
|
|
56
|
-
export const PlaywrightBrowser = mod.PlaywrightBrowser;
|
|
57
|
-
export const PlaywrightController = mod.PlaywrightController;
|
|
58
|
-
export const PlaywrightCrawler = mod.PlaywrightCrawler;
|
|
59
|
-
export const PlaywrightLauncher = mod.PlaywrightLauncher;
|
|
60
|
-
export const PlaywrightPlugin = mod.PlaywrightPlugin;
|
|
61
|
-
export const ProxyConfiguration = mod.ProxyConfiguration;
|
|
62
|
-
export const PseudoUrl = mod.PseudoUrl;
|
|
63
|
-
export const PuppeteerController = mod.PuppeteerController;
|
|
64
|
-
export const PuppeteerCrawler = mod.PuppeteerCrawler;
|
|
65
|
-
export const PuppeteerLauncher = mod.PuppeteerLauncher;
|
|
66
|
-
export const PuppeteerPlugin = mod.PuppeteerPlugin;
|
|
67
|
-
export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
|
|
68
|
-
export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
|
|
69
|
-
export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
70
|
-
export const RETRY_CSS_SELECTORS = mod.RETRY_CSS_SELECTORS;
|
|
71
|
-
export const ROTATE_PROXY_ERRORS = mod.ROTATE_PROXY_ERRORS;
|
|
72
|
-
export const RenderingTypePredictor = mod.RenderingTypePredictor;
|
|
73
|
-
export const Request = mod.Request;
|
|
74
|
-
export const RequestHandlerResult = mod.RequestHandlerResult;
|
|
75
|
-
export const RequestList = mod.RequestList;
|
|
76
|
-
export const RequestProvider = mod.RequestProvider;
|
|
77
|
-
export const RequestQueue = mod.RequestQueue;
|
|
78
|
-
export const RequestQueueV1 = mod.RequestQueueV1;
|
|
79
|
-
export const RequestQueueV2 = mod.RequestQueueV2;
|
|
80
|
-
export const RequestState = mod.RequestState;
|
|
81
|
-
export const RetryRequestError = mod.RetryRequestError;
|
|
82
|
-
export const RobotsFile = mod.RobotsFile;
|
|
83
|
-
export const RobotsTxtFile = mod.RobotsTxtFile;
|
|
84
|
-
export const Router = mod.Router;
|
|
85
|
-
export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
|
|
86
|
-
export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
|
|
87
|
-
export const Session = mod.Session;
|
|
88
|
-
export const SessionError = mod.SessionError;
|
|
89
|
-
export const SessionPool = mod.SessionPool;
|
|
90
|
-
export const Sitemap = mod.Sitemap;
|
|
91
|
-
export const SitemapRequestList = mod.SitemapRequestList;
|
|
92
|
-
export const Snapshotter = mod.Snapshotter;
|
|
93
|
-
export const Statistics = mod.Statistics;
|
|
94
|
-
export const StorageManager = mod.StorageManager;
|
|
95
|
-
export const SystemStatus = mod.SystemStatus;
|
|
96
|
-
export const URL_NO_COMMAS_REGEX = mod.URL_NO_COMMAS_REGEX;
|
|
97
|
-
export const URL_WITH_COMMAS_REGEX = mod.URL_WITH_COMMAS_REGEX;
|
|
98
|
-
export const applySearchParams = mod.applySearchParams;
|
|
99
|
-
export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
|
|
100
|
-
export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
|
|
101
|
-
export const checkAndSerialize = mod.checkAndSerialize;
|
|
102
|
-
export const checkStorageAccess = mod.checkStorageAccess;
|
|
103
|
-
export const cheerioCrawlerEnqueueLinks = mod.cheerioCrawlerEnqueueLinks;
|
|
104
|
-
export const chunk = mod.chunk;
|
|
105
|
-
export const chunkBySize = mod.chunkBySize;
|
|
106
|
-
export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
|
|
107
|
-
export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
|
|
108
|
-
export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
|
|
109
|
-
export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
|
|
110
|
-
export const createAdaptivePlaywrightRouter = mod.createAdaptivePlaywrightRouter;
|
|
111
|
-
export const createBasicRouter = mod.createBasicRouter;
|
|
112
|
-
export const createCheerioRouter = mod.createCheerioRouter;
|
|
113
|
-
export const createDeserialize = mod.createDeserialize;
|
|
114
|
-
export const createFileRouter = mod.createFileRouter;
|
|
115
|
-
export const createHttpRouter = mod.createHttpRouter;
|
|
116
|
-
export const createJSDOMRouter = mod.createJSDOMRouter;
|
|
117
|
-
export const createLinkeDOMRouter = mod.createLinkeDOMRouter;
|
|
118
|
-
export const createPlaywrightRouter = mod.createPlaywrightRouter;
|
|
119
|
-
export const createPuppeteerRouter = mod.createPuppeteerRouter;
|
|
120
|
-
export const createRequestDebugInfo = mod.createRequestDebugInfo;
|
|
121
|
-
export const createRequestOptions = mod.createRequestOptions;
|
|
122
|
-
export const createRequests = mod.createRequests;
|
|
123
|
-
export const deserializeArray = mod.deserializeArray;
|
|
124
|
-
export const domCrawlerEnqueueLinks = mod.domCrawlerEnqueueLinks;
|
|
125
|
-
export const downloadListOfUrls = mod.downloadListOfUrls;
|
|
126
|
-
export const enqueueLinks = mod.enqueueLinks;
|
|
127
|
-
export const entries = mod.entries;
|
|
128
|
-
export const expandShadowRoots = mod.expandShadowRoots;
|
|
129
|
-
export const extractUrls = mod.extractUrls;
|
|
130
|
-
export const extractUrlsFromCheerio = mod.extractUrlsFromCheerio;
|
|
131
|
-
export const extractUrlsFromPage = mod.extractUrlsFromPage;
|
|
132
|
-
export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
|
|
133
|
-
export const getCgroupsVersion = mod.getCgroupsVersion;
|
|
134
|
-
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
135
|
-
export const getCurrentCpuTicksV2 = mod.getCurrentCpuTicksV2;
|
|
136
|
-
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
137
|
-
export const getMemoryInfo = mod.getMemoryInfo;
|
|
138
|
-
export const getMemoryInfoV2 = mod.getMemoryInfoV2;
|
|
139
|
-
export const getRequestId = mod.getRequestId;
|
|
140
|
-
export const gotScraping = mod.gotScraping;
|
|
141
|
-
export const handleRequestTimeout = mod.handleRequestTimeout;
|
|
142
|
-
export const htmlToText = mod.htmlToText;
|
|
143
|
-
export const isContainerized = mod.isContainerized;
|
|
144
|
-
export const isDocker = mod.isDocker;
|
|
145
|
-
export const isLambda = mod.isLambda;
|
|
146
|
-
export const keys = mod.keys;
|
|
147
|
-
export const launchPlaywright = mod.launchPlaywright;
|
|
148
|
-
export const launchPuppeteer = mod.launchPuppeteer;
|
|
149
|
-
export const linkedomCrawlerEnqueueLinks = mod.linkedomCrawlerEnqueueLinks;
|
|
150
|
-
export const log = mod.log;
|
|
151
|
-
export const maybeStringify = mod.maybeStringify;
|
|
152
|
-
export const mergeCookies = mod.mergeCookies;
|
|
153
|
-
export const parseOpenGraph = mod.parseOpenGraph;
|
|
154
|
-
export const parseSitemap = mod.parseSitemap;
|
|
155
|
-
export const playwrightClickElements = mod.playwrightClickElements;
|
|
156
|
-
export const playwrightUtils = mod.playwrightUtils;
|
|
157
|
-
export const processHttpRequestOptions = mod.processHttpRequestOptions;
|
|
158
|
-
export const puppeteerClickElements = mod.puppeteerClickElements;
|
|
159
|
-
export const puppeteerRequestInterception = mod.puppeteerRequestInterception;
|
|
160
|
-
export const puppeteerUtils = mod.puppeteerUtils;
|
|
161
|
-
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
162
|
-
export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
|
|
163
|
-
export const serializeArray = mod.serializeArray;
|
|
164
|
-
export const sleep = mod.sleep;
|
|
165
|
-
export const snakeCaseToCamelCase = mod.snakeCaseToCamelCase;
|
|
166
|
-
export const social = mod.social;
|
|
167
|
-
export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
|
|
168
|
-
export const tryAbsoluteURL = mod.tryAbsoluteURL;
|
|
169
|
-
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
|
170
|
-
export const useState = mod.useState;
|
|
171
|
-
export const utils = mod.utils;
|
|
172
|
-
export const validateGlobPattern = mod.validateGlobPattern;
|
|
173
|
-
export const validators = mod.validators;
|
|
174
|
-
export const weightedAvg = mod.weightedAvg;
|
|
175
|
-
export const withCheckedStorageAccess = mod.withCheckedStorageAccess;
|