@crawlee/utils 3.0.0-alpha.1 → 3.0.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +7 -0
- package/index.d.ts.map +1 -0
- package/index.js.map +1 -0
- package/index.mjs +18 -0
- package/internals/cheerio.d.ts +32 -0
- package/internals/cheerio.d.ts.map +1 -0
- package/internals/cheerio.js +94 -0
- package/internals/cheerio.js.map +1 -0
- package/internals/extract-urls.d.ts +39 -0
- package/internals/extract-urls.d.ts.map +1 -0
- package/internals/extract-urls.js +41 -0
- package/internals/extract-urls.js.map +1 -0
- package/internals/general.d.ts +62 -0
- package/internals/general.d.ts.map +1 -0
- package/internals/general.js +143 -0
- package/internals/general.js.map +1 -0
- package/internals/memory-info.d.ts +26 -0
- package/internals/memory-info.d.ts.map +1 -0
- package/internals/memory-info.js +142 -0
- package/internals/memory-info.js.map +1 -0
- package/internals/social.d.ts +493 -0
- package/internals/social.d.ts.map +1 -0
- package/internals/social.js +661 -0
- package/internals/social.js.map +1 -0
- package/internals/typedefs.d.ts +11 -0
- package/internals/typedefs.d.ts.map +1 -0
- package/internals/typedefs.js +15 -0
- package/internals/typedefs.js.map +1 -0
- package/package.json +1 -4
- package/tsconfig.build.tsbuildinfo +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-info.js","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,6DAA6D;AAC7D,qEAAoC;AACpC,2DAA8C;AAC9C,wEAAkC;AAClC,8DAAyB;AACzB,kEAA6B;AAC7B,uCAAqC;AAGrC,MAAM,iBAAiB,GAAG;IACtB,KAAK,EAAE;QACH,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,2BAA2B;KAClC;IACD,IAAI,EAAE;QACF,EAAE,EAAE,6CAA6C;QACjD,EAAE,EAAE,+BAA+B;KACtC;CACJ,CAAC;AAsBF;;;;;;;;GAQG;AACI,KAAK,UAAU,aAAa;IAC/B,MAAM,cAAc,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAM,CAAC,CAAC;IAE9C,2DAA2D;IAC3D,oCAAoC;IACpC,MAAM,mBAAmB,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO;WACjD,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IAErD,MAAM,WAAW,GAAG,CAAC,mBAAmB,IAAI,MAAM,IAAA,kBAAQ,GAAE,CAAC;IAE7D,IAAI,gBAAgB,GAAG,CAAC,CAAC,CAAC;IAC1B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,IAAI,mBAAmB,EAAE;QACrB,oBAAoB;QACpB,gBAAgB,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC;QAE7C,gEAAgE;QAChE,8CAA8C;QAC9C,mBAAmB,GAAG,CAAC,IAAA,6BAAQ,EAAC,mBAAmB,CAAC;aAC/C,QAAQ,EAAE;aACV,KAAK,CAAC,QAAQ,CAAC;aACf,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;cAC9B,IAAI,CAAC,mCAAmC;YAC1C,+CAA+C;YAC/C,gDAAgD;YAChD,uDAAuD;cACrD,gBAAgB,CAAC;KAC1B;SAAM;QACH,sCAAsC;QACtC,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE1D,SAAS,CAAC,OAAO,CAAC,CAAC,GAAuB,EAAE,EAAE;YAC1C,0EAA0E;YAC1E,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,KAAK,UAAU,EAAE;gBACpD,OAAO;aACV;YACD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,yCAAyC;YACzC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,OAAO,CAAC,GAAG,EAAE,EAAE;gBAC9B,gBAAgB,GAAG,KAAK,CAAC;gBACzB,OAAO;aACV;YACD,mBAAmB,IAAI,KAAK,CAAC;QACjC,CAAC,CAAC,CAAC;KACN;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,SAAiB,CAAC;IACtB,IAAI,SAAiB,CAAC;IAEtB,IAAI,mBAAmB,EAAE;QACrB,sCAAsC;QACtC,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,+BAAgC,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC;QAClF,SAAS,GAAG,gBAAgB,GAAG,mBAAmB,CAAC;QACnD,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;QAEnC,aAAG,CAAC,KAAK,CAAC,kBAAkB,UAAU,SAAS,SAAS,aAAa,CAAC,CAAC;KAC1E;SAAM,IAAI,WAAW,EAAE;QACpB,oEAAoE;QAEpE,yCAAyC;QACzC,IAAI,cAAc,GAAyC,IAAI,CAAC;QAChE,IAAI;YACA,uEAAuE;YACvE,MAAM,kBAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;SAC7C;QAAC,MAAM;YACJ,cAAc,GAAG,IAAI,CAAC;SACzB;QAED,IAAI;YACA,IAAI,CAAC,aAAa,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAClD,kBAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;gBAC5D,kBAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;aAC9D,CAAC,CAAC;YAEH,gHAAgH;YAChH,aAAa,GAAG,aAAa,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;YAE1D,4DAA4D;YAC5D,oIAAoI;YACpI,IAAI,aAAa,KAAK,KAAK,EAAE;gBACzB,UAAU,GAAG,iBAAE,CAAC,QAAQ,EAAE,CAAC;gBAC3B,yFAAyF;gBACzF,8CAA8C;aACjD;iBAAM;gBACH,UAAU,GAAG,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;gBACzC,MAAM,gCAAgC,GAAG,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBAC9E,IAAI,gCAAgC;oBAAE,UAAU,GAAG,iBAAE,CAAC,QAAQ,EAAE,CAAC;aACpE;YACD,SAAS,GAAG,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YACvC,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;SACtC;QAAC,OAAO,GAAG,EAAE;YACV,0CAA0C;YAC1C,aAAG,CAAC,UAAU,CAAC,+EAA+E;kBACxF,oGAAoG;kBACpG,UAAW,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1C,UAAU,GAAG,iBAAE,CAAC,QAAQ,EAAE,CAAC;YAC3B,SAAS,GAAG,iBAAE,CAAC,OAAO,EAAE,CAAC;YACzB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;SACtC;KACJ;SAAM;QACH,UAAU,GAAG,iBAAE,CAAC,QAAQ,EAAE,CAAC;QAC3B,SAAS,GAAG,iBAAE,CAAC,OAAO,EAAE,CAAC;QACzB,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;KACtC;IAED,OAAO;QACH,UAAU;QACV,SAAS;QACT,SAAS;QACT,gBAAgB;QAChB,mBAAmB;KACtB,CAAC;AACN,CAAC;AAnHD,sCAmHC"}
|
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regular expression to exactly match a single email address.
|
|
3
|
+
* It has the following form: `/^...$/i`.
|
|
4
|
+
*/
|
|
5
|
+
export declare const EMAIL_REGEX: RegExp;
|
|
6
|
+
/**
|
|
7
|
+
* Regular expression to find multiple email addresses in a text.
|
|
8
|
+
* It has the following form: `/.../ig`.
|
|
9
|
+
*/
|
|
10
|
+
export declare const EMAIL_REGEX_GLOBAL: RegExp;
|
|
11
|
+
/**
|
|
12
|
+
* The function extracts email addresses from a plain text.
|
|
13
|
+
* Note that the function preserves the order of emails and keep duplicates.
|
|
14
|
+
* @param text Text to search in.
|
|
15
|
+
* @return Array of emails addresses found.
|
|
16
|
+
* If no emails are found, the function returns an empty array.
|
|
17
|
+
*/
|
|
18
|
+
export declare function emailsFromText(text: string): string[];
|
|
19
|
+
/**
|
|
20
|
+
* The function extracts email addresses from a list of URLs.
|
|
21
|
+
* Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
|
|
22
|
+
* Note that the function preserves the order of emails and keep duplicates.
|
|
23
|
+
* @param urls Array of URLs.
|
|
24
|
+
* @return Array of emails addresses found.
|
|
25
|
+
* If no emails are found, the function returns an empty array.
|
|
26
|
+
*/
|
|
27
|
+
export declare function emailsFromUrls(urls: string[]): string[];
|
|
28
|
+
/**
|
|
29
|
+
* The function attempts to extract phone numbers from a text. Please note that
|
|
30
|
+
* the results might not be accurate, since phone numbers appear in a large variety of formats and conventions.
|
|
31
|
+
* If you encounter some problems, please [file an issue](https://github.com/apify/apify-js/issues).
|
|
32
|
+
* @param text Text to search the phone numbers in.
|
|
33
|
+
* @return Array of phone numbers found.
|
|
34
|
+
* If no phone numbers are found, the function returns an empty array.
|
|
35
|
+
*/
|
|
36
|
+
export declare function phonesFromText(text: string): string[];
|
|
37
|
+
/**
|
|
38
|
+
* Finds phone number links in an array of URLs and extracts the phone numbers from them.
|
|
39
|
+
* Note that the phone number links look like `tel://123456789`, `tel:/123456789` or `tel:123456789`.
|
|
40
|
+
* @param urls Array of URLs.
|
|
41
|
+
* @return Array of phone numbers found.
|
|
42
|
+
* If no phone numbers are found, the function returns an empty array.
|
|
43
|
+
*/
|
|
44
|
+
export declare function phonesFromUrls(urls: string[]): string[];
|
|
45
|
+
/**
|
|
46
|
+
* Representation of social handles parsed from a HTML page.
|
|
47
|
+
*/
|
|
48
|
+
export interface SocialHandles {
|
|
49
|
+
emails: string[];
|
|
50
|
+
phones: string[];
|
|
51
|
+
phonesUncertain: string[];
|
|
52
|
+
linkedIns: string[];
|
|
53
|
+
twitters: string[];
|
|
54
|
+
instagrams: string[];
|
|
55
|
+
facebooks: string[];
|
|
56
|
+
youtubes: string[];
|
|
57
|
+
tiktoks: string[];
|
|
58
|
+
pinterests: string[];
|
|
59
|
+
discords: string[];
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The function attempts to extract emails, phone numbers and social profile URLs from a HTML document,
|
|
63
|
+
* specifically LinkedIn, Twitter, Instagram and Facebook profile URLs.
|
|
64
|
+
* The function removes duplicates from the resulting arrays and sorts the items alphabetically.
|
|
65
|
+
*
|
|
66
|
+
* Note that the `phones` field contains phone numbers extracted from the special phone links
|
|
67
|
+
* such as `[call us](tel:+1234556789)` (see {@link phonesFromUrls})
|
|
68
|
+
* and potentially other sources with high certainty, while `phonesUncertain` contains phone numbers
|
|
69
|
+
* extracted from the plain text, which might be very inaccurate.
|
|
70
|
+
*
|
|
71
|
+
* **Example usage:**
|
|
72
|
+
* ```typescript
|
|
73
|
+
* import { launchPuppeteer } from '@crawlee/puppeteer';
|
|
74
|
+
* import { social } from '@crawlee/utils';
|
|
75
|
+
*
|
|
76
|
+
* const browser = await launchPuppeteer();
|
|
77
|
+
* const page = await browser.newPage();
|
|
78
|
+
* await page.goto('http://www.example.com');
|
|
79
|
+
* const html = await page.content();
|
|
80
|
+
*
|
|
81
|
+
* const result = social.parseHandlesFromHtml(html);
|
|
82
|
+
* console.log('Social handles:');
|
|
83
|
+
* console.dir(result);
|
|
84
|
+
* ```
|
|
85
|
+
*
|
|
86
|
+
* @param html HTML text
|
|
87
|
+
* @param [data] Optional object which will receive the `text` and `$` properties
|
|
88
|
+
* that contain text content of the HTML and `cheerio` object, respectively. This is an optimization
|
|
89
|
+
* so that the caller doesn't need to parse the HTML document again, if needed.
|
|
90
|
+
* @return An object with the social handles.
|
|
91
|
+
*/
|
|
92
|
+
export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
|
|
93
|
+
/**
|
|
94
|
+
* Regular expression to exactly match a single LinkedIn profile URL.
|
|
95
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
96
|
+
* ```
|
|
97
|
+
* https://www.linkedin.com/in/alan-turing
|
|
98
|
+
* en.linkedin.com/in/alan-turing
|
|
99
|
+
* linkedin.com/in/alan-turing
|
|
100
|
+
* https://www.linkedin.com/company/linkedin/
|
|
101
|
+
* ```
|
|
102
|
+
*
|
|
103
|
+
* The regular expression does NOT match URLs with additional
|
|
104
|
+
* subdirectories or query parameters, such as:
|
|
105
|
+
* ```
|
|
106
|
+
* https://www.linkedin.com/in/linus-torvalds/latest-activity
|
|
107
|
+
* ```
|
|
108
|
+
*
|
|
109
|
+
* Example usage:
|
|
110
|
+
* ```
|
|
111
|
+
* import { social } from '@crawlee/utils';
|
|
112
|
+
*
|
|
113
|
+
* if (social.LINKEDIN_REGEX.test('https://www.linkedin.com/in/alan-turing')) {
|
|
114
|
+
* console.log('Match!');
|
|
115
|
+
* }
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
export declare const LINKEDIN_REGEX: RegExp;
|
|
119
|
+
/**
|
|
120
|
+
* Regular expression to find multiple LinkedIn profile URLs in a text or HTML.
|
|
121
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
122
|
+
* ```
|
|
123
|
+
* https://www.linkedin.com/in/alan-turing
|
|
124
|
+
* en.linkedin.com/in/alan-turing
|
|
125
|
+
* linkedin.com/in/alan-turing
|
|
126
|
+
* https://www.linkedin.com/company/linkedin/
|
|
127
|
+
* ```
|
|
128
|
+
*
|
|
129
|
+
* If the profile URL contains subdirectories or query parameters, the regular expression
|
|
130
|
+
* extracts just the base part of the profile URL. For example, from text such as:
|
|
131
|
+
* ```
|
|
132
|
+
* https://www.linkedin.com/in/linus-torvalds/latest-activity
|
|
133
|
+
* ```
|
|
134
|
+
* the expression extracts just the following base URL:
|
|
135
|
+
* ```
|
|
136
|
+
* https://www.linkedin.com/in/linus-torvalds
|
|
137
|
+
* ```
|
|
138
|
+
*
|
|
139
|
+
* Example usage:
|
|
140
|
+
* ```
|
|
141
|
+
* import { social } from '@crawlee/utils';
|
|
142
|
+
*
|
|
143
|
+
* const matches = text.match(social.LINKEDIN_REGEX_GLOBAL);
|
|
144
|
+
* if (matches) console.log(`${matches.length} LinkedIn profiles found!`);
|
|
145
|
+
* ```
|
|
146
|
+
*/
|
|
147
|
+
export declare const LINKEDIN_REGEX_GLOBAL: RegExp;
|
|
148
|
+
/**
|
|
149
|
+
* Regular expression to exactly match a single Instagram profile URL.
|
|
150
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
151
|
+
* ```
|
|
152
|
+
* https://www.instagram.com/old_prague
|
|
153
|
+
* www.instagram.com/old_prague/
|
|
154
|
+
* instagr.am/old_prague
|
|
155
|
+
* ```
|
|
156
|
+
*
|
|
157
|
+
* The regular expression does NOT match URLs with additional
|
|
158
|
+
* subdirectories or query parameters, such as:
|
|
159
|
+
* ```
|
|
160
|
+
* https://www.instagram.com/cristiano/followers
|
|
161
|
+
* ```
|
|
162
|
+
*
|
|
163
|
+
* It also does NOT match the following URLs:
|
|
164
|
+
* ```
|
|
165
|
+
* https://www.instagram.com/explore/
|
|
166
|
+
* https://www.instagram.com/_n/
|
|
167
|
+
* https://www.instagram.com/_u/
|
|
168
|
+
*
|
|
169
|
+
* Example usage:
|
|
170
|
+
* ```
|
|
171
|
+
* import { social } from '@crawlee/utils';
|
|
172
|
+
*
|
|
173
|
+
* if (social.INSTAGRAM_REGEX.test('https://www.instagram.com/old_prague')) {
|
|
174
|
+
* console.log('Match!');
|
|
175
|
+
* }
|
|
176
|
+
* ```
|
|
177
|
+
*/
|
|
178
|
+
export declare const INSTAGRAM_REGEX: RegExp;
|
|
179
|
+
/**
|
|
180
|
+
* Regular expression to find multiple Instagram profile URLs in a text or HTML.
|
|
181
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
182
|
+
* ```
|
|
183
|
+
* https://www.instagram.com/old_prague
|
|
184
|
+
* www.instagram.com/old_prague/
|
|
185
|
+
* instagr.am/old_prague
|
|
186
|
+
* ```
|
|
187
|
+
*
|
|
188
|
+
* If the profile URL contains subdirectories or query parameters, the regular expression
|
|
189
|
+
* extracts just the base part of the profile URL. For example, from text such as:
|
|
190
|
+
* ```
|
|
191
|
+
* https://www.instagram.com/cristiano/followers
|
|
192
|
+
* ```
|
|
193
|
+
* the expression extracts just the following base URL:
|
|
194
|
+
* ```
|
|
195
|
+
* https://www.instagram.com/cristiano
|
|
196
|
+
* ```
|
|
197
|
+
*
|
|
198
|
+
* The regular expression does NOT match the following URLs:
|
|
199
|
+
* ```
|
|
200
|
+
* https://www.instagram.com/explore/
|
|
201
|
+
* https://www.instagram.com/_n/
|
|
202
|
+
* https://www.instagram.com/_u/
|
|
203
|
+
* ```
|
|
204
|
+
*
|
|
205
|
+
* Example usage:
|
|
206
|
+
* ```
|
|
207
|
+
* import { social } from '@crawlee/utils';
|
|
208
|
+
*
|
|
209
|
+
* const matches = text.match(social.INSTAGRAM_REGEX_GLOBAL);
|
|
210
|
+
* if (matches) console.log(`${matches.length} Instagram profiles found!`);
|
|
211
|
+
* ```
|
|
212
|
+
*/
|
|
213
|
+
export declare const INSTAGRAM_REGEX_GLOBAL: RegExp;
|
|
214
|
+
/**
|
|
215
|
+
* Regular expression to exactly match a single Twitter profile URL.
|
|
216
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
217
|
+
* ```
|
|
218
|
+
* https://www.twitter.com/apify
|
|
219
|
+
* twitter.com/apify
|
|
220
|
+
* ```
|
|
221
|
+
*
|
|
222
|
+
* The regular expression does NOT match URLs with additional
|
|
223
|
+
* subdirectories or query parameters, such as:
|
|
224
|
+
* ```
|
|
225
|
+
* https://www.twitter.com/realdonaldtrump/following
|
|
226
|
+
* ```
|
|
227
|
+
*
|
|
228
|
+
* Example usage:
|
|
229
|
+
* ```
|
|
230
|
+
* import { social } from '@crawlee/utils';
|
|
231
|
+
*
|
|
232
|
+
* if (social.TWITTER_REGEX.test('https://www.twitter.com/apify')) {
|
|
233
|
+
* console.log('Match!');
|
|
234
|
+
* }
|
|
235
|
+
* ```
|
|
236
|
+
*/
|
|
237
|
+
export declare const TWITTER_REGEX: RegExp;
|
|
238
|
+
/**
|
|
239
|
+
* Regular expression to find multiple Twitter profile URLs in a text or HTML.
|
|
240
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
241
|
+
* ```
|
|
242
|
+
* https://www.twitter.com/apify
|
|
243
|
+
* twitter.com/apify
|
|
244
|
+
* ```
|
|
245
|
+
*
|
|
246
|
+
* If the profile URL contains subdirectories or query parameters, the regular expression
|
|
247
|
+
* extracts just the base part of the profile URL. For example, from text such as:
|
|
248
|
+
* ```
|
|
249
|
+
* https://www.twitter.com/realdonaldtrump/following
|
|
250
|
+
* ```
|
|
251
|
+
* the expression extracts only the following base URL:
|
|
252
|
+
* ```
|
|
253
|
+
* https://www.twitter.com/realdonaldtrump
|
|
254
|
+
* ```
|
|
255
|
+
*
|
|
256
|
+
* Example usage:
|
|
257
|
+
* ```
|
|
258
|
+
* import { social } from '@crawlee/utils';
|
|
259
|
+
*
|
|
260
|
+
* const matches = text.match(social.TWITTER_REGEX_STRING);
|
|
261
|
+
* if (matches) console.log(`${matches.length} Twitter profiles found!`);
|
|
262
|
+
* ```
|
|
263
|
+
*/
|
|
264
|
+
export declare const TWITTER_REGEX_GLOBAL: RegExp;
|
|
265
|
+
/**
|
|
266
|
+
* Regular expression to exactly match a single Facebook profile URL.
|
|
267
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
268
|
+
* ```
|
|
269
|
+
* https://www.facebook.com/apifytech
|
|
270
|
+
* facebook.com/apifytech
|
|
271
|
+
* fb.com/apifytech
|
|
272
|
+
* https://www.facebook.com/profile.php?id=123456789
|
|
273
|
+
* ```
|
|
274
|
+
*
|
|
275
|
+
* The regular expression does NOT match URLs with additional
|
|
276
|
+
* subdirectories or query parameters, such as:
|
|
277
|
+
* ```
|
|
278
|
+
* https://www.facebook.com/apifytech/photos
|
|
279
|
+
* ```
|
|
280
|
+
*
|
|
281
|
+
* Example usage:
|
|
282
|
+
* ```
|
|
283
|
+
* import { social } from '@crawlee/utils';
|
|
284
|
+
*
|
|
285
|
+
* if (social.FACEBOOK_REGEX.test('https://www.facebook.com/apifytech')) {
|
|
286
|
+
* console.log('Match!');
|
|
287
|
+
* }
|
|
288
|
+
* ```
|
|
289
|
+
*/
|
|
290
|
+
export declare const FACEBOOK_REGEX: RegExp;
|
|
291
|
+
/**
|
|
292
|
+
* Regular expression to find multiple Facebook profile URLs in a text or HTML.
|
|
293
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
294
|
+
* ```
|
|
295
|
+
* https://www.facebook.com/apifytech
|
|
296
|
+
* facebook.com/apifytech
|
|
297
|
+
* fb.com/apifytech
|
|
298
|
+
* ```
|
|
299
|
+
*
|
|
300
|
+
* If the profile URL contains subdirectories or query parameters, the regular expression
|
|
301
|
+
* extracts just the base part of the profile URL. For example, from text such as:
|
|
302
|
+
* ```
|
|
303
|
+
* https://www.facebook.com/apifytech/photos
|
|
304
|
+
* ```
|
|
305
|
+
* the expression extracts only the following base URL:
|
|
306
|
+
* ```
|
|
307
|
+
* https://www.facebook.com/apifytech
|
|
308
|
+
* ```
|
|
309
|
+
*
|
|
310
|
+
* Example usage:
|
|
311
|
+
* ```
|
|
312
|
+
* import { social } from '@crawlee/utils';
|
|
313
|
+
*
|
|
314
|
+
* const matches = text.match(social.FACEBOOK_REGEX_GLOBAL);
|
|
315
|
+
* if (matches) console.log(`${matches.length} Facebook profiles found!`);
|
|
316
|
+
* ```
|
|
317
|
+
*/
|
|
318
|
+
export declare const FACEBOOK_REGEX_GLOBAL: RegExp;
|
|
319
|
+
/**
|
|
320
|
+
* Regular expression to exactly match a single Youtube channel, user or video URL.
|
|
321
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
322
|
+
* ```
|
|
323
|
+
* https://www.youtube.com/watch?v=kM7YfhfkiEE
|
|
324
|
+
* https://youtu.be/kM7YfhfkiEE
|
|
325
|
+
* https://www.youtube.com/c/TrapNation
|
|
326
|
+
* https://www.youtube.com/channel/UCklie6BM0fhFvzWYqQVoCTA
|
|
327
|
+
* https://www.youtube.com/user/pewdiepie
|
|
328
|
+
* ```
|
|
329
|
+
*
|
|
330
|
+
* Please note that this won't match URLs like https://www.youtube.com/pewdiepie that redirect to /user or /channel.
|
|
331
|
+
*
|
|
332
|
+
* Example usage:
|
|
333
|
+
* ```
|
|
334
|
+
* import { social } from '@crawlee/utils';
|
|
335
|
+
*
|
|
336
|
+
* if (social.YOUTUBE_REGEX.test('https://www.youtube.com/watch?v=kM7YfhfkiEE')) {
|
|
337
|
+
* console.log('Match!');
|
|
338
|
+
* }
|
|
339
|
+
* ```
|
|
340
|
+
*/
|
|
341
|
+
export declare const YOUTUBE_REGEX: RegExp;
|
|
342
|
+
/**
|
|
343
|
+
* Regular expression to find multiple Youtube channel, user or video URLs in a text or HTML.
|
|
344
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
345
|
+
* ```
|
|
346
|
+
* https://www.youtube.com/watch?v=kM7YfhfkiEE
|
|
347
|
+
* https://youtu.be/kM7YfhfkiEE
|
|
348
|
+
* https://www.youtube.com/c/TrapNation
|
|
349
|
+
* https://www.youtube.com/channel/UCklie6BM0fhFvzWYqQVoCTA
|
|
350
|
+
* https://www.youtube.com/user/pewdiepie
|
|
351
|
+
* ```
|
|
352
|
+
*
|
|
353
|
+
* Please note that this won't match URLs like https://www.youtube.com/pewdiepie that redirect to /user or /channel.
|
|
354
|
+
*
|
|
355
|
+
* Example usage:
|
|
356
|
+
* ```
|
|
357
|
+
* import { social } from '@crawlee/utils';
|
|
358
|
+
*
|
|
359
|
+
* const matches = text.match(social.YOUTUBE_REGEX_GLOBAL);
|
|
360
|
+
* if (matches) console.log(`${matches.length} Youtube videos found!`);
|
|
361
|
+
* ```
|
|
362
|
+
*/
|
|
363
|
+
export declare const YOUTUBE_REGEX_GLOBAL: RegExp;
|
|
364
|
+
/**
|
|
365
|
+
* Regular expression to exactly match a Tiktok video or user account.
|
|
366
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
367
|
+
* ```
|
|
368
|
+
* https://www.tiktok.com/trending?shareId=123456789
|
|
369
|
+
* https://www.tiktok.com/embed/123456789
|
|
370
|
+
* https://m.tiktok.com/v/123456789
|
|
371
|
+
* https://www.tiktok.com/@user
|
|
372
|
+
* https://www.tiktok.com/@user-account.pro
|
|
373
|
+
* https://www.tiktok.com/@user/video/123456789
|
|
374
|
+
* ```
|
|
375
|
+
*
|
|
376
|
+
* Example usage:
|
|
377
|
+
* ```
|
|
378
|
+
* import { social } from '@crawlee/utils';
|
|
379
|
+
*
|
|
380
|
+
* if (social.TIKTOK_REGEX.test('https://www.tiktok.com/trending?shareId=123456789')) {
|
|
381
|
+
* console.log('Match!');
|
|
382
|
+
* }
|
|
383
|
+
* ```
|
|
384
|
+
*/
|
|
385
|
+
export declare const TIKTOK_REGEX: RegExp;
|
|
386
|
+
/**
|
|
387
|
+
* Regular expression to find multiple Tiktok videos or user accounts in a text or HTML.
|
|
388
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
389
|
+
* ```
|
|
390
|
+
* https://www.tiktok.com/trending?shareId=123456789
|
|
391
|
+
* https://www.tiktok.com/embed/123456789
|
|
392
|
+
* https://m.tiktok.com/v/123456789
|
|
393
|
+
* https://www.tiktok.com/@user
|
|
394
|
+
* https://www.tiktok.com/@user-account.pro
|
|
395
|
+
* https://www.tiktok.com/@user/video/123456789
|
|
396
|
+
* ```
|
|
397
|
+
*
|
|
398
|
+
* Example usage:
|
|
399
|
+
* ```
|
|
400
|
+
* import { social } from '@crawlee/utils';
|
|
401
|
+
*
|
|
402
|
+
* const matches = text.match(social.TIKTOK_REGEX_GLOBAL);
|
|
403
|
+
* if (matches) console.log(`${matches.length} Tiktok profiles/videos found!`);
|
|
404
|
+
* ```
|
|
405
|
+
*/
|
|
406
|
+
export declare const TIKTOK_REGEX_GLOBAL: RegExp;
|
|
407
|
+
/**
|
|
408
|
+
* Regular expression to exactly match a Pinterest pin, user or user's board.
|
|
409
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
410
|
+
* ```
|
|
411
|
+
* https://pinterest.com/pin/123456789
|
|
412
|
+
* https://www.pinterest.cz/pin/123456789
|
|
413
|
+
* https://www.pinterest.com/user
|
|
414
|
+
* https://uk.pinterest.com/user
|
|
415
|
+
* https://www.pinterest.co.uk/user
|
|
416
|
+
* pinterest.com/user_name.gold
|
|
417
|
+
* https://cz.pinterest.com/user/board
|
|
418
|
+
* ```
|
|
419
|
+
*
|
|
420
|
+
* Example usage:
|
|
421
|
+
* ```
|
|
422
|
+
* import { social } from '@crawlee/utils';
|
|
423
|
+
*
|
|
424
|
+
* if (social.PINTEREST_REGEX.test('https://pinterest.com/pin/123456789')) {
|
|
425
|
+
* console.log('Match!');
|
|
426
|
+
* }
|
|
427
|
+
* ```
|
|
428
|
+
*/
|
|
429
|
+
export declare const PINTEREST_REGEX: RegExp;
|
|
430
|
+
/**
|
|
431
|
+
* Regular expression to find multiple Pinterest pins, users or boards in a text or HTML.
|
|
432
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
433
|
+
* ```
|
|
434
|
+
* https://pinterest.com/pin/123456789
|
|
435
|
+
* https://www.pinterest.cz/pin/123456789
|
|
436
|
+
* https://www.pinterest.com/user
|
|
437
|
+
* https://uk.pinterest.com/user
|
|
438
|
+
* https://www.pinterest.co.uk/user
|
|
439
|
+
* pinterest.com/user_name.gold
|
|
440
|
+
* https://cz.pinterest.com/user/board
|
|
441
|
+
* ```
|
|
442
|
+
*
|
|
443
|
+
* Example usage:
|
|
444
|
+
* ```
|
|
445
|
+
* import { social } from '@crawlee/utils';
|
|
446
|
+
*
|
|
447
|
+
* const matches = text.match(social.PINTEREST_REGEX_GLOBAL);
|
|
448
|
+
* if (matches) console.log(`${matches.length} Pinterest pins found!`);
|
|
449
|
+
* ```
|
|
450
|
+
*/
|
|
451
|
+
export declare const PINTEREST_REGEX_GLOBAL: RegExp;
|
|
452
|
+
/**
|
|
453
|
+
* Regular expression to exactly match a Discord invite or channel.
|
|
454
|
+
* It has the following form: `/^...$/i` and matches URLs such as:
|
|
455
|
+
* ```
|
|
456
|
+
* https://discord.gg/discord-developers
|
|
457
|
+
* https://discord.com/invite/jyEM2PRvMU
|
|
458
|
+
* https://discordapp.com/channels/1234
|
|
459
|
+
* https://discord.com/channels/1234/1234
|
|
460
|
+
* discord.gg/discord-developers
|
|
461
|
+
* ```
|
|
462
|
+
*
|
|
463
|
+
* Example usage:
|
|
464
|
+
* ```
|
|
465
|
+
* import { social } from '@crawlee/utils';
|
|
466
|
+
*
|
|
467
|
+
* if (social.DISCORD_REGEX.test('https://discord.gg/discord-developers')) {
|
|
468
|
+
* console.log('Match!');
|
|
469
|
+
* }
|
|
470
|
+
* ```
|
|
471
|
+
*/
|
|
472
|
+
export declare const DISCORD_REGEX: RegExp;
|
|
473
|
+
/**
|
|
474
|
+
* Regular expression to find multiple Discord channels or invites in a text or HTML.
|
|
475
|
+
* It has the following form: `/.../ig` and matches URLs such as:
|
|
476
|
+
* ```
|
|
477
|
+
* https://discord.gg/discord-developers
|
|
478
|
+
* https://discord.com/invite/jyEM2PRvMU
|
|
479
|
+
* https://discordapp.com/channels/1234
|
|
480
|
+
* https://discord.com/channels/1234/1234
|
|
481
|
+
* discord.gg/discord-developers
|
|
482
|
+
* ```
|
|
483
|
+
*
|
|
484
|
+
* Example usage:
|
|
485
|
+
* ```
|
|
486
|
+
* import { social } from '@crawlee/utils';
|
|
487
|
+
*
|
|
488
|
+
* const matches = text.match(social.DISCORD_REGEX_GLOBAL);
|
|
489
|
+
* if (matches) console.log(`${matches.length} Discord channels found!`);
|
|
490
|
+
* ```
|
|
491
|
+
*/
|
|
492
|
+
export declare const DISCORD_REGEX_GLOBAL: RegExp;
|
|
493
|
+
//# sourceMappingURL=social.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"social.d.ts","sourceRoot":"","sources":["../../src/internals/social.ts"],"names":[],"mappings":"AAOA;;;GAGG;AACH,eAAO,MAAM,WAAW,QAA6C,CAAC;AAEtE;;;GAGG;AACH,eAAO,MAAM,kBAAkB,QAAuC,CAAC;AAIvE;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAGrD;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAavD;AA4DD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAiBrD;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAYvD;AAmCD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAW,GAAG,aAAa,CAuD7G;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,cAAc,QAAgD,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,eAAO,MAAM,qBAAqB,QAA0C,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,eAAO,MAAM,eAAe,QAAiD,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,sBAAsB,QAA2C,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,aAAa,QAA+C,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,oBAAoB,QAAyC,CAAC;AAE3E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,eAAO,MAAM,cAAc,QAAgD,CAAC;AAE5E;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,eAAO,MAAM,qBAAqB,QAA0C,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,aAAa,QAA+C,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,oBAAoB,QAAyC,CAAC;AAC3E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,YAAY,QAA8C,CAAC;AAExE;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,mBAAmB,QAAwC,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,eAAO,MAAM,eAAe,QAAiD,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,sBAAsB,QAA2C,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,aAAa,QAA+C,CAAC;AAE1E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,oBAAoB,QAAyC,CAAC"}
|