imperium-crawl 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -77
- package/dist/cli-tui.d.ts +1 -1
- package/dist/cli-tui.js +1 -1
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +0 -11
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +0 -18
- package/dist/config.js.map +1 -1
- package/dist/constants.d.ts +1 -1
- package/dist/constants.js +1 -1
- package/dist/formatters.d.ts +2 -2
- package/dist/formatters.js +2 -2
- package/dist/index.js +6 -28
- package/dist/index.js.map +1 -1
- package/dist/recipes/index.d.ts.map +1 -1
- package/dist/recipes/index.js +8 -0
- package/dist/recipes/index.js.map +1 -1
- package/dist/recipes/influencer-competitor-spy.json +14 -0
- package/dist/recipes/influencer-content-scout.json +14 -0
- package/dist/recipes/influencer-hashtag-scout.json +14 -0
- package/dist/recipes/influencer-niche-discovery.json +14 -0
- package/dist/skills/manager.d.ts +11 -2
- package/dist/skills/manager.d.ts.map +1 -1
- package/dist/skills/manager.js.map +1 -1
- package/dist/social/ai-fallback.d.ts +22 -0
- package/dist/social/ai-fallback.d.ts.map +1 -0
- package/dist/social/ai-fallback.js +156 -0
- package/dist/social/ai-fallback.js.map +1 -0
- package/dist/social/parsers.d.ts +28 -0
- package/dist/social/parsers.d.ts.map +1 -0
- package/dist/social/parsers.js +146 -0
- package/dist/social/parsers.js.map +1 -0
- package/dist/social/types.d.ts +86 -0
- package/dist/social/types.d.ts.map +1 -0
- package/dist/social/types.js +5 -0
- package/dist/social/types.js.map +1 -0
- package/dist/social/whisper.d.ts +29 -0
- package/dist/social/whisper.d.ts.map +1 -0
- package/dist/social/whisper.js +88 -0
- package/dist/social/whisper.js.map +1 -0
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +7 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/instagram.d.ts +51 -0
- package/dist/tools/instagram.d.ts.map +1 -0
- package/dist/tools/instagram.js +462 -0
- package/dist/tools/instagram.js.map +1 -0
- package/dist/tools/interact.d.ts +4 -4
- package/dist/tools/interact.js +1 -1
- package/dist/tools/interact.js.map +1 -1
- package/dist/tools/manifest.d.ts +0 -1
- package/dist/tools/manifest.d.ts.map +1 -1
- package/dist/tools/manifest.js +17 -1
- package/dist/tools/manifest.js.map +1 -1
- package/dist/tools/reddit.d.ts +36 -0
- package/dist/tools/reddit.d.ts.map +1 -0
- package/dist/tools/reddit.js +190 -0
- package/dist/tools/reddit.js.map +1 -0
- package/dist/tools/run-skill.d.ts +24 -0
- package/dist/tools/run-skill.d.ts.map +1 -1
- package/dist/tools/run-skill.js +1015 -12
- package/dist/tools/run-skill.js.map +1 -1
- package/dist/tools/tiktok.d.ts +30 -0
- package/dist/tools/tiktok.d.ts.map +1 -0
- package/dist/tools/tiktok.js +246 -0
- package/dist/tools/tiktok.js.map +1 -0
- package/dist/tools/youtube.d.ts +33 -0
- package/dist/tools/youtube.d.ts.map +1 -0
- package/dist/tools/youtube.js +489 -0
- package/dist/tools/youtube.js.map +1 -0
- package/dist/utils/fetcher.d.ts.map +1 -1
- package/dist/utils/fetcher.js +1 -3
- package/dist/utils/fetcher.js.map +1 -1
- package/package.json +3 -9
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { smartFetch } from "../stealth/index.js";
|
|
3
|
+
import { isPlaywrightAvailable } from "../stealth/browser.js";
|
|
4
|
+
import { acquirePage } from "../stealth/chrome-profile.js";
|
|
5
|
+
import { MAX_QUERY_LENGTH, MAX_ITEMS } from "../constants.js";
|
|
6
|
+
import { extractScriptJson, parseCompactNumber } from "../social/parsers.js";
|
|
7
|
+
import { hasWhisperConfigured, transcribeAudio } from "../social/whisper.js";
|
|
8
|
+
import { socialAiFallback } from "../social/ai-fallback.js";
|
|
9
|
+
export const name = "youtube";
|
|
10
|
+
export const description = "Search YouTube videos, get video details, comments, transcripts, and channel info. No API key needed.";
|
|
11
|
+
const ActionEnum = z.enum(["search", "video", "comments", "transcript", "channel"]);
|
|
12
|
+
export const schema = z.object({
|
|
13
|
+
action: ActionEnum.describe("Action to perform"),
|
|
14
|
+
query: z.string().max(MAX_QUERY_LENGTH).optional().describe("Search query (for action: search)"),
|
|
15
|
+
url: z.string().max(8192).optional().describe("Video URL or ID (for action: video, comments, transcript)"),
|
|
16
|
+
channel_url: z.string().max(8192).optional().describe("Channel URL like youtube.com/@name (for action: channel)"),
|
|
17
|
+
limit: z.number().min(1).max(MAX_ITEMS).default(10).describe("Max results to return"),
|
|
18
|
+
sort: z.enum(["relevance", "date", "views"]).default("relevance").describe("Sort order for search"),
|
|
19
|
+
});
|
|
20
|
+
// ── Helpers ──
|
|
21
|
+
function extractVideoId(input) {
|
|
22
|
+
// Full URL: youtube.com/watch?v=xxx, youtu.be/xxx, youtube.com/shorts/xxx
|
|
23
|
+
try {
|
|
24
|
+
const url = new URL(input.startsWith("http") ? input : `https://${input}`);
|
|
25
|
+
if (url.hostname.includes("youtu.be"))
|
|
26
|
+
return url.pathname.slice(1);
|
|
27
|
+
if (url.pathname.startsWith("/shorts/"))
|
|
28
|
+
return url.pathname.split("/")[2];
|
|
29
|
+
return url.searchParams.get("v");
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// Maybe just a video ID
|
|
33
|
+
if (/^[\w-]{11}$/.test(input))
|
|
34
|
+
return input;
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function sortParam(sort) {
|
|
39
|
+
switch (sort) {
|
|
40
|
+
case "date": return "&sp=CAI%253D";
|
|
41
|
+
case "views": return "&sp=CAM%253D";
|
|
42
|
+
default: return "";
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// ── ytInitialData Extractors ──
|
|
46
|
+
function extractSearchResults(data, limit) {
|
|
47
|
+
const results = [];
|
|
48
|
+
try {
|
|
49
|
+
// Navigate the ytInitialData structure
|
|
50
|
+
const contents = data?.contents?.twoColumnSearchResultsRenderer
|
|
51
|
+
?.primaryContents?.sectionListRenderer?.contents;
|
|
52
|
+
if (!Array.isArray(contents))
|
|
53
|
+
return results;
|
|
54
|
+
for (const section of contents) {
|
|
55
|
+
const items = section?.itemSectionRenderer?.contents;
|
|
56
|
+
if (!Array.isArray(items))
|
|
57
|
+
continue;
|
|
58
|
+
for (const item of items) {
|
|
59
|
+
if (results.length >= limit)
|
|
60
|
+
break;
|
|
61
|
+
const v = item?.videoRenderer;
|
|
62
|
+
if (!v)
|
|
63
|
+
continue;
|
|
64
|
+
results.push({
|
|
65
|
+
id: v.videoId || "",
|
|
66
|
+
title: v.title?.runs?.[0]?.text || "",
|
|
67
|
+
url: `https://www.youtube.com/watch?v=${v.videoId}`,
|
|
68
|
+
thumbnail: v.thumbnail?.thumbnails?.slice(-1)?.[0]?.url,
|
|
69
|
+
duration: v.lengthText?.simpleText,
|
|
70
|
+
views: parseCompactNumber(v.viewCountText?.simpleText || v.shortViewCountText?.simpleText || ""),
|
|
71
|
+
author: v.ownerText?.runs?.[0]?.text || "",
|
|
72
|
+
author_url: v.ownerText?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl
|
|
73
|
+
? `https://www.youtube.com${v.ownerText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl}`
|
|
74
|
+
: undefined,
|
|
75
|
+
published: v.publishedTimeText?.simpleText,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
// Malformed data
|
|
82
|
+
}
|
|
83
|
+
return results;
|
|
84
|
+
}
|
|
85
|
+
function extractVideoDetails(data) {
|
|
86
|
+
try {
|
|
87
|
+
const vd = data?.contents?.twoColumnWatchNextResults?.results
|
|
88
|
+
?.results?.contents;
|
|
89
|
+
if (!Array.isArray(vd))
|
|
90
|
+
return null;
|
|
91
|
+
const primary = vd.find((c) => c.videoPrimaryInfoRenderer);
|
|
92
|
+
const secondary = vd.find((c) => c.videoSecondaryInfoRenderer);
|
|
93
|
+
const pi = primary?.videoPrimaryInfoRenderer;
|
|
94
|
+
const si = secondary?.videoSecondaryInfoRenderer;
|
|
95
|
+
const videoId = data?.currentVideoEndpoint?.watchEndpoint?.videoId || "";
|
|
96
|
+
const title = pi?.title?.runs?.map((r) => r.text).join("") || "";
|
|
97
|
+
const viewCountText = pi?.viewCount?.videoViewCountRenderer?.viewCount?.simpleText || "";
|
|
98
|
+
const views = parseCompactNumber(viewCountText);
|
|
99
|
+
const published = pi?.dateText?.simpleText;
|
|
100
|
+
// Like count from toggle buttons
|
|
101
|
+
let likes;
|
|
102
|
+
const buttons = pi?.videoActions?.menuRenderer?.topLevelButtons;
|
|
103
|
+
if (Array.isArray(buttons)) {
|
|
104
|
+
for (const btn of buttons) {
|
|
105
|
+
const seg = btn?.segmentedLikeDislikeButtonViewModel;
|
|
106
|
+
const likeText = seg?.likeButtonViewModel?.likeButtonViewModel
|
|
107
|
+
?.toggleButtonViewModel?.toggleButtonViewModel?.defaultButtonViewModel
|
|
108
|
+
?.buttonViewModel?.title;
|
|
109
|
+
if (likeText) {
|
|
110
|
+
likes = parseCompactNumber(likeText);
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const author = si?.owner?.videoOwnerRenderer?.title?.runs?.[0]?.text || "";
|
|
116
|
+
const authorUrl = si?.owner?.videoOwnerRenderer?.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl;
|
|
117
|
+
const description = si?.attributedDescription?.content || "";
|
|
118
|
+
return {
|
|
119
|
+
id: videoId,
|
|
120
|
+
title,
|
|
121
|
+
url: `https://www.youtube.com/watch?v=${videoId}`,
|
|
122
|
+
views: isNaN(views) ? undefined : views,
|
|
123
|
+
likes: likes && !isNaN(likes) ? likes : undefined,
|
|
124
|
+
author,
|
|
125
|
+
author_url: authorUrl ? `https://www.youtube.com${authorUrl}` : undefined,
|
|
126
|
+
published,
|
|
127
|
+
description: description.substring(0, 2000),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
function extractChannelInfo(data) {
|
|
135
|
+
try {
|
|
136
|
+
const header = data?.header;
|
|
137
|
+
const c4 = header?.c4TabbedHeaderRenderer;
|
|
138
|
+
const ph = header?.pageHeaderRenderer;
|
|
139
|
+
const metadata = data?.metadata?.channelMetadataRenderer;
|
|
140
|
+
if (!metadata)
|
|
141
|
+
return null;
|
|
142
|
+
// Subscriber count: try c4 header first, then pageHeaderRenderer
|
|
143
|
+
let subscriberText;
|
|
144
|
+
if (c4?.subscriberCountText?.simpleText) {
|
|
145
|
+
subscriberText = c4.subscriberCountText.simpleText;
|
|
146
|
+
}
|
|
147
|
+
else if (ph) {
|
|
148
|
+
// New YouTube layout: pageHeaderViewModel → metadataRows
|
|
149
|
+
const rows = ph?.content?.pageHeaderViewModel?.metadata?.contentMetadataViewModel?.metadataRows;
|
|
150
|
+
if (Array.isArray(rows)) {
|
|
151
|
+
for (const row of rows) {
|
|
152
|
+
for (const part of row?.metadataParts || []) {
|
|
153
|
+
const text = part?.text?.content || "";
|
|
154
|
+
if (text.includes("subscriber")) {
|
|
155
|
+
subscriberText = text;
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (subscriberText)
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// Verified badge: check c4 badges or pageHeaderViewModel
|
|
165
|
+
let verified = false;
|
|
166
|
+
if (c4?.badges) {
|
|
167
|
+
verified = c4.badges.some((b) => b.metadataBadgeRenderer?.style?.includes("VERIFIED"));
|
|
168
|
+
}
|
|
169
|
+
return {
|
|
170
|
+
name: metadata.title || "",
|
|
171
|
+
url: metadata.channelUrl || "",
|
|
172
|
+
description: (metadata.description || "").substring(0, 1000),
|
|
173
|
+
subscribers: subscriberText ? parseCompactNumber(subscriberText) : undefined,
|
|
174
|
+
avatar: metadata.avatar?.thumbnails?.slice(-1)?.[0]?.url,
|
|
175
|
+
verified,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
function extractTranscriptUrl(data) {
|
|
183
|
+
try {
|
|
184
|
+
// Look for captions in playerResponse
|
|
185
|
+
const captions = data?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
|
|
186
|
+
if (Array.isArray(captions) && captions.length > 0) {
|
|
187
|
+
// Prefer English, fallback to first
|
|
188
|
+
const en = captions.find((c) => c.languageCode === "en");
|
|
189
|
+
return (en || captions[0])?.baseUrl || null;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// No captions
|
|
194
|
+
}
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
// ── Actions ──
|
|
198
|
+
async function searchYouTube(query, limit, sort) {
|
|
199
|
+
const encoded = encodeURIComponent(query);
|
|
200
|
+
const result = await smartFetch(`https://www.youtube.com/results?search_query=${encoded}${sortParam(sort)}`, {
|
|
201
|
+
maxLevel: 2,
|
|
202
|
+
});
|
|
203
|
+
const data = extractScriptJson(result.html, "ytInitialData");
|
|
204
|
+
const videos = data ? extractSearchResults(data, limit) : [];
|
|
205
|
+
if (videos.length === 0) {
|
|
206
|
+
const url = `https://www.youtube.com/results?search_query=${encoded}${sortParam(sort)}`;
|
|
207
|
+
const fb = await socialAiFallback({ action: "youtube:search", url });
|
|
208
|
+
if (fb?.data?.results?.length)
|
|
209
|
+
return { query, platform: "youtube", results: fb.data.results.slice(0, limit) };
|
|
210
|
+
}
|
|
211
|
+
return { query, platform: "youtube", results: videos };
|
|
212
|
+
}
|
|
213
|
+
async function getVideoDetails(videoUrl) {
|
|
214
|
+
const videoId = extractVideoId(videoUrl);
|
|
215
|
+
if (!videoId)
|
|
216
|
+
return null;
|
|
217
|
+
const result = await smartFetch(`https://www.youtube.com/watch?v=${videoId}`, { maxLevel: 2 });
|
|
218
|
+
const data = extractScriptJson(result.html, "ytInitialData");
|
|
219
|
+
const video = data ? extractVideoDetails(data) : null;
|
|
220
|
+
if (!video) {
|
|
221
|
+
const fb = await socialAiFallback({ action: "youtube:video", url: `https://www.youtube.com/watch?v=${videoId}` });
|
|
222
|
+
return fb?.data ?? null;
|
|
223
|
+
}
|
|
224
|
+
// Also try to get more info from ytInitialPlayerResponse
|
|
225
|
+
if (video) {
|
|
226
|
+
const playerData = extractScriptJson(result.html, "ytInitialPlayerResponse");
|
|
227
|
+
if (playerData) {
|
|
228
|
+
const pd = playerData;
|
|
229
|
+
video.duration = video.duration || pd?.videoDetails?.lengthSeconds
|
|
230
|
+
? `${Math.floor(Number(pd.videoDetails.lengthSeconds) / 60)}:${String(Number(pd.videoDetails.lengthSeconds) % 60).padStart(2, "0")}`
|
|
231
|
+
: undefined;
|
|
232
|
+
video.thumbnail = video.thumbnail || pd?.videoDetails?.thumbnail?.thumbnails?.slice(-1)?.[0]?.url;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
return video;
|
|
236
|
+
}
|
|
237
|
+
async function getComments(videoUrl, limit) {
|
|
238
|
+
if (!(await isPlaywrightAvailable())) {
|
|
239
|
+
throw new Error("rebrowser-playwright is required for YouTube comments. Install with: npm i rebrowser-playwright");
|
|
240
|
+
}
|
|
241
|
+
const videoId = extractVideoId(videoUrl);
|
|
242
|
+
if (!videoId)
|
|
243
|
+
return [];
|
|
244
|
+
const handle = await acquirePage();
|
|
245
|
+
try {
|
|
246
|
+
const { page } = handle;
|
|
247
|
+
await page.goto(`https://www.youtube.com/watch?v=${videoId}`, {
|
|
248
|
+
waitUntil: "load",
|
|
249
|
+
timeout: 30_000,
|
|
250
|
+
});
|
|
251
|
+
// Scroll down to load comments
|
|
252
|
+
for (let i = 0; i < 3; i++) {
|
|
253
|
+
await page.evaluate(() => window.scrollBy(0, window.innerHeight));
|
|
254
|
+
await page.waitForTimeout(1500);
|
|
255
|
+
}
|
|
256
|
+
// Wait for comment section
|
|
257
|
+
await page.waitForSelector("ytd-comment-thread-renderer", { timeout: 10_000 }).catch(() => { });
|
|
258
|
+
const comments = await page.evaluate((lim) => {
|
|
259
|
+
const elements = document.querySelectorAll("ytd-comment-thread-renderer");
|
|
260
|
+
const result = [];
|
|
261
|
+
for (let i = 0; i < Math.min(elements.length, lim); i++) {
|
|
262
|
+
const el = elements[i];
|
|
263
|
+
const authorEl = el.querySelector("#author-text span");
|
|
264
|
+
const textEl = el.querySelector("#content-text");
|
|
265
|
+
const likesEl = el.querySelector("#vote-count-middle");
|
|
266
|
+
const timeEl = el.querySelector(".published-time-text a");
|
|
267
|
+
result.push({
|
|
268
|
+
id: `comment-${i}`,
|
|
269
|
+
author: authorEl?.textContent?.trim() || "Unknown",
|
|
270
|
+
text: textEl?.textContent?.trim() || "",
|
|
271
|
+
score: parseInt(likesEl?.textContent?.trim() || "0", 10) || 0,
|
|
272
|
+
published: timeEl?.textContent?.trim(),
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
return result;
|
|
276
|
+
}, limit);
|
|
277
|
+
if (comments.length === 0) {
|
|
278
|
+
const fb = await socialAiFallback({ action: "youtube:comments", page });
|
|
279
|
+
if (fb?.data?.comments?.length)
|
|
280
|
+
return fb.data.comments.slice(0, limit);
|
|
281
|
+
}
|
|
282
|
+
return comments;
|
|
283
|
+
}
|
|
284
|
+
finally {
|
|
285
|
+
await handle.cleanup();
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
async function getTranscript(videoUrl) {
|
|
289
|
+
const videoId = extractVideoId(videoUrl);
|
|
290
|
+
if (!videoId)
|
|
291
|
+
return null;
|
|
292
|
+
// YouTube transcript requires Playwright — click "Show transcript" and parse DOM.
|
|
293
|
+
if (!(await isPlaywrightAvailable())) {
|
|
294
|
+
throw new Error("rebrowser-playwright is required for YouTube transcripts. Install with: npm i rebrowser-playwright");
|
|
295
|
+
}
|
|
296
|
+
const handle = await acquirePage();
|
|
297
|
+
try {
|
|
298
|
+
const { page } = handle;
|
|
299
|
+
await page.goto(`https://www.youtube.com/watch?v=${videoId}`, {
|
|
300
|
+
waitUntil: "load",
|
|
301
|
+
timeout: 30_000,
|
|
302
|
+
});
|
|
303
|
+
await page.waitForTimeout(3000);
|
|
304
|
+
// Scroll down to reveal description area
|
|
305
|
+
await page.evaluate(() => window.scrollBy(0, 400));
|
|
306
|
+
await page.waitForTimeout(1000);
|
|
307
|
+
// Expand description to reveal "Show transcript" button
|
|
308
|
+
const expandBtn = page.locator("#expand, tp-yt-paper-button#expand").first();
|
|
309
|
+
if (await expandBtn.isVisible().catch(() => false)) {
|
|
310
|
+
await expandBtn.click();
|
|
311
|
+
await page.waitForTimeout(1500);
|
|
312
|
+
}
|
|
313
|
+
// Try captions first — click "Show transcript"
|
|
314
|
+
const transcriptBtn = page.locator('button:has-text("Show transcript")').first();
|
|
315
|
+
const hasTranscriptBtn = await transcriptBtn.isVisible({ timeout: 5000 }).catch(() => false);
|
|
316
|
+
if (hasTranscriptBtn) {
|
|
317
|
+
await transcriptBtn.click();
|
|
318
|
+
await page.waitForTimeout(5000);
|
|
319
|
+
const segments = await page.evaluate(() => {
|
|
320
|
+
const panel = document.querySelector('ytd-engagement-panel-section-list-renderer[target-id="engagement-panel-searchable-transcript"]');
|
|
321
|
+
if (!panel)
|
|
322
|
+
return [];
|
|
323
|
+
const renderers = panel.querySelectorAll("ytd-transcript-segment-renderer");
|
|
324
|
+
const result = [];
|
|
325
|
+
for (const renderer of renderers) {
|
|
326
|
+
const textNodes = [];
|
|
327
|
+
const walker = document.createTreeWalker(renderer, NodeFilter.SHOW_TEXT, null);
|
|
328
|
+
let node;
|
|
329
|
+
while ((node = walker.nextNode())) {
|
|
330
|
+
const t = node.textContent?.trim();
|
|
331
|
+
if (t && t.length > 0)
|
|
332
|
+
textNodes.push(t);
|
|
333
|
+
}
|
|
334
|
+
if (textNodes.length >= 2) {
|
|
335
|
+
result.push({ start: textNodes[0], text: textNodes.slice(1).join(" ") });
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
return result;
|
|
339
|
+
});
|
|
340
|
+
if (segments.length > 0) {
|
|
341
|
+
return { segments, total: segments.length, source: "captions" };
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
// No captions — try AI fallback for on-screen transcript data
|
|
345
|
+
if (!hasWhisperConfigured()) {
|
|
346
|
+
const fb = await socialAiFallback({ action: "youtube:transcript", page });
|
|
347
|
+
if (fb?.data?.segments?.length)
|
|
348
|
+
return { segments: fb.data.segments, total: fb.data.segments.length, source: "captions" };
|
|
349
|
+
return null;
|
|
350
|
+
}
|
|
351
|
+
// Extract audio URL from player data
|
|
352
|
+
const audioInfo = await page.evaluate(() => {
|
|
353
|
+
const pd = window.ytInitialPlayerResponse;
|
|
354
|
+
if (!pd?.streamingData?.adaptiveFormats)
|
|
355
|
+
return null;
|
|
356
|
+
// Find audio-only format — prefer lowest bitrate (smallest file)
|
|
357
|
+
const audioFormats = pd.streamingData.adaptiveFormats
|
|
358
|
+
.filter((f) => f.mimeType?.startsWith("audio/") && f.url)
|
|
359
|
+
.sort((a, b) => (a.bitrate || 0) - (b.bitrate || 0));
|
|
360
|
+
if (audioFormats.length === 0)
|
|
361
|
+
return null;
|
|
362
|
+
const fmt = audioFormats[0];
|
|
363
|
+
return {
|
|
364
|
+
url: fmt.url,
|
|
365
|
+
mimeType: fmt.mimeType,
|
|
366
|
+
contentLength: parseInt(fmt.contentLength || "0", 10),
|
|
367
|
+
bitrate: fmt.bitrate,
|
|
368
|
+
duration: parseInt(pd.videoDetails?.lengthSeconds || "0", 10),
|
|
369
|
+
};
|
|
370
|
+
});
|
|
371
|
+
if (!audioInfo?.url)
|
|
372
|
+
return null;
|
|
373
|
+
// Check file size (Whisper limit: 25MB)
|
|
374
|
+
if (audioInfo.contentLength > 25 * 1024 * 1024) {
|
|
375
|
+
throw new Error(`Video audio too large for Whisper (${Math.round(audioInfo.contentLength / 1024 / 1024)}MB). Limit is 25MB. Try a shorter video.`);
|
|
376
|
+
}
|
|
377
|
+
// Download audio from within browser context (URLs are session-signed)
|
|
378
|
+
const audioBase64 = await page.evaluate(async (url) => {
|
|
379
|
+
const resp = await fetch(url);
|
|
380
|
+
if (!resp.ok)
|
|
381
|
+
return null;
|
|
382
|
+
const buffer = await resp.arrayBuffer();
|
|
383
|
+
// Convert to base64
|
|
384
|
+
const bytes = new Uint8Array(buffer);
|
|
385
|
+
let binary = "";
|
|
386
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
387
|
+
binary += String.fromCharCode(bytes[i]);
|
|
388
|
+
}
|
|
389
|
+
return btoa(binary);
|
|
390
|
+
}, audioInfo.url);
|
|
391
|
+
if (!audioBase64)
|
|
392
|
+
return null;
|
|
393
|
+
const audioBuffer = Buffer.from(audioBase64, "base64");
|
|
394
|
+
// Transcribe with Whisper
|
|
395
|
+
const ext = audioInfo.mimeType?.includes("webm") ? "webm" : "mp4";
|
|
396
|
+
const whisperResult = await transcribeAudio(audioBuffer, { filename: `video.${ext}` });
|
|
397
|
+
// Convert Whisper segments to our format
|
|
398
|
+
const segments = whisperResult.segments?.map((s) => ({
|
|
399
|
+
start: formatSeconds(s.start),
|
|
400
|
+
text: s.text,
|
|
401
|
+
})) || [{ start: "0:00", text: whisperResult.text }];
|
|
402
|
+
return { segments, total: segments.length, source: "whisper" };
|
|
403
|
+
}
|
|
404
|
+
finally {
|
|
405
|
+
await handle.cleanup();
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
function formatSeconds(seconds) {
|
|
409
|
+
const mins = Math.floor(seconds / 60);
|
|
410
|
+
const secs = Math.floor(seconds % 60);
|
|
411
|
+
return `${mins}:${String(secs).padStart(2, "0")}`;
|
|
412
|
+
}
|
|
413
|
+
async function getChannel(channelUrl) {
|
|
414
|
+
// Normalize URL
|
|
415
|
+
let url = channelUrl;
|
|
416
|
+
if (!url.startsWith("http")) {
|
|
417
|
+
url = url.startsWith("@") ? `https://www.youtube.com/${url}` : `https://www.youtube.com/@${url}`;
|
|
418
|
+
}
|
|
419
|
+
const result = await smartFetch(url, { maxLevel: 2 });
|
|
420
|
+
const data = extractScriptJson(result.html, "ytInitialData");
|
|
421
|
+
const channel = data ? extractChannelInfo(data) : null;
|
|
422
|
+
if (!channel) {
|
|
423
|
+
const fb = await socialAiFallback({ action: "youtube:channel", url });
|
|
424
|
+
return fb?.data ?? null;
|
|
425
|
+
}
|
|
426
|
+
return channel;
|
|
427
|
+
}
|
|
428
|
+
// ── Execute ──
|
|
429
|
+
export async function execute(input) {
|
|
430
|
+
try {
|
|
431
|
+
let result;
|
|
432
|
+
switch (input.action) {
|
|
433
|
+
case "search": {
|
|
434
|
+
if (!input.query) {
|
|
435
|
+
return { content: [{ type: "text", text: JSON.stringify({ error: "query is required for search action" }) }] };
|
|
436
|
+
}
|
|
437
|
+
result = await searchYouTube(input.query, input.limit, input.sort);
|
|
438
|
+
break;
|
|
439
|
+
}
|
|
440
|
+
case "video": {
|
|
441
|
+
if (!input.url) {
|
|
442
|
+
return { content: [{ type: "text", text: JSON.stringify({ error: "url is required for video action" }) }] };
|
|
443
|
+
}
|
|
444
|
+
const video = await getVideoDetails(input.url);
|
|
445
|
+
result = video || { error: "Could not extract video details" };
|
|
446
|
+
break;
|
|
447
|
+
}
|
|
448
|
+
case "comments": {
|
|
449
|
+
if (!input.url) {
|
|
450
|
+
return { content: [{ type: "text", text: JSON.stringify({ error: "url is required for comments action" }) }] };
|
|
451
|
+
}
|
|
452
|
+
const comments = await getComments(input.url, input.limit);
|
|
453
|
+
result = { video_url: input.url, comments_count: comments.length, comments };
|
|
454
|
+
break;
|
|
455
|
+
}
|
|
456
|
+
case "transcript": {
|
|
457
|
+
if (!input.url) {
|
|
458
|
+
return { content: [{ type: "text", text: JSON.stringify({ error: "url is required for transcript action" }) }] };
|
|
459
|
+
}
|
|
460
|
+
const transcript = await getTranscript(input.url);
|
|
461
|
+
result = transcript || {
|
|
462
|
+
error: "No transcript available for this video",
|
|
463
|
+
hint: hasWhisperConfigured()
|
|
464
|
+
? undefined
|
|
465
|
+
: "Set OPENAI_API_KEY to enable Whisper AI transcription for videos without captions.",
|
|
466
|
+
};
|
|
467
|
+
break;
|
|
468
|
+
}
|
|
469
|
+
case "channel": {
|
|
470
|
+
const url = input.channel_url || input.url;
|
|
471
|
+
if (!url) {
|
|
472
|
+
return { content: [{ type: "text", text: JSON.stringify({ error: "channel_url or url is required for channel action" }) }] };
|
|
473
|
+
}
|
|
474
|
+
const channel = await getChannel(url);
|
|
475
|
+
result = channel || { error: "Could not extract channel info" };
|
|
476
|
+
break;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return {
|
|
480
|
+
content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
catch (err) {
|
|
484
|
+
return {
|
|
485
|
+
content: [{ type: "text", text: JSON.stringify({ error: err instanceof Error ? err.message : String(err) }) }],
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
//# sourceMappingURL=youtube.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../src/tools/youtube.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAgB,MAAM,sBAAsB,CAAC;AAC3F,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAG5D,MAAM,CAAC,MAAM,IAAI,GAAG,SAAS,CAAC;AAE9B,MAAM,CAAC,MAAM,WAAW,GACtB,uGAAuG,CAAC;AAE1G,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC;AAEpF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAChD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAChG,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2DAA2D,CAAC;IAC1G,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0DAA0D,CAAC;IACjH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrF,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;CACpG,CAAC,CAAC;AAIH,gBAAgB;AAEhB,SAAS,cAAc,CAAC,KAAa;IACnC,0EAA0E;IAC1E,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC;QAC3E,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpE,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAAE,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;QACxB,IAAI,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC;QACnC,KAAK,OAAO,CAAC,CAAC,OAAO,cAAc,CAAC;QACpC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,iCAAiC;AAEjC,SAAS,oBAAoB,CAAC,IAAa,EAAE,KAAa;IACxD,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,uCAAuC;QACvC,MAAM,QAAQ,GAAI,IAAY,EAAE,QAAQ,EAAE,8BAA8B;YACtE,EAAE,eAAe,EAAE,mBAAmB,EAAE,QAAQ,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,OAAO,OAAO,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,OAAO,EAAE,mBAAmB,EAAE,QAAQ,CAAC;YACrD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,SAAS;YAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAM;gBACnC,MAAM,CAAC,GAAG,IAAI,EAAE,aAAa,CAAC;gBAC9B,IAAI,CAAC,CAAC;oBAAE,SAAS;gBAEjB,OAAO,CAAC,IAAI,CAAC;oBACX,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;oBACnB,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE;oBACrC,GAAG,EAAE,mCAAmC,CAAC,CAAC,OAAO,EAAE;oBACnD,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG;oBACvD,QAAQ,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU;oBAClC,KAAK,EAAE,kBAAkB,CAAC,CAAC,CAAC,aAAa,EAAE,UAAU,IAAI,CAAC,CAAC,kBAAkB,EAAE,UAAU,IAAI,EAAE,CAAC;oBAChG,MAAM,EAAE,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE;oBAC1C,UAAU,EAAE,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,cAAc,EAAE,gBAAgB;wBACtF,CAAC,CAAC,0BAA0B,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,cAAc,CAAC,gBAAgB,EAAE;wBACpG,CAAC,CAAC,SAAS;oBACb,SAAS,EAAE,CAAC,CAAC,iBAAiB,EAAE,UAAU;iBAC3C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,iBAAiB;IACnB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAa;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,GAAI,IAAY,EAAE,QAAQ,EAAE,yBAAyB,EAAE,OAAO;YACpE,EAAE,OAAO,EAAE,QAAQ,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpC,MAAM,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;QAEpE,MAAM,EAAE,GAAG,OAAO,EAAE,wBAAwB,CAAC;QAC7C,MAAM,EAAE,GAAG,SAAS,EAAE,0BAA0B,CAAC;QAEjD,MAAM,OAAO,GAAI,IAAY,EAAE,oBAAoB,EAAE,aAAa,EAAE,OAAO,IAAI,EAAE,CAAC;QAClF,MAAM,KAAK,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACtE,MAAM,aAAa,GAAG,EAAE,EAAE,SAAS,EAAE,sBAAsB,EAAE,SAAS,EAAE,UAAU,IAAI,EAAE,CAAC;QACzF,MAAM,KAAK,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC;QAE3C,iCAAiC;QACjC,IAAI,KAAyB,CAAC;QAC9B,MAAM,OAAO,GAAG,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,CAAC;QAChE,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAAG,GAAG,EAAE,mCAAmC,CAAC;gBACrD,MAAM,QAAQ,GAAG,GAAG,EAAE,mBAAmB,EAAE,mBAAmB;oBAC5D,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,sBAAsB;oBACtE,EAAE,eAAe,EAAE,KAAK,CAAC;gBAC3B,IAAI,QAAQ,EAAE,CAAC;oBACb,KAAK,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;oBACrC,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3E,MAAM,SAAS,GAAG,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,cAAc,EAAE,gBAAgB,CAAC;QACtG,MAAM,WAAW,GAAG,EAAE,EAAE,qBAAqB,EAAE,OAAO,IAAI,EAAE,CAAC;QAE7D,OAAO;YACL,EAAE,EAAE,OAAO;YACX,KAAK;YACL,GAAG,EAAE,mCAAmC,OAAO,EAAE;YACjD,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK;YACvC,KAAK,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YACjD,MAAM;YACN,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,0BAA0B,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;YACzE,SAAS;YACT,WAAW,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;SAC5C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAa;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAI,IAAY,EAAE,MAAM,CAAC;QACrC,MAAM,EAAE,GAAG,MAAM,EAAE,sBAAsB,CAAC;QAC1C,MAAM,EAAE,GAAG,MAAM,EAAE,kBAAkB,CAAC;QACtC,MAAM,QAAQ,GAAI,IAAY,EAAE,QAAQ,EAAE,uBAAuB,CAAC;QAElE,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,iEAAiE;QACjE,IAAI,cAAkC,CAAC;QACvC,IAAI,EAAE,EAAE,mBAAmB,EAAE,UAAU,EAAE,CAAC;YACxC,cAAc,GAAG,EAAE,CAAC,mBAAmB,CAAC,UAAU,CAAC;QACrD,CAAC;aAAM,IAAI,EAAE,EAAE,CAAC;YACd,yDAAyD;YACzD,MAAM,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,wBAAwB,EAAE,YAAY,CAAC;YAChG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,aAAa,IAAI,EAAE,EAAE,CAAC;wBAC5C,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;wBACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;4BAChC,cAAc,GAAG,IAAI,CAAC;4BACtB,MAAM;wBACR,CAAC;oBACH,CAAC;oBACD,IAAI,cAAc;wBAAE,MAAM;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,yDAAyD;QACzD,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC;YACf,QAAQ,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,qBAAqB,EAAE,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;QAC9F,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;YAC1B,GAAG,EAAE,QAAQ,CAAC,UAAU,IAAI,EAAE;YAC9B,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;YAC5D,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS;YAC5E,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG;YACxD,QAAQ;SACT,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAa;IACzC,IAAI,CAAC;QACH,sCAAsC;QACtC,MAAM,QAAQ,GAAI,IAAY,EAAE,QAAQ,EAAE,+BAA+B,EAAE,aAAa,CAAC;QACzF,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,oCAAoC;YACpC,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC;YAC9D,OAAO,CAAC,EAAE,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC;QAC9C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;IAChB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,gBAAgB;AAEhB,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,KAAa,EAAE,IAAY;IACrE,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gDAAgD,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE;QAC3G,QAAQ,EAAE,CAAC;KACZ,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,gDAAgD,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxF,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAkC,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC;QACtG,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;IACjH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,mCAAmC,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IAE/F,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEtD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAc,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,mCAAmC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC/H,OAAO,EAAE,EAAE,IAAI,IAAI,IAAI,CAAC;IAC1B,CAAC;IAED,yDAAyD;IACzD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAC7E,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,EAAE,GAAG,UAAiB,CAAC;YAC7B,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,EAAE,YAAY,EAAE,aAAa;gBAChE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;gBACpI,CAAC,CAAC,SAAS,CAAC;YACd,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;QACpG,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,KAAa;IACxD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;IACrH,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAExB,MAAM,MAAM,GAAG,MAAM,WAAW,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QACxB,MAAM,IAAI,CAAC,IAAI,CAAC,mCAAmC,OAAO,EAAE,EAAE;YAC5D,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QAEH,+BAA+B;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YAClE,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,2BAA2B;QAC3B,MAAM,IAAI,CAAC,eAAe,CAAC,6BAA6B,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE/F,MAAM,QAAQ,GAAoB,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5D,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CAAC;YAC1E,MAAM,MAAM,GAAU,EAAE,CAAC;YACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxD,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,QAAQ,GAAG,EAAE,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC,wBAAwB,CAAC,CAAC;gBAE1D,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,WAAW,CAAC,EAAE;oBAClB,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS;oBAClD,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE;oBACvC,KAAK,EAAE,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;oBAC7D,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE;iBACvC,CAAC,CAAC;YACL,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,EAAE,KAAK,CAAC,CAAC;QAEV,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAgC,EAAE,MAAM,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;YACvG,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM;gBAAE,OAAO,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,QAAgB;IAEhB,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,kFAAkF;IAClF,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,oGAAoG,CAAC,CAAC;IACxH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,MAAM,IAAI,CAAC,IAAI,CAAC,mCAAmC,OAAO,EAAE,EAAE;YAC5D,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,yCAAyC;QACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,wDAAwD;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;QAC7E,IAAI,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACnD,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QAED,+CAA+C;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,KAAK,EAAE,CAAC;QACjF,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QAE7F,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAEhC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAClC,gGAAgG,CACjG,CAAC;gBACF,IAAI,CAAC,KAAK;oBAAE,OAAO,EAAE,CAAC;gBAEtB,MAAM,SAAS,GAAG,KAAK,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC;gBAC5E,MAAM,MAAM,GAA2C,EAAE,CAAC;gBAE1D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,SAAS,GAAa,EAAE,CAAC;oBAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;oBAC/E,IAAI,IAAI,CAAC;oBACT,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;wBAClC,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;wBACnC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;4BAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAC3C,CAAC;oBACD,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC3E,CAAC;gBACH,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;YAClE,CAAC;QACH,CAAC;QAED,8DAA8D;QAC9D,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAuD,EAAE,MAAM,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAC;YAChI,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM;gBAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;YAC1H,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qCAAqC;QACrC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YACzC,MAAM,EAAE,GAAI,MAAc,CAAC,uBAAuB,CAAC;YACnD,IAAI,CAAC,EAAE,EAAE,aAAa,EAAE,eAAe;gBAAE,OAAO,IAAI,CAAC;YAErD,iEAAiE;YACjE,MAAM,YAAY,GAAG,EAAE,CAAC,aAAa,CAAC,eAAe;iBAClD,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;iBAC7D,IAAI,CAAC,CAAC,CAAM,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;YAEjE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAE3C,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAC5B,OAAO;gBACL,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC;gBACrD,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,YAAY,EAAE,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC;aAC9D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,EAAE,GAAG;YAAE,OAAO,IAAI,CAAC;QAEjC,wCAAwC;QACxC,IAAI,SAAS,CAAC,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,sCAAsC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,GAAG,IAAI,GAAG,IAAI,CAAC,0CAA0C,CAClI,CAAC;QACJ,CAAC;QAED,uEAAuE;QACvE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;YAC5D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,EAAE;gBAAE,OAAO,IAAI,CAAC;YAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACxC,oBAAoB;YACpB,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;YACrC,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAE9B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAEvD,0BAA0B;QAC1B,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;QAClE,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,SAAS,GAAG,EAAE,EAAE,CAAC,CAAC;QAEvF,yCAAyC;QACzC,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnD,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7B,IAAI,EAAE,CAAC,CAAC,IAAI;SACb,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAErD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACjE,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACtC,OAAO,GAAG,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,UAAkB;IAC1C,gBAAgB;IAChB,IAAI,GAAG,GAAG,UAAU,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5B,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC,CAAC,4BAA4B,GAAG,EAAE,CAAC;IACnG,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAgB,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;QACrF,OAAO,EAAE,EAAE,IAAI,IAAI,IAAI,CAAC;IAC1B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,gBAAgB;AAEhB,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAmB;IAC/C,IAAI,CAAC;QACH,IAAI,MAAe,CAAC;QAEpB,QAAQ,KAAK,CAAC,MAAM,EAAE,CAAC;YACrB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;oBACjB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1H,CAAC;gBACD,MAAM,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBACnE,MAAM;YACR,CAAC;YACD,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;oBACf,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBACvH,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC/C,MAAM,GAAG,KAAK,IAAI,EAAE,KAAK,EAAE,iCAAiC,EAAE,CAAC;gBAC/D,MAAM;YACR,CAAC;YACD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;oBACf,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1H,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3D,MAAM,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC;gBAC7E,MAAM;YACR,CAAC;YACD,KAAK,YAAY,CAAC,CAAC,CAAC;gBAClB,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;oBACf,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,uCAAuC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC5H,CAAC;gBACD,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAClD,MAAM,GAAG,UAAU,IAAI;oBACrB,KAAK,EAAE,wCAAwC;oBAC/C,IAAI,EAAE,oBAAoB,EAAE;wBAC1B,CAAC,CAAC,SAAS;wBACX,CAAC,CAAC,oFAAoF;iBACzF,CAAC;gBACF,MAAM;YACR,CAAC;YACD,KAAK,SAAS,CAAC,CAAC,CAAC;gBACf,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,GAAG,CAAC;gBAC3C,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,mDAAmD,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;gBACtC,MAAM,GAAG,OAAO,IAAI,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;gBAChE,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC5E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;SACxH,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOtG,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAElC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC,iBAAS,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAgBlD;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAY3C;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAc3C;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAc9J,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA+C9F"}
|
package/dist/utils/fetcher.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { smartFetch, StealthError } from "../stealth/index.js";
|
|
2
2
|
import { isAllowed } from "./robots.js";
|
|
3
3
|
import { getDomain } from "./url.js";
|
|
4
|
-
import { getOptions } from "../config.js";
|
|
5
4
|
import { DEFAULT_CONCURRENCY } from "../constants.js";
|
|
6
5
|
// ── Concurrency Limiter ──
|
|
7
6
|
export class ConcurrencyLimiter {
|
|
@@ -95,8 +94,7 @@ function fullJitterBackoff(attempt) {
|
|
|
95
94
|
return Math.random() * expDelay;
|
|
96
95
|
}
|
|
97
96
|
export async function fetchPage(url, options) {
|
|
98
|
-
const
|
|
99
|
-
const respectRobots = options?.respectRobots ?? config.respectRobots;
|
|
97
|
+
const respectRobots = options?.respectRobots ?? (process.env.RESPECT_ROBOTS !== "false");
|
|
100
98
|
if (respectRobots) {
|
|
101
99
|
const allowed = await isAllowed(url);
|
|
102
100
|
if (!allowed) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAE9J,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;AAClC,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,yDAAyD,CAAC,CAAC;IAC/G,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,MAAM,CAAC,CAAC;YAEtB,+BAA+B;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/G,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "imperium-crawl",
|
|
3
|
-
"version": "2.
|
|
4
|
-
"description": "Open-source
|
|
3
|
+
"version": "2.2.0",
|
|
4
|
+
"description": "Open-source CLI tool for web scraping, crawling, search, and custom skills",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"imperium-crawl": "dist/index.js",
|
|
@@ -20,34 +20,29 @@
|
|
|
20
20
|
"build": "tsc && shx chmod +x dist/index.js",
|
|
21
21
|
"dev": "tsc --watch",
|
|
22
22
|
"start": "node dist/index.js",
|
|
23
|
-
"inspect": "npx @modelcontextprotocol/inspector node dist/index.js",
|
|
24
23
|
"test": "vitest run",
|
|
25
24
|
"test:watch": "vitest",
|
|
26
25
|
"prepublishOnly": "npm run build"
|
|
27
26
|
},
|
|
28
27
|
"keywords": [
|
|
29
|
-
"mcp",
|
|
30
28
|
"scraping",
|
|
31
29
|
"crawling",
|
|
32
30
|
"web-search",
|
|
33
31
|
"brave-search",
|
|
34
32
|
"firecrawl",
|
|
35
|
-
"
|
|
33
|
+
"cli"
|
|
36
34
|
],
|
|
37
35
|
"author": "ImperiumTech",
|
|
38
36
|
"license": "MIT",
|
|
39
37
|
"dependencies": {
|
|
40
38
|
"@clack/prompts": "^1.1.0",
|
|
41
39
|
"@inquirer/prompts": "^8.3.0",
|
|
42
|
-
"@modelcontextprotocol/sdk": "^1.12.1",
|
|
43
40
|
"@mozilla/readability": "^0.5.0",
|
|
44
41
|
"chalk": "^5.6.2",
|
|
45
42
|
"cheerio": "^1.0.0",
|
|
46
43
|
"cli-table3": "^0.6.5",
|
|
47
44
|
"commander": "^13.1.0",
|
|
48
45
|
"dotenv": "^16.4.7",
|
|
49
|
-
"express": "^4.21.2",
|
|
50
|
-
"express-rate-limit": "^7.5.1",
|
|
51
46
|
"header-generator": "^2.1.66",
|
|
52
47
|
"impit": "^0.9.2",
|
|
53
48
|
"jsdom": "^26.0.0",
|
|
@@ -66,7 +61,6 @@
|
|
|
66
61
|
"rebrowser-playwright": "^1.51.1"
|
|
67
62
|
},
|
|
68
63
|
"devDependencies": {
|
|
69
|
-
"@types/express": "^5.0.0",
|
|
70
64
|
"@types/jsdom": "^21.1.7",
|
|
71
65
|
"@types/node": "^22.13.5",
|
|
72
66
|
"@types/turndown": "^5.0.5",
|