imperium-crawl 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -11
- package/dist/cli-explore.d.ts +30 -0
- package/dist/cli-explore.d.ts.map +1 -0
- package/dist/cli-explore.js +427 -0
- package/dist/cli-explore.js.map +1 -0
- package/dist/cli-recorder.d.ts +44 -0
- package/dist/cli-recorder.d.ts.map +1 -0
- package/dist/cli-recorder.js +67 -0
- package/dist/cli-recorder.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +28 -0
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +3 -0
- package/dist/config.js.map +1 -1
- package/dist/knowledge/index.d.ts +1 -0
- package/dist/knowledge/index.d.ts.map +1 -1
- package/dist/knowledge/index.js +1 -0
- package/dist/knowledge/index.js.map +1 -1
- package/dist/knowledge/record-browser.d.ts +17 -0
- package/dist/knowledge/record-browser.d.ts.map +1 -0
- package/dist/knowledge/record-browser.js +29 -0
- package/dist/knowledge/record-browser.js.map +1 -0
- package/dist/llm/retry.d.ts +4 -2
- package/dist/llm/retry.d.ts.map +1 -1
- package/dist/llm/retry.js +15 -4
- package/dist/llm/retry.js.map +1 -1
- package/dist/sessions/index.d.ts +1 -1
- package/dist/sessions/index.d.ts.map +1 -1
- package/dist/sessions/index.js +1 -1
- package/dist/sessions/index.js.map +1 -1
- package/dist/sessions/manager.d.ts +20 -0
- package/dist/sessions/manager.d.ts.map +1 -1
- package/dist/sessions/manager.js +57 -0
- package/dist/sessions/manager.js.map +1 -1
- package/dist/sessions/types.d.ts +2 -0
- package/dist/sessions/types.d.ts.map +1 -1
- package/dist/skills/chain.d.ts +61 -0
- package/dist/skills/chain.d.ts.map +1 -0
- package/dist/skills/chain.js +182 -0
- package/dist/skills/chain.js.map +1 -0
- package/dist/skills/conditions.d.ts +14 -0
- package/dist/skills/conditions.d.ts.map +1 -0
- package/dist/skills/conditions.js +208 -0
- package/dist/skills/conditions.js.map +1 -0
- package/dist/skills/manager.d.ts +47 -2
- package/dist/skills/manager.d.ts.map +1 -1
- package/dist/skills/manager.js.map +1 -1
- package/dist/skills/parameters.d.ts +49 -0
- package/dist/skills/parameters.d.ts.map +1 -0
- package/dist/skills/parameters.js +157 -0
- package/dist/skills/parameters.js.map +1 -0
- package/dist/stealth/index.d.ts +1 -0
- package/dist/stealth/index.d.ts.map +1 -1
- package/dist/stealth/index.js +61 -13
- package/dist/stealth/index.js.map +1 -1
- package/dist/tools/action-executor.d.ts +64 -0
- package/dist/tools/action-executor.d.ts.map +1 -0
- package/dist/tools/action-executor.js +365 -0
- package/dist/tools/action-executor.js.map +1 -0
- package/dist/tools/batch-scrape.d.ts +2 -2
- package/dist/tools/crawl.d.ts +2 -2
- package/dist/tools/create-skill.d.ts +2 -2
- package/dist/tools/discover-apis.d.ts +1 -1
- package/dist/tools/discover-apis.d.ts.map +1 -1
- package/dist/tools/discover-apis.js +3 -0
- package/dist/tools/discover-apis.js.map +1 -1
- package/dist/tools/download.d.ts +4 -4
- package/dist/tools/download.d.ts.map +1 -1
- package/dist/tools/download.js +3 -0
- package/dist/tools/download.js.map +1 -1
- package/dist/tools/extract.d.ts +1 -1
- package/dist/tools/image-search.d.ts +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +3 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/instagram.d.ts +2 -2
- package/dist/tools/interact.d.ts +72 -44
- package/dist/tools/interact.d.ts.map +1 -1
- package/dist/tools/interact.js +33 -299
- package/dist/tools/interact.js.map +1 -1
- package/dist/tools/knowledge.d.ts +24 -0
- package/dist/tools/knowledge.d.ts.map +1 -0
- package/dist/tools/knowledge.js +99 -0
- package/dist/tools/knowledge.js.map +1 -0
- package/dist/tools/list-skills.js +1 -1
- package/dist/tools/list-skills.js.map +1 -1
- package/dist/tools/manifest.d.ts.map +1 -1
- package/dist/tools/manifest.js +5 -0
- package/dist/tools/manifest.js.map +1 -1
- package/dist/tools/monitor-websocket.d.ts +1 -1
- package/dist/tools/news-search.d.ts +1 -1
- package/dist/tools/query-api.d.ts +6 -6
- package/dist/tools/readability.d.ts +2 -2
- package/dist/tools/reddit.d.ts +4 -4
- package/dist/tools/run-skill.d.ts +14 -4
- package/dist/tools/run-skill.d.ts.map +1 -1
- package/dist/tools/run-skill.js +73 -0
- package/dist/tools/run-skill.js.map +1 -1
- package/dist/tools/scrape.d.ts +9 -6
- package/dist/tools/scrape.d.ts.map +1 -1
- package/dist/tools/scrape.js +2 -0
- package/dist/tools/scrape.js.map +1 -1
- package/dist/tools/screenshot.d.ts.map +1 -1
- package/dist/tools/screenshot.js +6 -0
- package/dist/tools/screenshot.js.map +1 -1
- package/dist/tools/search.d.ts +1 -1
- package/dist/tools/snapshot.d.ts +5 -5
- package/dist/tools/snapshot.d.ts.map +1 -1
- package/dist/tools/snapshot.js +3 -0
- package/dist/tools/snapshot.js.map +1 -1
- package/dist/tools/video-search.d.ts +1 -1
- package/dist/tools/youtube.d.ts +2 -2
- package/dist/utils/fetcher.d.ts.map +1 -1
- package/dist/utils/fetcher.js +33 -0
- package/dist/utils/fetcher.js.map +1 -1
- package/package.json +5 -2
package/dist/tools/screenshot.js
CHANGED
|
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import { normalizeUrl } from "../utils/url.js";
|
|
3
3
|
import { isPlaywrightAvailable } from "../stealth/index.js";
|
|
4
4
|
import { browserFetch } from "../stealth/browser.js";
|
|
5
|
+
import { recordBrowserOutcome } from "../knowledge/index.js";
|
|
5
6
|
import { MAX_URL_LENGTH } from "../constants.js";
|
|
6
7
|
export const name = "screenshot";
|
|
7
8
|
export const description = "Take a screenshot of a web page. Requires rebrowser-playwright to be installed.";
|
|
@@ -25,7 +26,12 @@ export async function execute(input) {
|
|
|
25
26
|
],
|
|
26
27
|
};
|
|
27
28
|
}
|
|
29
|
+
const fetchStart = Date.now();
|
|
28
30
|
const result = await browserFetch(url, { screenshot: true, proxyUrl: input.proxy, chromeProfile: input.chrome_profile });
|
|
31
|
+
recordBrowserOutcome({
|
|
32
|
+
url, success: result.status < 400, responseTimeMs: Date.now() - fetchStart,
|
|
33
|
+
httpStatus: result.status, captchaSolved: result.captchaSolved, proxyUsed: !!input.proxy,
|
|
34
|
+
});
|
|
29
35
|
if (!result.screenshot) {
|
|
30
36
|
return {
|
|
31
37
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,YAAY,CAAC;AAEjC,MAAM,CAAC,MAAM,WAAW,GACtB,iFAAiF,CAAC;AAEpF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACnF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oEAAoE,CAAC;IAC/H,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+HAA+H,CAAC;CAC1L,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAsB;IAClD,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEpC,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,KAAK,EACH,uHAAuH;qBAC1H,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,YAAY,CAAC;AAEjC,MAAM,CAAC,MAAM,WAAW,GACtB,iFAAiF,CAAC;AAEpF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACnF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oEAAoE,CAAC;IAC/H,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+HAA+H,CAAC;CAC1L,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAsB;IAClD,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEpC,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,KAAK,EACH,uHAAuH;qBAC1H,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IACzH,oBAAoB,CAAC;QACnB,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;QAC1E,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK;KACzF,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;iBACzE;aACF;SACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,OAAgB;gBACtB,IAAI,EAAE,MAAM,CAAC,UAAU;gBACvB,QAAQ,EAAE,WAAoB;aAC/B;SACF;KACF,CAAC;AACJ,CAAC"}
|
package/dist/tools/search.d.ts
CHANGED
|
@@ -13,8 +13,8 @@ export declare const schema: z.ZodObject<{
|
|
|
13
13
|
freshness?: "pd" | "pw" | "pm" | "py" | undefined;
|
|
14
14
|
}, {
|
|
15
15
|
query: string;
|
|
16
|
-
country?: string | undefined;
|
|
17
16
|
count?: number | undefined;
|
|
17
|
+
country?: string | undefined;
|
|
18
18
|
freshness?: "pd" | "pw" | "pm" | "py" | undefined;
|
|
19
19
|
}>;
|
|
20
20
|
export type SearchInput = z.infer<typeof schema>;
|
package/dist/tools/snapshot.d.ts
CHANGED
|
@@ -15,11 +15,11 @@ export declare const schema: z.ZodObject<{
|
|
|
15
15
|
timeout: z.ZodDefault<z.ZodNumber>;
|
|
16
16
|
allowed_domains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
17
17
|
}, "strip", z.ZodTypeAny, {
|
|
18
|
-
url: string;
|
|
19
|
-
timeout: number;
|
|
20
18
|
interactive: boolean;
|
|
21
19
|
cursor: boolean;
|
|
22
20
|
compact: boolean;
|
|
21
|
+
url: string;
|
|
22
|
+
timeout: number;
|
|
23
23
|
return_screenshot: boolean;
|
|
24
24
|
annotate: boolean;
|
|
25
25
|
proxy?: string | undefined;
|
|
@@ -29,12 +29,12 @@ export declare const schema: z.ZodObject<{
|
|
|
29
29
|
scope_selector?: string | undefined;
|
|
30
30
|
}, {
|
|
31
31
|
url: string;
|
|
32
|
-
proxy?: string | undefined;
|
|
33
|
-
timeout?: number | undefined;
|
|
34
|
-
chrome_profile?: string | undefined;
|
|
35
32
|
interactive?: boolean | undefined;
|
|
36
33
|
cursor?: boolean | undefined;
|
|
37
34
|
compact?: boolean | undefined;
|
|
35
|
+
timeout?: number | undefined;
|
|
36
|
+
proxy?: string | undefined;
|
|
37
|
+
chrome_profile?: string | undefined;
|
|
38
38
|
return_screenshot?: boolean | undefined;
|
|
39
39
|
session_id?: string | undefined;
|
|
40
40
|
allowed_domains?: string[] | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,eAAO,MAAM,IAAI,aAAa,CAAC;AAE/B,eAAO,MAAM,WAAW,qTAC6Q,CAAC;AAEtS,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqDjB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEnD,wBAAsB,OAAO,CAAC,KAAK,EAAE,aAAa;;cAkEjB,MAAM;eAAS,MAAM;eAAS,MAAM;mBAAa,MAAM;;GAwCvF"}
|
package/dist/tools/snapshot.js
CHANGED
|
@@ -3,6 +3,7 @@ import { isPlaywrightAvailable } from "../stealth/browser.js";
|
|
|
3
3
|
import { acquirePage } from "../stealth/chrome-profile.js";
|
|
4
4
|
import { resolveProxy } from "../stealth/proxy.js";
|
|
5
5
|
import { normalizeUrl } from "../utils/url.js";
|
|
6
|
+
import { recordBrowserOutcome } from "../knowledge/index.js";
|
|
6
7
|
import { getSessionManager } from "../sessions/index.js";
|
|
7
8
|
import { getEnhancedSnapshot, getSnapshotStore, annotateScreenshot } from "../snapshot/index.js";
|
|
8
9
|
import { installDomainFilter } from "../security/domain-filter.js";
|
|
@@ -77,6 +78,7 @@ export async function execute(input) {
|
|
|
77
78
|
const url = normalizeUrl(input.url);
|
|
78
79
|
const proxyUrl = resolveProxy(input.proxy);
|
|
79
80
|
const snapshotId = input.session_id ?? `snap_${Date.now()}`;
|
|
81
|
+
const fetchStart = Date.now();
|
|
80
82
|
const handle = await acquirePage({
|
|
81
83
|
chromeProfile: input.chrome_profile,
|
|
82
84
|
proxyUrl,
|
|
@@ -96,6 +98,7 @@ export async function execute(input) {
|
|
|
96
98
|
}
|
|
97
99
|
// Navigate
|
|
98
100
|
await page.goto(url, { waitUntil: "load", timeout: input.timeout });
|
|
101
|
+
recordBrowserOutcome({ url, success: true, responseTimeMs: Date.now() - fetchStart, proxyUsed: !!proxyUrl });
|
|
99
102
|
// Take snapshot
|
|
100
103
|
const snapshot = await getEnhancedSnapshot(page, {
|
|
101
104
|
interactive: input.interactive,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,oBAAoB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE7G,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
|
|
@@ -13,8 +13,8 @@ export declare const schema: z.ZodObject<{
|
|
|
13
13
|
freshness?: "pd" | "pw" | "pm" | "py" | undefined;
|
|
14
14
|
}, {
|
|
15
15
|
query: string;
|
|
16
|
-
country?: string | undefined;
|
|
17
16
|
count?: number | undefined;
|
|
17
|
+
country?: string | undefined;
|
|
18
18
|
freshness?: "pd" | "pw" | "pm" | "py" | undefined;
|
|
19
19
|
}>;
|
|
20
20
|
export type VideoSearchInput = z.infer<typeof schema>;
|
package/dist/tools/youtube.d.ts
CHANGED
|
@@ -9,8 +9,8 @@ export declare const schema: z.ZodObject<{
|
|
|
9
9
|
limit: z.ZodDefault<z.ZodNumber>;
|
|
10
10
|
sort: z.ZodDefault<z.ZodEnum<["relevance", "date", "views"]>>;
|
|
11
11
|
}, "strip", z.ZodTypeAny, {
|
|
12
|
-
action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
|
|
13
12
|
sort: "date" | "relevance" | "views";
|
|
13
|
+
action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
|
|
14
14
|
limit: number;
|
|
15
15
|
url?: string | undefined;
|
|
16
16
|
query?: string | undefined;
|
|
@@ -18,8 +18,8 @@ export declare const schema: z.ZodObject<{
|
|
|
18
18
|
}, {
|
|
19
19
|
action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
|
|
20
20
|
url?: string | undefined;
|
|
21
|
-
sort?: "date" | "relevance" | "views" | undefined;
|
|
22
21
|
query?: string | undefined;
|
|
22
|
+
sort?: "date" | "relevance" | "views" | undefined;
|
|
23
23
|
channel_url?: string | undefined;
|
|
24
24
|
limit?: number | undefined;
|
|
25
25
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQtG,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAElC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC,iBAAS,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAgBlD;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAY3C;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAc3C;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAgD9J,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAsD9F"}
|
package/dist/utils/fetcher.js
CHANGED
|
@@ -2,6 +2,7 @@ import { smartFetch, StealthError } from "../stealth/index.js";
|
|
|
2
2
|
import { isAllowed } from "./robots.js";
|
|
3
3
|
import { getDomain } from "./url.js";
|
|
4
4
|
import { DEFAULT_CONCURRENCY } from "../constants.js";
|
|
5
|
+
import { getKnowledgeEngine } from "../knowledge/index.js";
|
|
5
6
|
// ── Concurrency Limiter ──
|
|
6
7
|
export class ConcurrencyLimiter {
|
|
7
8
|
maxConcurrent;
|
|
@@ -93,6 +94,32 @@ function fullJitterBackoff(attempt) {
|
|
|
93
94
|
const expDelay = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * Math.pow(2, attempt));
|
|
94
95
|
return Math.random() * expDelay;
|
|
95
96
|
}
|
|
97
|
+
// ── Per-Domain Rate Limiter ──
|
|
98
|
+
const DEFAULT_DOMAIN_RATE_MS = parseInt(process.env.DOMAIN_RATE_LIMIT_MS || "500", 10);
|
|
99
|
+
class DomainThrottle {
|
|
100
|
+
lastRequest = new Map();
|
|
101
|
+
defaultDelay;
|
|
102
|
+
constructor(defaultDelayMs = DEFAULT_DOMAIN_RATE_MS) {
|
|
103
|
+
this.defaultDelay = defaultDelayMs;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Wait until enough time has passed since the last request to this domain.
|
|
107
|
+
* Uses knowledge engine's safe_rate_limit if available, else default delay.
|
|
108
|
+
*/
|
|
109
|
+
async throttle(domain, knowledgeDelayMs) {
|
|
110
|
+
const delay = knowledgeDelayMs ?? this.defaultDelay;
|
|
111
|
+
if (delay <= 0)
|
|
112
|
+
return;
|
|
113
|
+
const now = Date.now();
|
|
114
|
+
const last = this.lastRequest.get(domain) ?? 0;
|
|
115
|
+
const elapsed = now - last;
|
|
116
|
+
if (elapsed < delay) {
|
|
117
|
+
await new Promise((r) => setTimeout(r, delay - elapsed));
|
|
118
|
+
}
|
|
119
|
+
this.lastRequest.set(domain, Date.now());
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const domainThrottle = new DomainThrottle();
|
|
96
123
|
export async function fetchPage(url, options) {
|
|
97
124
|
const respectRobots = options?.respectRobots ?? (process.env.RESPECT_ROBOTS !== "false");
|
|
98
125
|
if (respectRobots) {
|
|
@@ -107,6 +134,12 @@ export async function fetchPage(url, options) {
|
|
|
107
134
|
if (circuit.state === "open") {
|
|
108
135
|
throw new Error(`Circuit breaker open for ${domain} — too many consecutive failures. Retry after cooldown.`);
|
|
109
136
|
}
|
|
137
|
+
// Per-domain rate limiting — use knowledge engine's safe_rate_limit if available
|
|
138
|
+
const knowledge = await getKnowledgeEngine().get(domain);
|
|
139
|
+
const knowledgeDelayMs = knowledge?.safe_rate_limit
|
|
140
|
+
? Math.round(60_000 / knowledge.safe_rate_limit)
|
|
141
|
+
: undefined;
|
|
142
|
+
await domainThrottle.throttle(domain, knowledgeDelayMs);
|
|
110
143
|
const retries = options?.retries ?? 2;
|
|
111
144
|
let lastError;
|
|
112
145
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAE9J,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;AAClC,CAAC;AAED,gCAAgC;AAEhC,MAAM,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AAEvF,MAAM,cAAc;IACV,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,YAAY,CAAS;IAE7B,YAAY,iBAAyB,sBAAsB;QACzD,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,gBAAyB;QACtD,MAAM,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC;QACpD,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAE3B,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;AAS5C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,yDAAyD,CAAC,CAAC;IAC/G,CAAC;IAED,iFAAiF;IACjF,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,SAAS,EAAE,eAAe;QACjD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAExD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,MAAM,CAAC,CAAC;YAEtB,+BAA+B;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/G,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "imperium-crawl",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Open-source CLI tool for web scraping, crawling, search, and custom skills",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -11,7 +11,10 @@
|
|
|
11
11
|
"exports": {
|
|
12
12
|
".": "./dist/index.js",
|
|
13
13
|
"./tools/*": "./dist/tools/*.js",
|
|
14
|
-
"./stealth/*": "./dist/stealth/*.js"
|
|
14
|
+
"./stealth/*": "./dist/stealth/*.js",
|
|
15
|
+
"./sessions": "./dist/sessions/index.js",
|
|
16
|
+
"./sessions/*": "./dist/sessions/*.js",
|
|
17
|
+
"./utils/*": "./dist/utils/*.js"
|
|
15
18
|
},
|
|
16
19
|
"files": [
|
|
17
20
|
"dist",
|