imperium-crawl 2.3.1 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/README.md +69 -11
  2. package/dist/cli-explore.d.ts +30 -0
  3. package/dist/cli-explore.d.ts.map +1 -0
  4. package/dist/cli-explore.js +427 -0
  5. package/dist/cli-explore.js.map +1 -0
  6. package/dist/cli-recorder.d.ts +44 -0
  7. package/dist/cli-recorder.d.ts.map +1 -0
  8. package/dist/cli-recorder.js +67 -0
  9. package/dist/cli-recorder.js.map +1 -0
  10. package/dist/cli.d.ts.map +1 -1
  11. package/dist/cli.js +28 -0
  12. package/dist/cli.js.map +1 -1
  13. package/dist/config.d.ts.map +1 -1
  14. package/dist/config.js +3 -0
  15. package/dist/config.js.map +1 -1
  16. package/dist/knowledge/index.d.ts +1 -0
  17. package/dist/knowledge/index.d.ts.map +1 -1
  18. package/dist/knowledge/index.js +1 -0
  19. package/dist/knowledge/index.js.map +1 -1
  20. package/dist/knowledge/record-browser.d.ts +17 -0
  21. package/dist/knowledge/record-browser.d.ts.map +1 -0
  22. package/dist/knowledge/record-browser.js +29 -0
  23. package/dist/knowledge/record-browser.js.map +1 -0
  24. package/dist/llm/retry.d.ts +4 -2
  25. package/dist/llm/retry.d.ts.map +1 -1
  26. package/dist/llm/retry.js +15 -4
  27. package/dist/llm/retry.js.map +1 -1
  28. package/dist/sessions/index.d.ts +1 -1
  29. package/dist/sessions/index.d.ts.map +1 -1
  30. package/dist/sessions/index.js +1 -1
  31. package/dist/sessions/index.js.map +1 -1
  32. package/dist/sessions/manager.d.ts +20 -0
  33. package/dist/sessions/manager.d.ts.map +1 -1
  34. package/dist/sessions/manager.js +57 -0
  35. package/dist/sessions/manager.js.map +1 -1
  36. package/dist/sessions/types.d.ts +2 -0
  37. package/dist/sessions/types.d.ts.map +1 -1
  38. package/dist/skills/chain.d.ts +61 -0
  39. package/dist/skills/chain.d.ts.map +1 -0
  40. package/dist/skills/chain.js +182 -0
  41. package/dist/skills/chain.js.map +1 -0
  42. package/dist/skills/conditions.d.ts +14 -0
  43. package/dist/skills/conditions.d.ts.map +1 -0
  44. package/dist/skills/conditions.js +208 -0
  45. package/dist/skills/conditions.js.map +1 -0
  46. package/dist/skills/manager.d.ts +47 -2
  47. package/dist/skills/manager.d.ts.map +1 -1
  48. package/dist/skills/manager.js.map +1 -1
  49. package/dist/skills/parameters.d.ts +49 -0
  50. package/dist/skills/parameters.d.ts.map +1 -0
  51. package/dist/skills/parameters.js +157 -0
  52. package/dist/skills/parameters.js.map +1 -0
  53. package/dist/stealth/index.d.ts +1 -0
  54. package/dist/stealth/index.d.ts.map +1 -1
  55. package/dist/stealth/index.js +61 -13
  56. package/dist/stealth/index.js.map +1 -1
  57. package/dist/tools/action-executor.d.ts +64 -0
  58. package/dist/tools/action-executor.d.ts.map +1 -0
  59. package/dist/tools/action-executor.js +365 -0
  60. package/dist/tools/action-executor.js.map +1 -0
  61. package/dist/tools/batch-scrape.d.ts +2 -2
  62. package/dist/tools/crawl.d.ts +2 -2
  63. package/dist/tools/create-skill.d.ts +2 -2
  64. package/dist/tools/discover-apis.d.ts +1 -1
  65. package/dist/tools/discover-apis.d.ts.map +1 -1
  66. package/dist/tools/discover-apis.js +3 -0
  67. package/dist/tools/discover-apis.js.map +1 -1
  68. package/dist/tools/download.d.ts +4 -4
  69. package/dist/tools/download.d.ts.map +1 -1
  70. package/dist/tools/download.js +3 -0
  71. package/dist/tools/download.js.map +1 -1
  72. package/dist/tools/extract.d.ts +1 -1
  73. package/dist/tools/image-search.d.ts +1 -1
  74. package/dist/tools/index.d.ts.map +1 -1
  75. package/dist/tools/index.js +3 -0
  76. package/dist/tools/index.js.map +1 -1
  77. package/dist/tools/instagram.d.ts +2 -2
  78. package/dist/tools/interact.d.ts +72 -44
  79. package/dist/tools/interact.d.ts.map +1 -1
  80. package/dist/tools/interact.js +33 -299
  81. package/dist/tools/interact.js.map +1 -1
  82. package/dist/tools/knowledge.d.ts +24 -0
  83. package/dist/tools/knowledge.d.ts.map +1 -0
  84. package/dist/tools/knowledge.js +99 -0
  85. package/dist/tools/knowledge.js.map +1 -0
  86. package/dist/tools/list-skills.js +1 -1
  87. package/dist/tools/list-skills.js.map +1 -1
  88. package/dist/tools/manifest.d.ts.map +1 -1
  89. package/dist/tools/manifest.js +5 -0
  90. package/dist/tools/manifest.js.map +1 -1
  91. package/dist/tools/monitor-websocket.d.ts +1 -1
  92. package/dist/tools/news-search.d.ts +1 -1
  93. package/dist/tools/query-api.d.ts +6 -6
  94. package/dist/tools/readability.d.ts +2 -2
  95. package/dist/tools/reddit.d.ts +4 -4
  96. package/dist/tools/run-skill.d.ts +14 -4
  97. package/dist/tools/run-skill.d.ts.map +1 -1
  98. package/dist/tools/run-skill.js +73 -0
  99. package/dist/tools/run-skill.js.map +1 -1
  100. package/dist/tools/scrape.d.ts +9 -6
  101. package/dist/tools/scrape.d.ts.map +1 -1
  102. package/dist/tools/scrape.js +2 -0
  103. package/dist/tools/scrape.js.map +1 -1
  104. package/dist/tools/screenshot.d.ts.map +1 -1
  105. package/dist/tools/screenshot.js +6 -0
  106. package/dist/tools/screenshot.js.map +1 -1
  107. package/dist/tools/search.d.ts +1 -1
  108. package/dist/tools/snapshot.d.ts +5 -5
  109. package/dist/tools/snapshot.d.ts.map +1 -1
  110. package/dist/tools/snapshot.js +3 -0
  111. package/dist/tools/snapshot.js.map +1 -1
  112. package/dist/tools/video-search.d.ts +1 -1
  113. package/dist/tools/youtube.d.ts +2 -2
  114. package/dist/utils/fetcher.d.ts.map +1 -1
  115. package/dist/utils/fetcher.js +33 -0
  116. package/dist/utils/fetcher.js.map +1 -1
  117. package/package.json +5 -2
@@ -2,6 +2,7 @@ import { z } from "zod";
2
2
  import { normalizeUrl } from "../utils/url.js";
3
3
  import { isPlaywrightAvailable } from "../stealth/index.js";
4
4
  import { browserFetch } from "../stealth/browser.js";
5
+ import { recordBrowserOutcome } from "../knowledge/index.js";
5
6
  import { MAX_URL_LENGTH } from "../constants.js";
6
7
  export const name = "screenshot";
7
8
  export const description = "Take a screenshot of a web page. Requires rebrowser-playwright to be installed.";
@@ -25,7 +26,12 @@ export async function execute(input) {
25
26
  ],
26
27
  };
27
28
  }
29
+ const fetchStart = Date.now();
28
30
  const result = await browserFetch(url, { screenshot: true, proxyUrl: input.proxy, chromeProfile: input.chrome_profile });
31
+ recordBrowserOutcome({
32
+ url, success: result.status < 400, responseTimeMs: Date.now() - fetchStart,
33
+ httpStatus: result.status, captchaSolved: result.captchaSolved, proxyUsed: !!input.proxy,
34
+ });
29
35
  if (!result.screenshot) {
30
36
  return {
31
37
  content: [
@@ -1 +1 @@
1
- {"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,YAAY,CAAC;AAEjC,MAAM,CAAC,MAAM,WAAW,GACtB,iFAAiF,CAAC;AAEpF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACnF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oEAAoE,CAAC;IAC/H,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+HAA+H,CAAC;CAC1L,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAsB;IAClD,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEpC,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,KAAK,EACH,uHAAuH;qBAC1H,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IAEzH,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;iBACzE;aACF;SACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,OAAgB;gBACtB,IAAI,EAAE,MAAM,CAAC,UAAU;gBACvB,QAAQ,EAAE,WAAoB;aAC/B;SACF;KACF,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,YAAY,CAAC;AAEjC,MAAM,CAAC,MAAM,WAAW,GACtB,iFAAiF,CAAC;AAEpF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IACnF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oEAAoE,CAAC;IAC/H,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+HAA+H,CAAC;CAC1L,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAsB;IAClD,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAEpC,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,KAAK,EACH,uHAAuH;qBAC1H,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IACzH,oBAAoB,CAAC;QACnB,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;QAC1E,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK;KACzF,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,8BAA8B,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;iBACzE;aACF;SACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,OAAgB;gBACtB,IAAI,EAAE,MAAM,CAAC,UAAU;gBACvB,QAAQ,EAAE,WAAoB;aAC/B;SACF;KACF,CAAC;AACJ,CAAC"}
@@ -13,8 +13,8 @@ export declare const schema: z.ZodObject<{
13
13
  freshness?: "pd" | "pw" | "pm" | "py" | undefined;
14
14
  }, {
15
15
  query: string;
16
- country?: string | undefined;
17
16
  count?: number | undefined;
17
+ country?: string | undefined;
18
18
  freshness?: "pd" | "pw" | "pm" | "py" | undefined;
19
19
  }>;
20
20
  export type SearchInput = z.infer<typeof schema>;
@@ -15,11 +15,11 @@ export declare const schema: z.ZodObject<{
15
15
  timeout: z.ZodDefault<z.ZodNumber>;
16
16
  allowed_domains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
17
17
  }, "strip", z.ZodTypeAny, {
18
- url: string;
19
- timeout: number;
20
18
  interactive: boolean;
21
19
  cursor: boolean;
22
20
  compact: boolean;
21
+ url: string;
22
+ timeout: number;
23
23
  return_screenshot: boolean;
24
24
  annotate: boolean;
25
25
  proxy?: string | undefined;
@@ -29,12 +29,12 @@ export declare const schema: z.ZodObject<{
29
29
  scope_selector?: string | undefined;
30
30
  }, {
31
31
  url: string;
32
- proxy?: string | undefined;
33
- timeout?: number | undefined;
34
- chrome_profile?: string | undefined;
35
32
  interactive?: boolean | undefined;
36
33
  cursor?: boolean | undefined;
37
34
  compact?: boolean | undefined;
35
+ timeout?: number | undefined;
36
+ proxy?: string | undefined;
37
+ chrome_profile?: string | undefined;
38
38
  return_screenshot?: boolean | undefined;
39
39
  session_id?: string | undefined;
40
40
  allowed_domains?: string[] | undefined;
@@ -1 +1 @@
1
- {"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,eAAO,MAAM,IAAI,aAAa,CAAC;AAE/B,eAAO,MAAM,WAAW,qTAC6Q,CAAC;AAEtS,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqDjB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEnD,wBAAsB,OAAO,CAAC,KAAK,EAAE,aAAa;;cAgEjB,MAAM;eAAS,MAAM;eAAS,MAAM;mBAAa,MAAM;;GAwCvF"}
1
+ {"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,eAAO,MAAM,IAAI,aAAa,CAAC;AAE/B,eAAO,MAAM,WAAW,qTAC6Q,CAAC;AAEtS,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqDjB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEnD,wBAAsB,OAAO,CAAC,KAAK,EAAE,aAAa;;cAkEjB,MAAM;eAAS,MAAM;eAAS,MAAM;mBAAa,MAAM;;GAwCvF"}
@@ -3,6 +3,7 @@ import { isPlaywrightAvailable } from "../stealth/browser.js";
3
3
  import { acquirePage } from "../stealth/chrome-profile.js";
4
4
  import { resolveProxy } from "../stealth/proxy.js";
5
5
  import { normalizeUrl } from "../utils/url.js";
6
+ import { recordBrowserOutcome } from "../knowledge/index.js";
6
7
  import { getSessionManager } from "../sessions/index.js";
7
8
  import { getEnhancedSnapshot, getSnapshotStore, annotateScreenshot } from "../snapshot/index.js";
8
9
  import { installDomainFilter } from "../security/domain-filter.js";
@@ -77,6 +78,7 @@ export async function execute(input) {
77
78
  const url = normalizeUrl(input.url);
78
79
  const proxyUrl = resolveProxy(input.proxy);
79
80
  const snapshotId = input.session_id ?? `snap_${Date.now()}`;
81
+ const fetchStart = Date.now();
80
82
  const handle = await acquirePage({
81
83
  chromeProfile: input.chrome_profile,
82
84
  proxyUrl,
@@ -96,6 +98,7 @@ export async function execute(input) {
96
98
  }
97
99
  // Navigate
98
100
  await page.goto(url, { waitUntil: "load", timeout: input.timeout });
101
+ recordBrowserOutcome({ url, success: true, responseTimeMs: Date.now() - fetchStart, proxyUsed: !!proxyUrl });
99
102
  // Take snapshot
100
103
  const snapshot = await getEnhancedSnapshot(page, {
101
104
  interactive: input.interactive,
@@ -1 +1 @@
1
- {"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAE5D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAEpE,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,oBAAoB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE7G,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
@@ -13,8 +13,8 @@ export declare const schema: z.ZodObject<{
13
13
  freshness?: "pd" | "pw" | "pm" | "py" | undefined;
14
14
  }, {
15
15
  query: string;
16
- country?: string | undefined;
17
16
  count?: number | undefined;
17
+ country?: string | undefined;
18
18
  freshness?: "pd" | "pw" | "pm" | "py" | undefined;
19
19
  }>;
20
20
  export type VideoSearchInput = z.infer<typeof schema>;
@@ -9,8 +9,8 @@ export declare const schema: z.ZodObject<{
9
9
  limit: z.ZodDefault<z.ZodNumber>;
10
10
  sort: z.ZodDefault<z.ZodEnum<["relevance", "date", "views"]>>;
11
11
  }, "strip", z.ZodTypeAny, {
12
- action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
13
12
  sort: "date" | "relevance" | "views";
13
+ action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
14
14
  limit: number;
15
15
  url?: string | undefined;
16
16
  query?: string | undefined;
@@ -18,8 +18,8 @@ export declare const schema: z.ZodObject<{
18
18
  }, {
19
19
  action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
20
20
  url?: string | undefined;
21
- sort?: "date" | "relevance" | "views" | undefined;
22
21
  query?: string | undefined;
22
+ sort?: "date" | "relevance" | "views" | undefined;
23
23
  channel_url?: string | undefined;
24
24
  limit?: number | undefined;
25
25
  }>;
@@ -1 +1 @@
1
- {"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOtG,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAElC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC,iBAAS,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAgBlD;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAY3C;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAc3C;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAc9J,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA+C9F"}
1
+ {"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQtG,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAElC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC,iBAAS,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAgBlD;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAY3C;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAc3C;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAgD9J,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAsD9F"}
@@ -2,6 +2,7 @@ import { smartFetch, StealthError } from "../stealth/index.js";
2
2
  import { isAllowed } from "./robots.js";
3
3
  import { getDomain } from "./url.js";
4
4
  import { DEFAULT_CONCURRENCY } from "../constants.js";
5
+ import { getKnowledgeEngine } from "../knowledge/index.js";
5
6
  // ── Concurrency Limiter ──
6
7
  export class ConcurrencyLimiter {
7
8
  maxConcurrent;
@@ -93,6 +94,32 @@ function fullJitterBackoff(attempt) {
93
94
  const expDelay = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * Math.pow(2, attempt));
94
95
  return Math.random() * expDelay;
95
96
  }
97
+ // ── Per-Domain Rate Limiter ──
98
+ const DEFAULT_DOMAIN_RATE_MS = parseInt(process.env.DOMAIN_RATE_LIMIT_MS || "500", 10);
99
+ class DomainThrottle {
100
+ lastRequest = new Map();
101
+ defaultDelay;
102
+ constructor(defaultDelayMs = DEFAULT_DOMAIN_RATE_MS) {
103
+ this.defaultDelay = defaultDelayMs;
104
+ }
105
+ /**
106
+ * Wait until enough time has passed since the last request to this domain.
107
+ * Uses knowledge engine's safe_rate_limit if available, else default delay.
108
+ */
109
+ async throttle(domain, knowledgeDelayMs) {
110
+ const delay = knowledgeDelayMs ?? this.defaultDelay;
111
+ if (delay <= 0)
112
+ return;
113
+ const now = Date.now();
114
+ const last = this.lastRequest.get(domain) ?? 0;
115
+ const elapsed = now - last;
116
+ if (elapsed < delay) {
117
+ await new Promise((r) => setTimeout(r, delay - elapsed));
118
+ }
119
+ this.lastRequest.set(domain, Date.now());
120
+ }
121
+ }
122
+ const domainThrottle = new DomainThrottle();
96
123
  export async function fetchPage(url, options) {
97
124
  const respectRobots = options?.respectRobots ?? (process.env.RESPECT_ROBOTS !== "false");
98
125
  if (respectRobots) {
@@ -107,6 +134,12 @@ export async function fetchPage(url, options) {
107
134
  if (circuit.state === "open") {
108
135
  throw new Error(`Circuit breaker open for ${domain} — too many consecutive failures. Retry after cooldown.`);
109
136
  }
137
+ // Per-domain rate limiting — use knowledge engine's safe_rate_limit if available
138
+ const knowledge = await getKnowledgeEngine().get(domain);
139
+ const knowledgeDelayMs = knowledge?.safe_rate_limit
140
+ ? Math.round(60_000 / knowledge.safe_rate_limit)
141
+ : undefined;
142
+ await domainThrottle.throttle(domain, knowledgeDelayMs);
110
143
  const retries = options?.retries ?? 2;
111
144
  let lastError;
112
145
  for (let attempt = 0; attempt <= retries; attempt++) {
@@ -1 +1 @@
1
- {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAE9J,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;AAClC,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,yDAAyD,CAAC,CAAC;IAC/G,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,MAAM,CAAC,CAAC;YAEtB,+BAA+B;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/G,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAyC,MAAM,qBAAqB,CAAC;AACtG,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,4BAA4B;AAE5B,MAAM,OAAO,kBAAkB;IAIT;IAHZ,OAAO,GAAG,CAAC,CAAC;IACZ,KAAK,GAAsB,EAAE,CAAC;IAEtC,YAAoB,gBAAwB,mBAAmB;QAA3C,kBAAa,GAAb,aAAa,CAA8B;IAAG,CAAC;IAEnE,KAAK,CAAC,GAAG,CAAI,EAAoB;QAC/B,OAAO,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;QACnB,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,kBAAkB,EAAE,CAAC;AAcvD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AAEnD,4DAA4D;AAC5D,MAAM,gBAAgB,GAAG,SAAS,CAAC;AACnC,WAAW,CAAC,GAAG,EAAE;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzC,IAAI,GAAG,GAAG,OAAO,CAAC,YAAY,GAAG,gBAAgB,EAAE,CAAC;YAClD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC,EAAE,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AAEpB,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACrG,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAElC,uDAAuD;IACvD,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,wBAAwB,EAAE,CAAC;QAC1F,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC;QAC5B,OAAO,CAAC,cAAc,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,cAAc,IAAI,uBAAuB,EAAE,CAAC;YACtD,kBAAkB;YAClB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,sDAAsD;IACtD,IAAI,OAAO,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,CAAC,QAAQ,GAAG,yBAAyB,CAAC;QAC7C,OAAO;IACT,CAAC;IACD,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnB,IAAI,OAAO,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;QAClD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,uBAAuB;AACvB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAE9J,2DAA2D;AAE3D,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,SAAS,iBAAiB,CAAC,OAAe;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC;AAClC,CAAC;AAED,gCAAgC;AAEhC,MAAM,sBAAsB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;AAEvF,MAAM,cAAc;IACV,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,YAAY,CAAS;IAE7B,YAAY,iBAAyB,sBAAsB;QACzD,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;IACrC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,MAAc,EAAE,gBAAyB;QACtD,MAAM,KAAK,GAAG,gBAAgB,IAAI,IAAI,CAAC,YAAY,CAAC;QACpD,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC;QAE3B,IAAI,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;AAS5C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAA2B;IACtE,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,OAAO,CAAC,CAAC;IAEzF,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,yDAAyD,CAAC,CAAC;IAC/G,CAAC;IAED,iFAAiF;IACjF,MAAM,SAAS,GAAG,MAAM,kBAAkB,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,SAAS,EAAE,eAAe;QACjD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,eAAe,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAExD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC;IACtC,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,aAAa,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,aAAa,CAAC,MAAM,CAAC,CAAC;YAEtB,+BAA+B;YAC/B,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,cAAc,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACpC,2DAA2D;gBAC3D,MAAM,MAAM,GAAG,GAAG,YAAY,YAAY;oBACxC,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,SAAS,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;oBACjG,CAAC,CAAC,EAAE,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YAC/G,CAAC;YAED,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAU,CAAC;AACnB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "imperium-crawl",
3
- "version": "2.3.1",
3
+ "version": "2.4.0",
4
4
  "description": "Open-source CLI tool for web scraping, crawling, search, and custom skills",
5
5
  "type": "module",
6
6
  "bin": {
@@ -11,7 +11,10 @@
11
11
  "exports": {
12
12
  ".": "./dist/index.js",
13
13
  "./tools/*": "./dist/tools/*.js",
14
- "./stealth/*": "./dist/stealth/*.js"
14
+ "./stealth/*": "./dist/stealth/*.js",
15
+ "./sessions": "./dist/sessions/index.js",
16
+ "./sessions/*": "./dist/sessions/*.js",
17
+ "./utils/*": "./dist/utils/*.js"
15
18
  },
16
19
  "files": [
17
20
  "dist",