imperium-crawl 2.3.1 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. package/README.md +146 -11
  2. package/dist/cli-explore.d.ts +30 -0
  3. package/dist/cli-explore.d.ts.map +1 -0
  4. package/dist/cli-explore.js +427 -0
  5. package/dist/cli-explore.js.map +1 -0
  6. package/dist/cli-recorder.d.ts +44 -0
  7. package/dist/cli-recorder.d.ts.map +1 -0
  8. package/dist/cli-recorder.js +67 -0
  9. package/dist/cli-recorder.js.map +1 -0
  10. package/dist/cli.d.ts.map +1 -1
  11. package/dist/cli.js +51 -3
  12. package/dist/cli.js.map +1 -1
  13. package/dist/config.d.ts.map +1 -1
  14. package/dist/config.js +3 -0
  15. package/dist/config.js.map +1 -1
  16. package/dist/constants.d.ts +1 -1
  17. package/dist/constants.d.ts.map +1 -1
  18. package/dist/constants.js +31 -1
  19. package/dist/constants.js.map +1 -1
  20. package/dist/flows/engine.d.ts +7 -0
  21. package/dist/flows/engine.d.ts.map +1 -0
  22. package/dist/flows/engine.js +183 -0
  23. package/dist/flows/engine.js.map +1 -0
  24. package/dist/flows/index.d.ts +6 -0
  25. package/dist/flows/index.d.ts.map +1 -0
  26. package/dist/flows/index.js +6 -0
  27. package/dist/flows/index.js.map +1 -0
  28. package/dist/flows/server.d.ts +11 -0
  29. package/dist/flows/server.d.ts.map +1 -0
  30. package/dist/flows/server.js +81 -0
  31. package/dist/flows/server.js.map +1 -0
  32. package/dist/flows/smart-target.d.ts +9 -0
  33. package/dist/flows/smart-target.d.ts.map +1 -0
  34. package/dist/flows/smart-target.js +84 -0
  35. package/dist/flows/smart-target.js.map +1 -0
  36. package/dist/flows/storage.d.ts +26 -0
  37. package/dist/flows/storage.d.ts.map +1 -0
  38. package/dist/flows/storage.js +118 -0
  39. package/dist/flows/storage.js.map +1 -0
  40. package/dist/flows/templates.d.ts +4 -0
  41. package/dist/flows/templates.d.ts.map +1 -0
  42. package/dist/flows/templates.js +35 -0
  43. package/dist/flows/templates.js.map +1 -0
  44. package/dist/flows/types.d.ts +3356 -0
  45. package/dist/flows/types.d.ts.map +1 -0
  46. package/dist/flows/types.js +133 -0
  47. package/dist/flows/types.js.map +1 -0
  48. package/dist/knowledge/index.d.ts +1 -0
  49. package/dist/knowledge/index.d.ts.map +1 -1
  50. package/dist/knowledge/index.js +1 -0
  51. package/dist/knowledge/index.js.map +1 -1
  52. package/dist/knowledge/record-browser.d.ts +17 -0
  53. package/dist/knowledge/record-browser.d.ts.map +1 -0
  54. package/dist/knowledge/record-browser.js +29 -0
  55. package/dist/knowledge/record-browser.js.map +1 -0
  56. package/dist/knowledge/store.d.ts +19 -0
  57. package/dist/knowledge/store.d.ts.map +1 -1
  58. package/dist/knowledge/store.js +63 -4
  59. package/dist/knowledge/store.js.map +1 -1
  60. package/dist/llm/retry.d.ts +4 -2
  61. package/dist/llm/retry.d.ts.map +1 -1
  62. package/dist/llm/retry.js +15 -4
  63. package/dist/llm/retry.js.map +1 -1
  64. package/dist/sessions/browser-connect.d.ts +30 -0
  65. package/dist/sessions/browser-connect.d.ts.map +1 -0
  66. package/dist/sessions/browser-connect.js +68 -0
  67. package/dist/sessions/browser-connect.js.map +1 -0
  68. package/dist/sessions/browser-state.d.ts +35 -0
  69. package/dist/sessions/browser-state.d.ts.map +1 -0
  70. package/dist/sessions/browser-state.js +74 -0
  71. package/dist/sessions/browser-state.js.map +1 -0
  72. package/dist/sessions/index.d.ts +1 -1
  73. package/dist/sessions/index.d.ts.map +1 -1
  74. package/dist/sessions/index.js +1 -1
  75. package/dist/sessions/index.js.map +1 -1
  76. package/dist/sessions/inject-cookies.d.ts +20 -0
  77. package/dist/sessions/inject-cookies.d.ts.map +1 -0
  78. package/dist/sessions/inject-cookies.js +57 -0
  79. package/dist/sessions/inject-cookies.js.map +1 -0
  80. package/dist/sessions/manager.d.ts +31 -1
  81. package/dist/sessions/manager.d.ts.map +1 -1
  82. package/dist/sessions/manager.js +97 -6
  83. package/dist/sessions/manager.js.map +1 -1
  84. package/dist/sessions/types.d.ts +2 -0
  85. package/dist/sessions/types.d.ts.map +1 -1
  86. package/dist/skills/chain.d.ts +61 -0
  87. package/dist/skills/chain.d.ts.map +1 -0
  88. package/dist/skills/chain.js +182 -0
  89. package/dist/skills/chain.js.map +1 -0
  90. package/dist/skills/conditions.d.ts +14 -0
  91. package/dist/skills/conditions.d.ts.map +1 -0
  92. package/dist/skills/conditions.js +208 -0
  93. package/dist/skills/conditions.js.map +1 -0
  94. package/dist/skills/manager.d.ts +47 -2
  95. package/dist/skills/manager.d.ts.map +1 -1
  96. package/dist/skills/manager.js.map +1 -1
  97. package/dist/skills/parameters.d.ts +49 -0
  98. package/dist/skills/parameters.d.ts.map +1 -0
  99. package/dist/skills/parameters.js +157 -0
  100. package/dist/skills/parameters.js.map +1 -0
  101. package/dist/snapshot/store.d.ts +8 -0
  102. package/dist/snapshot/store.d.ts.map +1 -1
  103. package/dist/snapshot/store.js +48 -0
  104. package/dist/snapshot/store.js.map +1 -1
  105. package/dist/stealth/antibot-detector.d.ts +1 -1
  106. package/dist/stealth/antibot-detector.d.ts.map +1 -1
  107. package/dist/stealth/antibot-detector.js +56 -0
  108. package/dist/stealth/antibot-detector.js.map +1 -1
  109. package/dist/stealth/browser-image-extract.d.ts +43 -0
  110. package/dist/stealth/browser-image-extract.d.ts.map +1 -0
  111. package/dist/stealth/browser-image-extract.js +268 -0
  112. package/dist/stealth/browser-image-extract.js.map +1 -0
  113. package/dist/stealth/browser.d.ts +5 -0
  114. package/dist/stealth/browser.d.ts.map +1 -1
  115. package/dist/stealth/browser.js +82 -1
  116. package/dist/stealth/browser.js.map +1 -1
  117. package/dist/stealth/chrome-profile.d.ts +1 -0
  118. package/dist/stealth/chrome-profile.d.ts.map +1 -1
  119. package/dist/stealth/chrome-profile.js +28 -5
  120. package/dist/stealth/chrome-profile.js.map +1 -1
  121. package/dist/stealth/detector.d.ts +10 -1
  122. package/dist/stealth/detector.d.ts.map +1 -1
  123. package/dist/stealth/detector.js +117 -25
  124. package/dist/stealth/detector.js.map +1 -1
  125. package/dist/stealth/headers.d.ts +1 -1
  126. package/dist/stealth/headers.d.ts.map +1 -1
  127. package/dist/stealth/headers.js +94 -2
  128. package/dist/stealth/headers.js.map +1 -1
  129. package/dist/stealth/index.d.ts +5 -0
  130. package/dist/stealth/index.d.ts.map +1 -1
  131. package/dist/stealth/index.js +257 -27
  132. package/dist/stealth/index.js.map +1 -1
  133. package/dist/stealth/proxy.d.ts +40 -1
  134. package/dist/stealth/proxy.d.ts.map +1 -1
  135. package/dist/stealth/proxy.js +90 -6
  136. package/dist/stealth/proxy.js.map +1 -1
  137. package/dist/tools/action-executor.d.ts +66 -0
  138. package/dist/tools/action-executor.d.ts.map +1 -0
  139. package/dist/tools/action-executor.js +403 -0
  140. package/dist/tools/action-executor.js.map +1 -0
  141. package/dist/tools/batch-download.d.ts +33 -0
  142. package/dist/tools/batch-download.d.ts.map +1 -0
  143. package/dist/tools/batch-download.js +208 -0
  144. package/dist/tools/batch-download.js.map +1 -0
  145. package/dist/tools/batch-scrape.d.ts +2 -2
  146. package/dist/tools/browser.d.ts +100 -0
  147. package/dist/tools/browser.d.ts.map +1 -0
  148. package/dist/tools/browser.js +448 -0
  149. package/dist/tools/browser.js.map +1 -0
  150. package/dist/tools/crawl.d.ts +2 -2
  151. package/dist/tools/create-skill.d.ts +2 -2
  152. package/dist/tools/discover-apis.d.ts +1 -1
  153. package/dist/tools/discover-apis.d.ts.map +1 -1
  154. package/dist/tools/discover-apis.js +3 -0
  155. package/dist/tools/discover-apis.js.map +1 -1
  156. package/dist/tools/download.d.ts +39 -6
  157. package/dist/tools/download.d.ts.map +1 -1
  158. package/dist/tools/download.js +248 -44
  159. package/dist/tools/download.js.map +1 -1
  160. package/dist/tools/extract.d.ts +1 -1
  161. package/dist/tools/image-search.d.ts +1 -1
  162. package/dist/tools/index.d.ts.map +1 -1
  163. package/dist/tools/index.js +26 -0
  164. package/dist/tools/index.js.map +1 -1
  165. package/dist/tools/inspect-flow.d.ts +24 -0
  166. package/dist/tools/inspect-flow.d.ts.map +1 -0
  167. package/dist/tools/inspect-flow.js +23 -0
  168. package/dist/tools/inspect-flow.js.map +1 -0
  169. package/dist/tools/instagram.d.ts +2 -2
  170. package/dist/tools/interact.d.ts +91 -50
  171. package/dist/tools/interact.d.ts.map +1 -1
  172. package/dist/tools/interact.js +80 -299
  173. package/dist/tools/interact.js.map +1 -1
  174. package/dist/tools/knowledge.d.ts +24 -0
  175. package/dist/tools/knowledge.d.ts.map +1 -0
  176. package/dist/tools/knowledge.js +99 -0
  177. package/dist/tools/knowledge.js.map +1 -0
  178. package/dist/tools/list-flows.d.ts +21 -0
  179. package/dist/tools/list-flows.d.ts.map +1 -0
  180. package/dist/tools/list-flows.js +18 -0
  181. package/dist/tools/list-flows.js.map +1 -0
  182. package/dist/tools/list-skills.js +1 -1
  183. package/dist/tools/list-skills.js.map +1 -1
  184. package/dist/tools/manifest.d.ts.map +1 -1
  185. package/dist/tools/manifest.js +48 -0
  186. package/dist/tools/manifest.js.map +1 -1
  187. package/dist/tools/monitor-websocket.d.ts +1 -1
  188. package/dist/tools/monitor.d.ts +46 -0
  189. package/dist/tools/monitor.d.ts.map +1 -0
  190. package/dist/tools/monitor.js +213 -0
  191. package/dist/tools/monitor.js.map +1 -0
  192. package/dist/tools/news-search.d.ts +1 -1
  193. package/dist/tools/pdf-extract.d.ts +38 -0
  194. package/dist/tools/pdf-extract.d.ts.map +1 -0
  195. package/dist/tools/pdf-extract.js +244 -0
  196. package/dist/tools/pdf-extract.js.map +1 -0
  197. package/dist/tools/query-api.d.ts +6 -6
  198. package/dist/tools/readability.d.ts +2 -2
  199. package/dist/tools/record-flow.d.ts +39 -0
  200. package/dist/tools/record-flow.d.ts.map +1 -0
  201. package/dist/tools/record-flow.js +406 -0
  202. package/dist/tools/record-flow.js.map +1 -0
  203. package/dist/tools/reddit.d.ts +4 -4
  204. package/dist/tools/run-flow.d.ts +54 -0
  205. package/dist/tools/run-flow.d.ts.map +1 -0
  206. package/dist/tools/run-flow.js +47 -0
  207. package/dist/tools/run-flow.js.map +1 -0
  208. package/dist/tools/run-skill.d.ts +14 -4
  209. package/dist/tools/run-skill.d.ts.map +1 -1
  210. package/dist/tools/run-skill.js +74 -0
  211. package/dist/tools/run-skill.js.map +1 -1
  212. package/dist/tools/scrape.d.ts +9 -6
  213. package/dist/tools/scrape.d.ts.map +1 -1
  214. package/dist/tools/scrape.js +19 -1
  215. package/dist/tools/scrape.js.map +1 -1
  216. package/dist/tools/screenshot.d.ts.map +1 -1
  217. package/dist/tools/screenshot.js +6 -0
  218. package/dist/tools/screenshot.js.map +1 -1
  219. package/dist/tools/search.d.ts +1 -1
  220. package/dist/tools/serve-flow.d.ts +36 -0
  221. package/dist/tools/serve-flow.d.ts.map +1 -0
  222. package/dist/tools/serve-flow.js +42 -0
  223. package/dist/tools/serve-flow.js.map +1 -0
  224. package/dist/tools/snapshot.d.ts +5 -5
  225. package/dist/tools/snapshot.d.ts.map +1 -1
  226. package/dist/tools/snapshot.js +3 -0
  227. package/dist/tools/snapshot.js.map +1 -1
  228. package/dist/tools/validate-flow.d.ts +24 -0
  229. package/dist/tools/validate-flow.d.ts.map +1 -0
  230. package/dist/tools/validate-flow.js +23 -0
  231. package/dist/tools/validate-flow.js.map +1 -0
  232. package/dist/tools/video-search.d.ts +1 -1
  233. package/dist/tools/watch.d.ts +68 -0
  234. package/dist/tools/watch.d.ts.map +1 -0
  235. package/dist/tools/watch.js +224 -0
  236. package/dist/tools/watch.js.map +1 -0
  237. package/dist/tools/youtube.d.ts +2 -2
  238. package/dist/utils/fetcher.d.ts +13 -4
  239. package/dist/utils/fetcher.d.ts.map +1 -1
  240. package/dist/utils/fetcher.js +153 -23
  241. package/dist/utils/fetcher.js.map +1 -1
  242. package/package.json +19 -5
@@ -15,11 +15,11 @@ export declare const schema: z.ZodObject<{
15
15
  timeout: z.ZodDefault<z.ZodNumber>;
16
16
  allowed_domains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
17
17
  }, "strip", z.ZodTypeAny, {
18
- url: string;
19
- timeout: number;
20
18
  interactive: boolean;
21
19
  cursor: boolean;
22
20
  compact: boolean;
21
+ url: string;
22
+ timeout: number;
23
23
  return_screenshot: boolean;
24
24
  annotate: boolean;
25
25
  proxy?: string | undefined;
@@ -29,12 +29,12 @@ export declare const schema: z.ZodObject<{
29
29
  scope_selector?: string | undefined;
30
30
  }, {
31
31
  url: string;
32
- proxy?: string | undefined;
33
- timeout?: number | undefined;
34
- chrome_profile?: string | undefined;
35
32
  interactive?: boolean | undefined;
36
33
  cursor?: boolean | undefined;
37
34
  compact?: boolean | undefined;
35
+ timeout?: number | undefined;
36
+ proxy?: string | undefined;
37
+ chrome_profile?: string | undefined;
38
38
  return_screenshot?: boolean | undefined;
39
39
  session_id?: string | undefined;
40
40
  allowed_domains?: string[] | undefined;
@@ -1 +1 @@
1
- {"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,eAAO,MAAM,IAAI,aAAa,CAAC;AAE/B,eAAO,MAAM,WAAW,qTAC6Q,CAAC;AAEtS,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqDjB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEnD,wBAAsB,OAAO,CAAC,KAAK,EAAE,aAAa;;cAgEjB,MAAM;eAAS,MAAM;eAAS,MAAM;mBAAa,MAAM;;GAwCvF"}
1
+ {"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,eAAO,MAAM,IAAI,aAAa,CAAC;AAE/B,eAAO,MAAM,WAAW,qTAC6Q,CAAC;AAEtS,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqDjB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEnD,wBAAsB,OAAO,CAAC,KAAK,EAAE,aAAa;;cAkEjB,MAAM;eAAS,MAAM;eAAS,MAAM;mBAAa,MAAM;;GAwCvF"}
@@ -3,6 +3,7 @@ import { isPlaywrightAvailable } from "../stealth/browser.js";
3
3
  import { acquirePage } from "../stealth/chrome-profile.js";
4
4
  import { resolveProxy } from "../stealth/proxy.js";
5
5
  import { normalizeUrl } from "../utils/url.js";
6
+ import { recordBrowserOutcome } from "../knowledge/index.js";
6
7
  import { getSessionManager } from "../sessions/index.js";
7
8
  import { getEnhancedSnapshot, getSnapshotStore, annotateScreenshot } from "../snapshot/index.js";
8
9
  import { installDomainFilter } from "../security/domain-filter.js";
@@ -77,6 +78,7 @@ export async function execute(input) {
77
78
  const url = normalizeUrl(input.url);
78
79
  const proxyUrl = resolveProxy(input.proxy);
79
80
  const snapshotId = input.session_id ?? `snap_${Date.now()}`;
81
+ const fetchStart = Date.now();
80
82
  const handle = await acquirePage({
81
83
  chromeProfile: input.chrome_profile,
82
84
  proxyUrl,
@@ -96,6 +98,7 @@ export async function execute(input) {
96
98
  }
97
99
  // Navigate
98
100
  await page.goto(url, { waitUntil: "load", timeout: input.timeout });
101
+ recordBrowserOutcome({ url, success: true, responseTimeMs: Date.now() - fetchStart, proxyUsed: !!proxyUrl });
99
102
  // Take snapshot
100
103
  const snapshot = await getEnhancedSnapshot(page, {
101
104
  interactive: input.interactive,
@@ -1 +1 @@
1
- {"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAE5D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAEpE,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,oBAAoB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE7G,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
@@ -0,0 +1,24 @@
1
+ import { z } from "zod";
2
+ export declare const name = "validate_flow";
3
+ export declare const description = "Validate an Imperium Flow schema and report its inputs, steps, and storage path.";
4
+ export declare const schema: z.ZodObject<{
5
+ flow: z.ZodOptional<z.ZodString>;
6
+ flows_dir: z.ZodOptional<z.ZodString>;
7
+ global: z.ZodDefault<z.ZodBoolean>;
8
+ }, "strip", z.ZodTypeAny, {
9
+ global: boolean;
10
+ flow?: string | undefined;
11
+ flows_dir?: string | undefined;
12
+ }, {
13
+ flow?: string | undefined;
14
+ flows_dir?: string | undefined;
15
+ global?: boolean | undefined;
16
+ }>;
17
+ export type ValidateFlowInput = z.infer<typeof schema>;
18
+ export declare function execute(input: ValidateFlowInput): Promise<{
19
+ content: {
20
+ type: "text";
21
+ text: string;
22
+ }[];
23
+ }>;
24
+ //# sourceMappingURL=validate-flow.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"validate-flow.d.ts","sourceRoot":"","sources":["../../src/tools/validate-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,IAAI,kBAAkB,CAAC;AACpC,eAAO,MAAM,WAAW,qFAAqF,CAAC;AAE9G,eAAO,MAAM,MAAM;;;;;;;;;;;;EAIjB,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEvD,wBAAsB,OAAO,CAAC,KAAK,EAAE,iBAAiB;;;;;GASrD"}
@@ -0,0 +1,23 @@
1
+ import { z } from "zod";
2
+ import { toolResult, errorResult } from "../utils/tool-response.js";
3
+ import { parseFlowRef, validateStoredFlow } from "../flows/index.js";
4
+ export const name = "validate_flow";
5
+ export const description = "Validate an Imperium Flow schema and report its inputs, steps, and storage path.";
6
+ export const schema = z.object({
7
+ flow: z.string().optional().describe("Flow reference in '<family>/<variant>' form"),
8
+ flows_dir: z.string().optional().describe("Flow storage directory override"),
9
+ global: z.boolean().default(false).describe("Use ~/.imperium-crawl/flows instead of ./flows"),
10
+ });
11
+ export async function execute(input) {
12
+ try {
13
+ if (!input.flow)
14
+ throw new Error("Flow reference is required. Use validate-flow <family>/<variant> or --flow <family>/<variant>.");
15
+ const ref = parseFlowRef(input.flow);
16
+ const validation = await validateStoredFlow(ref, { flowsDir: input.flows_dir, global: input.global });
17
+ return toolResult(validation);
18
+ }
19
+ catch (err) {
20
+ return errorResult(err instanceof Error ? err.message : String(err));
21
+ }
22
+ }
23
+ //# sourceMappingURL=validate-flow.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"validate-flow.js","sourceRoot":"","sources":["../../src/tools/validate-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAErE,MAAM,CAAC,MAAM,IAAI,GAAG,eAAe,CAAC;AACpC,MAAM,CAAC,MAAM,WAAW,GAAG,kFAAkF,CAAC;AAE9G,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;IACnF,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IAC5E,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,gDAAgD,CAAC;CAC9F,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAwB;IACpD,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,gGAAgG,CAAC,CAAC;QACnI,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACtG,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,WAAW,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
@@ -13,8 +13,8 @@ export declare const schema: z.ZodObject<{
13
13
  freshness?: "pd" | "pw" | "pm" | "py" | undefined;
14
14
  }, {
15
15
  query: string;
16
- country?: string | undefined;
17
16
  count?: number | undefined;
17
+ country?: string | undefined;
18
18
  freshness?: "pd" | "pw" | "pm" | "py" | undefined;
19
19
  }>;
20
20
  export type VideoSearchInput = z.infer<typeof schema>;
@@ -0,0 +1,68 @@
1
+ /**
2
+ * watch — one-shot change detector for URLs.
3
+ *
4
+ * v2.5.0: one-shot mode only. Snapshots content, hashes it, diffs against
5
+ * the previous snapshot for the same URL. Fires a webhook on change.
6
+ *
7
+ * Daemon mode (SIGINT loop) is deferred to v2.6.0 — use cron externally:
8
+ * * /30 * * * * imperium-crawl watch --url X --output-dir /var/watch
9
+ */
10
+ import { z } from "zod";
11
+ export declare const name = "watch";
12
+ export declare const description = "One-shot change detector: scrape a URL, hash its content, and compare against the last snapshot. Fires a webhook on change. Run via cron for periodic monitoring.";
13
+ export declare const schema: z.ZodObject<{
14
+ url: z.ZodString;
15
+ output_dir: z.ZodDefault<z.ZodString>;
16
+ hash_on: z.ZodDefault<z.ZodEnum<["content", "readability", "markdown"]>>;
17
+ webhook: z.ZodOptional<z.ZodString>;
18
+ diff_format: z.ZodDefault<z.ZodEnum<["unified", "json"]>>;
19
+ one_shot: z.ZodDefault<z.ZodBoolean>;
20
+ }, "strip", z.ZodTypeAny, {
21
+ url: string;
22
+ output_dir: string;
23
+ hash_on: "readability" | "markdown" | "content";
24
+ diff_format: "json" | "unified";
25
+ one_shot: boolean;
26
+ webhook?: string | undefined;
27
+ }, {
28
+ url: string;
29
+ output_dir?: string | undefined;
30
+ hash_on?: "readability" | "markdown" | "content" | undefined;
31
+ webhook?: string | undefined;
32
+ diff_format?: "json" | "unified" | undefined;
33
+ one_shot?: boolean | undefined;
34
+ }>;
35
+ export type WatchInput = z.infer<typeof schema>;
36
+ interface WatchState {
37
+ url: string;
38
+ last_hash: string;
39
+ last_checked: string;
40
+ last_changed: string | null;
41
+ hash_on: string;
42
+ check_count: number;
43
+ change_count: number;
44
+ }
45
+ interface WatchResult {
46
+ url: string;
47
+ changed: boolean;
48
+ first_run: boolean;
49
+ previous_hash: string | null;
50
+ current_hash: string;
51
+ hash_on: string;
52
+ snapshot_file: string;
53
+ diff: string | null;
54
+ webhook_fired: boolean;
55
+ webhook_status?: number;
56
+ state: WatchState;
57
+ checked_at: string;
58
+ }
59
+ export declare function computeSignature(html: string, url: string, hashOn: WatchInput["hash_on"]): Promise<string>;
60
+ export declare function runWatchOnce(input: WatchInput): Promise<WatchResult>;
61
+ export declare function execute(input: WatchInput): Promise<{
62
+ content: {
63
+ type: "text";
64
+ text: string;
65
+ }[];
66
+ }>;
67
+ export {};
68
+ //# sourceMappingURL=watch.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"watch.d.ts","sourceRoot":"","sources":["../../src/tools/watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,IAAI,UAAU,CAAC;AAE5B,eAAO,MAAM,WAAW,sKAC6I,CAAC;AAEtK,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;EA0BjB,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEhD,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,WAAW;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,UAAU,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AA+CD,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC,CAcjB;AAiCD,wBAAsB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAoF1E;AAED,wBAAsB,OAAO,CAAC,KAAK,EAAE,UAAU;;;;;GAQ9C"}
@@ -0,0 +1,224 @@
1
+ /**
2
+ * watch — one-shot change detector for URLs.
3
+ *
4
+ * v2.5.0: one-shot mode only. Snapshots content, hashes it, diffs against
5
+ * the previous snapshot for the same URL. Fires a webhook on change.
6
+ *
7
+ * Daemon mode (SIGINT loop) is deferred to v2.6.0 — use cron externally:
8
+ * * /30 * * * * imperium-crawl watch --url X --output-dir /var/watch
9
+ */
10
+ import { z } from "zod";
11
+ import { createHash } from "node:crypto";
12
+ import { mkdir, readFile, writeFile } from "node:fs/promises";
13
+ import { existsSync } from "node:fs";
14
+ import { join, resolve as resolvePath } from "node:path";
15
+ import { JSDOM } from "jsdom";
16
+ import { Readability } from "@mozilla/readability";
17
+ import { fetchPage } from "../utils/fetcher.js";
18
+ import { htmlToMarkdown } from "../utils/markdown.js";
19
+ import { toolResult, errorResult } from "../utils/tool-response.js";
20
+ import { debugLog } from "../utils/debug.js";
21
+ import { MAX_URL_LENGTH } from "../constants.js";
22
+ export const name = "watch";
23
+ export const description = "One-shot change detector: scrape a URL, hash its content, and compare against the last snapshot. Fires a webhook on change. Run via cron for periodic monitoring.";
24
+ export const schema = z.object({
25
+ url: z
26
+ .string()
27
+ .max(MAX_URL_LENGTH)
28
+ .describe("URL to watch"),
29
+ output_dir: z
30
+ .string()
31
+ .default("./data/watch")
32
+ .describe("Directory to persist snapshots and state"),
33
+ hash_on: z
34
+ .enum(["content", "readability", "markdown"])
35
+ .default("readability")
36
+ .describe("What to hash: full HTML, readability main content, or markdown"),
37
+ webhook: z
38
+ .string()
39
+ .max(MAX_URL_LENGTH)
40
+ .optional()
41
+ .describe("If set, POST a JSON payload to this URL on detected change"),
42
+ diff_format: z
43
+ .enum(["unified", "json"])
44
+ .default("unified")
45
+ .describe("Diff representation in the result"),
46
+ one_shot: z
47
+ .boolean()
48
+ .default(true)
49
+ .describe("v2.5.0: always true. Daemon mode lands in v2.6.0."),
50
+ });
51
+ function slugify(url) {
52
+ return createHash("sha1").update(url).digest("hex").slice(0, 16);
53
+ }
54
+ function hashString(s) {
55
+ return createHash("sha256").update(s).digest("hex");
56
+ }
57
+ function simpleUnifiedDiff(prev, next, maxLines = 200) {
58
+ const prevLines = prev.split("\n");
59
+ const nextLines = next.split("\n");
60
+ const prevSet = new Set(prevLines);
61
+ const nextSet = new Set(nextLines);
62
+ const out = [];
63
+ let removed = 0;
64
+ let added = 0;
65
+ for (const line of prevLines) {
66
+ if (!nextSet.has(line)) {
67
+ out.push(`- ${line}`);
68
+ removed++;
69
+ }
70
+ }
71
+ for (const line of nextLines) {
72
+ if (!prevSet.has(line)) {
73
+ out.push(`+ ${line}`);
74
+ added++;
75
+ }
76
+ }
77
+ const header = `@@ -${prevLines.length} +${nextLines.length} @@ (${removed} removed, ${added} added)`;
78
+ const body = out.slice(0, maxLines).join("\n");
79
+ const truncated = out.length > maxLines ? `\n... (${out.length - maxLines} more lines)` : "";
80
+ return `${header}\n${body}${truncated}`;
81
+ }
82
+ function jsonDiff(prev, next) {
83
+ const prevLines = prev.split("\n");
84
+ const nextLines = next.split("\n");
85
+ const prevSet = new Set(prevLines);
86
+ const nextSet = new Set(nextLines);
87
+ const removed = prevLines.filter((l) => !nextSet.has(l));
88
+ const added = nextLines.filter((l) => !prevSet.has(l));
89
+ return JSON.stringify({ removed, added, prev_lines: prevLines.length, next_lines: nextLines.length });
90
+ }
91
+ export async function computeSignature(html, url, hashOn) {
92
+ if (hashOn === "content")
93
+ return html;
94
+ if (hashOn === "markdown")
95
+ return htmlToMarkdown(html);
96
+ // readability
97
+ try {
98
+ const dom = new JSDOM(html, { url });
99
+ const reader = new Readability(dom.window.document);
100
+ const article = reader.parse();
101
+ if (article?.textContent)
102
+ return article.textContent.trim();
103
+ return htmlToMarkdown(html);
104
+ }
105
+ catch {
106
+ return htmlToMarkdown(html);
107
+ }
108
+ }
109
+ async function loadState(stateFile) {
110
+ if (!existsSync(stateFile))
111
+ return {};
112
+ try {
113
+ const raw = await readFile(stateFile, "utf-8");
114
+ return JSON.parse(raw);
115
+ }
116
+ catch {
117
+ return {};
118
+ }
119
+ }
120
+ async function saveState(stateFile, state) {
121
+ await writeFile(stateFile, JSON.stringify(state, null, 2), "utf-8");
122
+ }
123
+ async function fireWebhook(webhook, payload) {
124
+ try {
125
+ const res = await fetch(webhook, {
126
+ method: "POST",
127
+ headers: { "content-type": "application/json" },
128
+ body: JSON.stringify(payload),
129
+ });
130
+ return { fired: true, status: res.status };
131
+ }
132
+ catch (err) {
133
+ debugLog("watch", "webhook failed", err);
134
+ return { fired: false };
135
+ }
136
+ }
137
+ export async function runWatchOnce(input) {
138
+ const outDir = resolvePath(input.output_dir);
139
+ await mkdir(outDir, { recursive: true });
140
+ const stateFile = join(outDir, ".state.json");
141
+ const state = await loadState(stateFile);
142
+ const fetched = await fetchPage(input.url);
143
+ const signature = await computeSignature(fetched.html, input.url, input.hash_on);
144
+ const currentHash = hashString(signature);
145
+ const slug = slugify(input.url);
146
+ const snapshotFile = join(outDir, `${slug}.snapshot.txt`);
147
+ const prevSnapshotFile = join(outDir, `${slug}.previous.txt`);
148
+ const existing = state[input.url];
149
+ const firstRun = !existing;
150
+ const changed = !firstRun && existing.last_hash !== currentHash;
151
+ let previousSig = null;
152
+ if (existsSync(snapshotFile)) {
153
+ try {
154
+ previousSig = await readFile(snapshotFile, "utf-8");
155
+ }
156
+ catch {
157
+ previousSig = null;
158
+ }
159
+ }
160
+ // Rotate previous snapshot only when content changed
161
+ if (changed && previousSig !== null) {
162
+ await writeFile(prevSnapshotFile, previousSig, "utf-8");
163
+ }
164
+ await writeFile(snapshotFile, signature, "utf-8");
165
+ const nowIso = new Date().toISOString();
166
+ const newState = {
167
+ url: input.url,
168
+ last_hash: currentHash,
169
+ last_checked: nowIso,
170
+ last_changed: changed ? nowIso : existing?.last_changed ?? null,
171
+ hash_on: input.hash_on,
172
+ check_count: (existing?.check_count ?? 0) + 1,
173
+ change_count: (existing?.change_count ?? 0) + (changed ? 1 : 0),
174
+ };
175
+ state[input.url] = newState;
176
+ await saveState(stateFile, state);
177
+ let diff = null;
178
+ if (changed && previousSig !== null) {
179
+ diff =
180
+ input.diff_format === "unified"
181
+ ? simpleUnifiedDiff(previousSig, signature)
182
+ : jsonDiff(previousSig, signature);
183
+ }
184
+ let webhookFired = false;
185
+ let webhookStatus;
186
+ if (changed && input.webhook) {
187
+ const payload = {
188
+ event: "watch.change",
189
+ url: input.url,
190
+ previous_hash: existing?.last_hash ?? null,
191
+ current_hash: currentHash,
192
+ detected_at: nowIso,
193
+ diff,
194
+ };
195
+ const res = await fireWebhook(input.webhook, payload);
196
+ webhookFired = res.fired;
197
+ webhookStatus = res.status;
198
+ }
199
+ return {
200
+ url: input.url,
201
+ changed,
202
+ first_run: firstRun,
203
+ previous_hash: existing?.last_hash ?? null,
204
+ current_hash: currentHash,
205
+ hash_on: input.hash_on,
206
+ snapshot_file: snapshotFile,
207
+ diff,
208
+ webhook_fired: webhookFired,
209
+ webhook_status: webhookStatus,
210
+ state: newState,
211
+ checked_at: nowIso,
212
+ };
213
+ }
214
+ export async function execute(input) {
215
+ try {
216
+ const result = await runWatchOnce(input);
217
+ return toolResult(result);
218
+ }
219
+ catch (err) {
220
+ debugLog("watch", "failed", err);
221
+ return errorResult(err instanceof Error ? err.message : String(err));
222
+ }
223
+ }
224
+ //# sourceMappingURL=watch.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"watch.js","sourceRoot":"","sources":["../../src/tools/watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC;AAE5B,MAAM,CAAC,MAAM,WAAW,GACtB,mKAAmK,CAAC;AAEtK,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,CAAC,cAAc,CAAC;IAC3B,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,OAAO,CAAC,cAAc,CAAC;SACvB,QAAQ,CAAC,0CAA0C,CAAC;IACvD,OAAO,EAAE,CAAC;SACP,IAAI,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;SAC5C,OAAO,CAAC,aAAa,CAAC;SACtB,QAAQ,CAAC,gEAAgE,CAAC;IAC7E,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,4DAA4D,CAAC;IACzE,WAAW,EAAE,CAAC;SACX,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;SACzB,OAAO,CAAC,SAAS,CAAC;SAClB,QAAQ,CAAC,mCAAmC,CAAC;IAChD,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,mDAAmD,CAAC;CACjE,CAAC,CAAC;AA6BH,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,IAAY,EAAE,QAAQ,GAAG,GAAG;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAEnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACtB,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,QAAQ,OAAO,aAAa,KAAK,SAAS,CAAC;IACtG,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,MAAM,GAAG,QAAQ,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7F,OAAO,GAAG,MAAM,KAAK,IAAI,GAAG,SAAS,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAY;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;AACxG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,GAAW,EACX,MAA6B;IAE7B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAEvD,cAAc;IACd,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,OAAO,EAAE,WAAW;YAAE,OAAO,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAC5D,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB;IACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA+B,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB,EAAE,KAAiC;IAC3E,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAAe,EACf,OAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB;IAClD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACjF,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,WAAW,CAAC;IAEhE,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,SAAS,CAAC,gBAAgB,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,SAAS,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAElD,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAe;QAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,SAAS,EAAE,WAAW;QACtB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,YAAY,IAAI,IAAI;QAC/D,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,WAAW,EAAE,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;QAC7C,YAAY,EAAE,CAAC,QAAQ,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAChE,CAAC;IACF,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;IAC5B,MAAM,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAElC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,IAAI,OAAO,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,IAAI;YACF,KAAK,CAAC,WAAW,KAAK,SAAS;gBAC7B,CAAC,CAAC,iBAAiB,CAAC,WAAW,EAAE,SAAS,CAAC;gBAC3C,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,aAAiC,CAAC;IACtC,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,cAAc;YACrB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,aAAa,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;YAC1C,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,MAAM;YACnB,IAAI;SACL,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtD,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC;QACzB,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,OAAO;QACP,SAAS,EAAE,QAAQ;QACnB,aAAa,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QAC1C,YAAY,EAAE,WAAW;QACzB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,aAAa,EAAE,YAAY;QAC3B,IAAI;QACJ,aAAa,EAAE,YAAY;QAC3B,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACjC,OAAO,WAAW,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
@@ -9,8 +9,8 @@ export declare const schema: z.ZodObject<{
9
9
  limit: z.ZodDefault<z.ZodNumber>;
10
10
  sort: z.ZodDefault<z.ZodEnum<["relevance", "date", "views"]>>;
11
11
  }, "strip", z.ZodTypeAny, {
12
- action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
13
12
  sort: "date" | "relevance" | "views";
13
+ action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
14
14
  limit: number;
15
15
  url?: string | undefined;
16
16
  query?: string | undefined;
@@ -18,8 +18,8 @@ export declare const schema: z.ZodObject<{
18
18
  }, {
19
19
  action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
20
20
  url?: string | undefined;
21
- sort?: "date" | "relevance" | "views" | undefined;
22
21
  query?: string | undefined;
22
+ sort?: "date" | "relevance" | "views" | undefined;
23
23
  channel_url?: string | undefined;
24
24
  limit?: number | undefined;
25
25
  }>;
@@ -20,10 +20,19 @@ declare const CIRCUIT_OPEN_DURATION_MS = 60000;
20
20
  declare const CIRCUIT_PROBE_SUCCESSES = 3;
21
21
  declare const circuits: Map<string, CircuitBreaker>;
22
22
  declare const CIRCUIT_STALE_MS = 3600000;
23
- declare function getCircuit(domain: string): CircuitBreaker;
24
- declare function recordSuccess(domain: string): void;
25
- declare function recordFailure(domain: string): void;
26
- export { circuits, getCircuit, recordSuccess, recordFailure, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
23
+ /**
24
+ * Get circuit breaker key for a URL.
25
+ * Uses domain + first 2 path segments for endpoint-level granularity.
26
+ */
27
+ declare function getCircuitKey(url: string): string;
28
+ declare function getCircuit(key: string): CircuitBreaker;
29
+ declare function recordSuccess(key: string): void;
30
+ declare function recordFailure(key: string): void;
31
+ /**
32
+ * Check domain-level circuit: opens when 3+ endpoint circuits are open for this domain.
33
+ */
34
+ declare function isDomainCircuitOpen(domain: string): boolean;
35
+ export { circuits, getCircuit, getCircuitKey, recordSuccess, recordFailure, isDomainCircuitOpen, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
27
36
  export interface SmartFetchOptions extends StealthOptions {
28
37
  respectRobots?: boolean;
29
38
  retries?: number;
@@ -1 +1 @@
1
- {"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAE,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAOtG,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAElC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC,iBAAS,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAgBlD;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAY3C;AAED,iBAAS,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAc3C;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAc9J,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA+C9F"}
1
+ {"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAqB,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQzH,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAIlC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC;;;GAGG;AACH,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAS1C;AAED,iBAAS,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAgB/C;AAED,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAYxC;AAED,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAcxC;AAED;;GAEG;AACH,iBAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAQpD;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAsDlM,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AA+BD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiG9F"}