imperium-crawl 2.3.1 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -11
- package/dist/cli-explore.d.ts +30 -0
- package/dist/cli-explore.d.ts.map +1 -0
- package/dist/cli-explore.js +427 -0
- package/dist/cli-explore.js.map +1 -0
- package/dist/cli-recorder.d.ts +44 -0
- package/dist/cli-recorder.d.ts.map +1 -0
- package/dist/cli-recorder.js +67 -0
- package/dist/cli-recorder.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +51 -3
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +3 -0
- package/dist/config.js.map +1 -1
- package/dist/constants.d.ts +1 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +31 -1
- package/dist/constants.js.map +1 -1
- package/dist/flows/engine.d.ts +7 -0
- package/dist/flows/engine.d.ts.map +1 -0
- package/dist/flows/engine.js +183 -0
- package/dist/flows/engine.js.map +1 -0
- package/dist/flows/index.d.ts +6 -0
- package/dist/flows/index.d.ts.map +1 -0
- package/dist/flows/index.js +6 -0
- package/dist/flows/index.js.map +1 -0
- package/dist/flows/server.d.ts +11 -0
- package/dist/flows/server.d.ts.map +1 -0
- package/dist/flows/server.js +81 -0
- package/dist/flows/server.js.map +1 -0
- package/dist/flows/smart-target.d.ts +9 -0
- package/dist/flows/smart-target.d.ts.map +1 -0
- package/dist/flows/smart-target.js +84 -0
- package/dist/flows/smart-target.js.map +1 -0
- package/dist/flows/storage.d.ts +26 -0
- package/dist/flows/storage.d.ts.map +1 -0
- package/dist/flows/storage.js +118 -0
- package/dist/flows/storage.js.map +1 -0
- package/dist/flows/templates.d.ts +4 -0
- package/dist/flows/templates.d.ts.map +1 -0
- package/dist/flows/templates.js +35 -0
- package/dist/flows/templates.js.map +1 -0
- package/dist/flows/types.d.ts +3356 -0
- package/dist/flows/types.d.ts.map +1 -0
- package/dist/flows/types.js +133 -0
- package/dist/flows/types.js.map +1 -0
- package/dist/knowledge/index.d.ts +1 -0
- package/dist/knowledge/index.d.ts.map +1 -1
- package/dist/knowledge/index.js +1 -0
- package/dist/knowledge/index.js.map +1 -1
- package/dist/knowledge/record-browser.d.ts +17 -0
- package/dist/knowledge/record-browser.d.ts.map +1 -0
- package/dist/knowledge/record-browser.js +29 -0
- package/dist/knowledge/record-browser.js.map +1 -0
- package/dist/knowledge/store.d.ts +19 -0
- package/dist/knowledge/store.d.ts.map +1 -1
- package/dist/knowledge/store.js +63 -4
- package/dist/knowledge/store.js.map +1 -1
- package/dist/llm/retry.d.ts +4 -2
- package/dist/llm/retry.d.ts.map +1 -1
- package/dist/llm/retry.js +15 -4
- package/dist/llm/retry.js.map +1 -1
- package/dist/sessions/browser-connect.d.ts +30 -0
- package/dist/sessions/browser-connect.d.ts.map +1 -0
- package/dist/sessions/browser-connect.js +68 -0
- package/dist/sessions/browser-connect.js.map +1 -0
- package/dist/sessions/browser-state.d.ts +35 -0
- package/dist/sessions/browser-state.d.ts.map +1 -0
- package/dist/sessions/browser-state.js +74 -0
- package/dist/sessions/browser-state.js.map +1 -0
- package/dist/sessions/index.d.ts +1 -1
- package/dist/sessions/index.d.ts.map +1 -1
- package/dist/sessions/index.js +1 -1
- package/dist/sessions/index.js.map +1 -1
- package/dist/sessions/inject-cookies.d.ts +20 -0
- package/dist/sessions/inject-cookies.d.ts.map +1 -0
- package/dist/sessions/inject-cookies.js +57 -0
- package/dist/sessions/inject-cookies.js.map +1 -0
- package/dist/sessions/manager.d.ts +31 -1
- package/dist/sessions/manager.d.ts.map +1 -1
- package/dist/sessions/manager.js +97 -6
- package/dist/sessions/manager.js.map +1 -1
- package/dist/sessions/types.d.ts +2 -0
- package/dist/sessions/types.d.ts.map +1 -1
- package/dist/skills/chain.d.ts +61 -0
- package/dist/skills/chain.d.ts.map +1 -0
- package/dist/skills/chain.js +182 -0
- package/dist/skills/chain.js.map +1 -0
- package/dist/skills/conditions.d.ts +14 -0
- package/dist/skills/conditions.d.ts.map +1 -0
- package/dist/skills/conditions.js +208 -0
- package/dist/skills/conditions.js.map +1 -0
- package/dist/skills/manager.d.ts +47 -2
- package/dist/skills/manager.d.ts.map +1 -1
- package/dist/skills/manager.js.map +1 -1
- package/dist/skills/parameters.d.ts +49 -0
- package/dist/skills/parameters.d.ts.map +1 -0
- package/dist/skills/parameters.js +157 -0
- package/dist/skills/parameters.js.map +1 -0
- package/dist/snapshot/store.d.ts +8 -0
- package/dist/snapshot/store.d.ts.map +1 -1
- package/dist/snapshot/store.js +48 -0
- package/dist/snapshot/store.js.map +1 -1
- package/dist/stealth/antibot-detector.d.ts +1 -1
- package/dist/stealth/antibot-detector.d.ts.map +1 -1
- package/dist/stealth/antibot-detector.js +56 -0
- package/dist/stealth/antibot-detector.js.map +1 -1
- package/dist/stealth/browser-image-extract.d.ts +43 -0
- package/dist/stealth/browser-image-extract.d.ts.map +1 -0
- package/dist/stealth/browser-image-extract.js +268 -0
- package/dist/stealth/browser-image-extract.js.map +1 -0
- package/dist/stealth/browser.d.ts +5 -0
- package/dist/stealth/browser.d.ts.map +1 -1
- package/dist/stealth/browser.js +82 -1
- package/dist/stealth/browser.js.map +1 -1
- package/dist/stealth/chrome-profile.d.ts +1 -0
- package/dist/stealth/chrome-profile.d.ts.map +1 -1
- package/dist/stealth/chrome-profile.js +28 -5
- package/dist/stealth/chrome-profile.js.map +1 -1
- package/dist/stealth/detector.d.ts +10 -1
- package/dist/stealth/detector.d.ts.map +1 -1
- package/dist/stealth/detector.js +117 -25
- package/dist/stealth/detector.js.map +1 -1
- package/dist/stealth/headers.d.ts +1 -1
- package/dist/stealth/headers.d.ts.map +1 -1
- package/dist/stealth/headers.js +94 -2
- package/dist/stealth/headers.js.map +1 -1
- package/dist/stealth/index.d.ts +5 -0
- package/dist/stealth/index.d.ts.map +1 -1
- package/dist/stealth/index.js +257 -27
- package/dist/stealth/index.js.map +1 -1
- package/dist/stealth/proxy.d.ts +40 -1
- package/dist/stealth/proxy.d.ts.map +1 -1
- package/dist/stealth/proxy.js +90 -6
- package/dist/stealth/proxy.js.map +1 -1
- package/dist/tools/action-executor.d.ts +66 -0
- package/dist/tools/action-executor.d.ts.map +1 -0
- package/dist/tools/action-executor.js +403 -0
- package/dist/tools/action-executor.js.map +1 -0
- package/dist/tools/batch-download.d.ts +33 -0
- package/dist/tools/batch-download.d.ts.map +1 -0
- package/dist/tools/batch-download.js +208 -0
- package/dist/tools/batch-download.js.map +1 -0
- package/dist/tools/batch-scrape.d.ts +2 -2
- package/dist/tools/browser.d.ts +100 -0
- package/dist/tools/browser.d.ts.map +1 -0
- package/dist/tools/browser.js +448 -0
- package/dist/tools/browser.js.map +1 -0
- package/dist/tools/crawl.d.ts +2 -2
- package/dist/tools/create-skill.d.ts +2 -2
- package/dist/tools/discover-apis.d.ts +1 -1
- package/dist/tools/discover-apis.d.ts.map +1 -1
- package/dist/tools/discover-apis.js +3 -0
- package/dist/tools/discover-apis.js.map +1 -1
- package/dist/tools/download.d.ts +39 -6
- package/dist/tools/download.d.ts.map +1 -1
- package/dist/tools/download.js +248 -44
- package/dist/tools/download.js.map +1 -1
- package/dist/tools/extract.d.ts +1 -1
- package/dist/tools/image-search.d.ts +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +26 -0
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/inspect-flow.d.ts +24 -0
- package/dist/tools/inspect-flow.d.ts.map +1 -0
- package/dist/tools/inspect-flow.js +23 -0
- package/dist/tools/inspect-flow.js.map +1 -0
- package/dist/tools/instagram.d.ts +2 -2
- package/dist/tools/interact.d.ts +91 -50
- package/dist/tools/interact.d.ts.map +1 -1
- package/dist/tools/interact.js +80 -299
- package/dist/tools/interact.js.map +1 -1
- package/dist/tools/knowledge.d.ts +24 -0
- package/dist/tools/knowledge.d.ts.map +1 -0
- package/dist/tools/knowledge.js +99 -0
- package/dist/tools/knowledge.js.map +1 -0
- package/dist/tools/list-flows.d.ts +21 -0
- package/dist/tools/list-flows.d.ts.map +1 -0
- package/dist/tools/list-flows.js +18 -0
- package/dist/tools/list-flows.js.map +1 -0
- package/dist/tools/list-skills.js +1 -1
- package/dist/tools/list-skills.js.map +1 -1
- package/dist/tools/manifest.d.ts.map +1 -1
- package/dist/tools/manifest.js +48 -0
- package/dist/tools/manifest.js.map +1 -1
- package/dist/tools/monitor-websocket.d.ts +1 -1
- package/dist/tools/monitor.d.ts +46 -0
- package/dist/tools/monitor.d.ts.map +1 -0
- package/dist/tools/monitor.js +213 -0
- package/dist/tools/monitor.js.map +1 -0
- package/dist/tools/news-search.d.ts +1 -1
- package/dist/tools/pdf-extract.d.ts +38 -0
- package/dist/tools/pdf-extract.d.ts.map +1 -0
- package/dist/tools/pdf-extract.js +244 -0
- package/dist/tools/pdf-extract.js.map +1 -0
- package/dist/tools/query-api.d.ts +6 -6
- package/dist/tools/readability.d.ts +2 -2
- package/dist/tools/record-flow.d.ts +39 -0
- package/dist/tools/record-flow.d.ts.map +1 -0
- package/dist/tools/record-flow.js +406 -0
- package/dist/tools/record-flow.js.map +1 -0
- package/dist/tools/reddit.d.ts +4 -4
- package/dist/tools/run-flow.d.ts +54 -0
- package/dist/tools/run-flow.d.ts.map +1 -0
- package/dist/tools/run-flow.js +47 -0
- package/dist/tools/run-flow.js.map +1 -0
- package/dist/tools/run-skill.d.ts +14 -4
- package/dist/tools/run-skill.d.ts.map +1 -1
- package/dist/tools/run-skill.js +74 -0
- package/dist/tools/run-skill.js.map +1 -1
- package/dist/tools/scrape.d.ts +9 -6
- package/dist/tools/scrape.d.ts.map +1 -1
- package/dist/tools/scrape.js +19 -1
- package/dist/tools/scrape.js.map +1 -1
- package/dist/tools/screenshot.d.ts.map +1 -1
- package/dist/tools/screenshot.js +6 -0
- package/dist/tools/screenshot.js.map +1 -1
- package/dist/tools/search.d.ts +1 -1
- package/dist/tools/serve-flow.d.ts +36 -0
- package/dist/tools/serve-flow.d.ts.map +1 -0
- package/dist/tools/serve-flow.js +42 -0
- package/dist/tools/serve-flow.js.map +1 -0
- package/dist/tools/snapshot.d.ts +5 -5
- package/dist/tools/snapshot.d.ts.map +1 -1
- package/dist/tools/snapshot.js +3 -0
- package/dist/tools/snapshot.js.map +1 -1
- package/dist/tools/validate-flow.d.ts +24 -0
- package/dist/tools/validate-flow.d.ts.map +1 -0
- package/dist/tools/validate-flow.js +23 -0
- package/dist/tools/validate-flow.js.map +1 -0
- package/dist/tools/video-search.d.ts +1 -1
- package/dist/tools/watch.d.ts +68 -0
- package/dist/tools/watch.d.ts.map +1 -0
- package/dist/tools/watch.js +224 -0
- package/dist/tools/watch.js.map +1 -0
- package/dist/tools/youtube.d.ts +2 -2
- package/dist/utils/fetcher.d.ts +13 -4
- package/dist/utils/fetcher.d.ts.map +1 -1
- package/dist/utils/fetcher.js +153 -23
- package/dist/utils/fetcher.js.map +1 -1
- package/package.json +19 -5
package/dist/tools/snapshot.d.ts
CHANGED
|
@@ -15,11 +15,11 @@ export declare const schema: z.ZodObject<{
|
|
|
15
15
|
timeout: z.ZodDefault<z.ZodNumber>;
|
|
16
16
|
allowed_domains: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
17
17
|
}, "strip", z.ZodTypeAny, {
|
|
18
|
-
url: string;
|
|
19
|
-
timeout: number;
|
|
20
18
|
interactive: boolean;
|
|
21
19
|
cursor: boolean;
|
|
22
20
|
compact: boolean;
|
|
21
|
+
url: string;
|
|
22
|
+
timeout: number;
|
|
23
23
|
return_screenshot: boolean;
|
|
24
24
|
annotate: boolean;
|
|
25
25
|
proxy?: string | undefined;
|
|
@@ -29,12 +29,12 @@ export declare const schema: z.ZodObject<{
|
|
|
29
29
|
scope_selector?: string | undefined;
|
|
30
30
|
}, {
|
|
31
31
|
url: string;
|
|
32
|
-
proxy?: string | undefined;
|
|
33
|
-
timeout?: number | undefined;
|
|
34
|
-
chrome_profile?: string | undefined;
|
|
35
32
|
interactive?: boolean | undefined;
|
|
36
33
|
cursor?: boolean | undefined;
|
|
37
34
|
compact?: boolean | undefined;
|
|
35
|
+
timeout?: number | undefined;
|
|
36
|
+
proxy?: string | undefined;
|
|
37
|
+
chrome_profile?: string | undefined;
|
|
38
38
|
return_screenshot?: boolean | undefined;
|
|
39
39
|
session_id?: string | undefined;
|
|
40
40
|
allowed_domains?: string[] | undefined;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,eAAO,MAAM,IAAI,aAAa,CAAC;AAE/B,eAAO,MAAM,WAAW,qTAC6Q,CAAC;AAEtS,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqDjB,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEnD,wBAAsB,OAAO,CAAC,KAAK,EAAE,aAAa;;cAkEjB,MAAM;eAAS,MAAM;eAAS,MAAM;mBAAa,MAAM;;GAwCvF"}
|
package/dist/tools/snapshot.js
CHANGED
|
@@ -3,6 +3,7 @@ import { isPlaywrightAvailable } from "../stealth/browser.js";
|
|
|
3
3
|
import { acquirePage } from "../stealth/chrome-profile.js";
|
|
4
4
|
import { resolveProxy } from "../stealth/proxy.js";
|
|
5
5
|
import { normalizeUrl } from "../utils/url.js";
|
|
6
|
+
import { recordBrowserOutcome } from "../knowledge/index.js";
|
|
6
7
|
import { getSessionManager } from "../sessions/index.js";
|
|
7
8
|
import { getEnhancedSnapshot, getSnapshotStore, annotateScreenshot } from "../snapshot/index.js";
|
|
8
9
|
import { installDomainFilter } from "../security/domain-filter.js";
|
|
@@ -77,6 +78,7 @@ export async function execute(input) {
|
|
|
77
78
|
const url = normalizeUrl(input.url);
|
|
78
79
|
const proxyUrl = resolveProxy(input.proxy);
|
|
79
80
|
const snapshotId = input.session_id ?? `snap_${Date.now()}`;
|
|
81
|
+
const fetchStart = Date.now();
|
|
80
82
|
const handle = await acquirePage({
|
|
81
83
|
chromeProfile: input.chrome_profile,
|
|
82
84
|
proxyUrl,
|
|
@@ -96,6 +98,7 @@ export async function execute(input) {
|
|
|
96
98
|
}
|
|
97
99
|
// Navigate
|
|
98
100
|
await page.goto(url, { waitUntil: "load", timeout: input.timeout });
|
|
101
|
+
recordBrowserOutcome({ url, success: true, responseTimeMs: Date.now() - fetchStart, proxyUsed: !!proxyUrl });
|
|
99
102
|
// Take snapshot
|
|
100
103
|
const snapshot = await getEnhancedSnapshot(page, {
|
|
101
104
|
interactive: input.interactive,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,oBAAoB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAE7G,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const name = "validate_flow";
|
|
3
|
+
export declare const description = "Validate an Imperium Flow schema and report its inputs, steps, and storage path.";
|
|
4
|
+
export declare const schema: z.ZodObject<{
|
|
5
|
+
flow: z.ZodOptional<z.ZodString>;
|
|
6
|
+
flows_dir: z.ZodOptional<z.ZodString>;
|
|
7
|
+
global: z.ZodDefault<z.ZodBoolean>;
|
|
8
|
+
}, "strip", z.ZodTypeAny, {
|
|
9
|
+
global: boolean;
|
|
10
|
+
flow?: string | undefined;
|
|
11
|
+
flows_dir?: string | undefined;
|
|
12
|
+
}, {
|
|
13
|
+
flow?: string | undefined;
|
|
14
|
+
flows_dir?: string | undefined;
|
|
15
|
+
global?: boolean | undefined;
|
|
16
|
+
}>;
|
|
17
|
+
export type ValidateFlowInput = z.infer<typeof schema>;
|
|
18
|
+
export declare function execute(input: ValidateFlowInput): Promise<{
|
|
19
|
+
content: {
|
|
20
|
+
type: "text";
|
|
21
|
+
text: string;
|
|
22
|
+
}[];
|
|
23
|
+
}>;
|
|
24
|
+
//# sourceMappingURL=validate-flow.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-flow.d.ts","sourceRoot":"","sources":["../../src/tools/validate-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,IAAI,kBAAkB,CAAC;AACpC,eAAO,MAAM,WAAW,qFAAqF,CAAC;AAE9G,eAAO,MAAM,MAAM;;;;;;;;;;;;EAIjB,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEvD,wBAAsB,OAAO,CAAC,KAAK,EAAE,iBAAiB;;;;;GASrD"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { toolResult, errorResult } from "../utils/tool-response.js";
|
|
3
|
+
import { parseFlowRef, validateStoredFlow } from "../flows/index.js";
|
|
4
|
+
export const name = "validate_flow";
|
|
5
|
+
export const description = "Validate an Imperium Flow schema and report its inputs, steps, and storage path.";
|
|
6
|
+
export const schema = z.object({
|
|
7
|
+
flow: z.string().optional().describe("Flow reference in '<family>/<variant>' form"),
|
|
8
|
+
flows_dir: z.string().optional().describe("Flow storage directory override"),
|
|
9
|
+
global: z.boolean().default(false).describe("Use ~/.imperium-crawl/flows instead of ./flows"),
|
|
10
|
+
});
|
|
11
|
+
export async function execute(input) {
|
|
12
|
+
try {
|
|
13
|
+
if (!input.flow)
|
|
14
|
+
throw new Error("Flow reference is required. Use validate-flow <family>/<variant> or --flow <family>/<variant>.");
|
|
15
|
+
const ref = parseFlowRef(input.flow);
|
|
16
|
+
const validation = await validateStoredFlow(ref, { flowsDir: input.flows_dir, global: input.global });
|
|
17
|
+
return toolResult(validation);
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=validate-flow.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-flow.js","sourceRoot":"","sources":["../../src/tools/validate-flow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAErE,MAAM,CAAC,MAAM,IAAI,GAAG,eAAe,CAAC;AACpC,MAAM,CAAC,MAAM,WAAW,GAAG,kFAAkF,CAAC;AAE9G,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;IACnF,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IAC5E,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,gDAAgD,CAAC;CAC9F,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAwB;IACpD,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,gGAAgG,CAAC,CAAC;QACnI,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACtG,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,WAAW,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
|
@@ -13,8 +13,8 @@ export declare const schema: z.ZodObject<{
|
|
|
13
13
|
freshness?: "pd" | "pw" | "pm" | "py" | undefined;
|
|
14
14
|
}, {
|
|
15
15
|
query: string;
|
|
16
|
-
country?: string | undefined;
|
|
17
16
|
count?: number | undefined;
|
|
17
|
+
country?: string | undefined;
|
|
18
18
|
freshness?: "pd" | "pw" | "pm" | "py" | undefined;
|
|
19
19
|
}>;
|
|
20
20
|
export type VideoSearchInput = z.infer<typeof schema>;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* watch — one-shot change detector for URLs.
|
|
3
|
+
*
|
|
4
|
+
* v2.5.0: one-shot mode only. Snapshots content, hashes it, diffs against
|
|
5
|
+
* the previous snapshot for the same URL. Fires a webhook on change.
|
|
6
|
+
*
|
|
7
|
+
* Daemon mode (SIGINT loop) is deferred to v2.6.0 — use cron externally:
|
|
8
|
+
* * /30 * * * * imperium-crawl watch --url X --output-dir /var/watch
|
|
9
|
+
*/
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
export declare const name = "watch";
|
|
12
|
+
export declare const description = "One-shot change detector: scrape a URL, hash its content, and compare against the last snapshot. Fires a webhook on change. Run via cron for periodic monitoring.";
|
|
13
|
+
export declare const schema: z.ZodObject<{
|
|
14
|
+
url: z.ZodString;
|
|
15
|
+
output_dir: z.ZodDefault<z.ZodString>;
|
|
16
|
+
hash_on: z.ZodDefault<z.ZodEnum<["content", "readability", "markdown"]>>;
|
|
17
|
+
webhook: z.ZodOptional<z.ZodString>;
|
|
18
|
+
diff_format: z.ZodDefault<z.ZodEnum<["unified", "json"]>>;
|
|
19
|
+
one_shot: z.ZodDefault<z.ZodBoolean>;
|
|
20
|
+
}, "strip", z.ZodTypeAny, {
|
|
21
|
+
url: string;
|
|
22
|
+
output_dir: string;
|
|
23
|
+
hash_on: "readability" | "markdown" | "content";
|
|
24
|
+
diff_format: "json" | "unified";
|
|
25
|
+
one_shot: boolean;
|
|
26
|
+
webhook?: string | undefined;
|
|
27
|
+
}, {
|
|
28
|
+
url: string;
|
|
29
|
+
output_dir?: string | undefined;
|
|
30
|
+
hash_on?: "readability" | "markdown" | "content" | undefined;
|
|
31
|
+
webhook?: string | undefined;
|
|
32
|
+
diff_format?: "json" | "unified" | undefined;
|
|
33
|
+
one_shot?: boolean | undefined;
|
|
34
|
+
}>;
|
|
35
|
+
export type WatchInput = z.infer<typeof schema>;
|
|
36
|
+
interface WatchState {
|
|
37
|
+
url: string;
|
|
38
|
+
last_hash: string;
|
|
39
|
+
last_checked: string;
|
|
40
|
+
last_changed: string | null;
|
|
41
|
+
hash_on: string;
|
|
42
|
+
check_count: number;
|
|
43
|
+
change_count: number;
|
|
44
|
+
}
|
|
45
|
+
interface WatchResult {
|
|
46
|
+
url: string;
|
|
47
|
+
changed: boolean;
|
|
48
|
+
first_run: boolean;
|
|
49
|
+
previous_hash: string | null;
|
|
50
|
+
current_hash: string;
|
|
51
|
+
hash_on: string;
|
|
52
|
+
snapshot_file: string;
|
|
53
|
+
diff: string | null;
|
|
54
|
+
webhook_fired: boolean;
|
|
55
|
+
webhook_status?: number;
|
|
56
|
+
state: WatchState;
|
|
57
|
+
checked_at: string;
|
|
58
|
+
}
|
|
59
|
+
export declare function computeSignature(html: string, url: string, hashOn: WatchInput["hash_on"]): Promise<string>;
|
|
60
|
+
export declare function runWatchOnce(input: WatchInput): Promise<WatchResult>;
|
|
61
|
+
export declare function execute(input: WatchInput): Promise<{
|
|
62
|
+
content: {
|
|
63
|
+
type: "text";
|
|
64
|
+
text: string;
|
|
65
|
+
}[];
|
|
66
|
+
}>;
|
|
67
|
+
export {};
|
|
68
|
+
//# sourceMappingURL=watch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watch.d.ts","sourceRoot":"","sources":["../../src/tools/watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,IAAI,UAAU,CAAC;AAE5B,eAAO,MAAM,WAAW,sKAC6I,CAAC;AAEtK,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;EA0BjB,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAEhD,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,WAAW;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,UAAU,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AA+CD,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC,GAC5B,OAAO,CAAC,MAAM,CAAC,CAcjB;AAiCD,wBAAsB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAoF1E;AAED,wBAAsB,OAAO,CAAC,KAAK,EAAE,UAAU;;;;;GAQ9C"}
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* watch — one-shot change detector for URLs.
|
|
3
|
+
*
|
|
4
|
+
* v2.5.0: one-shot mode only. Snapshots content, hashes it, diffs against
|
|
5
|
+
* the previous snapshot for the same URL. Fires a webhook on change.
|
|
6
|
+
*
|
|
7
|
+
* Daemon mode (SIGINT loop) is deferred to v2.6.0 — use cron externally:
|
|
8
|
+
* * /30 * * * * imperium-crawl watch --url X --output-dir /var/watch
|
|
9
|
+
*/
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
13
|
+
import { existsSync } from "node:fs";
|
|
14
|
+
import { join, resolve as resolvePath } from "node:path";
|
|
15
|
+
import { JSDOM } from "jsdom";
|
|
16
|
+
import { Readability } from "@mozilla/readability";
|
|
17
|
+
import { fetchPage } from "../utils/fetcher.js";
|
|
18
|
+
import { htmlToMarkdown } from "../utils/markdown.js";
|
|
19
|
+
import { toolResult, errorResult } from "../utils/tool-response.js";
|
|
20
|
+
import { debugLog } from "../utils/debug.js";
|
|
21
|
+
import { MAX_URL_LENGTH } from "../constants.js";
|
|
22
|
+
export const name = "watch";
|
|
23
|
+
export const description = "One-shot change detector: scrape a URL, hash its content, and compare against the last snapshot. Fires a webhook on change. Run via cron for periodic monitoring.";
|
|
24
|
+
export const schema = z.object({
|
|
25
|
+
url: z
|
|
26
|
+
.string()
|
|
27
|
+
.max(MAX_URL_LENGTH)
|
|
28
|
+
.describe("URL to watch"),
|
|
29
|
+
output_dir: z
|
|
30
|
+
.string()
|
|
31
|
+
.default("./data/watch")
|
|
32
|
+
.describe("Directory to persist snapshots and state"),
|
|
33
|
+
hash_on: z
|
|
34
|
+
.enum(["content", "readability", "markdown"])
|
|
35
|
+
.default("readability")
|
|
36
|
+
.describe("What to hash: full HTML, readability main content, or markdown"),
|
|
37
|
+
webhook: z
|
|
38
|
+
.string()
|
|
39
|
+
.max(MAX_URL_LENGTH)
|
|
40
|
+
.optional()
|
|
41
|
+
.describe("If set, POST a JSON payload to this URL on detected change"),
|
|
42
|
+
diff_format: z
|
|
43
|
+
.enum(["unified", "json"])
|
|
44
|
+
.default("unified")
|
|
45
|
+
.describe("Diff representation in the result"),
|
|
46
|
+
one_shot: z
|
|
47
|
+
.boolean()
|
|
48
|
+
.default(true)
|
|
49
|
+
.describe("v2.5.0: always true. Daemon mode lands in v2.6.0."),
|
|
50
|
+
});
|
|
51
|
+
function slugify(url) {
|
|
52
|
+
return createHash("sha1").update(url).digest("hex").slice(0, 16);
|
|
53
|
+
}
|
|
54
|
+
function hashString(s) {
|
|
55
|
+
return createHash("sha256").update(s).digest("hex");
|
|
56
|
+
}
|
|
57
|
+
function simpleUnifiedDiff(prev, next, maxLines = 200) {
|
|
58
|
+
const prevLines = prev.split("\n");
|
|
59
|
+
const nextLines = next.split("\n");
|
|
60
|
+
const prevSet = new Set(prevLines);
|
|
61
|
+
const nextSet = new Set(nextLines);
|
|
62
|
+
const out = [];
|
|
63
|
+
let removed = 0;
|
|
64
|
+
let added = 0;
|
|
65
|
+
for (const line of prevLines) {
|
|
66
|
+
if (!nextSet.has(line)) {
|
|
67
|
+
out.push(`- ${line}`);
|
|
68
|
+
removed++;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
for (const line of nextLines) {
|
|
72
|
+
if (!prevSet.has(line)) {
|
|
73
|
+
out.push(`+ ${line}`);
|
|
74
|
+
added++;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const header = `@@ -${prevLines.length} +${nextLines.length} @@ (${removed} removed, ${added} added)`;
|
|
78
|
+
const body = out.slice(0, maxLines).join("\n");
|
|
79
|
+
const truncated = out.length > maxLines ? `\n... (${out.length - maxLines} more lines)` : "";
|
|
80
|
+
return `${header}\n${body}${truncated}`;
|
|
81
|
+
}
|
|
82
|
+
function jsonDiff(prev, next) {
|
|
83
|
+
const prevLines = prev.split("\n");
|
|
84
|
+
const nextLines = next.split("\n");
|
|
85
|
+
const prevSet = new Set(prevLines);
|
|
86
|
+
const nextSet = new Set(nextLines);
|
|
87
|
+
const removed = prevLines.filter((l) => !nextSet.has(l));
|
|
88
|
+
const added = nextLines.filter((l) => !prevSet.has(l));
|
|
89
|
+
return JSON.stringify({ removed, added, prev_lines: prevLines.length, next_lines: nextLines.length });
|
|
90
|
+
}
|
|
91
|
+
export async function computeSignature(html, url, hashOn) {
|
|
92
|
+
if (hashOn === "content")
|
|
93
|
+
return html;
|
|
94
|
+
if (hashOn === "markdown")
|
|
95
|
+
return htmlToMarkdown(html);
|
|
96
|
+
// readability
|
|
97
|
+
try {
|
|
98
|
+
const dom = new JSDOM(html, { url });
|
|
99
|
+
const reader = new Readability(dom.window.document);
|
|
100
|
+
const article = reader.parse();
|
|
101
|
+
if (article?.textContent)
|
|
102
|
+
return article.textContent.trim();
|
|
103
|
+
return htmlToMarkdown(html);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return htmlToMarkdown(html);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function loadState(stateFile) {
|
|
110
|
+
if (!existsSync(stateFile))
|
|
111
|
+
return {};
|
|
112
|
+
try {
|
|
113
|
+
const raw = await readFile(stateFile, "utf-8");
|
|
114
|
+
return JSON.parse(raw);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return {};
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async function saveState(stateFile, state) {
|
|
121
|
+
await writeFile(stateFile, JSON.stringify(state, null, 2), "utf-8");
|
|
122
|
+
}
|
|
123
|
+
async function fireWebhook(webhook, payload) {
|
|
124
|
+
try {
|
|
125
|
+
const res = await fetch(webhook, {
|
|
126
|
+
method: "POST",
|
|
127
|
+
headers: { "content-type": "application/json" },
|
|
128
|
+
body: JSON.stringify(payload),
|
|
129
|
+
});
|
|
130
|
+
return { fired: true, status: res.status };
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
debugLog("watch", "webhook failed", err);
|
|
134
|
+
return { fired: false };
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
export async function runWatchOnce(input) {
|
|
138
|
+
const outDir = resolvePath(input.output_dir);
|
|
139
|
+
await mkdir(outDir, { recursive: true });
|
|
140
|
+
const stateFile = join(outDir, ".state.json");
|
|
141
|
+
const state = await loadState(stateFile);
|
|
142
|
+
const fetched = await fetchPage(input.url);
|
|
143
|
+
const signature = await computeSignature(fetched.html, input.url, input.hash_on);
|
|
144
|
+
const currentHash = hashString(signature);
|
|
145
|
+
const slug = slugify(input.url);
|
|
146
|
+
const snapshotFile = join(outDir, `${slug}.snapshot.txt`);
|
|
147
|
+
const prevSnapshotFile = join(outDir, `${slug}.previous.txt`);
|
|
148
|
+
const existing = state[input.url];
|
|
149
|
+
const firstRun = !existing;
|
|
150
|
+
const changed = !firstRun && existing.last_hash !== currentHash;
|
|
151
|
+
let previousSig = null;
|
|
152
|
+
if (existsSync(snapshotFile)) {
|
|
153
|
+
try {
|
|
154
|
+
previousSig = await readFile(snapshotFile, "utf-8");
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
previousSig = null;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
// Rotate previous snapshot only when content changed
|
|
161
|
+
if (changed && previousSig !== null) {
|
|
162
|
+
await writeFile(prevSnapshotFile, previousSig, "utf-8");
|
|
163
|
+
}
|
|
164
|
+
await writeFile(snapshotFile, signature, "utf-8");
|
|
165
|
+
const nowIso = new Date().toISOString();
|
|
166
|
+
const newState = {
|
|
167
|
+
url: input.url,
|
|
168
|
+
last_hash: currentHash,
|
|
169
|
+
last_checked: nowIso,
|
|
170
|
+
last_changed: changed ? nowIso : existing?.last_changed ?? null,
|
|
171
|
+
hash_on: input.hash_on,
|
|
172
|
+
check_count: (existing?.check_count ?? 0) + 1,
|
|
173
|
+
change_count: (existing?.change_count ?? 0) + (changed ? 1 : 0),
|
|
174
|
+
};
|
|
175
|
+
state[input.url] = newState;
|
|
176
|
+
await saveState(stateFile, state);
|
|
177
|
+
let diff = null;
|
|
178
|
+
if (changed && previousSig !== null) {
|
|
179
|
+
diff =
|
|
180
|
+
input.diff_format === "unified"
|
|
181
|
+
? simpleUnifiedDiff(previousSig, signature)
|
|
182
|
+
: jsonDiff(previousSig, signature);
|
|
183
|
+
}
|
|
184
|
+
let webhookFired = false;
|
|
185
|
+
let webhookStatus;
|
|
186
|
+
if (changed && input.webhook) {
|
|
187
|
+
const payload = {
|
|
188
|
+
event: "watch.change",
|
|
189
|
+
url: input.url,
|
|
190
|
+
previous_hash: existing?.last_hash ?? null,
|
|
191
|
+
current_hash: currentHash,
|
|
192
|
+
detected_at: nowIso,
|
|
193
|
+
diff,
|
|
194
|
+
};
|
|
195
|
+
const res = await fireWebhook(input.webhook, payload);
|
|
196
|
+
webhookFired = res.fired;
|
|
197
|
+
webhookStatus = res.status;
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
url: input.url,
|
|
201
|
+
changed,
|
|
202
|
+
first_run: firstRun,
|
|
203
|
+
previous_hash: existing?.last_hash ?? null,
|
|
204
|
+
current_hash: currentHash,
|
|
205
|
+
hash_on: input.hash_on,
|
|
206
|
+
snapshot_file: snapshotFile,
|
|
207
|
+
diff,
|
|
208
|
+
webhook_fired: webhookFired,
|
|
209
|
+
webhook_status: webhookStatus,
|
|
210
|
+
state: newState,
|
|
211
|
+
checked_at: nowIso,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
export async function execute(input) {
|
|
215
|
+
try {
|
|
216
|
+
const result = await runWatchOnce(input);
|
|
217
|
+
return toolResult(result);
|
|
218
|
+
}
|
|
219
|
+
catch (err) {
|
|
220
|
+
debugLog("watch", "failed", err);
|
|
221
|
+
return errorResult(err instanceof Error ? err.message : String(err));
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
//# sourceMappingURL=watch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watch.js","sourceRoot":"","sources":["../../src/tools/watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,MAAM,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC;AAE5B,MAAM,CAAC,MAAM,WAAW,GACtB,mKAAmK,CAAC;AAEtK,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC;SACH,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,CAAC,cAAc,CAAC;IAC3B,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,OAAO,CAAC,cAAc,CAAC;SACvB,QAAQ,CAAC,0CAA0C,CAAC;IACvD,OAAO,EAAE,CAAC;SACP,IAAI,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;SAC5C,OAAO,CAAC,aAAa,CAAC;SACtB,QAAQ,CAAC,gEAAgE,CAAC;IAC7E,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,4DAA4D,CAAC;IACzE,WAAW,EAAE,CAAC;SACX,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;SACzB,OAAO,CAAC,SAAS,CAAC;SAClB,QAAQ,CAAC,mCAAmC,CAAC;IAChD,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,mDAAmD,CAAC;CACjE,CAAC,CAAC;AA6BH,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,IAAY,EAAE,QAAQ,GAAG,GAAG;IACnE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAEnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACtB,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,QAAQ,OAAO,aAAa,KAAK,SAAS,CAAC;IACtG,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,MAAM,GAAG,QAAQ,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7F,OAAO,GAAG,MAAM,KAAK,IAAI,GAAG,SAAS,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAY;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;AACxG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAY,EACZ,GAAW,EACX,MAA6B;IAE7B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,MAAM,KAAK,UAAU;QAAE,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAEvD,cAAc;IACd,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,OAAO,EAAE,WAAW;YAAE,OAAO,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAC5D,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB;IACxC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA+B,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB,EAAE,KAAiC;IAC3E,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAAe,EACf,OAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB;IAClD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACjF,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;IAC1D,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,CAAC,QAAQ,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,WAAW,CAAC;IAEhE,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,OAAO,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,MAAM,SAAS,CAAC,gBAAgB,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,SAAS,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAElD,MAAM,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAe;QAC3B,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,SAAS,EAAE,WAAW;QACtB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,YAAY,IAAI,IAAI;QAC/D,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,WAAW,EAAE,CAAC,QAAQ,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;QAC7C,YAAY,EAAE,CAAC,QAAQ,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAChE,CAAC;IACF,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC;IAC5B,MAAM,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAElC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,IAAI,OAAO,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACpC,IAAI;YACF,KAAK,CAAC,WAAW,KAAK,SAAS;gBAC7B,CAAC,CAAC,iBAAiB,CAAC,WAAW,EAAE,SAAS,CAAC;gBAC3C,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,aAAiC,CAAC;IACtC,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,cAAc;YACrB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,aAAa,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;YAC1C,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,MAAM;YACnB,IAAI;SACL,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtD,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC;QACzB,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,OAAO;QACP,SAAS,EAAE,QAAQ;QACnB,aAAa,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QAC1C,YAAY,EAAE,WAAW;QACzB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,aAAa,EAAE,YAAY;QAC3B,IAAI;QACJ,aAAa,EAAE,YAAY;QAC3B,cAAc,EAAE,aAAa;QAC7B,KAAK,EAAE,QAAQ;QACf,UAAU,EAAE,MAAM;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAiB;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACjC,OAAO,WAAW,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
package/dist/tools/youtube.d.ts
CHANGED
|
@@ -9,8 +9,8 @@ export declare const schema: z.ZodObject<{
|
|
|
9
9
|
limit: z.ZodDefault<z.ZodNumber>;
|
|
10
10
|
sort: z.ZodDefault<z.ZodEnum<["relevance", "date", "views"]>>;
|
|
11
11
|
}, "strip", z.ZodTypeAny, {
|
|
12
|
-
action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
|
|
13
12
|
sort: "date" | "relevance" | "views";
|
|
13
|
+
action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
|
|
14
14
|
limit: number;
|
|
15
15
|
url?: string | undefined;
|
|
16
16
|
query?: string | undefined;
|
|
@@ -18,8 +18,8 @@ export declare const schema: z.ZodObject<{
|
|
|
18
18
|
}, {
|
|
19
19
|
action: "search" | "video" | "channel" | "comments" | "transcript" | "chapters";
|
|
20
20
|
url?: string | undefined;
|
|
21
|
-
sort?: "date" | "relevance" | "views" | undefined;
|
|
22
21
|
query?: string | undefined;
|
|
22
|
+
sort?: "date" | "relevance" | "views" | undefined;
|
|
23
23
|
channel_url?: string | undefined;
|
|
24
24
|
limit?: number | undefined;
|
|
25
25
|
}>;
|
package/dist/utils/fetcher.d.ts
CHANGED
|
@@ -20,10 +20,19 @@ declare const CIRCUIT_OPEN_DURATION_MS = 60000;
|
|
|
20
20
|
declare const CIRCUIT_PROBE_SUCCESSES = 3;
|
|
21
21
|
declare const circuits: Map<string, CircuitBreaker>;
|
|
22
22
|
declare const CIRCUIT_STALE_MS = 3600000;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Get circuit breaker key for a URL.
|
|
25
|
+
* Uses domain + first 2 path segments for endpoint-level granularity.
|
|
26
|
+
*/
|
|
27
|
+
declare function getCircuitKey(url: string): string;
|
|
28
|
+
declare function getCircuit(key: string): CircuitBreaker;
|
|
29
|
+
declare function recordSuccess(key: string): void;
|
|
30
|
+
declare function recordFailure(key: string): void;
|
|
31
|
+
/**
|
|
32
|
+
* Check domain-level circuit: opens when 3+ endpoint circuits are open for this domain.
|
|
33
|
+
*/
|
|
34
|
+
declare function isDomainCircuitOpen(domain: string): boolean;
|
|
35
|
+
export { circuits, getCircuit, getCircuitKey, recordSuccess, recordFailure, isDomainCircuitOpen, CIRCUIT_FAILURE_THRESHOLD, CIRCUIT_OPEN_DURATION_MS, CIRCUIT_PROBE_SUCCESSES, CIRCUIT_STALE_MS };
|
|
27
36
|
export interface SmartFetchOptions extends StealthOptions {
|
|
28
37
|
respectRobots?: boolean;
|
|
29
38
|
retries?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,
|
|
1
|
+
{"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../src/utils/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,KAAK,WAAW,EAAqB,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAQzH,qBAAa,kBAAkB;IAIjB,OAAO,CAAC,aAAa;IAHjC,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,KAAK,CAAyB;gBAElB,aAAa,GAAE,MAA4B;IAEzD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;CAa/C;AAED,eAAO,MAAM,cAAc,oBAA2B,CAAC;AAIvD,KAAK,YAAY,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAEpD,UAAU,cAAc;IACtB,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,QAAA,MAAM,yBAAyB,IAAI,CAAC;AACpC,QAAA,MAAM,wBAAwB,QAAS,CAAC;AACxC,QAAA,MAAM,uBAAuB,IAAI,CAAC;AAIlC,QAAA,MAAM,QAAQ,6BAAoC,CAAC;AAGnD,QAAA,MAAM,gBAAgB,UAAY,CAAC;AAUnC;;;GAGG;AACH,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAS1C;AAED,iBAAS,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAgB/C;AAED,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAYxC;AAED,iBAAS,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAcxC;AAED;;GAEG;AACH,iBAAS,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAQpD;AAGD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC;AAsDlM,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACvD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AA+BD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiG9F"}
|