auspex 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/actions.d.ts.map +1 -1
- package/dist/agent/actions.js +7 -1
- package/dist/agent/actions.js.map +1 -1
- package/dist/agent/agent.d.ts +20 -6
- package/dist/agent/agent.d.ts.map +1 -1
- package/dist/agent/agent.js +197 -74
- package/dist/agent/agent.js.map +1 -1
- package/dist/agent/logger.d.ts +15 -0
- package/dist/agent/logger.d.ts.map +1 -0
- package/dist/agent/logger.js +70 -0
- package/dist/agent/logger.js.map +1 -0
- package/dist/agent/loop.d.ts +18 -3
- package/dist/agent/loop.d.ts.map +1 -1
- package/dist/agent/loop.js +123 -48
- package/dist/agent/loop.js.map +1 -1
- package/dist/agent/report.d.ts.map +1 -1
- package/dist/agent/report.js +41 -29
- package/dist/agent/report.js.map +1 -1
- package/dist/browser/executor.d.ts.map +1 -1
- package/dist/browser/executor.js +65 -11
- package/dist/browser/executor.js.map +1 -1
- package/dist/browser/pool.d.ts +33 -0
- package/dist/browser/pool.d.ts.map +1 -0
- package/dist/browser/pool.js +101 -0
- package/dist/browser/pool.js.map +1 -0
- package/dist/browser/snapshot.d.ts +1 -0
- package/dist/browser/snapshot.d.ts.map +1 -1
- package/dist/browser/snapshot.js +104 -48
- package/dist/browser/snapshot.js.map +1 -1
- package/dist/config/defaults.d.ts +6 -0
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +7 -1
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/schema.d.ts +107 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +29 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/llm/client.d.ts +1 -1
- package/dist/llm/client.d.ts.map +1 -1
- package/dist/llm/client.js +74 -37
- package/dist/llm/client.js.map +1 -1
- package/dist/llm/prompt.d.ts +14 -2
- package/dist/llm/prompt.d.ts.map +1 -1
- package/dist/llm/prompt.js +52 -6
- package/dist/llm/prompt.js.map +1 -1
- package/dist/llm/vision-models.d.ts +3 -0
- package/dist/llm/vision-models.d.ts.map +1 -0
- package/dist/llm/vision-models.js +30 -0
- package/dist/llm/vision-models.js.map +1 -0
- package/dist/scraper/extractors/content.d.ts +3 -2
- package/dist/scraper/extractors/content.d.ts.map +1 -1
- package/dist/scraper/extractors/content.js +4 -4
- package/dist/scraper/extractors/content.js.map +1 -1
- package/dist/scraper/extractors/ssr.d.ts +3 -2
- package/dist/scraper/extractors/ssr.d.ts.map +1 -1
- package/dist/scraper/extractors/ssr.js +4 -4
- package/dist/scraper/extractors/ssr.js.map +1 -1
- package/dist/scraper/tiers/tier1-http.d.ts.map +1 -1
- package/dist/scraper/tiers/tier1-http.js +7 -13
- package/dist/scraper/tiers/tier1-http.js.map +1 -1
- package/dist/scraper/tiers/tier2-stealth.d.ts.map +1 -1
- package/dist/scraper/tiers/tier2-stealth.js +6 -5
- package/dist/scraper/tiers/tier2-stealth.js.map +1 -1
- package/dist/scraper/tiers/tier3-browser.d.ts +1 -0
- package/dist/scraper/tiers/tier3-browser.d.ts.map +1 -1
- package/dist/scraper/tiers/tier3-browser.js +31 -26
- package/dist/scraper/tiers/tier3-browser.js.map +1 -1
- package/dist/security/action-validator.d.ts +35 -20
- package/dist/security/action-validator.d.ts.map +1 -1
- package/dist/security/action-validator.js +39 -3
- package/dist/security/action-validator.js.map +1 -1
- package/dist/security/url-validator.d.ts.map +1 -1
- package/dist/security/url-validator.js +10 -1
- package/dist/security/url-validator.js.map +1 -1
- package/dist/types.d.ts +97 -25
- package/dist/types.d.ts.map +1 -1
- package/package.json +8 -4
- package/readme.md +169 -35
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { gotScraping } from "got-scraping";
|
|
2
|
+
import { load } from "cheerio";
|
|
2
3
|
import { extractSSRData, hasEnoughContent } from "../extractors/ssr.js";
|
|
3
4
|
import { extractContent } from "../extractors/content.js";
|
|
4
5
|
import { htmlToMarkdown } from "../extractors/to-markdown.js";
|
|
@@ -58,7 +59,6 @@ export class Tier1HTTP {
|
|
|
58
59
|
timeout: { request: options.timeout ?? 15_000 },
|
|
59
60
|
// Retorna o corpo como string (HTML)
|
|
60
61
|
responseType: "text",
|
|
61
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
62
62
|
}));
|
|
63
63
|
}
|
|
64
64
|
catch (err) {
|
|
@@ -79,24 +79,18 @@ export class Tier1HTTP {
|
|
|
79
79
|
if (!contentType.includes("text/html") && !contentType.includes("text/plain")) {
|
|
80
80
|
throw new Error(`Tier1 HTTP: Content-Type "${contentType}" inesperado — esperava text/html`);
|
|
81
81
|
}
|
|
82
|
-
// got retorna o body como string quando responseType: 'text'
|
|
83
82
|
const html = response.body;
|
|
84
|
-
// response.url é a URL final após redirecionamentos
|
|
85
83
|
const finalUrl = response.url;
|
|
86
|
-
//
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
//
|
|
91
|
-
// Detecta: página vazia de SPA, Cloudflare challenge, "enable JavaScript", etc.
|
|
84
|
+
// Single Cheerio parse shared across SSR detection, content check, and extraction
|
|
85
|
+
const $ = load(html);
|
|
86
|
+
const ssrData = extractSSRData(html, $);
|
|
87
|
+
// NOTE: Do NOT pass shared $ to hasEnoughContent — it destructively removes
|
|
88
|
+
// <img>, <svg>, <iframe> etc. which would corrupt $ for extractContent below.
|
|
92
89
|
if (!hasEnoughContent(html) && !ssrData) {
|
|
93
90
|
throw new Error("Tier1 HTTP: conteúdo insuficiente — provavelmente SPA sem SSR ou anti-bot");
|
|
94
91
|
}
|
|
95
|
-
// ── Extrair conteúdo principal ──────────────────────────────────────
|
|
96
|
-
// 1. Mozilla Readability (mesmo algoritmo do Firefox Reader Mode)
|
|
97
|
-
// 2. Cheerio + heurísticas (fallback quando Readability falha)
|
|
98
92
|
const formats = options.formats ?? ["markdown", "text"];
|
|
99
|
-
const extracted = extractContent(html, options.onlyMainContent ?? true, finalUrl);
|
|
93
|
+
const extracted = extractContent(html, options.onlyMainContent ?? true, finalUrl, $);
|
|
100
94
|
const result = {
|
|
101
95
|
url: finalUrl,
|
|
102
96
|
statusCode: response.statusCode,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tier1-http.js","sourceRoot":"","sources":["../../../src/scraper/tiers/tier1-http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"tier1-http.js","sourceRoot":"","sources":["../../../src/scraper/tiers/tier1-http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG/B,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAS9D,8EAA8E;AAC9E,EAAE;AACF,YAAY;AACZ,0EAA0E;AAC1E,kEAAkE;AAClE,8CAA8C;AAC9C,0DAA0D;AAC1D,EAAE;AACF,iDAAiD;AACjD,gFAAgF;AAChF,qEAAqE;AACrE,iFAAiF;AACjF,2DAA2D;AAC3D,EAAE;AACF,cAAc;AACd,uDAAuD;AACvD,4EAA4E;AAC5E,6EAA6E;AAE7E,iEAAiE;AACjE,6DAA6D;AAC7D,mEAAmE;AACnE,gEAAgE;AAChE,EAAE;AACF,sEAAsE;AACtE,0CAA0C;AAC1C,sDAAsD;AACtD,qEAAqE;AACrE,gCAAgC;AAChC,MAAM,aAAa,GAA2B;IAC5C,iBAAiB,EAAE,qCAAqC;IACxD,eAAe,EAAE,UAAU;IAC3B,QAAQ,EAAE,UAAU;CACrB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEhD,MAAM,OAAO,SAAS;IACpB,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAyB,EAAE;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,uEAAuE;QACvE,4EAA4E;QAC5E,yEAAyE;QACzE,sEAAsE;QACtE,2DAA2D;QAC3D,IAAI,QAAqB,CAAC;QAE1B,IAAI,CAAC;YACH,QAAQ,GAAG,CAAC,MAAM,WAAW,CAAC;gBAC5B,GAAG;gBACH,qEAAqE;gBACrE,iEAAiE;gBACjE,OAAO,EAAE,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE;gBAEjD,6DAA6D;gBAC7D,eAAe,EAAE,KAAK;gBAEtB,kFAAkF;gBAClF,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,MAAM,EAAE;gBAE/C,qCAAqC;gBACrC,YAAY,EAAE,MAAM;aACrB,CAAC,CAAgB,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,uDAAuD;YACvD,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,uEAAuE;QACvE,IAAI,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACb,sBAAsB,QAAQ,CAAC,UAAU,2BAA2B,CACrE,CAAC;QACJ,CAAC;QAED,IAAI,QAAQ,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,wEAAwE;QACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAC/C,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAE5E,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9E,MAAM,IAAI,KAAK,CACb,6BAA6B,WAAW,mCAAmC,CAC5E,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAc,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;QAE9B,kFAAkF;QAClF,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAErB,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExC,4EAA4E;QAC5E,8EAA8E;QAC9E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,cAAc,CAC9B,IAAI,EACJ,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,QAAQ,EACR,CAAC,CACF,CAAC;QAEF,MAAM,MAAM,GAAiB;YAC3B,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,SAAS;YAC/C,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SAChE,CAAC;QAEF,IAAI,OAAO,CAAC,UAAU;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QAE9C,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnF,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAM,MAAM,CAAC,IAAI,GAAO,SAAS,CAAC,IAAI,CAAC;QACnE,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAM,MAAM,CAAC,IAAI,GAAO,SAAS,CAAC,IAAI,CAAC;QACnE,IAAI,OAAO;YAAuB,MAAM,CAAC,OAAO,GAAI,OAAO,CAAC;QAE5D,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tier2-stealth.d.ts","sourceRoot":"","sources":["../../../src/scraper/tiers/tier2-stealth.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tier2-stealth.d.ts","sourceRoot":"","sources":["../../../src/scraper/tiers/tier2-stealth.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA+B/D,qBAAa,YAAY;IACjB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC;CA0G9E"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { gotScraping } from "got-scraping";
|
|
2
|
+
import { load } from "cheerio";
|
|
2
3
|
import { extractSSRData, hasEnoughContent } from "../extractors/ssr.js";
|
|
3
4
|
import { extractContent } from "../extractors/content.js";
|
|
4
5
|
import { htmlToMarkdown } from "../extractors/to-markdown.js";
|
|
@@ -62,6 +63,7 @@ export class Tier2Stealth {
|
|
|
62
63
|
const html = response.body;
|
|
63
64
|
const statusCode = response.statusCode;
|
|
64
65
|
const finalUrl = response.url ?? url;
|
|
66
|
+
const $ = load(html);
|
|
65
67
|
// ── Verificações de bloqueio ────────────────────────────────────────
|
|
66
68
|
if (ANTIBOT_STATUS.has(statusCode)) {
|
|
67
69
|
throw new Error(`Tier2 Stealth: status ${statusCode} — bloqueado por anti-bot`);
|
|
@@ -74,15 +76,14 @@ export class Tier2Stealth {
|
|
|
74
76
|
!String(contentType).includes("text/plain")) {
|
|
75
77
|
throw new Error(`Tier2 Stealth: Content-Type inesperado "${contentType}" — esperava text/html`);
|
|
76
78
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
//
|
|
79
|
+
const ssrData = extractSSRData(html, $);
|
|
80
|
+
// NOTE: Do NOT pass shared $ to hasEnoughContent — it destructively removes
|
|
81
|
+
// <img>, <svg>, <iframe> etc. which would corrupt $ for extractContent below.
|
|
80
82
|
if (!hasEnoughContent(html) && !ssrData) {
|
|
81
83
|
throw new Error("Tier2 Stealth: conteúdo insuficiente — página precisa de JavaScript para renderizar");
|
|
82
84
|
}
|
|
83
|
-
// ── Extrair conteúdo ────────────────────────────────────────────────
|
|
84
85
|
const formats = options.formats ?? ["markdown", "text"];
|
|
85
|
-
const extracted = extractContent(html, options.onlyMainContent ?? true, finalUrl);
|
|
86
|
+
const extracted = extractContent(html, options.onlyMainContent ?? true, finalUrl, $);
|
|
86
87
|
const result = {
|
|
87
88
|
url: finalUrl,
|
|
88
89
|
statusCode,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tier2-stealth.js","sourceRoot":"","sources":["../../../src/scraper/tiers/tier2-stealth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"tier2-stealth.js","sourceRoot":"","sources":["../../../src/scraper/tiers/tier2-stealth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAE/B,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAS9D,8EAA8E;AAC9E,EAAE;AACF,YAAY;AACZ,yEAAyE;AACzE,kFAAkF;AAClF,kEAAkE;AAClE,8CAA8C;AAC9C,2DAA2D;AAC3D,EAAE;AACF,4EAA4E;AAC5E,yDAAyD;AACzD,EAAE;AACF,wEAAwE;AACxE,uDAAuD;AACvD,6EAA6E;AAE7E,iDAAiD;AACjD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEhD,MAAM,OAAO,YAAY;IACvB,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAyB,EAAE;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,uEAAuE;QACvE,IAAI,QAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,QAAQ,GAAG,CAAC,MAAM,WAAW,CAAC;gBAC5B,GAAG;gBACH,MAAM,EAAE,KAAK;gBACb,mDAAmD;gBACnD,sBAAsB,EAAE;oBACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;oBAC/C,gBAAgB,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;oBACtC,OAAO,EAAE,CAAC,SAAS,CAAC;oBACpB,OAAO,EAAE,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC;iBAClC;gBACD,0CAA0C;gBAC1C,OAAO,EAAE;oBACP,MAAM,EACJ,kGAAkG;oBACpG,iBAAiB,EAAE,qCAAqC;oBACxD,eAAe,EAAE,UAAU;oBAC3B,MAAM,EAAE,UAAU;oBAClB,gBAAgB,EAAE,UAAU;oBAC5B,gBAAgB,EAAE,UAAU;oBAC5B,gBAAgB,EAAE,MAAM;oBACxB,gBAAgB,EAAE,IAAI;oBACtB,2BAA2B,EAAE,GAAG;oBAChC,GAAG,OAAO,CAAC,OAAO;iBACnB;gBACD,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,MAAM,EAAE;gBAC/C,cAAc,EAAE,IAAI;gBACpB,YAAY,EAAE,EAAE;gBAChB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE;gBACrC,eAAe,EAAE,KAAK;gBACtB,UAAU,EAAE,IAAI;aACjB,CAAC,CAAgB,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC3F,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAc,CAAC;QACrC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;QAErB,uEAAuE;QACvE,IAAI,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,yBAAyB,UAAU,2BAA2B,CAC/D,CAAC;QACJ,CAAC;QAED,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,UAAU,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC3D,IACE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC1C,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAC3C,CAAC;YACD,MAAM,IAAI,KAAK,CACb,2CAA2C,WAAW,wBAAwB,CAC/E,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAExC,4EAA4E;QAC5E,8EAA8E;QAC9E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CACb,qFAAqF,CACtF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,cAAc,CAC9B,IAAI,EACJ,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,QAAQ,EACR,CAAC,CACF,CAAC;QAEF,MAAM,MAAM,GAAiB;YAC3B,GAAG,EAAE,QAAQ;YACb,UAAU;YACV,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,SAAS;YAC/C,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SAChE,CAAC;QAEF,IAAI,OAAO,CAAC,UAAU;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;QAE9C,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnF,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAM,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;QAC/D,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAM,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;QAC/D,IAAI,OAAO;YAAuB,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAE3D,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ScrapeOptions, ScrapeResult, ScraperConfig } from "../types.js";
|
|
2
2
|
export declare class Tier3Browser {
|
|
3
3
|
private browser;
|
|
4
|
+
private browserPromise;
|
|
4
5
|
private readonly browserConfig;
|
|
5
6
|
constructor(browserConfig?: ScraperConfig["browserConfig"]);
|
|
6
7
|
private getBrowser;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tier3-browser.d.ts","sourceRoot":"","sources":["../../../src/scraper/tiers/tier3-browser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EAEZ,aAAa,EACd,MAAM,aAAa,CAAC;AA4RrB,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA8C;gBAEhE,aAAa,GAAE,aAAa,CAAC,eAAe,CAAM;YAMhD,UAAU;
|
|
1
|
+
{"version":3,"file":"tier3-browser.d.ts","sourceRoot":"","sources":["../../../src/scraper/tiers/tier3-browser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EAEZ,aAAa,EACd,MAAM,aAAa,CAAC;AA4RrB,qBAAa,YAAY;IACvB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,cAAc,CAAiC;IACvD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA8C;gBAEhE,aAAa,GAAE,aAAa,CAAC,eAAe,CAAM;YAMhD,UAAU;IAqClB,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,CAAC;IAoNvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}
|
|
@@ -276,40 +276,45 @@ const BLOCKED_URL_PATTERNS = [
|
|
|
276
276
|
];
|
|
277
277
|
export class Tier3Browser {
|
|
278
278
|
browser = null;
|
|
279
|
+
browserPromise = null;
|
|
279
280
|
browserConfig;
|
|
280
281
|
constructor(browserConfig = {}) {
|
|
281
282
|
this.browserConfig = browserConfig;
|
|
282
283
|
}
|
|
283
|
-
// ── Lifecycle do browser (singleton
|
|
284
|
+
// ── Lifecycle do browser (singleton with mutex) ────────────────────────
|
|
284
285
|
async getBrowser() {
|
|
285
286
|
if (this.browser?.isConnected())
|
|
286
287
|
return this.browser;
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
288
|
+
if (!this.browserPromise) {
|
|
289
|
+
this.browserPromise = (async () => {
|
|
290
|
+
const launchOptions = {
|
|
291
|
+
headless: this.browserConfig.headless ?? true,
|
|
292
|
+
args: STEALTH_ARGS,
|
|
293
|
+
};
|
|
294
|
+
if (this.browserConfig.executablePath) {
|
|
295
|
+
launchOptions.executablePath = this.browserConfig.executablePath;
|
|
296
|
+
}
|
|
297
|
+
else if (this.browserConfig.channel) {
|
|
298
|
+
launchOptions.channel = this.browserConfig.channel;
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
try {
|
|
302
|
+
const browser = await chromium.launch({ ...launchOptions, channel: "chrome" });
|
|
303
|
+
this.browser = browser;
|
|
304
|
+
this.browserPromise = null;
|
|
305
|
+
return browser;
|
|
306
|
+
}
|
|
307
|
+
catch {
|
|
308
|
+
// Chrome not found → use Playwright's bundled Chromium
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
const browser = await chromium.launch(launchOptions);
|
|
312
|
+
this.browser = browser;
|
|
313
|
+
this.browserPromise = null;
|
|
314
|
+
return browser;
|
|
315
|
+
})();
|
|
310
316
|
}
|
|
311
|
-
this.
|
|
312
|
-
return this.browser;
|
|
317
|
+
return this.browserPromise;
|
|
313
318
|
}
|
|
314
319
|
// ── Scraping principal ─────────────────────────────────────────────────
|
|
315
320
|
async scrape(url, options = {}) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tier3-browser.js","sourceRoot":"","sources":["../../../src/scraper/tiers/tier3-browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAqC,MAAM,YAAY,CAAC;AAOzE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,8EAA8E;AAC9E,EAAE;AACF,gEAAgE;AAChE,6EAA6E;AAC7E,EAAE;AACF,yBAAyB;AACzB,oEAAoE;AACpE,2EAA2E;AAC3E,kEAAkE;AAClE,kDAAkD;AAClD,sDAAsD;AACtD,6EAA6E;AAE7E,2EAA2E;AAC3E,gDAAgD;AAChD,MAAM,SAAS,GACb,iHAAiH,CAAC;AAEpH,mDAAmD;AACnD,MAAM,YAAY,GAAG;IACnB,+CAA+C;IAC/C,oDAAoD;IACpD,oBAAoB;IACpB,gBAAgB;IAChB,cAAc;IACd,0BAA0B;IAC1B,yBAAyB;IACzB,iCAAiC;IACjC,aAAa;IACb,eAAe;IACf,yBAAyB;IACzB,iCAAiC;IACjC,0CAA0C;IAC1C,4BAA4B;IAC5B,wBAAwB;IACxB,8BAA8B;IAC9B,sBAAsB;IACtB,wBAAwB;IACxB,0BAA0B;IAC1B,4BAA4B;IAC5B,gBAAgB;IAChB,0BAA0B;IAC1B,oCAAoC;CACrC,CAAC;AAEF,iFAAiF;AACjF,EAAE;AACF,6DAA6D;AAC7D,8DAA8D;AAC9D,EAAE;AACF,8DAA8D;AAC9D,yEAAyE;AACzE,2FAA2F;AAC3F,oFAAoF;AACpF,2EAA2E;AAC3E,iEAAiE;AACjE,yFAAyF;AACzF,2FAA2F;AAC3F,6CAA6C;AAC7C,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiMrD,CAAC;AAEF,uDAAuD;AACvD,4EAA4E;AAC5E,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAEnE,+CAA+C;AAC/C,MAAM,oBAAoB,GAAG;IAC3B,sBAAsB;IACtB,sBAAsB;IACtB,gCAAgC;IAChC,sBAAsB;IACtB,YAAY;IACZ,eAAe;IACf,aAAa;IACb,cAAc;IACd,eAAe;IACf,WAAW;IACX,YAAY;IACZ,iBAAiB;IACjB,WAAW;IACX,YAAY;IACZ,aAAa;IACb,cAAc;CACf,CAAC;AAEF,MAAM,OAAO,YAAY;IACf,OAAO,GAAmB,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"tier3-browser.js","sourceRoot":"","sources":["../../../src/scraper/tiers/tier3-browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAqC,MAAM,YAAY,CAAC;AAOzE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,8EAA8E;AAC9E,EAAE;AACF,gEAAgE;AAChE,6EAA6E;AAC7E,EAAE;AACF,yBAAyB;AACzB,oEAAoE;AACpE,2EAA2E;AAC3E,kEAAkE;AAClE,kDAAkD;AAClD,sDAAsD;AACtD,6EAA6E;AAE7E,2EAA2E;AAC3E,gDAAgD;AAChD,MAAM,SAAS,GACb,iHAAiH,CAAC;AAEpH,mDAAmD;AACnD,MAAM,YAAY,GAAG;IACnB,+CAA+C;IAC/C,oDAAoD;IACpD,oBAAoB;IACpB,gBAAgB;IAChB,cAAc;IACd,0BAA0B;IAC1B,yBAAyB;IACzB,iCAAiC;IACjC,aAAa;IACb,eAAe;IACf,yBAAyB;IACzB,iCAAiC;IACjC,0CAA0C;IAC1C,4BAA4B;IAC5B,wBAAwB;IACxB,8BAA8B;IAC9B,sBAAsB;IACtB,wBAAwB;IACxB,0BAA0B;IAC1B,4BAA4B;IAC5B,gBAAgB;IAChB,0BAA0B;IAC1B,oCAAoC;CACrC,CAAC;AAEF,iFAAiF;AACjF,EAAE;AACF,6DAA6D;AAC7D,8DAA8D;AAC9D,EAAE;AACF,8DAA8D;AAC9D,yEAAyE;AACzE,2FAA2F;AAC3F,oFAAoF;AACpF,2EAA2E;AAC3E,iEAAiE;AACjE,yFAAyF;AACzF,2FAA2F;AAC3F,6CAA6C;AAC7C,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiMrD,CAAC;AAEF,uDAAuD;AACvD,4EAA4E;AAC5E,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAEnE,+CAA+C;AAC/C,MAAM,oBAAoB,GAAG;IAC3B,sBAAsB;IACtB,sBAAsB;IACtB,gCAAgC;IAChC,sBAAsB;IACtB,YAAY;IACZ,eAAe;IACf,aAAa;IACb,cAAc;IACd,eAAe;IACf,WAAW;IACX,YAAY;IACZ,iBAAiB;IACjB,WAAW;IACX,YAAY;IACZ,aAAa;IACb,cAAc;CACf,CAAC;AAEF,MAAM,OAAO,YAAY;IACf,OAAO,GAAmB,IAAI,CAAC;IAC/B,cAAc,GAA4B,IAAI,CAAC;IACtC,aAAa,CAA8C;IAE5E,YAAY,gBAAgD,EAAE;QAC5D,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,0EAA0E;IAElE,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QAErD,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,CAAC,KAAK,IAAI,EAAE;gBAChC,MAAM,aAAa,GAA0C;oBAC3D,QAAQ,EAAE,IAAI,CAAC,aAAa,CAAC,QAAQ,IAAI,IAAI;oBAC7C,IAAI,EAAE,YAAY;iBACnB,CAAC;gBAEF,IAAI,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC;oBACtC,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC;gBACnE,CAAC;qBAAM,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;oBACtC,aAAa,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;gBACrD,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;wBAC/E,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;wBACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;wBAC3B,OAAO,OAAO,CAAC;oBACjB,CAAC;oBAAC,MAAM,CAAC;wBACP,uDAAuD;oBACzD,CAAC;gBACH,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBACrD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;gBACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QAED,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC;IAED,0EAA0E;IAE1E,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAyB,EAAE;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAExC,IAAI,OAAO,GAA0B,IAAI,CAAC;QAE1C,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;gBACjC,SAAS,EAAE,SAAS;gBACpB,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;gBACvC,MAAM,EAAE,OAAO;gBACf,UAAU,EAAE,mBAAmB;gBAC/B,gBAAgB,EAAE;oBAChB,iBAAiB,EAAE,qCAAqC;oBACxD,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;iBAC3B;gBACD,iBAAiB,EAAE,IAAI;gBACvB,4EAA4E;gBAC5E,6DAA6D;aAC9D,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAErC,qEAAqE;YACrE,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;YAE9C,qEAAqE;YACrE,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC;gBAEzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACrC,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;gBACvB,CAAC;gBACD,IACE,IAAI,KAAK,QAAQ;oBACjB,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EACpD,CAAC;oBACD,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,qEAAqE;YACrE,MAAM,eAAe,GAAqB,EAAE,CAAC;YAC7C,MAAM,eAAe,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC;YAExD,IAAI,eAAe,EAAE,CAAC;gBACpB,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;oBACrC,IAAI,CAAC;wBACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;wBAC7D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC;4BAAE,OAAO;wBAEtD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC;wBAC9B,qCAAqC;wBACrC,IAAI,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;4BAAE,OAAO;wBACjE,IAAI,iCAAiC,CAAC,IAAI,CAAC,MAAM,CAAC;4BAAE,OAAO;wBAE3D,uEAAuE;wBACvE,MAAM,aAAa,GAAG,QAAQ,CAC5B,QAAQ,CAAC,OAAO,EAAE,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAC3C,EAAE,CACH,CAAC;wBACF,IAAI,aAAa,GAAG,OAAO;4BAAE,OAAO;wBAEpC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;wBACrD,IAAI,CAAC,IAAI;4BAAE,OAAO;wBAElB,eAAe,CAAC,IAAI,CAAC;4BACnB,GAAG,EAAE,MAAM;4BACX,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE;4BACnC,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE;4BAC7B,WAAW;4BACX,IAAI;yBACL,CAAC,CAAC;oBACL,CAAC;oBAAC,MAAM,CAAC;wBACP,mEAAmE;oBACrE,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YAED,0EAA0E;YAC1E,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;YAEhE,sEAAsE;YACtE,6EAA6E;YAC7E,sEAAsE;YACtE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC;YAC1C,IAAI,UAAU,GAAG,GAAG,CAAC;YACrB,IAAI,YAAY,GAAiB,IAAI,CAAC;YAEtC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;gBAC9C,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;wBACvC,SAAS,EAAE,kBAAkB;wBAC7B,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC;qBACnC,CAAC,CAAC;oBACH,UAAU,GAAG,WAAW,EAAE,MAAM,EAAE,IAAI,GAAG,CAAC;oBAC1C,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,CAAC,wBAAwB;gBACjC,CAAC;gBAAC,OAAO,MAAM,EAAE,CAAC;oBAChB,YAAY,GAAG,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;oBAC5E,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;wBAChB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBACnD,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,uCAAuC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;YACjF,CAAC;YAED,qEAAqE;YACrE,sDAAsD;YACtD,MAAM,IAAI;iBACP,gBAAgB,CAAC,aAAa,EAAE;gBAC/B,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,EAAE,MAAM,CAAC;aACzC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE;gBACV,qDAAqD;YACvD,CAAC,CAAC,CAAC;YAEL,8DAA8D;YAC9D,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,MAAM,IAAI;qBACP,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE;oBACxC,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,MAAM;iBAChB,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE;oBACV,kDAAkD;gBACpD,CAAC,CAAC,CAAC;YACP,CAAC;YAED,qEAAqE;YACrE,uEAAuE;YACvE,2EAA2E;YAC3E,kEAAkE;YAClE,MAAM,IAAI;iBACP,QAAQ,CAAC,GAAG,EAAE;gBACb,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC;oBAC/C,IAAI,WAAW,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;wBACtC,OAAO,EAAE,CAAC;wBACV,OAAO;oBACT,CAAC;oBAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBACxD,IAAI,QAAQ,GAAG,CAAC,CAAC;oBAEjB,MAAM,IAAI,GAAG,GAAG,EAAE;wBAChB,QAAQ,IAAI,IAAI,CAAC;wBACjB,MAAM,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;wBACvD,IAAI,QAAQ,GAAG,WAAW,EAAE,CAAC;4BAC3B,8DAA8D;4BAC9D,uDAAuD;4BACvD,UAAU,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;wBAC1D,CAAC;6BAAM,CAAC;4BACN,MAAM,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;4BACjD,OAAO,EAAE,CAAC;wBACZ,CAAC;oBACH,CAAC,CAAC;oBAEF,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACxB,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE;gBACV,2DAA2D;YAC7D,CAAC,CAAC,CAAC;YAEL,qEAAqE;YACrE,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAC1C,IAAI,CAAC,OAAO,EAAE;gBACd,IAAI,CAAC,KAAK,EAAE;aACb,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACxD,MAAM,SAAS,GAAG,cAAc,CAC9B,IAAI,EACJ,OAAO,CAAC,eAAe,IAAI,IAAI,EAC/B,QAAQ,CACT,CAAC;YAEF,MAAM,MAAM,GAAiB;gBAC3B,GAAG,EAAE,QAAQ;gBACb,UAAU;gBACV,KAAK,EAAE,SAAS,IAAI,SAAS,CAAC,KAAK;gBACnC,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,SAAS;gBAC/C,IAAI,EAAE,SAAS;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBAClC,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;gBAC/D,eAAe,EACb,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;aAC3D,CAAC;YAEF,IAAI,OAAO,CAAC,UAAU;gBAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YAE9C,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,MAAM,CAAC,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACnF,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAM,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;YAC/D,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAM,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;YAE/D,OAAO,MAAM,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,0EAA0E;IAE1E,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import type { AgentAction } from "../types.js";
|
|
2
3
|
export declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
3
4
|
type: z.ZodLiteral<"click">;
|
|
4
5
|
selector: z.ZodEffects<z.ZodString, string, string>;
|
|
@@ -20,6 +21,36 @@ export declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
|
|
|
20
21
|
type: "type";
|
|
21
22
|
text: string;
|
|
22
23
|
selector: string;
|
|
24
|
+
}>, z.ZodObject<{
|
|
25
|
+
type: z.ZodLiteral<"select">;
|
|
26
|
+
selector: z.ZodEffects<z.ZodString, string, string>;
|
|
27
|
+
value: z.ZodString;
|
|
28
|
+
}, "strip", z.ZodTypeAny, {
|
|
29
|
+
type: "select";
|
|
30
|
+
value: string;
|
|
31
|
+
selector: string;
|
|
32
|
+
}, {
|
|
33
|
+
type: "select";
|
|
34
|
+
value: string;
|
|
35
|
+
selector: string;
|
|
36
|
+
}>, z.ZodObject<{
|
|
37
|
+
type: z.ZodLiteral<"pressKey">;
|
|
38
|
+
key: z.ZodEnum<["Enter", "Tab", "Escape", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "Space", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12"]>;
|
|
39
|
+
}, "strip", z.ZodTypeAny, {
|
|
40
|
+
type: "pressKey";
|
|
41
|
+
key: "Enter" | "Tab" | "Escape" | "Backspace" | "Delete" | "ArrowUp" | "ArrowDown" | "ArrowLeft" | "ArrowRight" | "Home" | "End" | "PageUp" | "PageDown" | "Space" | "F1" | "F2" | "F3" | "F4" | "F5" | "F6" | "F7" | "F8" | "F9" | "F10" | "F11" | "F12";
|
|
42
|
+
}, {
|
|
43
|
+
type: "pressKey";
|
|
44
|
+
key: "Enter" | "Tab" | "Escape" | "Backspace" | "Delete" | "ArrowUp" | "ArrowDown" | "ArrowLeft" | "ArrowRight" | "Home" | "End" | "PageUp" | "PageDown" | "Space" | "F1" | "F2" | "F3" | "F4" | "F5" | "F6" | "F7" | "F8" | "F9" | "F10" | "F11" | "F12";
|
|
45
|
+
}>, z.ZodObject<{
|
|
46
|
+
type: z.ZodLiteral<"hover">;
|
|
47
|
+
selector: z.ZodEffects<z.ZodString, string, string>;
|
|
48
|
+
}, "strip", z.ZodTypeAny, {
|
|
49
|
+
type: "hover";
|
|
50
|
+
selector: string;
|
|
51
|
+
}, {
|
|
52
|
+
type: "hover";
|
|
53
|
+
selector: string;
|
|
23
54
|
}>, z.ZodObject<{
|
|
24
55
|
type: z.ZodLiteral<"goto">;
|
|
25
56
|
url: z.ZodString;
|
|
@@ -41,12 +72,15 @@ export declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
|
|
|
41
72
|
}>, z.ZodObject<{
|
|
42
73
|
type: z.ZodLiteral<"scroll">;
|
|
43
74
|
direction: z.ZodEnum<["up", "down"]>;
|
|
75
|
+
amount: z.ZodOptional<z.ZodNumber>;
|
|
44
76
|
}, "strip", z.ZodTypeAny, {
|
|
45
77
|
type: "scroll";
|
|
46
78
|
direction: "up" | "down";
|
|
79
|
+
amount?: number | undefined;
|
|
47
80
|
}, {
|
|
48
81
|
type: "scroll";
|
|
49
82
|
direction: "up" | "down";
|
|
83
|
+
amount?: number | undefined;
|
|
50
84
|
}>, z.ZodObject<{
|
|
51
85
|
type: z.ZodLiteral<"done">;
|
|
52
86
|
result: z.ZodString;
|
|
@@ -60,24 +94,5 @@ export declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
|
|
|
60
94
|
export declare class ActionValidationError extends Error {
|
|
61
95
|
constructor(message: string);
|
|
62
96
|
}
|
|
63
|
-
export declare function validateAction(raw: unknown):
|
|
64
|
-
type: "click";
|
|
65
|
-
selector: string;
|
|
66
|
-
} | {
|
|
67
|
-
type: "type";
|
|
68
|
-
text: string;
|
|
69
|
-
selector: string;
|
|
70
|
-
} | {
|
|
71
|
-
type: "goto";
|
|
72
|
-
url: string;
|
|
73
|
-
} | {
|
|
74
|
-
type: "wait";
|
|
75
|
-
ms: number;
|
|
76
|
-
} | {
|
|
77
|
-
type: "scroll";
|
|
78
|
-
direction: "up" | "down";
|
|
79
|
-
} | {
|
|
80
|
-
type: "done";
|
|
81
|
-
result: string;
|
|
82
|
-
};
|
|
97
|
+
export declare function validateAction(raw: unknown): AgentAction;
|
|
83
98
|
//# sourceMappingURL=action-validator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action-validator.d.ts","sourceRoot":"","sources":["../../src/security/action-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"action-validator.d.ts","sourceRoot":"","sources":["../../src/security/action-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AA4B/C,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAkCvB,CAAC;AAEH,qBAAa,qBAAsB,SAAQ,KAAK;gBAClC,OAAO,EAAE,MAAM;CAI5B;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,WAAW,CAQxD"}
|
|
@@ -1,7 +1,26 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { DEFAULTS } from "../config/defaults.js";
|
|
3
|
-
const SELECTOR_BLACKLIST = [/javascript:/i, /on\w+\s*=/i, /<script/i];
|
|
4
|
-
const
|
|
3
|
+
const SELECTOR_BLACKLIST = [/javascript:/i, /on\w+\s*=/i, /<script/i, /data:/i];
|
|
4
|
+
const ROLE_SELECTOR_RE = /^role=\w+(\[name=".*"\])?$/;
|
|
5
|
+
const selectorSchema = z
|
|
6
|
+
.string()
|
|
7
|
+
.trim()
|
|
8
|
+
.min(1, "Selector must not be empty or whitespace-only")
|
|
9
|
+
.max(DEFAULTS.maxSelectorLength, `Selector exceeds max length of ${DEFAULTS.maxSelectorLength}`)
|
|
10
|
+
.refine((s) => {
|
|
11
|
+
// Role-based locators (from a11y tree) are always safe
|
|
12
|
+
if (ROLE_SELECTOR_RE.test(s))
|
|
13
|
+
return true;
|
|
14
|
+
return !SELECTOR_BLACKLIST.some((pattern) => pattern.test(s));
|
|
15
|
+
}, "Selector contains forbidden pattern");
|
|
16
|
+
// Allowed keyboard keys for pressKey action
|
|
17
|
+
const ALLOWED_KEYS = [
|
|
18
|
+
"Enter", "Tab", "Escape", "Backspace", "Delete",
|
|
19
|
+
"ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight",
|
|
20
|
+
"Home", "End", "PageUp", "PageDown",
|
|
21
|
+
"Space", "F1", "F2", "F3", "F4", "F5", "F6",
|
|
22
|
+
"F7", "F8", "F9", "F10", "F11", "F12",
|
|
23
|
+
];
|
|
5
24
|
export const actionSchema = z.discriminatedUnion("type", [
|
|
6
25
|
z.object({ type: z.literal("click"), selector: selectorSchema }),
|
|
7
26
|
z.object({
|
|
@@ -9,6 +28,19 @@ export const actionSchema = z.discriminatedUnion("type", [
|
|
|
9
28
|
selector: selectorSchema,
|
|
10
29
|
text: z.string().max(DEFAULTS.maxTypeLength),
|
|
11
30
|
}),
|
|
31
|
+
z.object({
|
|
32
|
+
type: z.literal("select"),
|
|
33
|
+
selector: selectorSchema,
|
|
34
|
+
value: z.string().max(500),
|
|
35
|
+
}),
|
|
36
|
+
z.object({
|
|
37
|
+
type: z.literal("pressKey"),
|
|
38
|
+
key: z.enum(ALLOWED_KEYS),
|
|
39
|
+
}),
|
|
40
|
+
z.object({
|
|
41
|
+
type: z.literal("hover"),
|
|
42
|
+
selector: selectorSchema,
|
|
43
|
+
}),
|
|
12
44
|
z.object({ type: z.literal("goto"), url: z.string().url() }),
|
|
13
45
|
z.object({
|
|
14
46
|
type: z.literal("wait"),
|
|
@@ -17,8 +49,12 @@ export const actionSchema = z.discriminatedUnion("type", [
|
|
|
17
49
|
z.object({
|
|
18
50
|
type: z.literal("scroll"),
|
|
19
51
|
direction: z.enum(["up", "down"]),
|
|
52
|
+
amount: z.number().int().positive().max(5000).optional(),
|
|
53
|
+
}),
|
|
54
|
+
z.object({
|
|
55
|
+
type: z.literal("done"),
|
|
56
|
+
result: z.string().max(DEFAULTS.maxResultLength),
|
|
20
57
|
}),
|
|
21
|
-
z.object({ type: z.literal("done"), result: z.string() }),
|
|
22
58
|
]);
|
|
23
59
|
export class ActionValidationError extends Error {
|
|
24
60
|
constructor(message) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action-validator.js","sourceRoot":"","sources":["../../src/security/action-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"action-validator.js","sourceRoot":"","sources":["../../src/security/action-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGjD,MAAM,kBAAkB,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAChF,MAAM,gBAAgB,GAAG,4BAA4B,CAAC;AAEtD,MAAM,cAAc,GAAG,CAAC;KACrB,MAAM,EAAE;KACR,IAAI,EAAE;KACN,GAAG,CAAC,CAAC,EAAE,+CAA+C,CAAC;KACvD,GAAG,CAAC,QAAQ,CAAC,iBAAiB,EAAE,kCAAkC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;KAC/F,MAAM,CACL,CAAC,CAAC,EAAE,EAAE;IACJ,uDAAuD;IACvD,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,EACD,qCAAqC,CACtC,CAAC;AAEJ,4CAA4C;AAC5C,MAAM,YAAY,GAAG;IACnB,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ;IAC/C,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY;IACjD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU;IACnC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAC3C,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;CAC7B,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,kBAAkB,CAAC,MAAM,EAAE;IACvD,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;IAChE,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACvB,QAAQ,EAAE,cAAc;QACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;KAC7C,CAAC;IACF,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QACzB,QAAQ,EAAE,cAAc;QACxB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC;KAC3B,CAAC;IACF,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3B,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;KAC1B,CAAC;IACF,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACxB,QAAQ,EAAE,cAAc;KACzB,CAAC;IACF,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;IAC5D,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACvB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC;KACxD,CAAC;IACF,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;QACzB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;KACzD,CAAC;IACF,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC;KACjD,CAAC;CACH,CAAC,CAAC;AAEH,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAC9C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAED,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,qBAAqB,CAC7B,mBAAmB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC1E,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,IAAmB,CAAC;AACpC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/security/url-validator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url-validator.d.ts","sourceRoot":"","sources":["../../src/security/url-validator.ts"],"names":[],"mappings":"AA6BA,MAAM,WAAW,oBAAoB;IACnC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,kBAAmB,SAAQ,KAAK;gBAC/B,OAAO,EAAE,MAAM;CAI5B;AAED,wBAAsB,WAAW,CAC/B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,MAAM,CAAC,CA+DjB"}
|
|
@@ -11,6 +11,13 @@ const PRIVATE_IP_RANGES = [
|
|
|
11
11
|
/^fc00:/i,
|
|
12
12
|
/^fe80:/i,
|
|
13
13
|
/^::1$/,
|
|
14
|
+
// IPv6-mapped IPv4 addresses (::ffff:127.0.0.1, ::ffff:10.0.0.1, etc.)
|
|
15
|
+
/^::ffff:127\./i,
|
|
16
|
+
/^::ffff:10\./i,
|
|
17
|
+
/^::ffff:192\.168\./i,
|
|
18
|
+
/^::ffff:172\.(1[6-9]|2\d|3[01])\./i,
|
|
19
|
+
/^::ffff:169\.254\./i,
|
|
20
|
+
/^::ffff:0\./i,
|
|
14
21
|
];
|
|
15
22
|
function isPrivateIp(ip) {
|
|
16
23
|
return PRIVATE_IP_RANGES.some((range) => range.test(ip));
|
|
@@ -51,6 +58,7 @@ export async function validateUrl(rawUrl, options = {}) {
|
|
|
51
58
|
throw new UrlValidationError(`Domain ${hostname} is in the blockedDomains list`);
|
|
52
59
|
}
|
|
53
60
|
}
|
|
61
|
+
// DNS rebinding protection — fail CLOSED on DNS errors
|
|
54
62
|
try {
|
|
55
63
|
const addresses = await resolve(hostname);
|
|
56
64
|
for (const addr of addresses) {
|
|
@@ -62,7 +70,8 @@ export async function validateUrl(rawUrl, options = {}) {
|
|
|
62
70
|
catch (err) {
|
|
63
71
|
if (err instanceof UrlValidationError)
|
|
64
72
|
throw err;
|
|
65
|
-
// DNS resolution failed —
|
|
73
|
+
// DNS resolution failed — reject for safety (fail closed)
|
|
74
|
+
throw new UrlValidationError(`DNS resolution failed for ${hostname} — rejecting for safety`);
|
|
66
75
|
}
|
|
67
76
|
return parsed.href;
|
|
68
77
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/security/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEvD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtD,MAAM,iBAAiB,GAAG;IACxB,QAAQ;IACR,OAAO;IACP,aAAa;IACb,4BAA4B;IAC5B,aAAa;IACb,MAAM;IACN,SAAS;IACT,SAAS;IACT,OAAO;
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/security/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE5C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEvD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtD,MAAM,iBAAiB,GAAG;IACxB,QAAQ;IACR,OAAO;IACP,aAAa;IACb,4BAA4B;IAC5B,aAAa;IACb,MAAM;IACN,SAAS;IACT,SAAS;IACT,OAAO;IACP,uEAAuE;IACvE,gBAAgB;IAChB,eAAe;IACf,qBAAqB;IACrB,oCAAoC;IACpC,qBAAqB;IACrB,cAAc;CACf,CAAC;AAEF,SAAS,WAAW,CAAC,EAAU;IAC7B,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAOD,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAC3C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,UAAgC,EAAE;IAElC,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,kBAAkB,CAAC,gBAAgB,MAAM,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,kBAAkB,CAC1B,qBAAqB,MAAM,CAAC,QAAQ,oCAAoC,CACzE,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAEjC,IAAI,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,kBAAkB,CAAC,uBAAuB,QAAQ,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CACpD,CAAC;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,kBAAkB,CAC1B,UAAU,QAAQ,oCAAoC,CACvD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CACpD,CAAC;QACF,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,kBAAkB,CAAC,UAAU,QAAQ,gCAAgC,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,kBAAkB,CAC1B,2BAA2B,QAAQ,2BAA2B,IAAI,EAAE,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,kBAAkB;YAAE,MAAM,GAAG,CAAC;QACjD,0DAA0D;QAC1D,MAAM,IAAI,kBAAkB,CAC1B,6BAA6B,QAAQ,yBAAyB,CAC/D,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|