@absolutejs/rag 0.22.13 → 0.22.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/changelog.json +15 -0
- package/dist/index.js +4 -1
- package/dist/index.js.map +3 -3
- package/dist/research/index.js +7 -4
- package/dist/research/index.js.map +5 -5
- package/dist/src/web/index.d.ts +1 -1
- package/dist/src/web/transport.d.ts +4 -0
- package/dist/web/index.js +7 -3
- package/dist/web/index.js.map +4 -4
- package/dist/web/playwright.js +4 -1
- package/dist/web/playwright.js.map +3 -3
- package/dist/web-index/index.js +7 -4
- package/dist/web-index/index.js.map +5 -5
- package/package.json +1 -1
package/dist/web/playwright.js
CHANGED
|
@@ -10,15 +10,18 @@ import {
|
|
|
10
10
|
import { lookup } from "dns/promises";
|
|
11
11
|
import { isIP } from "net";
|
|
12
12
|
import { brotliDecompressSync, gunzipSync, inflateSync } from "zlib";
|
|
13
|
+
var WEB_READ_ERROR = Symbol.for("@absolutejs/rag/WebReadError");
|
|
13
14
|
|
|
14
15
|
class WebReadError extends Error {
|
|
15
16
|
code;
|
|
17
|
+
[WEB_READ_ERROR] = true;
|
|
16
18
|
constructor(code, message) {
|
|
17
19
|
super(message);
|
|
18
20
|
this.code = code;
|
|
19
21
|
this.name = "WebReadError";
|
|
20
22
|
}
|
|
21
23
|
}
|
|
24
|
+
var isWebReadError = (error) => error instanceof Error && Reflect.get(error, WEB_READ_ERROR) === true && typeof Reflect.get(error, "code") === "string";
|
|
22
25
|
var isPublicWebAddress = (address) => isPublicNetworkAddress(address);
|
|
23
26
|
var validatePublicWebUrl = (raw) => {
|
|
24
27
|
let url;
|
|
@@ -248,5 +251,5 @@ export {
|
|
|
248
251
|
createPlaywrightWebRenderer
|
|
249
252
|
};
|
|
250
253
|
|
|
251
|
-
//# debugId=
|
|
254
|
+
//# debugId=D0DCB66F403B29EA64756E2164756E21
|
|
252
255
|
//# sourceMappingURL=playwright.js.map
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
"sources": ["../src/web/playwright.ts", "../src/web/transport.ts"],
|
|
4
4
|
"sourcesContent": [
|
|
5
5
|
"import { chromium, type Browser } from \"playwright-core\";\nimport {\n fetchPublicWebResource,\n validatePublicWebUrl,\n WebReadError,\n} from \"./transport\";\nimport type { WebRedirect } from \"./transport\";\nimport type { WebRenderer } from \"./index\";\n/** Fresh contexts share a browser, never cookies; all HTTP traffic uses pinned public DNS. */\nexport const createPlaywrightWebRenderer = (\n options: { executablePath?: string } = {},\n) => {\n let browser: Browser | undefined;\n let launching: Promise<Browser> | undefined;\n const launch = async () => {\n if (browser?.isConnected()) return browser;\n launching ??= chromium.launch({\n headless: true,\n executablePath: options.executablePath,\n args: [\"--disable-dev-shm-usage\"],\n });\n try {\n browser = await launching;\n return browser;\n } finally {\n launching = undefined;\n }\n };\n const renderOnce: WebRenderer = async (url, { signal }) => {\n validatePublicWebUrl(url);\n signal.throwIfAborted();\n const instance = await launch();\n const context = await instance.newContext({\n serviceWorkers: \"block\",\n acceptDownloads: false,\n });\n const abort = () => {\n void context.close().catch(() => {});\n };\n signal.addEventListener(\"abort\", abort, { once: true });\n const redirects: WebRedirect[] = [];\n let count = 0,\n bytes = 0;\n try {\n await context.routeWebSocket(\"**/*\", (socket) => socket.close());\n await context.route(\"**/*\", async (route) => {\n const request = route.request();\n try {\n if (\n ++count > 80 ||\n bytes > 20_000_000 ||\n [\"image\", \"media\", \"font\"].includes(request.resourceType())\n )\n return await route.abort();\n const body = request.postData() ?? undefined;\n if (body && body.length > 100_000) return await route.abort();\n const original = request.headers();\n const headers: Record<string, string> = {};\n for (const key of [\"accept\", \"content-type\", \"origin\", \"user-agent\"])\n if (original[key]) headers[key] = original[key];\n const response = await fetchPublicWebResource(request.url(), {\n signal,\n maxBytes: 5_000_000,\n method: request.method(),\n body,\n headers,\n redirect: \"manual\",\n });\n if (\n request.isNavigationRequest() &&\n request.frame() === context.pages()[0]?.mainFrame() &&\n [301, 302, 303, 307, 308].includes(response.status) &&\n response.headers.location\n ) {\n redirects.push({\n from: request.url(),\n to: new URL(response.headers.location, request.url()).href,\n kind: \"http\",\n status: response.status,\n });\n }\n bytes += response.body.byteLength;\n await route.fulfill({\n status: response.status,\n headers: response.headers,\n body: Buffer.from(response.body),\n });\n } catch {\n await route.abort().catch(() => {});\n }\n });\n const page = await context.newPage();\n let previousUrl = url;\n page.on(\"framenavigated\", (frame) => {\n if (frame !== page.mainFrame() || !/^https?:/u.test(frame.url()))\n return;\n const next = frame.url();\n if (next !== previousUrl && !redirects.some((hop) => hop.to === next))\n redirects.push({ from: previousUrl, to: next, kind: \"client\" });\n previousUrl = next;\n });\n const response = await page.goto(url, {\n waitUntil: \"domcontentloaded\",\n timeout: 25000,\n });\n const settled = await page\n .waitForFunction(\n () => {\n const root = document.querySelector(\n \"#root,#app,#__next,app-root,[ng-version]\",\n );\n const rootText = root?.textContent?.trim() ?? \"\";\n return (\n document.body.innerText.trim().length > 100 &&\n (!root ||\n (rootText.length > 20 &&\n !/^(loading|please wait)/i.test(rootText)))\n );\n },\n undefined,\n { timeout: 8000 },\n )\n .then(\n () => true,\n () => false,\n );\n // Allow hydration and async content to settle after the first readable frame.\n await page.waitForTimeout(1200);\n // Bounded scrolling exposes common lazy-loaded sections without clicking actions.\n for (let step = 0; step < 3; step++) {\n await page.evaluate(() => window.scrollBy(0, window.innerHeight));\n await page.waitForTimeout(350);\n }\n await page.evaluate(() => {\n document.querySelectorAll(\"details\").forEach((element) => {\n element.open = true;\n });\n window.scrollTo(0, 0);\n });\n signal.throwIfAborted();\n const html = await page.content();\n if (html.length > 5_000_000)\n throw new WebReadError(\n \"too_large\",\n \"Rendered page exceeded the content limit.\",\n );\n return {\n html,\n redirects,\n settled,\n text: await page.locator(\"body\").innerText(),\n url: page.url(),\n status: response?.status() ?? 200,\n };\n } finally {\n signal.removeEventListener(\"abort\", abort);\n await context.close().catch(() => {});\n }\n };\n const render: WebRenderer = async (url, options) => {\n try {\n return await renderOnce(url, options);\n } catch (error) {\n if (options.signal.aborted || browser?.isConnected()) throw error;\n // A crashed worker may be replaced once; the same total deadline still applies.\n return renderOnce(url, options);\n }\n };\n return {\n render,\n ready: async () => {\n await launch();\n },\n close: async () => {\n await browser?.close();\n browser = undefined;\n },\n };\n};\n",
|
|
6
|
-
"import {\n isPublicNetworkAddress,\n pinnedPublicRequest,\n} from \"@absolutejs/egress/transport\";\nimport { lookup } from \"node:dns/promises\";\nimport { isIP } from \"node:net\";\nimport { brotliDecompressSync, gunzipSync, inflateSync } from \"node:zlib\";\n\nexport class WebReadError extends Error {\n constructor(\n public code: string,\n message: string,\n ) {\n super(message);\n this.name = \"WebReadError\";\n }\n}\nexport const isPublicWebAddress = (address: string) =>\n isPublicNetworkAddress(address);\nexport type WebRedirect = {\n from: string;\n to: string;\n kind: \"http\" | \"client\";\n status?: number;\n};\nexport type WebFetchResult = {\n redirects?: WebRedirect[];\n url: string;\n status: number;\n headers: Record<string, string>;\n body: Uint8Array;\n};\nexport type WebFetchOptions = {\n signal?: AbortSignal;\n redirect?: \"follow\" | \"manual\";\n maxBytes?: number;\n method?: string;\n body?: string;\n headers?: Record<string, string>;\n};\nexport const validatePublicWebUrl = (raw: string) => {\n let url: URL;\n try {\n url = new URL(raw);\n } catch {\n throw new WebReadError(\n \"invalid_url\",\n \"Provide a valid public HTTP or HTTPS URL.\",\n );\n }\n if (\n ![\"http:\", \"https:\"].includes(url.protocol) ||\n url.username ||\n url.password ||\n (url.port && ![\"80\", \"443\"].includes(url.port))\n )\n throw new WebReadError(\n \"blocked_url\",\n \"Only public HTTP/HTTPS URLs on standard ports are supported.\",\n );\n const host = url.hostname.replace(/^\\[|\\]$/gu, \"\").toLowerCase();\n if (\n host === \"localhost\" ||\n /\\.(?:localhost|local|internal)$/u.test(host) ||\n (isIP(host) && !isPublicWebAddress(host))\n )\n throw new WebReadError(\n \"blocked_url\",\n \"Private and local network destinations are not allowed.\",\n );\n return url;\n};\n/** Pin the validated DNS answer into the connection, including every redirect. */\nexport const fetchPublicWebResource = async (\n raw: string,\n options: WebFetchOptions = {},\n): Promise<WebFetchResult> => {\n const signal = options.signal ?? AbortSignal.timeout(15000);\n const maxBytes = options.maxBytes ?? 5_000_000;\n let url = validatePublicWebUrl(raw);\n const redirects: WebRedirect[] = [];\n let method = options.method ?? \"GET\";\n let body = options.body;\n for (let hop = 0; hop <= 5; hop++) {\n signal.throwIfAborted();\n const host = url.hostname.replace(/^\\[|\\]$/gu, \"\");\n const addresses = await lookup(host, { all: true });\n if (\n !addresses.length ||\n addresses.some(({ address }) => !isPublicWebAddress(address))\n )\n throw new WebReadError(\n \"blocked_url\",\n \"The destination resolved to a private or reserved network.\",\n );\n signal.throwIfAborted();\n const selected =\n addresses.find(({ family }) => family === 4) ?? addresses[0]!;\n const incoming = await pinnedPublicRequest(\n new Request(url, {\n method,\n body,\n signal,\n headers: {\n \"user-agent\": \"AbsoluteJSReader/1.0\",\n accept: \"*/*\",\n \"accept-encoding\": \"identity\",\n ...options.headers,\n host: url.host,\n },\n }),\n { hostname: host, address: selected.address, maxResponseBytes: maxBytes },\n );\n let data: Uint8Array = new Uint8Array(await incoming.arrayBuffer());\n const encoding = incoming.headers.get(\"content-encoding\");\n const limits = { maxOutputLength: maxBytes };\n if (encoding === \"gzip\") data = gunzipSync(data, limits);\n else if (encoding === \"br\") data = brotliDecompressSync(data, limits);\n else if (encoding === \"deflate\") data = inflateSync(data, limits);\n const response: WebFetchResult = {\n url: url.href,\n status: incoming.status,\n headers: Object.fromEntries(\n [...incoming.headers].filter(\n ([key]) =>\n ![\n \"content-encoding\",\n \"content-length\",\n \"transfer-encoding\",\n \"set-cookie\",\n ].includes(key),\n ),\n ),\n body: data,\n };\n if (options.redirect === \"manual\") return response;\n if (\n [301, 302, 303, 307, 308].includes(response.status) &&\n response.headers.location\n ) {\n const next = validatePublicWebUrl(\n new URL(response.headers.location, url).href,\n );\n redirects.push({\n from: url.href,\n to: next.href,\n kind: \"http\",\n status: response.status,\n });\n url = next;\n if (\n response.status === 303 ||\n ([301, 302].includes(response.status) && method === \"POST\")\n ) {\n method = \"GET\";\n body = undefined;\n }\n continue;\n }\n return { ...response, redirects };\n }\n throw new WebReadError(\n \"redirect_limit\",\n \"The site exceeded the redirect limit.\",\n );\n};\n"
|
|
6
|
+
"import {\n isPublicNetworkAddress,\n pinnedPublicRequest,\n} from \"@absolutejs/egress/transport\";\nimport { lookup } from \"node:dns/promises\";\nimport { isIP } from \"node:net\";\nimport { brotliDecompressSync, gunzipSync, inflateSync } from \"node:zlib\";\n\n// Package entrypoints are bundled independently; constructor identity is not shared.\nconst WEB_READ_ERROR = Symbol.for(\"@absolutejs/rag/WebReadError\");\nexport class WebReadError extends Error {\n readonly [WEB_READ_ERROR] = true;\n constructor(\n public code: string,\n message: string,\n ) {\n super(message);\n this.name = \"WebReadError\";\n }\n}\nexport const isWebReadError = (error: unknown): error is WebReadError =>\n error instanceof Error &&\n Reflect.get(error, WEB_READ_ERROR) === true &&\n typeof Reflect.get(error, \"code\") === \"string\";\nexport const isPublicWebAddress = (address: string) =>\n isPublicNetworkAddress(address);\nexport type WebRedirect = {\n from: string;\n to: string;\n kind: \"http\" | \"client\";\n status?: number;\n};\nexport type WebFetchResult = {\n redirects?: WebRedirect[];\n url: string;\n status: number;\n headers: Record<string, string>;\n body: Uint8Array;\n};\nexport type WebFetchOptions = {\n signal?: AbortSignal;\n redirect?: \"follow\" | \"manual\";\n maxBytes?: number;\n method?: string;\n body?: string;\n headers?: Record<string, string>;\n};\nexport const validatePublicWebUrl = (raw: string) => {\n let url: URL;\n try {\n url = new URL(raw);\n } catch {\n throw new WebReadError(\n \"invalid_url\",\n \"Provide a valid public HTTP or HTTPS URL.\",\n );\n }\n if (\n ![\"http:\", \"https:\"].includes(url.protocol) ||\n url.username ||\n url.password ||\n (url.port && ![\"80\", \"443\"].includes(url.port))\n )\n throw new WebReadError(\n \"blocked_url\",\n \"Only public HTTP/HTTPS URLs on standard ports are supported.\",\n );\n const host = url.hostname.replace(/^\\[|\\]$/gu, \"\").toLowerCase();\n if (\n host === \"localhost\" ||\n /\\.(?:localhost|local|internal)$/u.test(host) ||\n (isIP(host) && !isPublicWebAddress(host))\n )\n throw new WebReadError(\n \"blocked_url\",\n \"Private and local network destinations are not allowed.\",\n );\n return url;\n};\n/** Pin the validated DNS answer into the connection, including every redirect. */\nexport const fetchPublicWebResource = async (\n raw: string,\n options: WebFetchOptions = {},\n): Promise<WebFetchResult> => {\n const signal = options.signal ?? AbortSignal.timeout(15000);\n const maxBytes = options.maxBytes ?? 5_000_000;\n let url = validatePublicWebUrl(raw);\n const redirects: WebRedirect[] = [];\n let method = options.method ?? \"GET\";\n let body = options.body;\n for (let hop = 0; hop <= 5; hop++) {\n signal.throwIfAborted();\n const host = url.hostname.replace(/^\\[|\\]$/gu, \"\");\n const addresses = await lookup(host, { all: true });\n if (\n !addresses.length ||\n addresses.some(({ address }) => !isPublicWebAddress(address))\n )\n throw new WebReadError(\n \"blocked_url\",\n \"The destination resolved to a private or reserved network.\",\n );\n signal.throwIfAborted();\n const selected =\n addresses.find(({ family }) => family === 4) ?? addresses[0]!;\n const incoming = await pinnedPublicRequest(\n new Request(url, {\n method,\n body,\n signal,\n headers: {\n \"user-agent\": \"AbsoluteJSReader/1.0\",\n accept: \"*/*\",\n \"accept-encoding\": \"identity\",\n ...options.headers,\n host: url.host,\n },\n }),\n { hostname: host, address: selected.address, maxResponseBytes: maxBytes },\n );\n let data: Uint8Array = new Uint8Array(await incoming.arrayBuffer());\n const encoding = incoming.headers.get(\"content-encoding\");\n const limits = { maxOutputLength: maxBytes };\n if (encoding === \"gzip\") data = gunzipSync(data, limits);\n else if (encoding === \"br\") data = brotliDecompressSync(data, limits);\n else if (encoding === \"deflate\") data = inflateSync(data, limits);\n const response: WebFetchResult = {\n url: url.href,\n status: incoming.status,\n headers: Object.fromEntries(\n [...incoming.headers].filter(\n ([key]) =>\n ![\n \"content-encoding\",\n \"content-length\",\n \"transfer-encoding\",\n \"set-cookie\",\n ].includes(key),\n ),\n ),\n body: data,\n };\n if (options.redirect === \"manual\") return response;\n if (\n [301, 302, 303, 307, 308].includes(response.status) &&\n response.headers.location\n ) {\n const next = validatePublicWebUrl(\n new URL(response.headers.location, url).href,\n );\n redirects.push({\n from: url.href,\n to: next.href,\n kind: \"http\",\n status: response.status,\n });\n url = next;\n if (\n response.status === 303 ||\n ([301, 302].includes(response.status) && method === \"POST\")\n ) {\n method = \"GET\";\n body = undefined;\n }\n continue;\n }\n return { ...response, redirects };\n }\n throw new WebReadError(\n \"redirect_limit\",\n \"The site exceeded the redirect limit.\",\n );\n};\n"
|
|
7
7
|
],
|
|
8
|
-
"mappings": ";;AAAA;;;ACAA;AAAA;AAAA;AAAA;AAIA;AACA;AACA;AAAA;
|
|
9
|
-
"debugId": "
|
|
8
|
+
"mappings": ";;AAAA;;;ACAA;AAAA;AAAA;AAAA;AAIA;AACA;AACA;AAGA,IAAM,iBAAiB,OAAO,IAAI,8BAA8B;AAAA;AACzD,MAAM,qBAAqB,MAAM;AAAA,EAG7B;AAAA,GAFC,kBAAkB;AAAA,EAC5B,WAAW,CACF,MACP,SACA;AAAA,IACA,MAAM,OAAO;AAAA,IAHN;AAAA,IAIP,KAAK,OAAO;AAAA;AAEhB;AACO,IAAM,iBAAiB,CAAC,UAC7B,iBAAiB,SACjB,QAAQ,IAAI,OAAO,cAAc,MAAM,QACvC,OAAO,QAAQ,IAAI,OAAO,MAAM,MAAM;AACjC,IAAM,qBAAqB,CAAC,YACjC,uBAAuB,OAAO;AAsBzB,IAAM,uBAAuB,CAAC,QAAgB;AAAA,EACnD,IAAI;AAAA,EACJ,IAAI;AAAA,IACF,MAAM,IAAI,IAAI,GAAG;AAAA,IACjB,MAAM;AAAA,IACN,MAAM,IAAI,aACR,eACA,2CACF;AAAA;AAAA,EAEF,IACE,CAAC,CAAC,SAAS,QAAQ,EAAE,SAAS,IAAI,QAAQ,KAC1C,IAAI,YACJ,IAAI,YACH,IAAI,QAAQ,CAAC,CAAC,MAAM,KAAK,EAAE,SAAS,IAAI,IAAI;AAAA,IAE7C,MAAM,IAAI,aACR,eACA,8DACF;AAAA,EACF,MAAM,OAAO,IAAI,SAAS,QAAQ,aAAa,EAAE,EAAE,YAAY;AAAA,EAC/D,IACE,SAAS,eACT,mCAAmC,KAAK,IAAI,KAC3C,KAAK,IAAI,KAAK,CAAC,mBAAmB,IAAI;AAAA,IAEvC,MAAM,IAAI,aACR,eACA,yDACF;AAAA,EACF,OAAO;AAAA;AAGF,IAAM,yBAAyB,OACpC,KACA,UAA2B,CAAC,MACA;AAAA,EAC5B,MAAM,SAAS,QAAQ,UAAU,YAAY,QAAQ,KAAK;AAAA,EAC1D,MAAM,WAAW,QAAQ,YAAY;AAAA,EACrC,IAAI,MAAM,qBAAqB,GAAG;AAAA,EAClC,MAAM,YAA2B,CAAC;AAAA,EAClC,IAAI,SAAS,QAAQ,UAAU;AAAA,EAC/B,IAAI,OAAO,QAAQ;AAAA,EACnB,SAAS,MAAM,EAAG,OAAO,GAAG,OAAO;AAAA,IACjC,OAAO,eAAe;AAAA,IACtB,MAAM,OAAO,IAAI,SAAS,QAAQ,aAAa,EAAE;AAAA,IACjD,MAAM,YAAY,MAAM,OAAO,MAAM,EAAE,KAAK,KAAK,CAAC;AAAA,IAClD,IACE,CAAC,UAAU,UACX,UAAU,KAAK,GAAG,cAAc,CAAC,mBAAmB,OAAO,CAAC;AAAA,MAE5D,MAAM,IAAI,aACR,eACA,4DACF;AAAA,IACF,OAAO,eAAe;AAAA,IACtB,MAAM,WACJ,UAAU,KAAK,GAAG,aAAa,WAAW,CAAC,KAAK,UAAU;AAAA,IAC5D,MAAM,WAAW,MAAM,oBACrB,IAAI,QAAQ,KAAK;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,QACP,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,mBAAmB;AAAA,WAChB,QAAQ;AAAA,QACX,MAAM,IAAI;AAAA,MACZ;AAAA,IACF,CAAC,GACD,EAAE,UAAU,MAAM,SAAS,SAAS,SAAS,kBAAkB,SAAS,CAC1E;AAAA,IACA,IAAI,OAAmB,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;AAAA,IAClE,MAAM,WAAW,SAAS,QAAQ,IAAI,kBAAkB;AAAA,IACxD,MAAM,SAAS,EAAE,iBAAiB,SAAS;AAAA,IAC3C,IAAI,aAAa;AAAA,MAAQ,OAAO,WAAW,MAAM,MAAM;AAAA,IAClD,SAAI,aAAa;AAAA,MAAM,OAAO,qBAAqB,MAAM,MAAM;AAAA,IAC/D,SAAI,aAAa;AAAA,MAAW,OAAO,YAAY,MAAM,MAAM;AAAA,IAChE,MAAM,WAA2B;AAAA,MAC/B,KAAK,IAAI;AAAA,MACT,QAAQ,SAAS;AAAA,MACjB,SAAS,OAAO,YACd,CAAC,GAAG,SAAS,OAAO,EAAE,OACpB,EAAE,SACA,CAAC;AAAA,QACC;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,SAAS,GAAG,CAClB,CACF;AAAA,MACA,MAAM;AAAA,IACR;AAAA,IACA,IAAI,QAAQ,aAAa;AAAA,MAAU,OAAO;AAAA,IAC1C,IACE,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,SAAS,MAAM,KAClD,SAAS,QAAQ,UACjB;AAAA,MACA,MAAM,OAAO,qBACX,IAAI,IAAI,SAAS,QAAQ,UAAU,GAAG,EAAE,IAC1C;AAAA,MACA,UAAU,KAAK;AAAA,QACb,MAAM,IAAI;AAAA,QACV,IAAI,KAAK;AAAA,QACT,MAAM;AAAA,QACN,QAAQ,SAAS;AAAA,MACnB,CAAC;AAAA,MACD,MAAM;AAAA,MACN,IACE,SAAS,WAAW,OACnB,CAAC,KAAK,GAAG,EAAE,SAAS,SAAS,MAAM,KAAK,WAAW,QACpD;AAAA,QACA,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,MACA;AAAA,IACF;AAAA,IACA,OAAO,KAAK,UAAU,UAAU;AAAA,EAClC;AAAA,EACA,MAAM,IAAI,aACR,kBACA,uCACF;AAAA;;;ADlKK,IAAM,8BAA8B,CACzC,UAAuC,CAAC,MACrC;AAAA,EACH,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,MAAM,SAAS,YAAY;AAAA,IACzB,IAAI,SAAS,YAAY;AAAA,MAAG,OAAO;AAAA,IACnC,cAAc,SAAS,OAAO;AAAA,MAC5B,UAAU;AAAA,MACV,gBAAgB,QAAQ;AAAA,MACxB,MAAM,CAAC,yBAAyB;AAAA,IAClC,CAAC;AAAA,IACD,IAAI;AAAA,MACF,UAAU,MAAM;AAAA,MAChB,OAAO;AAAA,cACP;AAAA,MACA,YAAY;AAAA;AAAA;AAAA,EAGhB,MAAM,aAA0B,OAAO,OAAO,aAAa;AAAA,IACzD,qBAAqB,GAAG;AAAA,IACxB,OAAO,eAAe;AAAA,IACtB,MAAM,WAAW,MAAM,OAAO;AAAA,IAC9B,MAAM,UAAU,MAAM,SAAS,WAAW;AAAA,MACxC,gBAAgB;AAAA,MAChB,iBAAiB;AAAA,IACnB,CAAC;AAAA,IACD,MAAM,QAAQ,MAAM;AAAA,MACb,QAAQ,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA;AAAA,IAErC,OAAO,iBAAiB,SAAS,OAAO,EAAE,MAAM,KAAK,CAAC;AAAA,IACtD,MAAM,YAA2B,CAAC;AAAA,IAClC,IAAI,QAAQ,GACV,QAAQ;AAAA,IACV,IAAI;AAAA,MACF,MAAM,QAAQ,eAAe,QAAQ,CAAC,WAAW,OAAO,MAAM,CAAC;AAAA,MAC/D,MAAM,QAAQ,MAAM,QAAQ,OAAO,UAAU;AAAA,QAC3C,MAAM,UAAU,MAAM,QAAQ;AAAA,QAC9B,IAAI;AAAA,UACF,IACE,EAAE,QAAQ,MACV,QAAQ,YACR,CAAC,SAAS,SAAS,MAAM,EAAE,SAAS,QAAQ,aAAa,CAAC;AAAA,YAE1D,OAAO,MAAM,MAAM,MAAM;AAAA,UAC3B,MAAM,OAAO,QAAQ,SAAS,KAAK;AAAA,UACnC,IAAI,QAAQ,KAAK,SAAS;AAAA,YAAS,OAAO,MAAM,MAAM,MAAM;AAAA,UAC5D,MAAM,WAAW,QAAQ,QAAQ;AAAA,UACjC,MAAM,UAAkC,CAAC;AAAA,UACzC,WAAW,OAAO,CAAC,UAAU,gBAAgB,UAAU,YAAY;AAAA,YACjE,IAAI,SAAS;AAAA,cAAM,QAAQ,OAAO,SAAS;AAAA,UAC7C,MAAM,WAAW,MAAM,uBAAuB,QAAQ,IAAI,GAAG;AAAA,YAC3D;AAAA,YACA,UAAU;AAAA,YACV,QAAQ,QAAQ,OAAO;AAAA,YACvB;AAAA,YACA;AAAA,YACA,UAAU;AAAA,UACZ,CAAC;AAAA,UACD,IACE,QAAQ,oBAAoB,KAC5B,QAAQ,MAAM,MAAM,QAAQ,MAAM,EAAE,IAAI,UAAU,KAClD,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,EAAE,SAAS,SAAS,MAAM,KAClD,SAAS,QAAQ,UACjB;AAAA,YACA,UAAU,KAAK;AAAA,cACb,MAAM,QAAQ,IAAI;AAAA,cAClB,IAAI,IAAI,IAAI,SAAS,QAAQ,UAAU,QAAQ,IAAI,CAAC,EAAE;AAAA,cACtD,MAAM;AAAA,cACN,QAAQ,SAAS;AAAA,YACnB,CAAC;AAAA,UACH;AAAA,UACA,SAAS,SAAS,KAAK;AAAA,UACvB,MAAM,MAAM,QAAQ;AAAA,YAClB,QAAQ,SAAS;AAAA,YACjB,SAAS,SAAS;AAAA,YAClB,MAAM,OAAO,KAAK,SAAS,IAAI;AAAA,UACjC,CAAC;AAAA,UACD,MAAM;AAAA,UACN,MAAM,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA;AAAA,OAErC;AAAA,MACD,MAAM,OAAO,MAAM,QAAQ,QAAQ;AAAA,MACnC,IAAI,cAAc;AAAA,MAClB,KAAK,GAAG,kBAAkB,CAAC,UAAU;AAAA,QACnC,IAAI,UAAU,KAAK,UAAU,KAAK,CAAC,YAAY,KAAK,MAAM,IAAI,CAAC;AAAA,UAC7D;AAAA,QACF,MAAM,OAAO,MAAM,IAAI;AAAA,QACvB,IAAI,SAAS,eAAe,CAAC,UAAU,KAAK,CAAC,QAAQ,IAAI,OAAO,IAAI;AAAA,UAClE,UAAU,KAAK,EAAE,MAAM,aAAa,IAAI,MAAM,MAAM,SAAS,CAAC;AAAA,QAChE,cAAc;AAAA,OACf;AAAA,MACD,MAAM,WAAW,MAAM,KAAK,KAAK,KAAK;AAAA,QACpC,WAAW;AAAA,QACX,SAAS;AAAA,MACX,CAAC;AAAA,MACD,MAAM,UAAU,MAAM,KACnB,gBACC,MAAM;AAAA,QACJ,MAAM,OAAO,SAAS,cACpB,0CACF;AAAA,QACA,MAAM,WAAW,MAAM,aAAa,KAAK,KAAK;AAAA,QAC9C,OACE,SAAS,KAAK,UAAU,KAAK,EAAE,SAAS,QACvC,CAAC,QACC,SAAS,SAAS,MACjB,CAAC,0BAA0B,KAAK,QAAQ;AAAA,SAGhD,WACA,EAAE,SAAS,KAAK,CAClB,EACC,KACC,MAAM,MACN,MAAM,KACR;AAAA,MAEF,MAAM,KAAK,eAAe,IAAI;AAAA,MAE9B,SAAS,OAAO,EAAG,OAAO,GAAG,QAAQ;AAAA,QACnC,MAAM,KAAK,SAAS,MAAM,OAAO,SAAS,GAAG,OAAO,WAAW,CAAC;AAAA,QAChE,MAAM,KAAK,eAAe,GAAG;AAAA,MAC/B;AAAA,MACA,MAAM,KAAK,SAAS,MAAM;AAAA,QACxB,SAAS,iBAAiB,SAAS,EAAE,QAAQ,CAAC,YAAY;AAAA,UACxD,QAAQ,OAAO;AAAA,SAChB;AAAA,QACD,OAAO,SAAS,GAAG,CAAC;AAAA,OACrB;AAAA,MACD,OAAO,eAAe;AAAA,MACtB,MAAM,OAAO,MAAM,KAAK,QAAQ;AAAA,MAChC,IAAI,KAAK,SAAS;AAAA,QAChB,MAAM,IAAI,aACR,aACA,2CACF;AAAA,MACF,OAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM,MAAM,KAAK,QAAQ,MAAM,EAAE,UAAU;AAAA,QAC3C,KAAK,KAAK,IAAI;AAAA,QACd,QAAQ,UAAU,OAAO,KAAK;AAAA,MAChC;AAAA,cACA;AAAA,MACA,OAAO,oBAAoB,SAAS,KAAK;AAAA,MACzC,MAAM,QAAQ,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA;AAAA;AAAA,EAGxC,MAAM,SAAsB,OAAO,KAAK,YAAY;AAAA,IAClD,IAAI;AAAA,MACF,OAAO,MAAM,WAAW,KAAK,OAAO;AAAA,MACpC,OAAO,OAAO;AAAA,MACd,IAAI,QAAQ,OAAO,WAAW,SAAS,YAAY;AAAA,QAAG,MAAM;AAAA,MAE5D,OAAO,WAAW,KAAK,OAAO;AAAA;AAAA;AAAA,EAGlC,OAAO;AAAA,IACL;AAAA,IACA,OAAO,YAAY;AAAA,MACjB,MAAM,OAAO;AAAA;AAAA,IAEf,OAAO,YAAY;AAAA,MACjB,MAAM,SAAS,MAAM;AAAA,MACrB,UAAU;AAAA;AAAA,EAEd;AAAA;",
|
|
9
|
+
"debugId": "D0DCB66F403B29EA64756E2164756E21",
|
|
10
10
|
"names": []
|
|
11
11
|
}
|
package/dist/web-index/index.js
CHANGED
|
@@ -101639,15 +101639,18 @@ import {
|
|
|
101639
101639
|
import { lookup } from "dns/promises";
|
|
101640
101640
|
import { isIP } from "net";
|
|
101641
101641
|
import { brotliDecompressSync, gunzipSync, inflateSync } from "zlib";
|
|
101642
|
+
var WEB_READ_ERROR = Symbol.for("@absolutejs/rag/WebReadError");
|
|
101642
101643
|
|
|
101643
101644
|
class WebReadError extends Error {
|
|
101644
101645
|
code;
|
|
101646
|
+
[WEB_READ_ERROR] = true;
|
|
101645
101647
|
constructor(code, message) {
|
|
101646
101648
|
super(message);
|
|
101647
101649
|
this.code = code;
|
|
101648
101650
|
this.name = "WebReadError";
|
|
101649
101651
|
}
|
|
101650
101652
|
}
|
|
101653
|
+
var isWebReadError = (error) => error instanceof Error && Reflect.get(error, WEB_READ_ERROR) === true && typeof Reflect.get(error, "code") === "string";
|
|
101651
101654
|
var isPublicWebAddress = (address) => isPublicNetworkAddress(address);
|
|
101652
101655
|
var validatePublicWebUrl = (raw) => {
|
|
101653
101656
|
let url;
|
|
@@ -123097,7 +123100,7 @@ var createWebpageReaderClient = (options) => {
|
|
|
123097
123100
|
|
|
123098
123101
|
// src/web/index.ts
|
|
123099
123102
|
var messageFor = (error) => error instanceof Error ? error.message : "Website retrieval failed.";
|
|
123100
|
-
var codeFor = (error) => error
|
|
123103
|
+
var codeFor = (error) => isWebReadError(error) ? error.code : error instanceof Error && /^(?:AbortError|TimeoutError)$/u.test(error.name) ? "timeout" : "fetch_failed";
|
|
123101
123104
|
var cleanHtml = websiteHtmlText;
|
|
123102
123105
|
var titleOf = (html) => {
|
|
123103
123106
|
const title = /<title\b[^>]*>([\s\S]*?)<\/title>/iu.exec(html)?.[1];
|
|
@@ -123144,7 +123147,7 @@ var readRAGWebpage = async (options) => {
|
|
|
123144
123147
|
response = await (options.fetchResource ?? fetchPublicWebResource)(options.url, { signal, maxBytes: 5000000 });
|
|
123145
123148
|
} catch (error) {
|
|
123146
123149
|
attempts.push({ method: "http", status: codeFor(error) });
|
|
123147
|
-
if (error
|
|
123150
|
+
if (isWebReadError(error) && error.code === "blocked_url")
|
|
123148
123151
|
throw error;
|
|
123149
123152
|
if (!options.render)
|
|
123150
123153
|
throw error;
|
|
@@ -123615,7 +123618,7 @@ Correct the previous schema-invalid response using the validation diagnostics. T
|
|
|
123615
123618
|
fetch_failed: "Page could not be retrieved",
|
|
123616
123619
|
timeout: "Page retrieval timed out"
|
|
123617
123620
|
};
|
|
123618
|
-
if (!(error
|
|
123621
|
+
if (!isWebReadError(error) || !messages[error.code])
|
|
123619
123622
|
throw error;
|
|
123620
123623
|
return {
|
|
123621
123624
|
status: "error",
|
|
@@ -123918,5 +123921,5 @@ export {
|
|
|
123918
123921
|
webIndexPostgresSchemaSql
|
|
123919
123922
|
};
|
|
123920
123923
|
|
|
123921
|
-
//# debugId=
|
|
123924
|
+
//# debugId=FEE9813D4409B5C064756E2164756E21
|
|
123922
123925
|
//# sourceMappingURL=index.js.map
|