@felan-ai/ext-web-access 0.0.0

package/dist/extract.js

@@ -0,0 +1,168 @@
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import { positiveInteger, positiveNumber } from './config.js';
+import { extractGitHubRepository, parseGitHubUrl } from './github.js';
+import { combinedSignal, readResponseBytes } from './http.js';
+import { fetchRemoteUrl, ssrfSettings, validateRemoteUrl } from './ssrf.js';
+const MAX_HTTP_RESPONSE_BYTES = 5 * 1024 * 1024;
+const MAX_IMAGE_BYTES = 3 * 1024 * 1024;
+const MAX_EXTRACTED_CHARACTERS = 750_000;
+const DEFAULT_PDF_MAX_SIZE_MB = 20;
+const MAX_PDF_MAX_SIZE_MB = 50;
+const DEFAULT_PDF_MAX_PAGES = 100;
+const HTTP_TIMEOUT_MS = 30_000;
+export async function extractContent(rawUrl, runtime, config, signal, options = {}) {
+    const settings = ssrfSettings(config);
+    const url = await validateRemoteUrl(rawUrl, settings);
+    const github = options.allowGitHub === false ? undefined : parseGitHubUrl(url.toString());
+    if (github && options.mode !== 'raw') {
+        return extractGitHubRepository(url.toString(), github, runtime, config, signal, options.forceClone);
+    }
+    const response = await fetchRemoteUrl(url, {
+        headers: {
+            Accept: 'text/html,application/xhtml+xml,application/json,text/plain,application/pdf,image/*;q=0.8,*/*;q=0.5',
+            'User-Agent': 'Felan-Web-Access/0.1',
+        },
+        signal: combinedSignal(signal, HTTP_TIMEOUT_MS),
+    }, settings);
+    if (!response.ok) {
+        return { url: url.toString(), title: url.hostname, content: '', error: `HTTP ${response.status}` };
+    }
+    const contentType = response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase() ?? '';
+    const contentLength = Number(response.headers.get('content-length'));
+    if (contentType.startsWith('image/') && Number.isFinite(contentLength) && contentLength > MAX_IMAGE_BYTES) {
+        throw new Error(`Image exceeds the ${MAX_IMAGE_BYTES}-byte limit`);
+    }
+    const pdfMaximumBytes = positiveNumber(config.pdf?.maxSizeMB, DEFAULT_PDF_MAX_SIZE_MB, MAX_PDF_MAX_SIZE_MB) * 1024 * 1024;
+    if (isPdf(url, contentType) && Number.isFinite(contentLength) && contentLength > pdfMaximumBytes) {
+        throw new Error(`PDF exceeds the configured ${pdfMaximumBytes}-byte limit`);
+    }
+    const maximumBytes = contentType.startsWith('image/')
+        ? MAX_IMAGE_BYTES
+        : isPdf(url, contentType) ? pdfMaximumBytes : MAX_HTTP_RESPONSE_BYTES;
+    const bytes = await readResponseBytes(response, maximumBytes);
+    if (contentType.startsWith('image/')) {
+        return {
+            url: url.toString(),
+            title: fileTitle(url, 'image'),
+            content: `[Remote image: ${contentType}, ${bytes.byteLength} bytes]`,
+            error: null,
+            contentType,
+            image: { data: Buffer.from(bytes).toString('base64'), mimeType: contentType },
+        };
+    }
+    if (isPdf(url, contentType))
+        return extractPdf(bytes, url, config, contentType);
+    const text = new TextDecoder().decode(bytes);
+    if (options.mode === 'raw') {
+        return { url: url.toString(), title: fileTitle(url, url.hostname), content: text, error: null, contentType };
+    }
+    if (contentType.includes('html') || looksLikeHtml(text))
+        return extractHtml(text, url, contentType);
+    if (contentType.includes('json') || looksLikeJson(text))
+        return extractJson(text, url, contentType);
+    return boundedExtracted({
+        url: url.toString(),
+        title: fileTitle(url, url.hostname),
+        content: text,
+        error: null,
+        contentType: contentType || 'text/plain',
+    });
+}
+export async function fetchWithConcurrency(urls, concurrency, fetchOne, captureErrors = true) {
+    const results = new Array(urls.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.min(concurrency, urls.length) }, async () => {
+        while (next < urls.length) {
+            const index = next;
+            next += 1;
+            const url = urls[index];
+            try {
+                results[index] = await fetchOne(url);
+            }
+            catch (error) {
+                if (!captureErrors)
+                    throw error;
+                results[index] = { url, title: '', content: '', error: errorMessage(error) };
+            }
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+function extractHtml(html, url, contentType) {
+    const { document } = parseHTML(html);
+    const article = new Readability(document, { charThreshold: 20 }).parse();
+    const turndown = new TurndownService({ codeBlockStyle: 'fenced', headingStyle: 'atx' });
+    const title = article?.title?.trim() || document.title?.trim() || fileTitle(url, url.hostname);
+    const content = article?.content ? turndown.turndown(article.content) : turndown.turndown(document.body?.innerHTML ?? html);
+    return boundedExtracted({ url: url.toString(), title, content: content.trim(), error: null, contentType });
+}
+function extractJson(text, url, contentType) {
+    let content = text;
+    try {
+        content = JSON.stringify(JSON.parse(text), null, 2);
+    }
+    catch {
+        // Invalid JSON is still useful as exact text.
+    }
+    return boundedExtracted({ url: url.toString(), title: fileTitle(url, url.hostname), content, error: null, contentType });
+}
+async function extractPdf(bytes, url, config, contentType) {
+    if (typeof Promise.try !== 'function') {
+        const { default: promiseTry } = await import('promise.try');
+        promiseTry.shim();
+    }
+    const [{ getDocumentProxy }, pdfjs] = await Promise.all([import('unpdf'), import('unpdf/pdfjs')]);
+    const verbosity = pdfjs.VerbosityLevel.ERRORS;
+    const pdf = await getDocumentProxy(bytes, { verbosity });
+    const metadata = await pdf.getMetadata();
+    const info = metadata.info && typeof metadata.info === 'object' ? metadata.info : {};
+    const title = typeof info.Title === 'string' && info.Title.trim() ? info.Title.trim() : fileTitle(url, 'document');
+    const maximumPages = positiveInteger(config.pdf?.maxPages, DEFAULT_PDF_MAX_PAGES, 500);
+    const pageCount = Math.min(pdf.numPages, maximumPages);
+    const sections = [`# ${title}`, '', `Source: ${url.toString()}`, `Pages: ${pdf.numPages}`, ''];
+    for (let pageNumber = 1; pageNumber <= pageCount; pageNumber += 1) {
+        const page = await pdf.getPage(pageNumber);
+        const textContent = await page.getTextContent();
+        const pageText = textContent.items.map((item) => typeof item.str === 'string' ? item.str : '').join(' ').replace(/\s+/gu, ' ').trim();
+        if (pageText)
+            sections.push(`<!-- Page ${pageNumber} -->`, '', pageText, '');
+    }
+    return boundedExtracted({
+        url: url.toString(),
+        title,
+        content: sections.join('\n').trim(),
+        error: null,
+        contentType,
+        ...(pdf.numPages > maximumPages ? { truncated: true } : {}),
+    });
+}
+function boundedExtracted(value) {
+    if (value.content.length <= MAX_EXTRACTED_CHARACTERS)
+        return value;
+    return {
+        ...value,
+        content: value.content.slice(0, MAX_EXTRACTED_CHARACTERS),
+        truncated: true,
+    };
+}
+function isPdf(url, contentType) {
+    return contentType === 'application/pdf' || url.pathname.toLowerCase().endsWith('.pdf');
+}
+function looksLikeHtml(text) {
+    return /<!doctype html|<html[\s>]|<body[\s>]/iu.test(text.slice(0, 2_000));
+}
+function looksLikeJson(text) {
+    const trimmed = text.trimStart();
+    return trimmed.startsWith('{') || trimmed.startsWith('[');
+}
+function fileTitle(url, fallback) {
+    const segment = url.pathname.split('/').filter(Boolean).at(-1);
+    return segment ? decodeURIComponent(segment) : fallback;
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=extract.js.map

package/dist/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG5E,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAChD,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACxC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,eAAe,GAAG,MAAM,CAAC;AAQ/B,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,OAAqB,EACrB,MAAuB,EACvB,MAAoB,EACpB,UAA0B,EAAE;IAE5B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1F,IAAI,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACrC,OAAO,uBAAuB,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACtG,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;QACzC,OAAO,EAAE;YACP,MAAM,EAAE,qGAAqG;YAC7G,YAAY,EAAE,sBAAsB;SACrC;QACD,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,eAAe,CAAC;KAChD,EAAE,QAAQ,CAAC,CAAC;IACb,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IACpG,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACrE,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,eAAe,EAAE,CAAC;QAC1G,MAAM,IAAI,KAAK,CAAC,qBAAqB,eAAe,aAAa,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE,uBAAuB,EAAE,mBAAmB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAC1H,IAAI,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,eAAe,EAAE,CAAC;QACjG,MAAM,IAAI,KAAK,CAAC,8BAA8B,eAAe,aAAa,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC;QACnD,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,uBAAuB,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE9D,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;YACnB,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;YAC9B,OAAO,EAAE,kBAAkB,WAAW,KAAK,KAAK,CAAC,UAAU,SAAS;YACpE,KAAK,EAAE,IAAI;YACX,WAAW;YACX,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE;SAC9E,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC;QAAE,OAAO,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAEhF,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3B,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IAC/G,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IACpG,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IACpG,OAAO,gBAAgB,CAAC;QACtB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC;QACnC,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,IAAI;QACX,WAAW,EAAE,WAAW,IAAI,YAAY;KACzC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAc,EACd,WAAmB,EACnB,QAAoD,EACpD,aAAa,GAAG,IAAI;IAEpB,MAAM,OAAO,GAAG,IAAI,KAAK,CAAmB,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACpF,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;YACzB,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,aAAa;oBAAE,MAAM,KAAK,CAAC;gBAChC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/E,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,WAAmB;IAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAA+B,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAChG,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC;IAC5H,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,WAAmB;IAC9D,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,8CAA8C;IAChD,CAAC;IACD,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAC3H,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAQ,EAAE,MAAuB,EAAE,WAAmB;IACjG,IAAI,OAAQ,OAAkD,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAClF,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5D,UAAU,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IACD,MAAM,CAAC,EAAE,gBAAgB,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,SAAS,GAAI,KAA+D,CAAC,cAAc,CAAC,MAAM,CAAC;IACzG,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAA+B,CAAC,CAAC,CAAC,EAAE,CAAC;IAChH,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACnH,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACvF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,CAAC,KAAK,KAAK,EAAE,EAAE,EAAE,EAAE,WAAW,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,UAAU,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/F,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,SAAS,EAAE,UAAU,IAAI,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,OAAQ,IAA0B,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3L,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,UAAU,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,gBAAgB,CAAC;QACtB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,KAAK;QACL,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QACnC,KAAK,EAAE,IAAI;QACX,WAAW;QACX,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAuB;IAC/C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,wBAAwB;QAAE,OAAO,KAAK,CAAC;IACnE,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC;QACzD,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,GAAQ,EAAE,WAAmB;IAC1C,OAAO,WAAW,KAAK,iBAAiB,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC1F,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,wCAAwC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IACjC,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,SAAS,CAAC,GAAQ,EAAE,QAAgB;IAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC1D,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}

package/dist/github.d.ts

@@ -0,0 +1,13 @@
+import type { AgentRuntime } from '@felan-ai/agent-core';
+import { type WebAccessConfig } from './config.js';
+import type { ExtractedContent } from './types.js';
+export interface GitHubUrlInfo {
+    owner: string;
+    repo: string;
+    ref?: string;
+    path?: string;
+    type: 'root' | 'tree' | 'blob';
+}
+export declare function parseGitHubUrl(rawUrl: string): GitHubUrlInfo | undefined;
+export declare function extractGitHubRepository(url: string, info: GitHubUrlInfo, runtime: AgentRuntime, config: WebAccessConfig, signal?: AbortSignal, forceClone?: boolean): Promise<ExtractedContent>;
+//# sourceMappingURL=github.d.ts.map

package/dist/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../src/github.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AAGnE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAenD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;CAChC;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAqBxE;AAED,wBAAsB,uBAAuB,CAC3C,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,aAAa,EACnB,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,eAAe,EACvB,MAAM,CAAC,EAAE,WAAW,EACpB,UAAU,UAAQ,GACjB,OAAO,CAAC,gBAAgB,CAAC,CAoE3B"}

package/dist/github.js

@@ -0,0 +1,166 @@
+import { createHash } from 'node:crypto';
+import { join, posix } from 'node:path';
+import { positiveNumber } from './config.js';
+import { combinedSignal, readJsonResponse } from './http.js';
+import { endpointSsrfSettings, fetchRemoteUrl } from './ssrf.js';
+const MAX_FILES = 200;
+const MAX_FILE_CHARACTERS = 100_000;
+const MAX_TOTAL_CHARACTERS = 750_000;
+const clonePromises = new Map();
+const textDecoder = new TextDecoder('utf8', { fatal: true });
+const BINARY_EXTENSIONS = new Set([
+    '.7z', '.avi', '.bin', '.bmp', '.class', '.dll', '.doc', '.docx', '.dylib', '.exe', '.gif', '.gz',
+    '.ico', '.jar', '.jpeg', '.jpg', '.lockb', '.mov', '.mp3', '.mp4', '.o', '.otf', '.pdf', '.png',
+    '.pyc', '.so', '.tar', '.ttf', '.wav', '.webm', '.webp', '.woff', '.woff2', '.xls', '.xlsx', '.zip',
+]);
+const NOISE_DIRECTORIES = new Set(['.git', '.next', '.turbo', 'build', 'coverage', 'dist', 'node_modules', 'target', 'vendor']);
+export function parseGitHubUrl(rawUrl) {
+    let url;
+    try {
+        url = new URL(rawUrl);
+    }
+    catch {
+        return undefined;
+    }
+    if (url.hostname.toLowerCase() !== 'github.com')
+        return undefined;
+    const segments = url.pathname.split('/').filter(Boolean).map((segment) => decodeURIComponent(segment));
+    const [owner, rawRepo, kind, ref, ...path] = segments;
+    const repo = rawRepo?.replace(/\.git$/u, '');
+    if (!owner || !repo || !validSegment(owner) || !validSegment(repo))
+        return undefined;
+    if (kind !== undefined && kind !== 'tree' && kind !== 'blob')
+        return undefined;
+    if (kind && (!ref || !validSegment(ref) || path.some((segment) => !validPathSegment(segment))))
+        return undefined;
+    return {
+        owner,
+        repo,
+        type: kind ?? 'root',
+        ...(ref ? { ref } : {}),
+        ...(path.length > 0 ? { path: path.join('/') } : {}),
+    };
+}
+export async function extractGitHubRepository(url, info, runtime, config, signal, forceClone = false) {
+    if (config.githubClone?.enabled === false)
+        throw new Error('GitHub repository extraction is disabled');
+    const maximumRepoSizeMb = positiveNumber(config.githubClone?.maxRepoSizeMB, 350, 2_000);
+    const timeoutMs = positiveNumber(config.githubClone?.cloneTimeoutSeconds, 30, 300) * 1_000;
+    if (!forceClone) {
+        const sizeMb = await githubRepositorySizeMb(info, config, signal);
+        if (sizeMb > maximumRepoSizeMb) {
+            throw new Error(`GitHub repository is ${Math.ceil(sizeMb)} MB; pass forceClone: true to exceed the configured ${maximumRepoSizeMb} MB threshold`);
+        }
+    }
+    const storage = runtime.storage('session');
+    const key = createHash('sha256').update(`${info.owner}/${info.repo}@${info.ref ?? 'HEAD'}`).digest('hex').slice(0, 20);
+    const repositoryPath = join(storage.root, 'web-access', 'repos', key);
+    await storage.mkdir(join(storage.root, 'web-access', 'repos'), { recursive: true });
+    if (forceClone) {
+        try {
+            await storage.remove(repositoryPath, { recursive: true });
+        }
+        catch {
+            // A missing clone is already in the requested state.
+        }
+    }
+    if (!await hasClone(storage, repositoryPath)) {
+        let clone = clonePromises.get(repositoryPath);
+        if (!clone) {
+            clone = cloneRepository(runtime, repositoryPath, info, timeoutMs, signal);
+            clonePromises.set(repositoryPath, clone);
+        }
+        try {
+            await clone;
+        }
+        finally {
+            clonePromises.delete(repositoryPath);
+        }
+    }
+    const files = (await storage.listFiles(repositoryPath, { recursive: true }))
+        .filter(isUsefulFile)
+        .filter((path) => info.type === 'root' || !info.path || path === info.path || path.startsWith(`${info.path}/`))
+        .slice(0, MAX_FILES);
+    if (info.type === 'blob' && info.path && !files.includes(info.path))
+        throw new Error(`GitHub repository path not found: ${info.path}`);
+    const sections = [`# ${info.owner}/${info.repo}`, '', `Source: ${url}`, ''];
+    let totalCharacters = sections.join('\n').length;
+    let includedFiles = 0;
+    for (const file of files) {
+        let text;
+        try {
+            text = textDecoder.decode(await storage.readFile(join(repositoryPath, file)));
+        }
+        catch {
+            continue;
+        }
+        if (text.includes('\0'))
+            continue;
+        const body = text.length > MAX_FILE_CHARACTERS ? `${text.slice(0, MAX_FILE_CHARACTERS)}\n[File truncated]` : text;
+        const fence = '```';
+        const rendered = `## ${file}\n\n${fence}${languageFor(file)}\n${body}\n${fence}`;
+        if (totalCharacters + rendered.length > MAX_TOTAL_CHARACTERS)
+            break;
+        sections.push(rendered);
+        totalCharacters += rendered.length;
+        includedFiles += 1;
+    }
+    return {
+        url,
+        title: info.path ? `${info.owner}/${info.repo} - ${info.path}` : `${info.owner}/${info.repo}`,
+        content: sections.join('\n'),
+        error: null,
+        contentType: 'text/markdown',
+        ...(includedFiles < files.length ? { truncated: true } : {}),
+    };
+}
+async function cloneRepository(runtime, destination, info, timeoutMs, signal) {
+    const args = ['clone', '--depth', '1', '--filter=blob:none'];
+    if (info.ref)
+        args.push('--branch', info.ref);
+    args.push(`https://github.com/${info.owner}/${info.repo}.git`, destination);
+    const result = await runtime.exec('git', args, { ...(signal ? { signal } : {}), timeout: timeoutMs });
+    if (signal?.aborted || result.killed)
+        throw new Error('GitHub clone was aborted');
+    if (result.code !== 0)
+        throw new Error('GitHub repository clone failed');
+}
+async function githubRepositorySizeMb(info, config, signal) {
+    const response = await fetchRemoteUrl(`https://api.github.com/repos/${info.owner}/${info.repo}`, {
+        headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'felan-web-access' },
+        signal: combinedSignal(signal, 15_000),
+    }, endpointSsrfSettings(config), { allowCrossOriginRedirects: false });
+    if (!response.ok)
+        throw new Error(`GitHub API request failed with HTTP ${response.status}`);
+    const data = await readJsonResponse(response, 512 * 1024, 'GitHub API');
+    if (typeof data.size !== 'number' || !Number.isFinite(data.size) || data.size < 0) {
+        throw new Error('GitHub API response did not include a valid repository size');
+    }
+    return data.size / 1_024;
+}
+async function hasClone(storage, path) {
+    try {
+        return (await storage.listFiles(path, { recursive: false })).length > 0
+            || (await storage.listFiles(join(path, '.git'), { recursive: false })).length > 0;
+    }
+    catch {
+        return false;
+    }
+}
+function isUsefulFile(path) {
+    const normalized = path.replace(/\\/gu, '/');
+    if (normalized.split('/').some((segment) => NOISE_DIRECTORIES.has(segment)))
+        return false;
+    const extension = posix.extname(normalized).toLowerCase();
+    return !BINARY_EXTENSIONS.has(extension);
+}
+function languageFor(path) {
+    return posix.extname(path).slice(1).replace(/[^a-z0-9_+-]/giu, '');
+}
+function validSegment(value) {
+    return /^[A-Za-z0-9_.-]+$/u.test(value) && value !== '.' && value !== '..';
+}
+function validPathSegment(value) {
+    return value !== '' && value !== '.' && value !== '..' && !value.includes('\0');
+}
+//# sourceMappingURL=github.js.map

package/dist/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../src/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGjE,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,mBAAmB,GAAG,OAAO,CAAC;AACpC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AACrC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAyB,CAAC;AACvD,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE7D,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK;IACjG,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC/F,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CACpG,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAUhI,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,YAAY;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC;IACvG,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC;IACtD,MAAM,IAAI,GAAG,OAAO,EAAE,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC;IAC/E,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACjH,OAAO;QACL,KAAK;QACL,IAAI;QACJ,IAAI,EAAE,IAAI,IAAI,MAAM;QACpB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAW,EACX,IAAmB,EACnB,OAAqB,EACrB,MAAuB,EACvB,MAAoB,EACpB,UAAU,GAAG,KAAK;IAElB,IAAI,MAAM,CAAC,WAAW,EAAE,OAAO,KAAK,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACvG,MAAM,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACxF,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC;IAC3F,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAClE,IAAI,MAAM,GAAG,iBAAiB,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,uDAAuD,iBAAiB,eAAe,CAAC,CAAC;QACpJ,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,IAAI,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACvH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;IACtE,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpF,IAAI,UAAU,EAAE,CAAC;QACf,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,qDAAqD;QACvD,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,aAAa,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACzE,MAAM,CAAC,YAAY,CAAC;SACpB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;SAC9G,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACvB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAEvI,MAAM,QAAQ,GAAa,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,WAAW,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACtF,IAAI,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACjD,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QAChF,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC;QAClH,MAAM,KAAK,GAAG,KAAK,CAAC;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,OAAO,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,EAAE,CAAC;QACjF,IAAI,eAAe,GAAG,QAAQ,CAAC,MAAM,GAAG,oBAAoB;YAAE,MAAM;QACpE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxB,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC;QACnC,aAAa,IAAI,CAAC,CAAC;IACrB,CAAC;IACD,OAAO;QACL,GAAG;QACH,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE;QAC7F,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QAC5B,KAAK,EAAE,IAAI;QACX,WAAW,EAAE,eAAe;QAC5B,GAAG,CAAC,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,OAAqB,EACrB,WAAmB,EACnB,IAAmB,EACnB,SAAiB,EACjB,MAAoB;IAEpB,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,oBAAoB,CAAC,CAAC;IAC7D,IAAI,IAAI,CAAC,GAAG;QAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,MAAM,EAAE,WAAW,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IACtG,IAAI,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAClF,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,IAAmB,EACnB,MAAuB,EACvB,MAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,gCAAgC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE;QAC/F,OAAO,EAAE,EAAE,MAAM,EAAE,6BAA6B,EAAE,YAAY,EAAE,kBAAkB,EAAE;QACpF,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC;KACvC,EAAE,oBAAoB,CAAC,MAAM,CAAC,EAAE,EAAE,yBAAyB,EAAE,KAAK,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAqB,QAAQ,EAAE,GAAG,GAAG,IAAI,EAAE,YAAY,CAAC,CAAC;IAC5F,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAA4C,EAAE,IAAY;IAChF,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC;eAClE,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7C,IAAI,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1F,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,IAAI,CAAC;AAC7E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa;IACrC,OAAO,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAClF,CAAC"}

package/dist/http.d.ts

@@ -0,0 +1,5 @@
+export declare function readResponseBytes(response: Response, maximumBytes: number): Promise<Uint8Array>;
+export declare function readResponseText(response: Response, maximumBytes: number): Promise<string>;
+export declare function readJsonResponse<T>(response: Response, maximumBytes: number, label: string): Promise<T>;
+export declare function combinedSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal;
+//# sourceMappingURL=http.d.ts.map

package/dist/http.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CA0BrG;AAED,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEhG;AAED,wBAAsB,gBAAgB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAO7G;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAG9F"}

package/dist/http.js

@@ -0,0 +1,47 @@
+export async function readResponseBytes(response, maximumBytes) {
+    const reader = response.body?.getReader();
+    if (!reader)
+        return new Uint8Array();
+    const chunks = [];
+    let total = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            total += value.byteLength;
+            if (total > maximumBytes) {
+                await reader.cancel();
+                throw new Error(`Response exceeds the ${maximumBytes}-byte limit`);
+            }
+            chunks.push(value);
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    const bytes = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return bytes;
+}
+export async function readResponseText(response, maximumBytes) {
+    return new TextDecoder().decode(await readResponseBytes(response, maximumBytes));
+}
+export async function readJsonResponse(response, maximumBytes, label) {
+    const text = await readResponseText(response, maximumBytes);
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        throw new Error(`${label} returned invalid JSON`);
+    }
+}
+export function combinedSignal(signal, timeoutMs) {
+    const timeout = AbortSignal.timeout(timeoutMs);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,QAAkB,EAAE,YAAoB;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAC1C,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,UAAU,EAAE,CAAC;IACrC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,MAAM;YAChB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAC1B,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;gBACzB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,wBAAwB,YAAY,aAAa,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,YAAoB;IAC7E,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAI,QAAkB,EAAE,YAAoB,EAAE,KAAa;IAC/F,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAA+B,EAAE,SAAiB;IAC/E,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC/D,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+import type { FelanExtension } from '@felan-ai/agent-core';
+declare const webAccessExtension: FelanExtension;
+export default webAccessExtension;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAGV,cAAc,EAGf,MAAM,sBAAsB,CAAC;AA4F9B,QAAA,MAAM,kBAAkB,EAAE,cA2LzB,CAAC;AAyQF,eAAe,kBAAkB,CAAC"}