@pellux/goodvibes-sdk 0.18.3 → 0.18.11

package/dist/platform/tools/fetch/extract.js

@@ -0,0 +1,300 @@
+function stripHtml(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/\s{2,}/g, ' ')
+        .trim();
+}
+function htmlToMarkdown(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        .replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '# $1\n')
+        .replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '## $1\n')
+        .replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '### $1\n')
+        .replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '#### $1\n')
+        .replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '##### $1\n')
+        .replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '###### $1\n')
+        .replace(/<(strong|b)[^>]*>([\s\S]*?)<\/(strong|b)>/gi, '**$2**')
+        .replace(/<(em|i)[^>]*>([\s\S]*?)<\/(em|i)>/gi, '_$2_')
+        .replace(/<a[^>]+href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, '[$2]($1)')
+        .replace(/<img[^>]*>/gi, (match) => {
+        const alt = match.match(/\balt=["']([^"']*)["']/i)?.[1] ?? '';
+        const src = match.match(/\bsrc=["']([^"']*)["']/i)?.[1] ?? '';
+        return alt ? `![${alt}](${src})` : `![](${src})`;
+    })
+        .replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1')
+        .replace(/<ul[^>]*>/gi, '\n').replace(/<\/ul>/gi, '\n')
+        .replace(/<ol[^>]*>/gi, '\n').replace(/<\/ol>/gi, '\n')
+        .replace(/<p[^>]*>/gi, '\n').replace(/<\/p>/gi, '\n')
+        .replace(/<br[^>]*>/gi, '\n')
+        .replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`')
+        .replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '```\n$1\n```')
+        .replace(/<hr[^>]*>/gi, '\n---\n')
+        .replace(/<[^>]+>/g, '')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+function extractReadable(html) {
+    const stripped = html
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        .replace(/<nav[\s\S]*?<\/nav>/gi, '')
+        .replace(/<aside[\s\S]*?<\/aside>/gi, '')
+        .replace(/<header[\s\S]*?<\/header>/gi, '')
+        .replace(/<footer[\s\S]*?<\/footer>/gi, '');
+    return stripHtml(stripped);
+}
+function extractCodeBlocks(html) {
+    const blocks = [];
+    const preRe = /<pre[^>]*>([\s\S]*?)<\/pre>/gi;
+    let m;
+    while ((m = preRe.exec(html)) !== null) {
+        blocks.push(stripHtml(m[1]));
+    }
+    const withoutPre = html.replace(/<pre[\s\S]*?<\/pre>/gi, '');
+    const codeRe = /<code[^>]*>([\s\S]*?)<\/code>/gi;
+    while ((m = codeRe.exec(withoutPre)) !== null) {
+        const code = stripHtml(m[1]);
+        if (code.trim())
+            blocks.push(code);
+    }
+    return blocks.join('\n\n');
+}
+function extractLinks(html) {
+    const links = [];
+    const re = /(?:href|src)=["']([^"'#][^"']*)["']/gi;
+    let m;
+    while ((m = re.exec(html)) !== null) {
+        links.push(m[1]);
+    }
+    return links.join('\n');
+}
+function extractStructured(html, selectors) {
+    const results = [];
+    for (const selector of selectors) {
+        const trimmed = selector.trim();
+        let tagPattern = null;
+        let classFilter = null;
+        let idFilter = null;
+        if (trimmed.startsWith('#')) {
+            idFilter = trimmed.slice(1);
+            tagPattern = '[a-z][a-z0-9]*';
+        }
+        else if (trimmed.startsWith('.')) {
+            classFilter = trimmed.slice(1);
+            tagPattern = '[a-z][a-z0-9]*';
+        }
+        else if (trimmed.includes('.')) {
+            const [tag, cls] = trimmed.split('.');
+            tagPattern = tag || '[a-z][a-z0-9]*';
+            classFilter = cls;
+        }
+        else {
+            tagPattern = trimmed || '[a-z][a-z0-9]*';
+        }
+        let attrClause = '';
+        if (idFilter) {
+            attrClause = `(?=[^>]*\\bid=["']${idFilter}["'])`;
+        }
+        else if (classFilter) {
+            attrClause = `(?=[^>]*\\bclass=["'][^"']*\\b${classFilter}\\b[^"']*["'])`;
+        }
+        const re = new RegExp(`<(${tagPattern})${attrClause}[^>]*>([\\s\\S]*?)<\\/\\1>`, 'gi');
+        let m;
+        while ((m = re.exec(html)) !== null) {
+            const text = stripHtml(m[2]).trim();
+            if (text)
+                results.push(text);
+        }
+    }
+    return JSON.stringify(results);
+}
+function extractTables(html) {
+    const tables = [];
+    const tableRe = /<table[^>]*>([\s\S]*?)<\/table>/gi;
+    let tableM;
+    while ((tableM = tableRe.exec(html)) !== null) {
+        const tableHtml = tableM[1];
+        const headers = [];
+        const rows = [];
+        const thRe = /<th[^>]*>([\s\S]*?)<\/th>/gi;
+        let thM;
+        while ((thM = thRe.exec(tableHtml)) !== null) {
+            headers.push(stripHtml(thM[1]).trim());
+        }
+        const trRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
+        let trM;
+        while ((trM = trRe.exec(tableHtml)) !== null) {
+            const rowHtml = trM[1];
+            if (/<th[^>]*>/i.test(rowHtml))
+                continue;
+            const cells = [];
+            const tdRe = /<td[^>]*>([\s\S]*?)<\/td>/gi;
+            let tdM;
+            while ((tdM = tdRe.exec(rowHtml)) !== null) {
+                cells.push(stripHtml(tdM[1]).trim());
+            }
+            if (cells.length > 0)
+                rows.push(cells);
+        }
+        tables.push({ headers, rows });
+    }
+    return JSON.stringify(tables);
+}
+function extractPdf(body) {
+    const texts = [];
+    const streamRe = /stream\r?\n([\s\S]*?)\r?\nendstream/g;
+    let m;
+    while ((m = streamRe.exec(body)) !== null) {
+        const chunk = m[1];
+        const parenRe = /\(([^)\\]*(?:\\.[^)\\]*)*)\)/g;
+        let pm;
+        while ((pm = parenRe.exec(chunk)) !== null) {
+            const text = pm[1]
+                .replace(/\\n/g, '\n')
+                .replace(/\\r/g, '\r')
+                .replace(/\\t/g, '\t')
+                .replace(/\\\\/g, '\\')
+                .replace(/\\\(/g, '(')
+                .replace(/\\\)/g, ')')
+                .trim();
+            if (text.length > 1)
+                texts.push(text);
+        }
+    }
+    if (texts.length === 0) {
+        return JSON.stringify({
+            note: 'PDF text extraction requires a dedicated library for complex PDFs. No readable text streams found.',
+            byteSize: Buffer.byteLength(body, 'utf-8'),
+        });
+    }
+    return texts.join(' ');
+}
+function extractMetadata(html) {
+    const result = {};
+    const titleM = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    if (titleM)
+        result['title'] = stripHtml(titleM[1]);
+    const metaRe = /<meta[^>]+>/gi;
+    let m;
+    while ((m = metaRe.exec(html)) !== null) {
+        const tag = m[0];
+        const nameM = tag.match(/name=["'](\w[^"']*)["']/i);
+        const contentM = tag.match(/content=["']([^"']*)["']/i);
+        if (nameM && contentM) {
+            result[nameM[1]] = contentM[1];
+        }
+        const propM = tag.match(/property=["'](og:[^"']*)["']/i);
+        if (propM && contentM) {
+            result[propM[1]] = contentM[1];
+        }
+    }
+    return JSON.stringify(result, null, 2);
+}
+function extractSummary(body, contentType) {
+    const isHtml = /text\/html/i.test(contentType);
+    if (!isHtml) {
+        const paragraphs = body.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
+        return paragraphs.slice(0, 2).join('\n\n');
+    }
+    const parts = [];
+    const headingM = body.match(/<h[1-3][^>]*>([\s\S]*?)<\/h[1-3]>/i);
+    if (headingM) {
+        parts.push(stripHtml(headingM[1]).trim());
+    }
+    const paraM = body.match(/<p[^>]*>([\s\S]*?)<\/p>/i);
+    if (paraM) {
+        const text = stripHtml(paraM[1]).trim();
+        if (text)
+            parts.push(text);
+    }
+    const headingRe = /<h([1-6])[^>]*>([\s\S]*?)<\/h[1-6]>/gi;
+    let hm;
+    const headings = [];
+    while ((hm = headingRe.exec(body)) !== null) {
+        const level = parseInt(hm[1], 10);
+        const text = stripHtml(hm[2]).trim();
+        if (text && !parts.includes(text)) {
+            headings.push(`${'#'.repeat(level)} ${text}`);
+        }
+    }
+    if (headings.length > 0) {
+        parts.push('\nHeadings:\n' + headings.join('\n'));
+    }
+    return parts.join('\n\n') || extractReadable(body).slice(0, 500);
+}
+export function sniffContentType(contentType, body) {
+    const isMissing = !contentType || /application\/octet-stream/i.test(contentType);
+    if (!isMissing)
+        return contentType;
+    const sample = body.slice(0, 512).trimStart();
+    if (/^<!DOCTYPE\s+html/i.test(sample) || /^<html/i.test(sample)) {
+        return 'text/html';
+    }
+    if (/^<\?xml/i.test(sample)) {
+        return 'application/xml';
+    }
+    if (/^[{[]/.test(sample)) {
+        return 'application/json';
+    }
+    return contentType;
+}
+export function applyExtract(body, contentType, mode, opts) {
+    const effectiveContentType = sniffContentType(contentType, body);
+    const isHtml = /text\/html/i.test(effectiveContentType);
+    switch (mode) {
+        case 'raw':
+            return body;
+        case 'text':
+            return isHtml ? stripHtml(body) : body;
+        case 'json':
+            try {
+                return JSON.stringify(JSON.parse(body), null, 2);
+            }
+            catch {
+                return body;
+            }
+        case 'markdown':
+            return isHtml ? htmlToMarkdown(body) : body;
+        case 'readable':
+            return isHtml ? extractReadable(body) : body;
+        case 'code_blocks':
+            return isHtml ? extractCodeBlocks(body) : body;
+        case 'links':
+            return isHtml ? extractLinks(body) : '';
+        case 'metadata':
+            return isHtml ? extractMetadata(body) : '{}';
+        case 'structured': {
+            const selectors = opts?.selectors ?? [];
+            if (selectors.length === 0)
+                return JSON.stringify([]);
+            return isHtml ? extractStructured(body, selectors) : JSON.stringify([]);
+        }
+        case 'tables':
+            return isHtml ? extractTables(body) : JSON.stringify([]);
+        case 'pdf': {
+            const isPdf = /application\/pdf/i.test(effectiveContentType);
+            return isPdf ? extractPdf(body) : JSON.stringify({
+                note: 'PDF extraction only applies to application/pdf responses.',
+            });
+        }
+        case 'summary':
+            return extractSummary(body, effectiveContentType);
+        default:
+            return body;
+    }
+}

package/dist/platform/tools/fetch/host-utils.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Shared host utility functions for fetch tool modules.
+ *
+ * Extracted to avoid duplication across trust-tiers.ts and any future
+ * network-scope modules.
+ */
+export declare function hostMatchesGlob(host: string, pattern: string): boolean;
+//# sourceMappingURL=host-utils.d.ts.map

package/dist/platform/tools/fetch/host-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"host-utils.d.ts","sourceRoot":"","sources":["../../../../src/platform/tools/fetch/host-utils.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA2BH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAGtE"}

package/dist/platform/tools/fetch/host-utils.js

@@ -0,0 +1,35 @@
+/**
+ * Shared host utility functions for fetch tool modules.
+ *
+ * Extracted to avoid duplication across trust-tiers.ts and any future
+ * network-scope modules.
+ */
+/**
+ * Glob-simple host pattern match (supports `*` and `**`).
+ * `*` matches one non-dot segment; `**` matches across dots.
+ *
+ * @param host    - Hostname string to test.
+ * @param pattern - Glob pattern (e.g. `*.example.com`, `**`).
+ * @returns       True when the host matches the pattern.
+ */
+const _globRegexCache = new Map();
+function getGlobRegex(pattern) {
+    let re = _globRegexCache.get(pattern);
+    if (!re) {
+        const escaped = pattern
+            .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+            .replace(/\*\*/g, '\u0001')
+            .replace(/\*/g, '[^.]*')
+            .replace(/\u0001/g, '.*');
+        re = new RegExp(`^${escaped}$`, 'i');
+        if (_globRegexCache.size > 100)
+            _globRegexCache.clear(); // simple eviction
+        _globRegexCache.set(pattern, re);
+    }
+    return re;
+}
+export function hostMatchesGlob(host, pattern) {
+    if (pattern === '*')
+        return true;
+    return getGlobRegex(pattern).test(host);
+}

package/dist/platform/tools/fetch/sanitizer.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Fetch response sanitizer.
+ *
+ * Implements three sanitization modes applied to HTTP response content before
+ * it is returned to the caller. The mode is deterministic and auditable:
+ * every result includes the `sanitization_tier` field indicating which mode
+ * was applied.
+ *
+ * Modes:
+ *   - `none`      — No sanitization. Content is returned as-is. Use only for
+ *                   trusted internal hosts.
+ *   - `safe-text` — Default. Strips HTML script/style blocks and control
+ *                   characters. Safe for general external content.
+ *   - `strict`    — Aggressive: allows only printable ASCII and common Unicode
+ *                   whitespace. Strips all HTML tags, script/style, and non-
+ *                   printable characters. Use for untrusted or unknown hosts.
+ */
+/**
+ * Sanitization mode applied to an HTTP response body.
+ *
+ * - `none`      — No sanitization; content returned verbatim.
+ * - `safe-text` — Strips script/style blocks and control characters.
+ * - `strict`    — Strips all HTML, allows only printable ASCII + whitespace.
+ */
+export type SanitizeMode = 'none' | 'safe-text' | 'strict';
+/**
+ * Result of applying a sanitizer to response content.
+ */
+export interface SanitizeResult {
+    /** Sanitized content string. */
+    content: string;
+    /** The sanitization mode that was applied. */
+    mode: SanitizeMode;
+    /** Whether the content was modified by sanitization. */
+    modified: boolean;
+}
+/**
+ * Apply the specified sanitization mode to response content.
+ *
+ * This function is deterministic: given the same `content` and `mode`, it
+ * always produces the same output. The `modified` field in the result
+ * indicates whether any transformation occurred.
+ *
+ * @param content - Raw response body string.
+ * @param mode    - Sanitization mode to apply.
+ * @returns       `SanitizeResult` with sanitized content, mode applied, and
+ *                whether the content was modified.
+ */
+export declare function applySanitizer(content: string, mode: SanitizeMode): SanitizeResult;
+/**
+ * Resolve the effective sanitize mode, applying the rollback default.
+ *
+ * When no explicit mode is requested, defaults to `'safe-text'` — the
+ * rollback-safe default for fetch sanitization.
+ *
+ * @param requested - Caller-supplied mode (may be undefined).
+ * @returns         Effective `SanitizeMode` to apply.
+ */
+export declare function resolveSanitizeMode(requested: SanitizeMode | undefined): SanitizeMode;
+//# sourceMappingURL=sanitizer.d.ts.map

package/dist/platform/tools/fetch/sanitizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sanitizer.d.ts","sourceRoot":"","sources":["../../../../src/platform/tools/fetch/sanitizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8CAA8C;IAC9C,IAAI,EAAE,YAAY,CAAC;IACnB,wDAAwD;IACxD,QAAQ,EAAE,OAAO,CAAC;CACnB;AA+ED;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,CAclF;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,YAAY,GAAG,SAAS,GAAG,YAAY,CAErF"}

package/dist/platform/tools/fetch/sanitizer.js

@@ -0,0 +1,124 @@
+/**
+ * Fetch response sanitizer.
+ *
+ * Implements three sanitization modes applied to HTTP response content before
+ * it is returned to the caller. The mode is deterministic and auditable:
+ * every result includes the `sanitization_tier` field indicating which mode
+ * was applied.
+ *
+ * Modes:
+ *   - `none`      — No sanitization. Content is returned as-is. Use only for
+ *                   trusted internal hosts.
+ *   - `safe-text` — Default. Strips HTML script/style blocks and control
+ *                   characters. Safe for general external content.
+ *   - `strict`    — Aggressive: allows only printable ASCII and common Unicode
+ *                   whitespace. Strips all HTML tags, script/style, and non-
+ *                   printable characters. Use for untrusted or unknown hosts.
+ */
+// ---------------------------------------------------------------------------
+// Sanitizer implementations
+// ---------------------------------------------------------------------------
+/**
+ * `none` mode — returns content verbatim.
+ */
+function sanitizeNone(content) {
+    return { content, mode: 'none', modified: false };
+}
+/**
+ * `safe-text` mode — strips script/style elements and C0/C1 control
+ * characters (except tab, newline, carriage return).
+ *
+ * This mode is appropriate for general external content where HTML injection
+ * or binary smuggling via control sequences is a concern.
+ */
+function sanitizeSafeText(content) {
+    const original = content;
+    let sanitized = content
+        // Remove script blocks (including event handlers encoded in attributes)
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        // Remove style blocks
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        // Strip C0 control characters (0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F) and DEL (0x7F)
+        // Keep: 0x09 (tab), 0x0A (LF), 0x0D (CR)
+        // eslint-disable-next-line no-control-regex
+        .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
+        // Strip C1 control characters (0x80-0x9F)
+        // eslint-disable-next-line no-control-regex
+        .replace(/[\x80-\x9F]/g, '');
+    return {
+        content: sanitized,
+        mode: 'safe-text',
+        modified: sanitized !== original,
+    };
+}
+/**
+ * `strict` mode — strips all HTML tags, script/style blocks, and any
+ * character outside printable ASCII (0x20-0x7E) plus tab (0x09),
+ * newline (0x0A), and carriage return (0x0D).
+ *
+ * Use for unknown or untrusted hosts where maximum reduction of attack
+ * surface is required.
+ */
+function sanitizeStrict(content) {
+    const original = content;
+    let sanitized = content
+        // Remove script blocks
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        // Remove style blocks
+        .replace(/<style[\s\S]*?<\/style>/gi, '')
+        // Strip all remaining HTML tags
+        .replace(/<[^>]+>/g, ' ')
+        // Allow only: printable ASCII (0x20-0x7E), tab (0x09), LF (0x0A), CR (0x0D)
+        // eslint-disable-next-line no-control-regex
+        .replace(/[^\x09\x0A\x0D\x20-\x7E]/g, '')
+        // Collapse multiple consecutive spaces to single space
+        .replace(/ {2,}/g, ' ')
+        .trim();
+    return {
+        content: sanitized,
+        mode: 'strict',
+        modified: sanitized !== original,
+    };
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Apply the specified sanitization mode to response content.
+ *
+ * This function is deterministic: given the same `content` and `mode`, it
+ * always produces the same output. The `modified` field in the result
+ * indicates whether any transformation occurred.
+ *
+ * @param content - Raw response body string.
+ * @param mode    - Sanitization mode to apply.
+ * @returns       `SanitizeResult` with sanitized content, mode applied, and
+ *                whether the content was modified.
+ */
+export function applySanitizer(content, mode) {
+    switch (mode) {
+        case 'none':
+            return sanitizeNone(content);
+        case 'safe-text':
+            return sanitizeSafeText(content);
+        case 'strict':
+            return sanitizeStrict(content);
+        default: {
+            // Exhaustiveness guard — TypeScript ensures this is unreachable
+            const _exhaustive = mode;
+            return sanitizeSafeText(content);
+        }
+    }
+}
+/**
+ * Resolve the effective sanitize mode, applying the rollback default.
+ *
+ * When no explicit mode is requested, defaults to `'safe-text'` — the
+ * rollback-safe default for fetch sanitization.
+ *
+ * @param requested - Caller-supplied mode (may be undefined).
+ * @returns         Effective `SanitizeMode` to apply.
+ */
+export function resolveSanitizeMode(requested) {
+    return requested ?? 'safe-text';
+}