@j0hanz/superfetch 2.1.3 → 2.1.5

package/dist/tools/utils/common.js

@@ -1,42 +0,0 @@
-import { TRUNCATION_MARKER } from '../../config/formatting.js';
-export function determineContentExtractionSource(extractMainContent, article) {
-    return extractMainContent && !!article;
-}
-export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
-    if (!includeMetadata)
-        return undefined;
-    const now = new Date().toISOString();
-    const metadata = {
-        type: 'metadata',
-        url,
-        fetchedAt: now,
-    };
-    if (shouldExtractFromArticle && article) {
-        if (article.title !== undefined)
-            metadata.title = article.title;
-        if (article.byline !== undefined)
-            metadata.author = article.byline;
-        return metadata;
-    }
-    if (extractedMeta.title !== undefined)
-        metadata.title = extractedMeta.title;
-    if (extractedMeta.description !== undefined) {
-        metadata.description = extractedMeta.description;
-    }
-    if (extractedMeta.author !== undefined)
-        metadata.author = extractedMeta.author;
-    return metadata;
-}
-export function truncateContent(content, maxLength, suffix = TRUNCATION_MARKER) {
-    if (maxLength === undefined ||
-        maxLength <= 0 ||
-        content.length <= maxLength) {
-        return { content, truncated: false };
-    }
-    const safeMax = Math.max(0, maxLength - suffix.length);
-    const marker = suffix.length > maxLength ? suffix.substring(0, maxLength) : suffix;
-    return {
-        content: `${content.substring(0, safeMax)}${marker}`,
-        truncated: true,
-    };
-}

package/dist/tools/utils/common.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"common.js","sourceRoot":"","sources":["../../../src/tools/utils/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAQ/D,MAAM,UAAU,gCAAgC,CAC9C,kBAA2B,EAC3B,OAAgC;IAEhC,OAAO,kBAAkB,IAAI,CAAC,CAAC,OAAO,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,GAAW,EACX,OAAgC,EAChC,aAAgC,EAChC,wBAAiC,EACjC,eAAwB;IAExB,IAAI,CAAC,eAAe;QAAE,OAAO,SAAS,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,OAAO,wBAAwB,IAAI,OAAO;QACxC,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG;YACH,SAAS,EAAE,GAAG;SACf;QACH,CAAC,CAAC;YACE,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,WAAW,EAAE,aAAa,CAAC,WAAW;YACtC,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,GAAG;YACH,SAAS,EAAE,GAAG;SACf,CAAC;AACR,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,SAAkB,EAClB,MAAM,GAAG,iBAAiB;IAE1B,MAAM,cAAc,GAClB,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAEzE,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,MAAM,EAAE;QACtD,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC"}

package/dist/tools/utils/content-shaping.d.ts

@@ -1,4 +0,0 @@
-import type { ExtractedArticle, ExtractedMetadata, MetadataBlock } from '../../config/types/content.js';
-export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
-export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
-export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;

package/dist/tools/utils/content-shaping.js

@@ -1,67 +0,0 @@
-const MIN_CONTENT_RATIO = 0.3;
-const MIN_HTML_LENGTH_FOR_GATE = 100;
-function stripHtmlTags(html) {
-    const parts = [];
-    let inTag = false;
-    for (const char of html) {
-        if (char === '<') {
-            inTag = true;
-            continue;
-        }
-        if (char === '>') {
-            inTag = false;
-            continue;
-        }
-        if (!inTag) {
-            parts.push(char);
-        }
-    }
-    return parts.join('');
-}
-function estimateTextLength(html) {
-    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
-}
-export function isExtractionSufficient(article, originalHtml) {
-    if (!article)
-        return false;
-    const articleLength = article.textContent.length;
-    const originalLength = estimateTextLength(originalHtml);
-    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
-        return true;
-    return articleLength / originalLength >= MIN_CONTENT_RATIO;
-}
-export function determineContentExtractionSource(article) {
-    return !!article;
-}
-function applyArticleMetadata(metadata, article) {
-    if (article.title !== undefined)
-        metadata.title = article.title;
-    if (article.byline !== undefined)
-        metadata.author = article.byline;
-}
-function applyExtractedMetadata(metadata, extractedMeta) {
-    if (extractedMeta.title !== undefined)
-        metadata.title = extractedMeta.title;
-    if (extractedMeta.description !== undefined) {
-        metadata.description = extractedMeta.description;
-    }
-    if (extractedMeta.author !== undefined) {
-        metadata.author = extractedMeta.author;
-    }
-}
-export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
-    if (!includeMetadata)
-        return undefined;
-    const now = new Date().toISOString();
-    const metadata = {
-        type: 'metadata',
-        url,
-        fetchedAt: now,
-    };
-    if (shouldExtractFromArticle && article) {
-        applyArticleMetadata(metadata, article);
-        return metadata;
-    }
-    applyExtractedMetadata(metadata, extractedMeta);
-    return metadata;
-}

package/dist/tools/utils/content-transform-async.d.ts

@@ -1,6 +0,0 @@
-import type { JsonlTransformResult, MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
-export declare function transformHtmlToJsonlAsync(html: string, url: string, options: TransformOptions): Promise<JsonlTransformResult>;
-export declare function transformHtmlToMarkdownAsync(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
-export declare function transformHtmlToMarkdownWithBlocksAsync(html: string, url: string, options: TransformOptions & {
-    includeContentBlocks?: boolean;
-}): Promise<JsonlTransformResult>;

package/dist/tools/utils/content-transform-async.js

@@ -1,33 +0,0 @@
-import { logWarn } from '../../services/logger.js';
-import { runTransformInWorker, } from '../../services/transform-worker-pool.js';
-import { transformHtmlToJsonl, transformHtmlToMarkdown, transformHtmlToMarkdownWithBlocks, } from './content-transform.js';
-async function runOrFallback(job, fallback) {
-    try {
-        const result = await runTransformInWorker(job);
-        if (result)
-            return result;
-    }
-    catch (error) {
-        logWarn('Transform worker unavailable; using main thread', {
-            error: error instanceof Error ? error.message : String(error),
-        });
-    }
-    return fallback();
-}
-export async function transformHtmlToJsonlAsync(html, url, options) {
-    const result = await runOrFallback({ mode: 'jsonl', html, url, options }, () => transformHtmlToJsonl(html, url, options));
-    return result;
-}
-export async function transformHtmlToMarkdownAsync(html, url, options) {
-    const result = await runOrFallback({ mode: 'markdown', html, url, options }, () => transformHtmlToMarkdown(html, url, options));
-    return result;
-}
-export async function transformHtmlToMarkdownWithBlocksAsync(html, url, options) {
-    const result = await runOrFallback({
-        mode: 'markdown-blocks',
-        html,
-        url,
-        options,
-    }, () => transformHtmlToMarkdownWithBlocks(html, url, options));
-    return result;
-}

package/dist/tools/utils/content-transform-core.d.ts

@@ -1,5 +0,0 @@
-import type { ExtractedArticle, ExtractedMetadata, MarkdownTransformResult, MetadataBlock, TransformOptions } from '../../config/types/content.js';
-export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
-export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
-export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
-export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;

package/dist/tools/utils/content-transform-core.js

@@ -1,180 +0,0 @@
-import { extractContent } from '../../services/extractor.js';
-import { logDebug } from '../../services/logger.js';
-import { endTransformStage, startTransformStage, } from '../../services/telemetry.js';
-import { throwIfAborted } from '../../utils/cancellation.js';
-import { htmlToMarkdown } from '../../transformers/markdown.js';
-import { tryTransformRawContent } from './raw-markdown.js';
-const MIN_CONTENT_RATIO = 0.3;
-const MIN_HTML_LENGTH_FOR_GATE = 100;
-function stripHtmlTags(html) {
-    const parts = [];
-    let inTag = false;
-    for (const char of html) {
-        if (char === '<') {
-            inTag = true;
-            continue;
-        }
-        if (char === '>') {
-            inTag = false;
-            continue;
-        }
-        if (!inTag) {
-            parts.push(char);
-        }
-    }
-    return parts.join('');
-}
-function estimateTextLength(html) {
-    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
-}
-export function isExtractionSufficient(article, originalHtml) {
-    if (!article)
-        return false;
-    const articleLength = article.textContent.length;
-    const originalLength = estimateTextLength(originalHtml);
-    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
-        return true;
-    return articleLength / originalLength >= MIN_CONTENT_RATIO;
-}
-export function determineContentExtractionSource(article) {
-    return !!article;
-}
-function applyArticleMetadata(metadata, article) {
-    if (article.title !== undefined)
-        metadata.title = article.title;
-    if (article.byline !== undefined)
-        metadata.author = article.byline;
-}
-function applyExtractedMetadata(metadata, extractedMeta) {
-    if (extractedMeta.title !== undefined)
-        metadata.title = extractedMeta.title;
-    if (extractedMeta.description !== undefined) {
-        metadata.description = extractedMeta.description;
-    }
-    if (extractedMeta.author !== undefined) {
-        metadata.author = extractedMeta.author;
-    }
-}
-export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
-    if (!includeMetadata)
-        return undefined;
-    const now = new Date().toISOString();
-    const metadata = {
-        type: 'metadata',
-        url,
-        fetchedAt: now,
-    };
-    if (shouldExtractFromArticle && article) {
-        applyArticleMetadata(metadata, article);
-        return metadata;
-    }
-    applyExtractedMetadata(metadata, extractedMeta);
-    return metadata;
-}
-function buildArticleContentSource({ url, article, extractedMeta, includeMetadata, }) {
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, true, includeMetadata);
-    return {
-        sourceHtml: article.content,
-        title: article.title,
-        metadata,
-    };
-}
-function buildFullHtmlContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, false, includeMetadata);
-    return {
-        sourceHtml: html,
-        title: extractedMeta.title,
-        metadata,
-    };
-}
-function logQualityGateFallback({ url, articleLength, }) {
-    logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
-        url: url.substring(0, 80),
-        articleLength,
-    });
-}
-function tryBuildExtractedArticleContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
-    if (!article)
-        return null;
-    const shouldExtractFromArticle = determineContentExtractionSource(article);
-    if (shouldExtractFromArticle && isExtractionSufficient(article, html)) {
-        return buildArticleContentSource({
-            url,
-            article,
-            extractedMeta,
-            includeMetadata,
-        });
-    }
-    if (shouldExtractFromArticle) {
-        logQualityGateFallback({
-            url,
-            articleLength: article.textContent.length,
-        });
-    }
-    return null;
-}
-function resolveContentSource({ html, url, includeMetadata, signal, }) {
-    const { article, metadata: extractedMeta } = extractContent(html, url, {
-        extractArticle: true,
-        ...(signal ? { signal } : {}),
-    });
-    const extracted = tryBuildExtractedArticleContentSource({
-        html,
-        url,
-        article,
-        extractedMeta,
-        includeMetadata,
-    });
-    if (extracted)
-        return extracted;
-    return buildFullHtmlContentSource({
-        html,
-        url,
-        article,
-        extractedMeta,
-        includeMetadata,
-    });
-}
-export function transformHtmlToMarkdownInProcess(html, url, options) {
-    const totalStage = startTransformStage(url, 'transform:total');
-    let success = false;
-    try {
-        throwIfAborted(options.signal, url, 'transform:begin');
-        const rawStage = startTransformStage(url, 'transform:raw');
-        const raw = tryTransformRawContent({
-            html,
-            url,
-            includeMetadata: options.includeMetadata,
-        });
-        endTransformStage(rawStage);
-        if (raw) {
-            success = true;
-            return raw;
-        }
-        const extractStage = startTransformStage(url, 'transform:extract');
-        const context = resolveContentSource({
-            html,
-            url,
-            includeMetadata: options.includeMetadata,
-            ...(options.signal ? { signal: options.signal } : {}),
-        });
-        endTransformStage(extractStage);
-        const markdownStage = startTransformStage(url, 'transform:markdown');
-        const content = htmlToMarkdown(context.sourceHtml, context.metadata, {
-            url,
-            ...(options.signal ? { signal: options.signal } : {}),
-        });
-        endTransformStage(markdownStage);
-        success = true;
-        return {
-            markdown: content,
-            title: context.title,
-            truncated: false,
-        };
-    }
-    finally {
-        if (success) {
-            endTransformStage(totalStage, { truncated: false });
-        }
-    }
-}

package/dist/tools/utils/content-transform-workers.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/tools/utils/content-transform-workers.js

@@ -1 +0,0 @@
-export {};

package/dist/tools/utils/content-transform.d.ts

@@ -1,3 +0,0 @@
-import type { MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
-export { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-transform-core.js';
-export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;

package/dist/tools/utils/content-transform.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"content-transform.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,oBAAoB,EACpB,uBAAuB,EACxB,MAAM,+BAA+B,CAAC;AAcvC,UAAU,iBAAiB;IACzB,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAQD,UAAU,oBAAoB;IAC5B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,UAAU,eAAgB,SAAQ,iBAAiB,EAAE,oBAAoB;CAAG;AA4D5E,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,GAChD,oBAAoB,CAatB;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,eAAe,GACvB,uBAAuB,CAYzB;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,iBAAiB,GAAG,oBAAoB,GAChD,oBAAoB,CActB"}

package/dist/tools/utils/content-transform.js

@@ -1,41 +0,0 @@
-import { FetchError } from '../../errors/app-error.js';
-import { endTransformStage, startTransformStage, } from '../../services/telemetry.js';
-import { getOrCreateTransformWorkerPool } from '../../services/transform-worker-pool.js';
-import { throwIfAborted } from '../../utils/cancellation.js';
-import { transformHtmlToMarkdownInProcess } from './content-transform-core.js';
-export { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-transform-core.js';
-export async function transformHtmlToMarkdown(html, url, options) {
-    const totalStage = startTransformStage(url, 'transform:total');
-    let success = false;
-    try {
-        throwIfAborted(options.signal, url, 'transform:begin');
-        const workerStage = startTransformStage(url, 'transform:worker');
-        try {
-            const pool = getOrCreateTransformWorkerPool();
-            const result = await pool.transform(html, url, {
-                includeMetadata: options.includeMetadata,
-                ...(options.signal ? { signal: options.signal } : {}),
-            });
-            success = true;
-            return result;
-        }
-        catch (error) {
-            if (error instanceof FetchError) {
-                throw error;
-            }
-            // Stability-first: if worker infrastructure fails, fall back to in-process.
-            throwIfAborted(options.signal, url, 'transform:worker-fallback');
-            const fallback = transformHtmlToMarkdownInProcess(html, url, options);
-            success = true;
-            return fallback;
-        }
-        finally {
-            endTransformStage(workerStage);
-        }
-    }
-    finally {
-        if (success) {
-            endTransformStage(totalStage, { truncated: false });
-        }
-    }
-}

package/dist/tools/utils/content-transform.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"content-transform.js","sourceRoot":"","sources":["../../../src/tools/utils/content-transform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAM/D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,yCAAyC,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE5E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,eAAe,GAChB,MAAM,aAAa,CAAC;AAmBrB,SAAS,oBAAoB,CAC3B,IAAY,EACZ,GAAW,EACX,OAA0B;IAE1B,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,EAAE;QACrE,cAAc,EAAE,OAAO,CAAC,kBAAkB;KAC3C,CAAC,CAAC;IAEH,MAAM,wBAAwB,GAAG,gCAAgC,CAC/D,OAAO,CAAC,kBAAkB,EAC1B,OAAO,CACR,CAAC;IAEF,MAAM,UAAU,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACrE,MAAM,QAAQ,GAAG,0BAA0B,CACzC,GAAG,EACH,OAAO,EACP,aAAa,EACb,wBAAwB,EACxB,OAAO,CAAC,eAAe,CACxB,CAAC;IACF,MAAM,KAAK,GAAG,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC;IAE7E,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,EACxC,gBAAgB,CACjB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAsB,EACtB,gBAAyB;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,eAAe,CAC5C,QAAQ,EACR,gBAAgB,EAChB,iBAAiB,CAClB,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAC7D,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa;QACb,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,GAAW,EACX,OAAwB;IAExB,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC/C,IAAY,EACZ,GAAW,EACX,OAAiD;IAEjD,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACzD,MAAM,aAAa,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,oBAAoB,CACjD,OAAO,EACP,OAAO,CAAC,gBAAgB,CACzB,CAAC;IAEF,OAAO;QACL,OAAO;QACP,aAAa,EAAE,aAAa,CAAC,MAAM;QACnC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,GAAG,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC"}

package/dist/tools/utils/fetch-pipeline.d.ts

@@ -1,2 +0,0 @@
-import type { FetchPipelineOptions, PipelineResult } from '../../config/types/runtime.js';
-export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;

package/dist/tools/utils/fetch-pipeline.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,oBAAoB,EACpB,cAAc,EACf,MAAM,+BAA+B,CAAC;AAuDvC;;;;;;;GAOG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAwB5B"}

package/dist/tools/utils/fetch-pipeline.js

@@ -1,98 +0,0 @@
-import * as cache from '../../services/cache.js';
-import { createCacheKey } from '../../services/cache-keys.js';
-import { fetchNormalizedUrl } from '../../services/fetcher.js';
-import { logDebug } from '../../services/logger.js';
-import { isRecord } from '../../utils/guards.js';
-import { transformToRawUrl } from '../../utils/url-transformer.js';
-import { normalizeUrl } from '../../utils/url-validator.js';
-function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
-    if (!cacheKey)
-        return null;
-    const cached = cache.get(cacheKey);
-    if (!cached)
-        return null;
-    if (!deserialize) {
-        logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
-        return null;
-    }
-    const data = deserialize(cached.content);
-    if (data === undefined) {
-        logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
-        return null;
-    }
-    logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
-    return {
-        data,
-        fromCache: true,
-        url: normalizedUrl,
-        fetchedAt: cached.fetchedAt,
-        cacheKey,
-    };
-}
-function resolveNormalizedUrl(url) {
-    const { normalizedUrl: validatedUrl } = normalizeUrl(url);
-    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
-    return { normalizedUrl, originalUrl: validatedUrl, transformed };
-}
-export async function executeFetchPipeline(options) {
-    const resolvedUrl = resolveNormalizedUrl(options.url);
-    logRawUrlTransformation(resolvedUrl);
-    const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
-    const cachedResult = attemptCacheRetrieval({
-        cacheKey,
-        deserialize: options.deserialize,
-        cacheNamespace: options.cacheNamespace,
-        normalizedUrl: resolvedUrl.normalizedUrl,
-    });
-    if (cachedResult)
-        return cachedResult;
-    logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
-    const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
-    const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
-    const data = await options.transform(html, resolvedUrl.normalizedUrl);
-    if (cache.isEnabled()) {
-        persistCache({
-            cacheKey,
-            data,
-            serialize: options.serialize,
-            normalizedUrl: resolvedUrl.normalizedUrl,
-        });
-    }
-    return {
-        data,
-        fromCache: false,
-        url: resolvedUrl.normalizedUrl,
-        fetchedAt: new Date().toISOString(),
-        cacheKey,
-    };
-}
-function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
-    if (!cacheKey)
-        return;
-    const serializer = serialize ?? JSON.stringify;
-    const title = extractTitle(data);
-    const metadata = {
-        url: normalizedUrl,
-        ...(title === undefined ? {} : { title }),
-    };
-    cache.set(cacheKey, serializer(data), metadata);
-}
-function extractTitle(value) {
-    if (!isRecord(value))
-        return undefined;
-    const { title } = value;
-    return typeof title === 'string' ? title : undefined;
-}
-function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
-    logDebug(`Cache miss due to ${reason}`, {
-        namespace: cacheNamespace,
-        url: normalizedUrl,
-    });
-}
-function logRawUrlTransformation(resolvedUrl) {
-    if (!resolvedUrl.transformed)
-        return;
-    logDebug('Using transformed raw content URL', {
-        original: resolvedUrl.originalUrl,
-    });
-}

package/dist/tools/utils/fetch-pipeline.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"fetch-pipeline.js","sourceRoot":"","sources":["../../../src/tools/utils/fetch-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,KAAK,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAE7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEnD,SAAS,aAAa,CAAC,MAAc,EAAE,QAAgB;IACrD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,4CAA4C,EAAE;YACpD,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC;SAChC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,QAAuB,EACvB,WAA4D,EAC5D,cAAsB,EACtB,aAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,QAAQ,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC;IAEzE,MAAM,IAAI,GAAG,WAAW;QACtB,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;QAC7B,CAAC,CAAE,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAmB,CAAC;IAE/D,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,QAAQ,CAAC,uCAAuC,EAAE;YAChD,SAAS,EAAE,cAAc;YACzB,GAAG,EAAE,aAAa;SACnB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,aAAa;QAClB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgC;IAEhC,MAAM,aAAa,GAAG,uBAAuB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEzD,MAAM,YAAY,GAAG,qBAAqB,CACxC,QAAQ,EACR,OAAO,CAAC,WAAW,EACnB,OAAO,CAAC,cAAc,EACtB,aAAa,CACd,CAAC;IACF,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IAEtC,MAAM,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAChD,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAE3E,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAClC,aAAa,EACb,YAAY,EACZ,OAAO,CAAC,OAAO,CAChB,CAAC;IACF,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IACpD,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAE/D,OAAO,mBAAmB,CAAC,aAAa,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,eAAe,CACtB,OAAgC,EAChC,aAAqB;IAErB,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC7E,OAAO,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAI,OAAgC;IAC5D,OAAO;QACL,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CACnB,QAAuB,EACvB,IAAO,EACP,SAA8C,EAC9C,aAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,UAAU,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;IAC/C,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE;QACpC,GAAG,EAAE,aAAa;QAClB,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC;KAC1B,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC1D,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,EAAE,KAAK,EAAE,GAAG,KAA4B,CAAC;IAC/C,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AAED,SAAS,mBAAmB,CAC1B,GAAW,EACX,IAAO,EACP,QAAuB;IAEvB,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,KAAK;QAChB,GAAG;QACH,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/tools/utils/frontmatter.d.ts

@@ -1,3 +0,0 @@
-export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
-export declare function addSourceToMarkdown(content: string, url: string): string;
-export declare function hasFrontmatter(trimmed: string): boolean;

package/dist/tools/utils/frontmatter.js

@@ -1,73 +0,0 @@
-function detectLineEnding(content) {
-    return content.includes('\r\n') ? '\r\n' : '\n';
-}
-function findFrontmatterLines(content) {
-    const lineEnding = detectLineEnding(content);
-    const lines = content.split(lineEnding);
-    if (lines[0] !== '---')
-        return null;
-    const endIndex = lines.indexOf('---', 1);
-    if (endIndex === -1)
-        return null;
-    return { lineEnding, lines, endIndex };
-}
-function stripOptionalQuotes(value) {
-    const trimmed = value.trim();
-    if (trimmed.length < 2)
-        return trimmed;
-    const first = trimmed[0];
-    const last = trimmed[trimmed.length - 1];
-    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
-        return trimmed.slice(1, -1).trim();
-    }
-    return trimmed;
-}
-function parseFrontmatterEntry(line) {
-    const trimmed = line.trim();
-    if (!trimmed)
-        return null;
-    const separatorIndex = trimmed.indexOf(':');
-    if (separatorIndex <= 0)
-        return null;
-    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
-    const value = trimmed.slice(separatorIndex + 1);
-    return { key, value };
-}
-function isTitleKey(key) {
-    return key === 'title' || key === 'name';
-}
-export function extractTitleFromRawMarkdown(content) {
-    const frontmatter = findFrontmatterLines(content);
-    if (!frontmatter)
-        return undefined;
-    const { lines, endIndex } = frontmatter;
-    const entry = lines
-        .slice(1, endIndex)
-        .map((line) => parseFrontmatterEntry(line))
-        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
-    if (!entry)
-        return undefined;
-    const value = stripOptionalQuotes(entry.value);
-    return value || undefined;
-}
-export function addSourceToMarkdown(content, url) {
-    const frontmatter = findFrontmatterLines(content);
-    if (!frontmatter) {
-        return `---\nsource: "${url}"\n---\n\n${content}`;
-    }
-    const { lineEnding, lines, endIndex } = frontmatter;
-    const bodyLines = lines.slice(1, endIndex);
-    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
-    if (hasSource)
-        return content;
-    const updatedLines = [
-        lines[0],
-        ...bodyLines,
-        `source: "${url}"`,
-        ...lines.slice(endIndex),
-    ];
-    return updatedLines.join(lineEnding);
-}
-export function hasFrontmatter(trimmed) {
-    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
-}

package/dist/tools/utils/index.d.ts

@@ -1,4 +0,0 @@
-export { executeFetchPipeline } from './fetch-pipeline.js';
-export type { FetchPipelineOptions, PipelineResult } from './fetch-pipeline.js';
-export { createSuccessResponse, createCachedResponse, createBatchResponse, } from './response-builder.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/tools/utils/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,YAAY,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}

package/dist/tools/utils/index.js

@@ -1,3 +0,0 @@
-export { executeFetchPipeline } from './fetch-pipeline.js';
-export { createSuccessResponse, createCachedResponse, createBatchResponse, } from './response-builder.js';
-//# sourceMappingURL=index.js.map

package/dist/tools/utils/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/tools/utils/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,uBAAuB,CAAC"}

package/dist/tools/utils/inline-content.d.ts

@@ -1,10 +0,0 @@
-interface InlineContentResult {
-    content?: string;
-    contentSize: number;
-    resourceUri?: string;
-    resourceMimeType?: string;
-    error?: string;
-    truncated?: boolean;
-}
-export declare function applyInlineContentLimit(content: string, cacheKey: string | null): InlineContentResult;
-export {};

package/dist/tools/utils/inline-content.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"inline-content.d.ts","sourceRoot":"","sources":["../../../src/tools/utils/inline-content.ts"],"names":[],"mappings":"AAKA,KAAK,mBAAmB,GAAG,OAAO,GAAG,UAAU,CAAC;AAEhD,UAAU,mBAAmB;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,MAAM,EAAE,mBAAmB,GAC1B,mBAAmB,CAkBrB"}