@steipete/summarize-core 0.7.0

package/README.md

@@ -0,0 +1,7 @@
+# @steipete/summarize-core
+
+Core library for Summarize (content extraction + prompt builders).
+
+- CLI package: `@steipete/summarize`
+- Recommended imports (library use): `@steipete/summarize-core/content`, `@steipete/summarize-core/prompts`
+

package/dist/esm/content/index.js

@@ -0,0 +1,5 @@
+export { createLinkPreviewClient, } from './link-preview/client.js';
+export { DEFAULT_CACHE_MODE, DEFAULT_MAX_CONTENT_CHARACTERS, DEFAULT_TIMEOUT_MS, } from './link-preview/content/types.js';
+export { ProgressKind } from './link-preview/deps.js';
+export { CACHE_MODES, } from './link-preview/types.js';
+//# sourceMappingURL=index.js.map

package/dist/esm/content/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/content/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,uBAAuB,GAGxB,MAAM,0BAA0B,CAAA;AACjC,OAAO,EACL,kBAAkB,EAClB,8BAA8B,EAC9B,kBAAkB,GAGnB,MAAM,iCAAiC,CAAA;AAUxC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EACL,WAAW,GAIZ,MAAM,yBAAyB,CAAA"}

package/dist/esm/content/link-preview/client.js

@@ -0,0 +1,28 @@
+import { fetchLinkContent } from './content/index.js';
+export function createLinkPreviewClient(options = {}) {
+    const fetchImpl = options.fetch ?? ((...args) => globalThis.fetch(...args));
+    const scrape = options.scrapeWithFirecrawl ?? null;
+    const apifyApiToken = typeof options.apifyApiToken === 'string' ? options.apifyApiToken : null;
+    const ytDlpPath = typeof options.ytDlpPath === 'string' ? options.ytDlpPath : null;
+    const falApiKey = typeof options.falApiKey === 'string' ? options.falApiKey : null;
+    const openaiApiKey = typeof options.openaiApiKey === 'string' ? options.openaiApiKey : null;
+    const convertHtmlToMarkdown = options.convertHtmlToMarkdown ?? null;
+    const transcriptCache = options.transcriptCache ?? null;
+    const readTweetWithBird = typeof options.readTweetWithBird === 'function' ? options.readTweetWithBird : null;
+    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
+    return {
+        fetchLinkContent: (url, contentOptions) => fetchLinkContent(url, contentOptions, {
+            fetch: fetchImpl,
+            scrapeWithFirecrawl: scrape,
+            apifyApiToken,
+            ytDlpPath,
+            falApiKey,
+            openaiApiKey,
+            convertHtmlToMarkdown,
+            transcriptCache,
+            readTweetWithBird,
+            onProgress,
+        }),
+    };
+}
+//# sourceMappingURL=client.js.map

package/dist/esm/content/link-preview/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../src/content/link-preview/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AA2BrD,MAAM,UAAU,uBAAuB,CAAC,UAAoC,EAAE;IAC5E,MAAM,SAAS,GACb,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;IACrF,MAAM,MAAM,GAA+B,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAA;IAC9E,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9F,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3F,MAAM,qBAAqB,GAAiC,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAA;IACjG,MAAM,eAAe,GAA2B,OAAO,CAAC,eAAe,IAAI,IAAI,CAAA;IAC/E,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAA;IACpF,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IAEvF,OAAO;QACL,gBAAgB,EAAE,CAAC,GAAW,EAAE,cAAwC,EAAE,EAAE,CAC1E,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE;YACpC,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,MAAM;YAC3B,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,qBAAqB;YACrB,eAAe;YACf,iBAAiB;YACjB,UAAU;SACX,CAAC;KACL,CAAA;AACH,CAAC"}

package/dist/esm/content/link-preview/content/article.js

@@ -0,0 +1,155 @@
+import { load } from 'cheerio';
+import sanitizeHtml from 'sanitize-html';
+import { decodeHtmlEntities, normalizeWhitespace } from './cleaner.js';
+const MIN_SEGMENT_LENGTH = 30;
+export function sanitizeHtmlForMarkdownConversion(html) {
+    return sanitizeHtml(html, {
+        allowedTags: [
+            'article',
+            'section',
+            'div',
+            'p',
+            'h1',
+            'h2',
+            'h3',
+            'h4',
+            'h5',
+            'h6',
+            'ol',
+            'ul',
+            'li',
+            'blockquote',
+            'pre',
+            'code',
+            'span',
+            'strong',
+            'em',
+            'br',
+            'a',
+        ],
+        allowedAttributes: {
+            a: ['href'],
+        },
+        nonTextTags: [
+            'style',
+            'script',
+            'noscript',
+            'template',
+            'svg',
+            'canvas',
+            'iframe',
+            'object',
+            'embed',
+        ],
+        textFilter(text) {
+            return decodeHtmlEntities(text);
+        },
+    });
+}
+export function extractArticleContent(html) {
+    const segments = collectSegmentsFromHtml(html);
+    if (segments.length > 0) {
+        return segments.join('\n');
+    }
+    const fallback = normalizeWhitespace(extractPlainText(html));
+    return fallback ?? '';
+}
+export function collectSegmentsFromHtml(html) {
+    const sanitized = sanitizeHtml(html, {
+        allowedTags: [
+            'article',
+            'section',
+            'div',
+            'p',
+            'h1',
+            'h2',
+            'h3',
+            'h4',
+            'h5',
+            'h6',
+            'ol',
+            'ul',
+            'li',
+            'blockquote',
+            'pre',
+            'code',
+            'span',
+            'strong',
+            'em',
+            'br',
+        ],
+        allowedAttributes: {},
+        nonTextTags: [
+            'style',
+            'script',
+            'noscript',
+            'template',
+            'svg',
+            'canvas',
+            'iframe',
+            'object',
+            'embed',
+        ],
+        textFilter(text) {
+            return decodeHtmlEntities(text);
+        },
+    });
+    const $ = load(sanitized);
+    const segments = [];
+    $('h1,h2,h3,h4,h5,h6,li,p,blockquote,pre').each((_, element) => {
+        if (!('tagName' in element) || typeof element.tagName !== 'string') {
+            return;
+        }
+        const tag = element.tagName.toLowerCase();
+        const raw = $(element).text();
+        const text = normalizeWhitespace(raw).replaceAll(/\n+/g, ' ');
+        if (!text || text.length === 0) {
+            return;
+        }
+        if (tag.startsWith('h')) {
+            if (text.length >= 10) {
+                segments.push(text);
+            }
+            return;
+        }
+        if (tag === 'li') {
+            if (text.length >= 20) {
+                segments.push(`• ${text}`);
+            }
+            return;
+        }
+        if (text.length < MIN_SEGMENT_LENGTH) {
+            return;
+        }
+        segments.push(text);
+    });
+    if (segments.length === 0) {
+        const fallback = normalizeWhitespace($('body').text() || sanitized);
+        return fallback ? [fallback] : [];
+    }
+    return mergeConsecutiveSegments(segments);
+}
+export function extractPlainText(html) {
+    const stripped = sanitizeHtml(html, {
+        allowedTags: [],
+        allowedAttributes: {},
+        nonTextTags: [
+            'style',
+            'script',
+            'noscript',
+            'template',
+            'svg',
+            'canvas',
+            'iframe',
+            'object',
+            'embed',
+        ],
+    });
+    return decodeHtmlEntities(stripped);
+}
+function mergeConsecutiveSegments(segments) {
+    // Keep headings as separate segments; merging short segments mostly collapses headings into the
+    // previous paragraph ("... Conclusion"), which reads worse than a standalone heading line.
+    return segments.filter(Boolean);
+}
+//# sourceMappingURL=article.js.map

package/dist/esm/content/link-preview/content/article.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/article.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,YAAY,MAAM,eAAe,CAAA;AAExC,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAEtE,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,MAAM,UAAU,iCAAiC,CAAC,IAAY;IAC5D,OAAO,YAAY,CAAC,IAAI,EAAE;QACxB,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;YACJ,GAAG;SACJ;QACD,iBAAiB,EAAE;YACjB,CAAC,EAAE,CAAC,MAAM,CAAC;SACZ;QACD,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IACD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,OAAO,QAAQ,IAAI,EAAE,CAAA;AACvB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE;QACnC,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;SACL;QACD,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAA;IACzB,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,CAAC,CAAC,uCAAuC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnE,OAAM;QACR,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEzC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7B,MAAM,IAAI,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAM;QACR,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACrB,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;YAC5B,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACrC,OAAM;QACR,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrB,CAAC,CAAC,CAAA;IAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,CAAA;QACnE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACnC,CAAC;IAED,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAA;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE;QAClC,WAAW,EAAE,EAAE;QACf,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;KACF,CAAC,CAAA;IACF,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAA;AACrC,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAkB;IAClD,gGAAgG;IAChG,2FAA2F;IAC3F,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AACjC,CAAC"}

package/dist/esm/content/link-preview/content/cleaner.js

@@ -0,0 +1,55 @@
+import { compact } from 'es-toolkit';
+const WORD_SPLIT_PATTERN = /\s+/g;
+export function normalizeForPrompt(input) {
+    return input
+        .replaceAll('\u00A0', ' ')
+        .replaceAll(/[\t ]+/g, ' ')
+        .replaceAll(/\s*\n\s*/g, '\n')
+        .replaceAll(/\n{3,}/g, '\n\n')
+        .trim();
+}
+export function normalizeWhitespace(input) {
+    return input
+        .replaceAll('\u00A0', ' ')
+        .replaceAll(/[\t ]+/g, ' ')
+        .replaceAll(/\s*\n\s*/g, '\n')
+        .trim();
+}
+export function decodeHtmlEntities(input) {
+    return input
+        .replaceAll('&', '&')
+        .replaceAll('&lt;', '<')
+        .replaceAll('&gt;', '>')
+        .replaceAll('&quot;', '"')
+        .replaceAll('&#39;', "'")
+        .replaceAll('&#x27;', "'")
+        .replaceAll('&#x2F;', '/')
+        .replaceAll('&nbsp;', ' ');
+}
+export function normalizeCandidate(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.replaceAll(/\s+/g, ' ').trim();
+    return trimmed.length > 0 ? trimmed : null;
+}
+export function clipAtSentenceBoundary(input, maxLength) {
+    if (input.length <= maxLength) {
+        return input;
+    }
+    const slice = input.slice(0, maxLength);
+    const lastSentenceBreak = Math.max(slice.lastIndexOf('. '), slice.lastIndexOf('! '), slice.lastIndexOf('? '), slice.lastIndexOf('\n\n'));
+    if (lastSentenceBreak > maxLength * 0.5) {
+        return slice.slice(0, lastSentenceBreak + 1);
+    }
+    return slice;
+}
+export function applyContentBudget(baseContent, maxCharacters) {
+    const totalCharacters = baseContent.length;
+    const truncated = totalCharacters > maxCharacters;
+    const clipped = truncated ? clipAtSentenceBoundary(baseContent, maxCharacters) : baseContent;
+    const content = clipped.trim();
+    const wordCount = content.length > 0 ? compact(content.split(WORD_SPLIT_PATTERN)).length : 0;
+    return { content, truncated, totalCharacters, wordCount };
+}
+//# sourceMappingURL=cleaner.js.map

package/dist/esm/content/link-preview/content/cleaner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/cleaner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAEpC,MAAM,kBAAkB,GAAG,MAAM,CAAA;AASjC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAgC;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAA;IACb,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa,EAAE,SAAiB;IACrE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAChC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAC1B,CAAA;IACD,IAAI,iBAAiB,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,WAAmB,EACnB,aAAqB;IAErB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAA;IAC1C,MAAM,SAAS,GAAG,eAAe,GAAG,aAAa,CAAA;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAA;IAC5F,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5F,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,CAAA;AAC3D,CAAC"}

package/dist/esm/content/link-preview/content/constants.js

@@ -0,0 +1,7 @@
+export const BLOCKED_HTML_HINT_PATTERN = /access denied|attention required|captcha|cloudflare|enable javascript|forbidden|please turn javascript on|verify you are human/i;
+export const MIN_HTML_CONTENT_CHARACTERS = 200;
+export const MIN_READABILITY_CONTENT_CHARACTERS = 200;
+export const MIN_METADATA_DESCRIPTION_CHARACTERS = 120;
+export const READABILITY_RELATIVE_THRESHOLD = 0.6;
+export const MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK = 5000;
+//# sourceMappingURL=constants.js.map

package/dist/esm/content/link-preview/content/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,yBAAyB,GACpC,iIAAiI,CAAA;AACnI,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAA;AAC9C,MAAM,CAAC,MAAM,kCAAkC,GAAG,GAAG,CAAA;AACrD,MAAM,CAAC,MAAM,mCAAmC,GAAG,GAAG,CAAA;AACtD,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAA;AACjD,MAAM,CAAC,MAAM,yCAAyC,GAAG,IAAI,CAAA"}

package/dist/esm/content/link-preview/content/fetcher.js

@@ -0,0 +1,124 @@
+import { isYouTubeUrl } from '../../transcript/utils.js';
+import { appendNote } from './utils.js';
+const REQUEST_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Cache-Control': 'no-cache',
+    Pragma: 'no-cache',
+};
+const DEFAULT_REQUEST_TIMEOUT_MS = 5000;
+export async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress, } = {}) {
+    onProgress?.({ kind: 'fetch-html-start', url });
+    const controller = new AbortController();
+    const effectiveTimeoutMs = typeof timeoutMs === 'number' && Number.isFinite(timeoutMs)
+        ? timeoutMs
+        : DEFAULT_REQUEST_TIMEOUT_MS;
+    const timeout = setTimeout(() => {
+        controller.abort();
+    }, effectiveTimeoutMs);
+    try {
+        const response = await fetchImpl(url, {
+            headers: REQUEST_HEADERS,
+            redirect: 'follow',
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch HTML document (status ${response.status})`);
+        }
+        const contentType = response.headers.get('content-type')?.toLowerCase() ?? null;
+        if (contentType &&
+            !contentType.includes('text/html') &&
+            !contentType.includes('application/xhtml+xml') &&
+            !contentType.includes('application/xml') &&
+            !contentType.includes('text/xml') &&
+            !contentType.includes('application/rss+xml') &&
+            !contentType.includes('application/atom+xml') &&
+            !contentType.startsWith('text/')) {
+            throw new Error(`Unsupported content-type for HTML document fetch: ${contentType}`);
+        }
+        const totalBytes = (() => {
+            const raw = response.headers.get('content-length');
+            if (!raw)
+                return null;
+            const parsed = Number(raw);
+            return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : null;
+        })();
+        const body = response.body;
+        if (!body) {
+            const text = await response.text();
+            const bytes = new TextEncoder().encode(text).byteLength;
+            onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes: bytes, totalBytes });
+            return text;
+        }
+        const reader = body.getReader();
+        const decoder = new TextDecoder();
+        let downloadedBytes = 0;
+        let text = '';
+        onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes: 0, totalBytes });
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            downloadedBytes += value.byteLength;
+            text += decoder.decode(value, { stream: true });
+            onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes, totalBytes });
+        }
+        text += decoder.decode();
+        onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes, totalBytes });
+        return text;
+    }
+    catch (error) {
+        if (error instanceof DOMException && error.name === 'AbortError') {
+            throw new Error('Fetching HTML document timed out');
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+export async function fetchWithFirecrawl(url, scrapeWithFirecrawl, options = {}) {
+    const timeoutMs = options.timeoutMs;
+    const cacheMode = options.cacheMode ?? 'default';
+    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
+    const reason = typeof options.reason === 'string' ? options.reason : null;
+    const diagnostics = {
+        attempted: false,
+        used: false,
+        cacheMode,
+        cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
+        notes: null,
+    };
+    if (isYouTubeUrl(url)) {
+        diagnostics.notes = appendNote(diagnostics.notes, 'Skipped Firecrawl for YouTube URL');
+        return { payload: null, diagnostics };
+    }
+    if (!scrapeWithFirecrawl) {
+        diagnostics.notes = appendNote(diagnostics.notes, 'Firecrawl is not configured');
+        return { payload: null, diagnostics };
+    }
+    diagnostics.attempted = true;
+    onProgress?.({ kind: 'firecrawl-start', url, reason: reason ?? 'firecrawl' });
+    try {
+        const payload = await scrapeWithFirecrawl(url, { timeoutMs, cacheMode });
+        if (!payload) {
+            diagnostics.notes = appendNote(diagnostics.notes, 'Firecrawl returned no content payload');
+            onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
+            return { payload: null, diagnostics };
+        }
+        const encoder = new TextEncoder();
+        const markdownBytes = typeof payload.markdown === 'string' ? encoder.encode(payload.markdown).byteLength : null;
+        const htmlBytes = typeof payload.html === 'string' ? encoder.encode(payload.html).byteLength : null;
+        onProgress?.({ kind: 'firecrawl-done', url, ok: true, markdownBytes, htmlBytes });
+        return { payload, diagnostics };
+    }
+    catch (error) {
+        diagnostics.notes = appendNote(diagnostics.notes, `Firecrawl error: ${error instanceof Error ? error.message : 'unknown error'}`);
+        onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
+        return { payload: null, diagnostics };
+    }
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/esm/content/link-preview/content/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAQxD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAEvC,MAAM,eAAe,GAA2B;IAC9C,YAAY,EACV,iHAAiH;IACnH,MAAM,EACJ,kGAAkG;IACpG,iBAAiB,EAAE,gBAAgB;IACnC,eAAe,EAAE,UAAU;IAC3B,MAAM,EAAE,UAAU;CACnB,CAAA;AAED,MAAM,0BAA0B,GAAG,IAAI,CAAA;AAOvC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,SAAuB,EACvB,GAAW,EACX,EACE,SAAS,EACT,UAAU,MACiF,EAAE;IAE/F,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAA;IAE/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,kBAAkB,GACtB,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;QACzD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,0BAA0B,CAAA;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC,EAAE,kBAAkB,CAAC,CAAA;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YACpC,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;QAC9E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAA;QAC/E,IACE,WAAW;YACX,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;YAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9C,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACxC,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC;YACjC,CAAC,WAAW,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YAC5C,CAAC,WAAW,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC7C,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,EAChC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,WAAW,EAAE,CAAC,CAAA;QACrF,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE;YACvB,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAA;YACrB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;YAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC1E,CAAC,CAAC,EAAE,CAAA;QAEJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YAClC,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAA;YACvD,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,eAAe,GAAG,CAAC,CAAA;QACvB,IAAI,IAAI,GAAG,EAAE,CAAA;QAEb,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAA;QAElF,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3C,IAAI,IAAI;gBAAE,MAAK;YACf,IAAI,CAAC,KAAK;gBAAE,SAAQ;YACpB,eAAe,IAAI,KAAK,CAAC,UAAU,CAAA;YACnC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/C,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAA;QACjF,CAAC;QAED,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;QACxB,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAA;QAC3E,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QACrD,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAA;IACvB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,mBAA+C,EAC/C,UAKI,EAAE;IAEN,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;IACnC,MAAM,SAAS,GAAc,OAAO,CAAC,SAAS,IAAI,SAAS,CAAA;IAC3D,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IACvF,MAAM,MAAM,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IACzE,MAAM,WAAW,GAAyB;QACxC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,KAAK;QACX,SAAS;QACT,WAAW,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC5D,KAAK,EAAE,IAAI;KACZ,CAAA;IAED,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,WAAW,CAAC,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,mCAAmC,CAAC,CAAA;QACtF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,WAAW,CAAC,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAA;QAChF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,WAAW,CAAC,SAAS,GAAG,IAAI,CAAA;IAC5B,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW,EAAE,CAAC,CAAA;IAE7E,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,WAAW,CAAC,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,uCAAuC,CAAC,CAAA;YAC1F,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACvC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,aAAa,GACjB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3F,MAAM,SAAS,GACb,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;QACnF,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAA;QAEjF,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,KAAK,GAAG,UAAU,CAC5B,WAAW,CAAC,KAAK,EACjB,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC/E,CAAA;QACD,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;AACH,CAAC"}

package/dist/esm/content/link-preview/content/firecrawl.js

@@ -0,0 +1,86 @@
+import { resolveTranscriptForLink } from '../../transcript/index.js';
+import { extractArticleContent, extractPlainText } from './article.js';
+import { normalizeForPrompt } from './cleaner.js';
+import { BLOCKED_HTML_HINT_PATTERN, MIN_HTML_CONTENT_CHARACTERS, MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK, MIN_METADATA_DESCRIPTION_CHARACTERS, READABILITY_RELATIVE_THRESHOLD, } from './constants.js';
+import { extractJsonLdContent } from './jsonld.js';
+import { extractMetadataFromFirecrawl, extractMetadataFromHtml } from './parsers.js';
+import { isPodcastHost, isPodcastLikeJsonLdType } from './podcast-utils.js';
+import { appendNote, ensureTranscriptDiagnostics, finalizeExtractedLinkContent, pickFirstText, safeHostname, selectBaseContent, } from './utils.js';
+import { detectPrimaryVideoFromHtml } from './video.js';
+export function shouldFallbackToFirecrawl(html) {
+    const plainText = normalizeForPrompt(extractPlainText(html));
+    if (BLOCKED_HTML_HINT_PATTERN.test(plainText))
+        return true;
+    const normalized = normalizeForPrompt(extractArticleContent(html));
+    if (normalized.length >= MIN_HTML_CONTENT_CHARACTERS) {
+        return false;
+    }
+    // Avoid spending Firecrawl on truly small/simple pages where the extracted HTML content is short but
+    // likely complete (e.g. https://example.com). Only treat "thin" content as a Firecrawl signal when
+    // the HTML document itself is large (SSR/app-shell pages, blocked pages without a match, etc.).
+    return html.length >= MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK;
+}
+export async function buildResultFromFirecrawl({ url, payload, cacheMode, maxCharacters, youtubeTranscriptMode, firecrawlDiagnostics, markdownRequested, deps, }) {
+    const normalizedMarkdown = normalizeForPrompt(payload.markdown ?? '');
+    if (normalizedMarkdown.length === 0) {
+        firecrawlDiagnostics.notes = appendNote(firecrawlDiagnostics.notes, 'Firecrawl markdown normalization yielded empty text');
+        return null;
+    }
+    const jsonLd = payload.html ? extractJsonLdContent(payload.html) : null;
+    const isPodcastJsonLd = isPodcastLikeJsonLdType(jsonLd?.type);
+    const transcriptResolution = await resolveTranscriptForLink(url, payload.html ?? null, deps, {
+        youtubeTranscriptMode,
+        cacheMode,
+    });
+    const htmlMetadata = payload.html
+        ? extractMetadataFromHtml(payload.html, url)
+        : { title: null, description: null, siteName: null };
+    const metadata = extractMetadataFromFirecrawl(payload.metadata ?? null);
+    const title = pickFirstText([jsonLd?.title, metadata.title, htmlMetadata.title]);
+    const description = pickFirstText([
+        jsonLd?.description,
+        metadata.description,
+        htmlMetadata.description,
+    ]);
+    const siteName = pickFirstText([metadata.siteName, htmlMetadata.siteName, safeHostname(url)]);
+    const descriptionCandidate = description ? normalizeForPrompt(description) : '';
+    const preferDescription = descriptionCandidate.length >= MIN_METADATA_DESCRIPTION_CHARACTERS &&
+        (isPodcastJsonLd ||
+            isPodcastHost(url) ||
+            normalizedMarkdown.length < MIN_HTML_CONTENT_CHARACTERS ||
+            descriptionCandidate.length >= normalizedMarkdown.length * READABILITY_RELATIVE_THRESHOLD);
+    const baseCandidate = preferDescription ? descriptionCandidate : normalizedMarkdown;
+    const baseContent = selectBaseContent(baseCandidate, transcriptResolution.text);
+    if (baseContent.length === 0) {
+        firecrawlDiagnostics.notes = appendNote(firecrawlDiagnostics.notes, 'Firecrawl produced content that normalized to an empty string');
+        return null;
+    }
+    firecrawlDiagnostics.used = true;
+    const transcriptDiagnostics = ensureTranscriptDiagnostics(transcriptResolution, cacheMode ?? 'default');
+    const video = payload.html ? detectPrimaryVideoFromHtml(payload.html, url) : null;
+    const isVideoOnly = !transcriptResolution.text &&
+        normalizedMarkdown.length < MIN_HTML_CONTENT_CHARACTERS &&
+        video !== null;
+    return finalizeExtractedLinkContent({
+        url,
+        baseContent,
+        maxCharacters,
+        title,
+        description,
+        siteName,
+        transcriptResolution,
+        video,
+        isVideoOnly,
+        diagnostics: {
+            strategy: 'firecrawl',
+            firecrawl: firecrawlDiagnostics,
+            markdown: {
+                requested: markdownRequested,
+                used: true,
+                provider: 'firecrawl',
+            },
+            transcript: transcriptDiagnostics,
+        },
+    });
+}
+//# sourceMappingURL=firecrawl.js.map

package/dist/esm/content/link-preview/content/firecrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAA;AAGpE,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,EACL,yBAAyB,EACzB,2BAA2B,EAC3B,yCAAyC,EACzC,mCAAmC,EACnC,8BAA8B,GAC/B,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACpF,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAA;AAE3E,OAAO,EACL,UAAU,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,aAAa,EACb,YAAY,EACZ,iBAAiB,GAClB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AAEvD,MAAM,UAAU,yBAAyB,CAAC,IAAY;IACpD,MAAM,SAAS,GAAG,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,yBAAyB,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAA;IAC1D,MAAM,UAAU,GAAG,kBAAkB,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAA;IAClE,IAAI,UAAU,CAAC,MAAM,IAAI,2BAA2B,EAAE,CAAC;QACrD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,qGAAqG;IACrG,mGAAmG;IACnG,gGAAgG;IAChG,OAAO,IAAI,CAAC,MAAM,IAAI,yCAAyC,CAAA;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,EAC7C,GAAG,EACH,OAAO,EACP,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,IAAI,GAUL;IACC,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACrE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,oBAAoB,CAAC,KAAK,GAAG,UAAU,CACrC,oBAAoB,CAAC,KAAK,EAC1B,qDAAqD,CACtD,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACvE,MAAM,eAAe,GAAG,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAE7D,MAAM,oBAAoB,GAAG,MAAM,wBAAwB,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE;QAC3F,qBAAqB;QACrB,SAAS;KACV,CAAC,CAAA;IACF,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI;QAC/B,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;QAC5C,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IACtD,MAAM,QAAQ,GAAG,4BAA4B,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAA;IAEvE,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChF,MAAM,WAAW,GAAG,aAAa,CAAC;QAChC,MAAM,EAAE,WAAW;QACnB,QAAQ,CAAC,WAAW;QACpB,YAAY,CAAC,WAAW;KACzB,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAE7F,MAAM,oBAAoB,GAAG,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/E,MAAM,iBAAiB,GACrB,oBAAoB,CAAC,MAAM,IAAI,mCAAmC;QAClE,CAAC,eAAe;YACd,aAAa,CAAC,GAAG,CAAC;YAClB,kBAAkB,CAAC,MAAM,GAAG,2BAA2B;YACvD,oBAAoB,CAAC,MAAM,IAAI,kBAAkB,CAAC,MAAM,GAAG,8BAA8B,CAAC,CAAA;IAC9F,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,kBAAkB,CAAA;IACnF,MAAM,WAAW,GAAG,iBAAiB,CAAC,aAAa,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAA;IAC/E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,oBAAoB,CAAC,KAAK,GAAG,UAAU,CACrC,oBAAoB,CAAC,KAAK,EAC1B,+DAA+D,CAChE,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,oBAAoB,CAAC,IAAI,GAAG,IAAI,CAAA;IAEhC,MAAM,qBAAqB,GAAG,2BAA2B,CACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACjF,MAAM,WAAW,GACf,CAAC,oBAAoB,CAAC,IAAI;QAC1B,kBAAkB,CAAC,MAAM,GAAG,2BAA2B;QACvD,KAAK,KAAK,IAAI,CAAA;IAEhB,OAAO,4BAA4B,CAAC;QAClC,GAAG;QACH,WAAW;QACX,aAAa;QACb,KAAK;QACL,WAAW;QACX,QAAQ;QACR,oBAAoB;QACpB,KAAK;QACL,WAAW;QACX,WAAW,EAAE;YACX,QAAQ,EAAE,WAAW;YACrB,SAAS,EAAE,oBAAoB;YAC/B,QAAQ,EAAE;gBACR,SAAS,EAAE,iBAAiB;gBAC5B,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,WAAW;aACtB;YACD,UAAU,EAAE,qBAAqB;SAClC;KACF,CAAC,CAAA;AACJ,CAAC"}