@steipete/summarize-core 0.7.0 → 0.7.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Peter Steinberger
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/cjs/content/index.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CACHE_MODES = exports.ProgressKind = exports.DEFAULT_TIMEOUT_MS = exports.DEFAULT_MAX_CONTENT_CHARACTERS = exports.DEFAULT_CACHE_MODE = exports.createLinkPreviewClient = void 0;
+var client_js_1 = require("./link-preview/client.js");
+Object.defineProperty(exports, "createLinkPreviewClient", { enumerable: true, get: function () { return client_js_1.createLinkPreviewClient; } });
+var types_js_1 = require("./link-preview/content/types.js");
+Object.defineProperty(exports, "DEFAULT_CACHE_MODE", { enumerable: true, get: function () { return types_js_1.DEFAULT_CACHE_MODE; } });
+Object.defineProperty(exports, "DEFAULT_MAX_CONTENT_CHARACTERS", { enumerable: true, get: function () { return types_js_1.DEFAULT_MAX_CONTENT_CHARACTERS; } });
+Object.defineProperty(exports, "DEFAULT_TIMEOUT_MS", { enumerable: true, get: function () { return types_js_1.DEFAULT_TIMEOUT_MS; } });
+var deps_js_1 = require("./link-preview/deps.js");
+Object.defineProperty(exports, "ProgressKind", { enumerable: true, get: function () { return deps_js_1.ProgressKind; } });
+var types_js_2 = require("./link-preview/types.js");
+Object.defineProperty(exports, "CACHE_MODES", { enumerable: true, get: function () { return types_js_2.CACHE_MODES; } });
+//# sourceMappingURL=index.js.map

package/dist/cjs/content/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/content/index.ts"],"names":[],"mappings":";;;AAAA,sDAIiC;AAH/B,oHAAA,uBAAuB,OAAA;AAIzB,4DAMwC;AALtC,8GAAA,kBAAkB,OAAA;AAClB,0HAAA,8BAA8B,OAAA;AAC9B,8GAAA,kBAAkB,OAAA;AAapB,kDAAqD;AAA5C,uGAAA,YAAY,OAAA;AACrB,oDAKgC;AAJ9B,uGAAA,WAAW,OAAA"}

package/dist/cjs/content/link-preview/client.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLinkPreviewClient = createLinkPreviewClient;
+const index_js_1 = require("./content/index.js");
+function createLinkPreviewClient(options = {}) {
+    const fetchImpl = options.fetch ?? ((...args) => globalThis.fetch(...args));
+    const scrape = options.scrapeWithFirecrawl ?? null;
+    const apifyApiToken = typeof options.apifyApiToken === 'string' ? options.apifyApiToken : null;
+    const ytDlpPath = typeof options.ytDlpPath === 'string' ? options.ytDlpPath : null;
+    const falApiKey = typeof options.falApiKey === 'string' ? options.falApiKey : null;
+    const openaiApiKey = typeof options.openaiApiKey === 'string' ? options.openaiApiKey : null;
+    const convertHtmlToMarkdown = options.convertHtmlToMarkdown ?? null;
+    const transcriptCache = options.transcriptCache ?? null;
+    const readTweetWithBird = typeof options.readTweetWithBird === 'function' ? options.readTweetWithBird : null;
+    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
+    return {
+        fetchLinkContent: (url, contentOptions) => (0, index_js_1.fetchLinkContent)(url, contentOptions, {
+            fetch: fetchImpl,
+            scrapeWithFirecrawl: scrape,
+            apifyApiToken,
+            ytDlpPath,
+            falApiKey,
+            openaiApiKey,
+            convertHtmlToMarkdown,
+            transcriptCache,
+            readTweetWithBird,
+            onProgress,
+        }),
+    };
+}
+//# sourceMappingURL=client.js.map

package/dist/cjs/content/link-preview/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../src/content/link-preview/client.ts"],"names":[],"mappings":";;AA2BA,0DA6BC;AAxDD,iDAAqD;AA2BrD,SAAgB,uBAAuB,CAAC,UAAoC,EAAE;IAC5E,MAAM,SAAS,GACb,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;IACrF,MAAM,MAAM,GAA+B,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAA;IAC9E,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9F,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3F,MAAM,qBAAqB,GAAiC,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAA;IACjG,MAAM,eAAe,GAA2B,OAAO,CAAC,eAAe,IAAI,IAAI,CAAA;IAC/E,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAA;IACpF,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IAEvF,OAAO;QACL,gBAAgB,EAAE,CAAC,GAAW,EAAE,cAAwC,EAAE,EAAE,CAC1E,IAAA,2BAAgB,EAAC,GAAG,EAAE,cAAc,EAAE;YACpC,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,MAAM;YAC3B,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,qBAAqB;YACrB,eAAe;YACf,iBAAiB;YACjB,UAAU;SACX,CAAC;KACL,CAAA;AACH,CAAC"}

package/dist/cjs/content/link-preview/content/article.js

@@ -0,0 +1,164 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeHtmlForMarkdownConversion = sanitizeHtmlForMarkdownConversion;
+exports.extractArticleContent = extractArticleContent;
+exports.collectSegmentsFromHtml = collectSegmentsFromHtml;
+exports.extractPlainText = extractPlainText;
+const cheerio_1 = require("cheerio");
+const sanitize_html_1 = __importDefault(require("sanitize-html"));
+const cleaner_js_1 = require("./cleaner.js");
+const MIN_SEGMENT_LENGTH = 30;
+function sanitizeHtmlForMarkdownConversion(html) {
+    return (0, sanitize_html_1.default)(html, {
+        allowedTags: [
+            'article',
+            'section',
+            'div',
+            'p',
+            'h1',
+            'h2',
+            'h3',
+            'h4',
+            'h5',
+            'h6',
+            'ol',
+            'ul',
+            'li',
+            'blockquote',
+            'pre',
+            'code',
+            'span',
+            'strong',
+            'em',
+            'br',
+            'a',
+        ],
+        allowedAttributes: {
+            a: ['href'],
+        },
+        nonTextTags: [
+            'style',
+            'script',
+            'noscript',
+            'template',
+            'svg',
+            'canvas',
+            'iframe',
+            'object',
+            'embed',
+        ],
+        textFilter(text) {
+            return (0, cleaner_js_1.decodeHtmlEntities)(text);
+        },
+    });
+}
+function extractArticleContent(html) {
+    const segments = collectSegmentsFromHtml(html);
+    if (segments.length > 0) {
+        return segments.join('\n');
+    }
+    const fallback = (0, cleaner_js_1.normalizeWhitespace)(extractPlainText(html));
+    return fallback ?? '';
+}
+function collectSegmentsFromHtml(html) {
+    const sanitized = (0, sanitize_html_1.default)(html, {
+        allowedTags: [
+            'article',
+            'section',
+            'div',
+            'p',
+            'h1',
+            'h2',
+            'h3',
+            'h4',
+            'h5',
+            'h6',
+            'ol',
+            'ul',
+            'li',
+            'blockquote',
+            'pre',
+            'code',
+            'span',
+            'strong',
+            'em',
+            'br',
+        ],
+        allowedAttributes: {},
+        nonTextTags: [
+            'style',
+            'script',
+            'noscript',
+            'template',
+            'svg',
+            'canvas',
+            'iframe',
+            'object',
+            'embed',
+        ],
+        textFilter(text) {
+            return (0, cleaner_js_1.decodeHtmlEntities)(text);
+        },
+    });
+    const $ = (0, cheerio_1.load)(sanitized);
+    const segments = [];
+    $('h1,h2,h3,h4,h5,h6,li,p,blockquote,pre').each((_, element) => {
+        if (!('tagName' in element) || typeof element.tagName !== 'string') {
+            return;
+        }
+        const tag = element.tagName.toLowerCase();
+        const raw = $(element).text();
+        const text = (0, cleaner_js_1.normalizeWhitespace)(raw).replaceAll(/\n+/g, ' ');
+        if (!text || text.length === 0) {
+            return;
+        }
+        if (tag.startsWith('h')) {
+            if (text.length >= 10) {
+                segments.push(text);
+            }
+            return;
+        }
+        if (tag === 'li') {
+            if (text.length >= 20) {
+                segments.push(`• ${text}`);
+            }
+            return;
+        }
+        if (text.length < MIN_SEGMENT_LENGTH) {
+            return;
+        }
+        segments.push(text);
+    });
+    if (segments.length === 0) {
+        const fallback = (0, cleaner_js_1.normalizeWhitespace)($('body').text() || sanitized);
+        return fallback ? [fallback] : [];
+    }
+    return mergeConsecutiveSegments(segments);
+}
+function extractPlainText(html) {
+    const stripped = (0, sanitize_html_1.default)(html, {
+        allowedTags: [],
+        allowedAttributes: {},
+        nonTextTags: [
+            'style',
+            'script',
+            'noscript',
+            'template',
+            'svg',
+            'canvas',
+            'iframe',
+            'object',
+            'embed',
+        ],
+    });
+    return (0, cleaner_js_1.decodeHtmlEntities)(stripped);
+}
+function mergeConsecutiveSegments(segments) {
+    // Keep headings as separate segments; merging short segments mostly collapses headings into the
+    // previous paragraph ("... Conclusion"), which reads worse than a standalone heading line.
+    return segments.filter(Boolean);
+}
+//# sourceMappingURL=article.js.map

package/dist/cjs/content/link-preview/content/article.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/article.ts"],"names":[],"mappings":";;;;;AAOA,8EA2CC;AAED,sDAOC;AAED,0DAoFC;AAED,4CAiBC;AApKD,qCAA8B;AAC9B,kEAAwC;AAExC,6CAAsE;AAEtE,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,SAAgB,iCAAiC,CAAC,IAAY;IAC5D,OAAO,IAAA,uBAAY,EAAC,IAAI,EAAE;QACxB,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;YACJ,GAAG;SACJ;QACD,iBAAiB,EAAE;YACjB,CAAC,EAAE,CAAC,MAAM,CAAC;SACZ;QACD,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,IAAA,+BAAkB,EAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;AACJ,CAAC;AAED,SAAgB,qBAAqB,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,gCAAmB,EAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,OAAO,QAAQ,IAAI,EAAE,CAAA;AACvB,CAAC;AAED,SAAgB,uBAAuB,CAAC,IAAY;IAClD,MAAM,SAAS,GAAG,IAAA,uBAAY,EAAC,IAAI,EAAE;QACnC,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;SACL;QACD,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,IAAA,+BAAkB,EAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,CAAC,GAAG,IAAA,cAAI,EAAC,SAAS,CAAC,CAAA;IACzB,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,CAAC,CAAC,uCAAuC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnE,OAAM;QACR,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEzC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7B,MAAM,IAAI,GAAG,IAAA,gCAAmB,EAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAM;QACR,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACrB,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;YAC5B,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACrC,OAAM;QACR,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrB,CAAC,CAAC,CAAA;IAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAA,gCAAmB,EAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,CAAA;QACnE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACnC,CAAC;IAED,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAA;AAC3C,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,IAAA,uBAAY,EAAC,IAAI,EAAE;QAClC,WAAW,EAAE,EAAE;QACf,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;KACF,CAAC,CAAA;IACF,OAAO,IAAA,+BAAkB,EAAC,QAAQ,CAAC,CAAA;AACrC,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAkB;IAClD,gGAAgG;IAChG,2FAA2F;IAC3F,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AACjC,CAAC"}

package/dist/cjs/content/link-preview/content/cleaner.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeForPrompt = normalizeForPrompt;
+exports.normalizeWhitespace = normalizeWhitespace;
+exports.decodeHtmlEntities = decodeHtmlEntities;
+exports.normalizeCandidate = normalizeCandidate;
+exports.clipAtSentenceBoundary = clipAtSentenceBoundary;
+exports.applyContentBudget = applyContentBudget;
+const es_toolkit_1 = require("es-toolkit");
+const WORD_SPLIT_PATTERN = /\s+/g;
+function normalizeForPrompt(input) {
+    return input
+        .replaceAll('\u00A0', ' ')
+        .replaceAll(/[\t ]+/g, ' ')
+        .replaceAll(/\s*\n\s*/g, '\n')
+        .replaceAll(/\n{3,}/g, '\n\n')
+        .trim();
+}
+function normalizeWhitespace(input) {
+    return input
+        .replaceAll('\u00A0', ' ')
+        .replaceAll(/[\t ]+/g, ' ')
+        .replaceAll(/\s*\n\s*/g, '\n')
+        .trim();
+}
+function decodeHtmlEntities(input) {
+    return input
+        .replaceAll('&', '&')
+        .replaceAll('&lt;', '<')
+        .replaceAll('&gt;', '>')
+        .replaceAll('&quot;', '"')
+        .replaceAll('&#39;', "'")
+        .replaceAll('&#x27;', "'")
+        .replaceAll('&#x2F;', '/')
+        .replaceAll('&nbsp;', ' ');
+}
+function normalizeCandidate(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.replaceAll(/\s+/g, ' ').trim();
+    return trimmed.length > 0 ? trimmed : null;
+}
+function clipAtSentenceBoundary(input, maxLength) {
+    if (input.length <= maxLength) {
+        return input;
+    }
+    const slice = input.slice(0, maxLength);
+    const lastSentenceBreak = Math.max(slice.lastIndexOf('. '), slice.lastIndexOf('! '), slice.lastIndexOf('? '), slice.lastIndexOf('\n\n'));
+    if (lastSentenceBreak > maxLength * 0.5) {
+        return slice.slice(0, lastSentenceBreak + 1);
+    }
+    return slice;
+}
+function applyContentBudget(baseContent, maxCharacters) {
+    const totalCharacters = baseContent.length;
+    const truncated = totalCharacters > maxCharacters;
+    const clipped = truncated ? clipAtSentenceBoundary(baseContent, maxCharacters) : baseContent;
+    const content = clipped.trim();
+    const wordCount = content.length > 0 ? (0, es_toolkit_1.compact)(content.split(WORD_SPLIT_PATTERN)).length : 0;
+    return { content, truncated, totalCharacters, wordCount };
+}
+//# sourceMappingURL=cleaner.js.map

package/dist/cjs/content/link-preview/content/cleaner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/cleaner.ts"],"names":[],"mappings":";;AAWA,gDAOC;AAED,kDAMC;AAED,gDAUC;AAED,gDAMC;AAED,wDAeC;AAED,gDAUC;AA3ED,2CAAoC;AAEpC,MAAM,kBAAkB,GAAG,MAAM,CAAA;AASjC,SAAgB,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,SAAgB,mBAAmB,CAAC,KAAa;IAC/C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC9B,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAAgC;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAA;IACb,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;AAC5C,CAAC;AAED,SAAgB,sBAAsB,CAAC,KAAa,EAAE,SAAiB;IACrE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAChC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAC1B,CAAA;IACD,IAAI,iBAAiB,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAgB,kBAAkB,CAChC,WAAmB,EACnB,aAAqB;IAErB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAA;IAC1C,MAAM,SAAS,GAAG,eAAe,GAAG,aAAa,CAAA;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAA;IAC5F,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAA,oBAAO,EAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5F,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,CAAA;AAC3D,CAAC"}

package/dist/cjs/content/link-preview/content/constants.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK = exports.READABILITY_RELATIVE_THRESHOLD = exports.MIN_METADATA_DESCRIPTION_CHARACTERS = exports.MIN_READABILITY_CONTENT_CHARACTERS = exports.MIN_HTML_CONTENT_CHARACTERS = exports.BLOCKED_HTML_HINT_PATTERN = void 0;
+exports.BLOCKED_HTML_HINT_PATTERN = /access denied|attention required|captcha|cloudflare|enable javascript|forbidden|please turn javascript on|verify you are human/i;
+exports.MIN_HTML_CONTENT_CHARACTERS = 200;
+exports.MIN_READABILITY_CONTENT_CHARACTERS = 200;
+exports.MIN_METADATA_DESCRIPTION_CHARACTERS = 120;
+exports.READABILITY_RELATIVE_THRESHOLD = 0.6;
+exports.MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK = 5000;
+//# sourceMappingURL=constants.js.map

package/dist/cjs/content/link-preview/content/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,yBAAyB,GACpC,iIAAiI,CAAA;AACtH,QAAA,2BAA2B,GAAG,GAAG,CAAA;AACjC,QAAA,kCAAkC,GAAG,GAAG,CAAA;AACxC,QAAA,mCAAmC,GAAG,GAAG,CAAA;AACzC,QAAA,8BAA8B,GAAG,GAAG,CAAA;AACpC,QAAA,yCAAyC,GAAG,IAAI,CAAA"}

package/dist/cjs/content/link-preview/content/fetcher.js

@@ -0,0 +1,128 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchHtmlDocument = fetchHtmlDocument;
+exports.fetchWithFirecrawl = fetchWithFirecrawl;
+const utils_js_1 = require("../../transcript/utils.js");
+const utils_js_2 = require("./utils.js");
+const REQUEST_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Cache-Control': 'no-cache',
+    Pragma: 'no-cache',
+};
+const DEFAULT_REQUEST_TIMEOUT_MS = 5000;
+async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress, } = {}) {
+    onProgress?.({ kind: 'fetch-html-start', url });
+    const controller = new AbortController();
+    const effectiveTimeoutMs = typeof timeoutMs === 'number' && Number.isFinite(timeoutMs)
+        ? timeoutMs
+        : DEFAULT_REQUEST_TIMEOUT_MS;
+    const timeout = setTimeout(() => {
+        controller.abort();
+    }, effectiveTimeoutMs);
+    try {
+        const response = await fetchImpl(url, {
+            headers: REQUEST_HEADERS,
+            redirect: 'follow',
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch HTML document (status ${response.status})`);
+        }
+        const contentType = response.headers.get('content-type')?.toLowerCase() ?? null;
+        if (contentType &&
+            !contentType.includes('text/html') &&
+            !contentType.includes('application/xhtml+xml') &&
+            !contentType.includes('application/xml') &&
+            !contentType.includes('text/xml') &&
+            !contentType.includes('application/rss+xml') &&
+            !contentType.includes('application/atom+xml') &&
+            !contentType.startsWith('text/')) {
+            throw new Error(`Unsupported content-type for HTML document fetch: ${contentType}`);
+        }
+        const totalBytes = (() => {
+            const raw = response.headers.get('content-length');
+            if (!raw)
+                return null;
+            const parsed = Number(raw);
+            return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : null;
+        })();
+        const body = response.body;
+        if (!body) {
+            const text = await response.text();
+            const bytes = new TextEncoder().encode(text).byteLength;
+            onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes: bytes, totalBytes });
+            return text;
+        }
+        const reader = body.getReader();
+        const decoder = new TextDecoder();
+        let downloadedBytes = 0;
+        let text = '';
+        onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes: 0, totalBytes });
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            if (!value)
+                continue;
+            downloadedBytes += value.byteLength;
+            text += decoder.decode(value, { stream: true });
+            onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes, totalBytes });
+        }
+        text += decoder.decode();
+        onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes, totalBytes });
+        return text;
+    }
+    catch (error) {
+        if (error instanceof DOMException && error.name === 'AbortError') {
+            throw new Error('Fetching HTML document timed out');
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+async function fetchWithFirecrawl(url, scrapeWithFirecrawl, options = {}) {
+    const timeoutMs = options.timeoutMs;
+    const cacheMode = options.cacheMode ?? 'default';
+    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
+    const reason = typeof options.reason === 'string' ? options.reason : null;
+    const diagnostics = {
+        attempted: false,
+        used: false,
+        cacheMode,
+        cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
+        notes: null,
+    };
+    if ((0, utils_js_1.isYouTubeUrl)(url)) {
+        diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, 'Skipped Firecrawl for YouTube URL');
+        return { payload: null, diagnostics };
+    }
+    if (!scrapeWithFirecrawl) {
+        diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, 'Firecrawl is not configured');
+        return { payload: null, diagnostics };
+    }
+    diagnostics.attempted = true;
+    onProgress?.({ kind: 'firecrawl-start', url, reason: reason ?? 'firecrawl' });
+    try {
+        const payload = await scrapeWithFirecrawl(url, { timeoutMs, cacheMode });
+        if (!payload) {
+            diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, 'Firecrawl returned no content payload');
+            onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
+            return { payload: null, diagnostics };
+        }
+        const encoder = new TextEncoder();
+        const markdownBytes = typeof payload.markdown === 'string' ? encoder.encode(payload.markdown).byteLength : null;
+        const htmlBytes = typeof payload.html === 'string' ? encoder.encode(payload.html).byteLength : null;
+        onProgress?.({ kind: 'firecrawl-done', url, ok: true, markdownBytes, htmlBytes });
+        return { payload, diagnostics };
+    }
+    catch (error) {
+        diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, `Firecrawl error: ${error instanceof Error ? error.message : 'unknown error'}`);
+        onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
+        return { payload: null, diagnostics };
+    }
+}
+//# sourceMappingURL=fetcher.js.map

package/dist/cjs/content/link-preview/content/fetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/fetcher.ts"],"names":[],"mappings":";;AA2BA,8CAsFC;AAED,gDA2DC;AA9KD,wDAAwD;AAQxD,yCAAuC;AAEvC,MAAM,eAAe,GAA2B;IAC9C,YAAY,EACV,iHAAiH;IACnH,MAAM,EACJ,kGAAkG;IACpG,iBAAiB,EAAE,gBAAgB;IACnC,eAAe,EAAE,UAAU;IAC3B,MAAM,EAAE,UAAU;CACnB,CAAA;AAED,MAAM,0BAA0B,GAAG,IAAI,CAAA;AAOhC,KAAK,UAAU,iBAAiB,CACrC,SAAuB,EACvB,GAAW,EACX,EACE,SAAS,EACT,UAAU,MACiF,EAAE;IAE/F,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAA;IAE/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,kBAAkB,GACtB,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;QACzD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,0BAA0B,CAAA;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC,EAAE,kBAAkB,CAAC,CAAA;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YACpC,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;QAC9E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAA;QAC/E,IACE,WAAW;YACX,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;YAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9C,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACxC,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC;YACjC,CAAC,WAAW,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YAC5C,CAAC,WAAW,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC7C,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,EAChC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,WAAW,EAAE,CAAC,CAAA;QACrF,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE;YACvB,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAA;YACrB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;YAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC1E,CAAC,CAAC,EAAE,CAAA;QAEJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YAClC,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAA;YACvD,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,eAAe,GAAG,CAAC,CAAA;QACvB,IAAI,IAAI,GAAG,EAAE,CAAA;QAEb,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAA;QAElF,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3C,IAAI,IAAI;gBAAE,MAAK;YACf,IAAI,CAAC,KAAK;gBAAE,SAAQ;YACpB,eAAe,IAAI,KAAK,CAAC,UAAU,CAAA;YACnC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/C,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAA;QACjF,CAAC;QAED,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;QACxB,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAA;QAC3E,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QACrD,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAA;IACvB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,mBAA+C,EAC/C,UAKI,EAAE;IAEN,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;IACnC,MAAM,SAAS,GAAc,OAAO,CAAC,SAAS,IAAI,SAAS,CAAA;IAC3D,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IACvF,MAAM,MAAM,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IACzE,MAAM,WAAW,GAAyB;QACxC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,KAAK;QACX,SAAS;QACT,WAAW,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC5D,KAAK,EAAE,IAAI;KACZ,CAAA;IAED,IAAI,IAAA,uBAAY,EAAC,GAAG,CAAC,EAAE,CAAC;QACtB,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAAC,WAAW,CAAC,KAAK,EAAE,mCAAmC,CAAC,CAAA;QACtF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAAC,WAAW,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAA;QAChF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,WAAW,CAAC,SAAS,GAAG,IAAI,CAAA;IAC5B,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW,EAAE,CAAC,CAAA;IAE7E,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAAC,WAAW,CAAC,KAAK,EAAE,uCAAuC,CAAC,CAAA;YAC1F,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACvC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,aAAa,GACjB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3F,MAAM,SAAS,GACb,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;QACnF,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAA;QAEjF,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAC5B,WAAW,CAAC,KAAK,EACjB,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC/E,CAAA;QACD,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;AACH,CAAC"}

package/dist/cjs/content/link-preview/content/firecrawl.js

@@ -0,0 +1,90 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shouldFallbackToFirecrawl = shouldFallbackToFirecrawl;
+exports.buildResultFromFirecrawl = buildResultFromFirecrawl;
+const index_js_1 = require("../../transcript/index.js");
+const article_js_1 = require("./article.js");
+const cleaner_js_1 = require("./cleaner.js");
+const constants_js_1 = require("./constants.js");
+const jsonld_js_1 = require("./jsonld.js");
+const parsers_js_1 = require("./parsers.js");
+const podcast_utils_js_1 = require("./podcast-utils.js");
+const utils_js_1 = require("./utils.js");
+const video_js_1 = require("./video.js");
+function shouldFallbackToFirecrawl(html) {
+    const plainText = (0, cleaner_js_1.normalizeForPrompt)((0, article_js_1.extractPlainText)(html));
+    if (constants_js_1.BLOCKED_HTML_HINT_PATTERN.test(plainText))
+        return true;
+    const normalized = (0, cleaner_js_1.normalizeForPrompt)((0, article_js_1.extractArticleContent)(html));
+    if (normalized.length >= constants_js_1.MIN_HTML_CONTENT_CHARACTERS) {
+        return false;
+    }
+    // Avoid spending Firecrawl on truly small/simple pages where the extracted HTML content is short but
+    // likely complete (e.g. https://example.com). Only treat "thin" content as a Firecrawl signal when
+    // the HTML document itself is large (SSR/app-shell pages, blocked pages without a match, etc.).
+    return html.length >= constants_js_1.MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK;
+}
+async function buildResultFromFirecrawl({ url, payload, cacheMode, maxCharacters, youtubeTranscriptMode, firecrawlDiagnostics, markdownRequested, deps, }) {
+    const normalizedMarkdown = (0, cleaner_js_1.normalizeForPrompt)(payload.markdown ?? '');
+    if (normalizedMarkdown.length === 0) {
+        firecrawlDiagnostics.notes = (0, utils_js_1.appendNote)(firecrawlDiagnostics.notes, 'Firecrawl markdown normalization yielded empty text');
+        return null;
+    }
+    const jsonLd = payload.html ? (0, jsonld_js_1.extractJsonLdContent)(payload.html) : null;
+    const isPodcastJsonLd = (0, podcast_utils_js_1.isPodcastLikeJsonLdType)(jsonLd?.type);
+    const transcriptResolution = await (0, index_js_1.resolveTranscriptForLink)(url, payload.html ?? null, deps, {
+        youtubeTranscriptMode,
+        cacheMode,
+    });
+    const htmlMetadata = payload.html
+        ? (0, parsers_js_1.extractMetadataFromHtml)(payload.html, url)
+        : { title: null, description: null, siteName: null };
+    const metadata = (0, parsers_js_1.extractMetadataFromFirecrawl)(payload.metadata ?? null);
+    const title = (0, utils_js_1.pickFirstText)([jsonLd?.title, metadata.title, htmlMetadata.title]);
+    const description = (0, utils_js_1.pickFirstText)([
+        jsonLd?.description,
+        metadata.description,
+        htmlMetadata.description,
+    ]);
+    const siteName = (0, utils_js_1.pickFirstText)([metadata.siteName, htmlMetadata.siteName, (0, utils_js_1.safeHostname)(url)]);
+    const descriptionCandidate = description ? (0, cleaner_js_1.normalizeForPrompt)(description) : '';
+    const preferDescription = descriptionCandidate.length >= constants_js_1.MIN_METADATA_DESCRIPTION_CHARACTERS &&
+        (isPodcastJsonLd ||
+            (0, podcast_utils_js_1.isPodcastHost)(url) ||
+            normalizedMarkdown.length < constants_js_1.MIN_HTML_CONTENT_CHARACTERS ||
+            descriptionCandidate.length >= normalizedMarkdown.length * constants_js_1.READABILITY_RELATIVE_THRESHOLD);
+    const baseCandidate = preferDescription ? descriptionCandidate : normalizedMarkdown;
+    const baseContent = (0, utils_js_1.selectBaseContent)(baseCandidate, transcriptResolution.text);
+    if (baseContent.length === 0) {
+        firecrawlDiagnostics.notes = (0, utils_js_1.appendNote)(firecrawlDiagnostics.notes, 'Firecrawl produced content that normalized to an empty string');
+        return null;
+    }
+    firecrawlDiagnostics.used = true;
+    const transcriptDiagnostics = (0, utils_js_1.ensureTranscriptDiagnostics)(transcriptResolution, cacheMode ?? 'default');
+    const video = payload.html ? (0, video_js_1.detectPrimaryVideoFromHtml)(payload.html, url) : null;
+    const isVideoOnly = !transcriptResolution.text &&
+        normalizedMarkdown.length < constants_js_1.MIN_HTML_CONTENT_CHARACTERS &&
+        video !== null;
+    return (0, utils_js_1.finalizeExtractedLinkContent)({
+        url,
+        baseContent,
+        maxCharacters,
+        title,
+        description,
+        siteName,
+        transcriptResolution,
+        video,
+        isVideoOnly,
+        diagnostics: {
+            strategy: 'firecrawl',
+            firecrawl: firecrawlDiagnostics,
+            markdown: {
+                requested: markdownRequested,
+                used: true,
+                provider: 'firecrawl',
+            },
+            transcript: transcriptDiagnostics,
+        },
+    });
+}
+//# sourceMappingURL=firecrawl.js.map

package/dist/cjs/content/link-preview/content/firecrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/firecrawl.ts"],"names":[],"mappings":";;AA0BA,8DAYC;AAED,4DAmGC;AA3ID,wDAAoE;AAGpE,6CAAsE;AACtE,6CAAiD;AACjD,iDAMuB;AACvB,2CAAkD;AAClD,6CAAoF;AACpF,yDAA2E;AAE3E,yCAOmB;AACnB,yCAAuD;AAEvD,SAAgB,yBAAyB,CAAC,IAAY;IACpD,MAAM,SAAS,GAAG,IAAA,+BAAkB,EAAC,IAAA,6BAAgB,EAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,wCAAyB,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAA;IAC1D,MAAM,UAAU,GAAG,IAAA,+BAAkB,EAAC,IAAA,kCAAqB,EAAC,IAAI,CAAC,CAAC,CAAA;IAClE,IAAI,UAAU,CAAC,MAAM,IAAI,0CAA2B,EAAE,CAAC;QACrD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,qGAAqG;IACrG,mGAAmG;IACnG,gGAAgG;IAChG,OAAO,IAAI,CAAC,MAAM,IAAI,wDAAyC,CAAA;AACjE,CAAC;AAEM,KAAK,UAAU,wBAAwB,CAAC,EAC7C,GAAG,EACH,OAAO,EACP,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,IAAI,GAUL;IACC,MAAM,kBAAkB,GAAG,IAAA,+BAAkB,EAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACrE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,oBAAoB,CAAC,KAAK,GAAG,IAAA,qBAAU,EACrC,oBAAoB,CAAC,KAAK,EAC1B,qDAAqD,CACtD,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,gCAAoB,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACvE,MAAM,eAAe,GAAG,IAAA,0CAAuB,EAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAE7D,MAAM,oBAAoB,GAAG,MAAM,IAAA,mCAAwB,EAAC,GAAG,EAAE,OAAO,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE;QAC3F,qBAAqB;QACrB,SAAS;KACV,CAAC,CAAA;IACF,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI;QAC/B,CAAC,CAAC,IAAA,oCAAuB,EAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;QAC5C,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IACtD,MAAM,QAAQ,GAAG,IAAA,yCAA4B,EAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAA;IAEvE,MAAM,KAAK,GAAG,IAAA,wBAAa,EAAC,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChF,MAAM,WAAW,GAAG,IAAA,wBAAa,EAAC;QAChC,MAAM,EAAE,WAAW;QACnB,QAAQ,CAAC,WAAW;QACpB,YAAY,CAAC,WAAW;KACzB,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,IAAA,wBAAa,EAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC,QAAQ,EAAE,IAAA,uBAAY,EAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAE7F,MAAM,oBAAoB,GAAG,WAAW,CAAC,CAAC,CAAC,IAAA,+BAAkB,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/E,MAAM,iBAAiB,GACrB,oBAAoB,CAAC,MAAM,IAAI,kDAAmC;QAClE,CAAC,eAAe;YACd,IAAA,gCAAa,EAAC,GAAG,CAAC;YAClB,kBAAkB,CAAC,MAAM,GAAG,0CAA2B;YACvD,oBAAoB,CAAC,MAAM,IAAI,kBAAkB,CAAC,MAAM,GAAG,6CAA8B,CAAC,CAAA;IAC9F,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,kBAAkB,CAAA;IACnF,MAAM,WAAW,GAAG,IAAA,4BAAiB,EAAC,aAAa,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAA;IAC/E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,oBAAoB,CAAC,KAAK,GAAG,IAAA,qBAAU,EACrC,oBAAoB,CAAC,KAAK,EAC1B,+DAA+D,CAChE,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,oBAAoB,CAAC,IAAI,GAAG,IAAI,CAAA;IAEhC,MAAM,qBAAqB,GAAG,IAAA,sCAA2B,EACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,qCAA0B,EAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACjF,MAAM,WAAW,GACf,CAAC,oBAAoB,CAAC,IAAI;QAC1B,kBAAkB,CAAC,MAAM,GAAG,0CAA2B;QACvD,KAAK,KAAK,IAAI,CAAA;IAEhB,OAAO,IAAA,uCAA4B,EAAC;QAClC,GAAG;QACH,WAAW;QACX,aAAa;QACb,KAAK;QACL,WAAW;QACX,QAAQ;QACR,oBAAoB;QACpB,KAAK;QACL,WAAW;QACX,WAAW,EAAE;YACX,QAAQ,EAAE,WAAW;YACrB,SAAS,EAAE,oBAAoB;YAC/B,QAAQ,EAAE;gBACR,SAAS,EAAE,iBAAiB;gBAC5B,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,WAAW;aACtB;YACD,UAAU,EAAE,qBAAqB;SAClC;KACF,CAAC,CAAA;AACJ,CAAC"}