@steipete/summarize-core 0.7.1 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steipete/summarize-core",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Summarize core library (content extraction + prompts).",
   "type": "module",
   "main": "./dist/esm/index.js",
@@ -9,23 +9,23 @@
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",
-      "import": "./dist/esm/index.js",
-      "require": "./dist/cjs/index.js"
+      "import": "./dist/esm/index.js"
     },
     "./content": {
       "types": "./dist/types/content/index.d.ts",
-      "import": "./dist/esm/content/index.js",
-      "require": "./dist/cjs/content/index.js"
+      "import": "./dist/esm/content/index.js"
+    },
+    "./content/url": {
+      "types": "./dist/types/content/url.d.ts",
+      "import": "./dist/esm/content/url.js"
     },
     "./prompts": {
       "types": "./dist/types/prompts/index.d.ts",
-      "import": "./dist/esm/prompts/index.js",
-      "require": "./dist/cjs/prompts/index.js"
+      "import": "./dist/esm/prompts/index.js"
     },
     "./language": {
       "types": "./dist/types/language.d.ts",
-      "import": "./dist/esm/language.js",
-      "require": "./dist/cjs/language.js"
+      "import": "./dist/esm/language.js"
     }
   },
   "files": [
@@ -51,7 +51,7 @@
   },
   "scripts": {
     "clean": "rm -rf dist",
-    "build": "pnpm clean && tsc -p tsconfig.build.json && tsc -p tsconfig.build.cjs.json && node scripts/postbuild-cjs.mjs",
+    "build": "pnpm clean && tsc -p tsconfig.build.json",
     "typecheck": "tsc -p tsconfig.build.json --noEmit"
   }
 }

package/dist/cjs/content/index.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CACHE_MODES = exports.ProgressKind = exports.DEFAULT_TIMEOUT_MS = exports.DEFAULT_MAX_CONTENT_CHARACTERS = exports.DEFAULT_CACHE_MODE = exports.createLinkPreviewClient = void 0;
-var client_js_1 = require("./link-preview/client.js");
-Object.defineProperty(exports, "createLinkPreviewClient", { enumerable: true, get: function () { return client_js_1.createLinkPreviewClient; } });
-var types_js_1 = require("./link-preview/content/types.js");
-Object.defineProperty(exports, "DEFAULT_CACHE_MODE", { enumerable: true, get: function () { return types_js_1.DEFAULT_CACHE_MODE; } });
-Object.defineProperty(exports, "DEFAULT_MAX_CONTENT_CHARACTERS", { enumerable: true, get: function () { return types_js_1.DEFAULT_MAX_CONTENT_CHARACTERS; } });
-Object.defineProperty(exports, "DEFAULT_TIMEOUT_MS", { enumerable: true, get: function () { return types_js_1.DEFAULT_TIMEOUT_MS; } });
-var deps_js_1 = require("./link-preview/deps.js");
-Object.defineProperty(exports, "ProgressKind", { enumerable: true, get: function () { return deps_js_1.ProgressKind; } });
-var types_js_2 = require("./link-preview/types.js");
-Object.defineProperty(exports, "CACHE_MODES", { enumerable: true, get: function () { return types_js_2.CACHE_MODES; } });
-//# sourceMappingURL=index.js.map

package/dist/cjs/content/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/content/index.ts"],"names":[],"mappings":";;;AAAA,sDAIiC;AAH/B,oHAAA,uBAAuB,OAAA;AAIzB,4DAMwC;AALtC,8GAAA,kBAAkB,OAAA;AAClB,0HAAA,8BAA8B,OAAA;AAC9B,8GAAA,kBAAkB,OAAA;AAapB,kDAAqD;AAA5C,uGAAA,YAAY,OAAA;AACrB,oDAKgC;AAJ9B,uGAAA,WAAW,OAAA"}

package/dist/cjs/content/link-preview/client.js

@@ -1,31 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createLinkPreviewClient = createLinkPreviewClient;
-const index_js_1 = require("./content/index.js");
-function createLinkPreviewClient(options = {}) {
-    const fetchImpl = options.fetch ?? ((...args) => globalThis.fetch(...args));
-    const scrape = options.scrapeWithFirecrawl ?? null;
-    const apifyApiToken = typeof options.apifyApiToken === 'string' ? options.apifyApiToken : null;
-    const ytDlpPath = typeof options.ytDlpPath === 'string' ? options.ytDlpPath : null;
-    const falApiKey = typeof options.falApiKey === 'string' ? options.falApiKey : null;
-    const openaiApiKey = typeof options.openaiApiKey === 'string' ? options.openaiApiKey : null;
-    const convertHtmlToMarkdown = options.convertHtmlToMarkdown ?? null;
-    const transcriptCache = options.transcriptCache ?? null;
-    const readTweetWithBird = typeof options.readTweetWithBird === 'function' ? options.readTweetWithBird : null;
-    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
-    return {
-        fetchLinkContent: (url, contentOptions) => (0, index_js_1.fetchLinkContent)(url, contentOptions, {
-            fetch: fetchImpl,
-            scrapeWithFirecrawl: scrape,
-            apifyApiToken,
-            ytDlpPath,
-            falApiKey,
-            openaiApiKey,
-            convertHtmlToMarkdown,
-            transcriptCache,
-            readTweetWithBird,
-            onProgress,
-        }),
-    };
-}
-//# sourceMappingURL=client.js.map

package/dist/cjs/content/link-preview/client.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../src/content/link-preview/client.ts"],"names":[],"mappings":";;AA2BA,0DA6BC;AAxDD,iDAAqD;AA2BrD,SAAgB,uBAAuB,CAAC,UAAoC,EAAE;IAC5E,MAAM,SAAS,GACb,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;IACrF,MAAM,MAAM,GAA+B,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAA;IAC9E,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9F,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3F,MAAM,qBAAqB,GAAiC,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAA;IACjG,MAAM,eAAe,GAA2B,OAAO,CAAC,eAAe,IAAI,IAAI,CAAA;IAC/E,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAA;IACpF,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IAEvF,OAAO;QACL,gBAAgB,EAAE,CAAC,GAAW,EAAE,cAAwC,EAAE,EAAE,CAC1E,IAAA,2BAAgB,EAAC,GAAG,EAAE,cAAc,EAAE;YACpC,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,MAAM;YAC3B,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,qBAAqB;YACrB,eAAe;YACf,iBAAiB;YACjB,UAAU;SACX,CAAC;KACL,CAAA;AACH,CAAC"}

package/dist/cjs/content/link-preview/content/article.js

@@ -1,164 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.sanitizeHtmlForMarkdownConversion = sanitizeHtmlForMarkdownConversion;
-exports.extractArticleContent = extractArticleContent;
-exports.collectSegmentsFromHtml = collectSegmentsFromHtml;
-exports.extractPlainText = extractPlainText;
-const cheerio_1 = require("cheerio");
-const sanitize_html_1 = __importDefault(require("sanitize-html"));
-const cleaner_js_1 = require("./cleaner.js");
-const MIN_SEGMENT_LENGTH = 30;
-function sanitizeHtmlForMarkdownConversion(html) {
-    return (0, sanitize_html_1.default)(html, {
-        allowedTags: [
-            'article',
-            'section',
-            'div',
-            'p',
-            'h1',
-            'h2',
-            'h3',
-            'h4',
-            'h5',
-            'h6',
-            'ol',
-            'ul',
-            'li',
-            'blockquote',
-            'pre',
-            'code',
-            'span',
-            'strong',
-            'em',
-            'br',
-            'a',
-        ],
-        allowedAttributes: {
-            a: ['href'],
-        },
-        nonTextTags: [
-            'style',
-            'script',
-            'noscript',
-            'template',
-            'svg',
-            'canvas',
-            'iframe',
-            'object',
-            'embed',
-        ],
-        textFilter(text) {
-            return (0, cleaner_js_1.decodeHtmlEntities)(text);
-        },
-    });
-}
-function extractArticleContent(html) {
-    const segments = collectSegmentsFromHtml(html);
-    if (segments.length > 0) {
-        return segments.join('\n');
-    }
-    const fallback = (0, cleaner_js_1.normalizeWhitespace)(extractPlainText(html));
-    return fallback ?? '';
-}
-function collectSegmentsFromHtml(html) {
-    const sanitized = (0, sanitize_html_1.default)(html, {
-        allowedTags: [
-            'article',
-            'section',
-            'div',
-            'p',
-            'h1',
-            'h2',
-            'h3',
-            'h4',
-            'h5',
-            'h6',
-            'ol',
-            'ul',
-            'li',
-            'blockquote',
-            'pre',
-            'code',
-            'span',
-            'strong',
-            'em',
-            'br',
-        ],
-        allowedAttributes: {},
-        nonTextTags: [
-            'style',
-            'script',
-            'noscript',
-            'template',
-            'svg',
-            'canvas',
-            'iframe',
-            'object',
-            'embed',
-        ],
-        textFilter(text) {
-            return (0, cleaner_js_1.decodeHtmlEntities)(text);
-        },
-    });
-    const $ = (0, cheerio_1.load)(sanitized);
-    const segments = [];
-    $('h1,h2,h3,h4,h5,h6,li,p,blockquote,pre').each((_, element) => {
-        if (!('tagName' in element) || typeof element.tagName !== 'string') {
-            return;
-        }
-        const tag = element.tagName.toLowerCase();
-        const raw = $(element).text();
-        const text = (0, cleaner_js_1.normalizeWhitespace)(raw).replaceAll(/\n+/g, ' ');
-        if (!text || text.length === 0) {
-            return;
-        }
-        if (tag.startsWith('h')) {
-            if (text.length >= 10) {
-                segments.push(text);
-            }
-            return;
-        }
-        if (tag === 'li') {
-            if (text.length >= 20) {
-                segments.push(`• ${text}`);
-            }
-            return;
-        }
-        if (text.length < MIN_SEGMENT_LENGTH) {
-            return;
-        }
-        segments.push(text);
-    });
-    if (segments.length === 0) {
-        const fallback = (0, cleaner_js_1.normalizeWhitespace)($('body').text() || sanitized);
-        return fallback ? [fallback] : [];
-    }
-    return mergeConsecutiveSegments(segments);
-}
-function extractPlainText(html) {
-    const stripped = (0, sanitize_html_1.default)(html, {
-        allowedTags: [],
-        allowedAttributes: {},
-        nonTextTags: [
-            'style',
-            'script',
-            'noscript',
-            'template',
-            'svg',
-            'canvas',
-            'iframe',
-            'object',
-            'embed',
-        ],
-    });
-    return (0, cleaner_js_1.decodeHtmlEntities)(stripped);
-}
-function mergeConsecutiveSegments(segments) {
-    // Keep headings as separate segments; merging short segments mostly collapses headings into the
-    // previous paragraph ("... Conclusion"), which reads worse than a standalone heading line.
-    return segments.filter(Boolean);
-}
-//# sourceMappingURL=article.js.map

package/dist/cjs/content/link-preview/content/article.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/article.ts"],"names":[],"mappings":";;;;;AAOA,8EA2CC;AAED,sDAOC;AAED,0DAoFC;AAED,4CAiBC;AApKD,qCAA8B;AAC9B,kEAAwC;AAExC,6CAAsE;AAEtE,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,SAAgB,iCAAiC,CAAC,IAAY;IAC5D,OAAO,IAAA,uBAAY,EAAC,IAAI,EAAE;QACxB,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;YACJ,GAAG;SACJ;QACD,iBAAiB,EAAE;YACjB,CAAC,EAAE,CAAC,MAAM,CAAC;SACZ;QACD,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,IAAA,+BAAkB,EAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;AACJ,CAAC;AAED,SAAgB,qBAAqB,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IACD,MAAM,QAAQ,GAAG,IAAA,gCAAmB,EAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,OAAO,QAAQ,IAAI,EAAE,CAAA;AACvB,CAAC;AAED,SAAgB,uBAAuB,CAAC,IAAY;IAClD,MAAM,SAAS,GAAG,IAAA,uBAAY,EAAC,IAAI,EAAE;QACnC,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;SACL;QACD,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,IAAA,+BAAkB,EAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,CAAC,GAAG,IAAA,cAAI,EAAC,SAAS,CAAC,CAAA;IACzB,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,CAAC,CAAC,uCAAuC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnE,OAAM;QACR,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEzC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7B,MAAM,IAAI,GAAG,IAAA,gCAAmB,EAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAM;QACR,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACrB,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;YAC5B,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACrC,OAAM;QACR,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrB,CAAC,CAAC,CAAA;IAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAA,gCAAmB,EAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,CAAA;QACnE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACnC,CAAC;IAED,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAA;AAC3C,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,IAAA,uBAAY,EAAC,IAAI,EAAE;QAClC,WAAW,EAAE,EAAE;QACf,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;KACF,CAAC,CAAA;IACF,OAAO,IAAA,+BAAkB,EAAC,QAAQ,CAAC,CAAA;AACrC,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAkB;IAClD,gGAAgG;IAChG,2FAA2F;IAC3F,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AACjC,CAAC"}

package/dist/cjs/content/link-preview/content/cleaner.js

@@ -1,63 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeForPrompt = normalizeForPrompt;
-exports.normalizeWhitespace = normalizeWhitespace;
-exports.decodeHtmlEntities = decodeHtmlEntities;
-exports.normalizeCandidate = normalizeCandidate;
-exports.clipAtSentenceBoundary = clipAtSentenceBoundary;
-exports.applyContentBudget = applyContentBudget;
-const es_toolkit_1 = require("es-toolkit");
-const WORD_SPLIT_PATTERN = /\s+/g;
-function normalizeForPrompt(input) {
-    return input
-        .replaceAll('\u00A0', ' ')
-        .replaceAll(/[\t ]+/g, ' ')
-        .replaceAll(/\s*\n\s*/g, '\n')
-        .replaceAll(/\n{3,}/g, '\n\n')
-        .trim();
-}
-function normalizeWhitespace(input) {
-    return input
-        .replaceAll('\u00A0', ' ')
-        .replaceAll(/[\t ]+/g, ' ')
-        .replaceAll(/\s*\n\s*/g, '\n')
-        .trim();
-}
-function decodeHtmlEntities(input) {
-    return input
-        .replaceAll('&', '&')
-        .replaceAll('&lt;', '<')
-        .replaceAll('&gt;', '>')
-        .replaceAll('&quot;', '"')
-        .replaceAll('&#39;', "'")
-        .replaceAll('&#x27;', "'")
-        .replaceAll('&#x2F;', '/')
-        .replaceAll('&nbsp;', ' ');
-}
-function normalizeCandidate(value) {
-    if (!value) {
-        return null;
-    }
-    const trimmed = value.replaceAll(/\s+/g, ' ').trim();
-    return trimmed.length > 0 ? trimmed : null;
-}
-function clipAtSentenceBoundary(input, maxLength) {
-    if (input.length <= maxLength) {
-        return input;
-    }
-    const slice = input.slice(0, maxLength);
-    const lastSentenceBreak = Math.max(slice.lastIndexOf('. '), slice.lastIndexOf('! '), slice.lastIndexOf('? '), slice.lastIndexOf('\n\n'));
-    if (lastSentenceBreak > maxLength * 0.5) {
-        return slice.slice(0, lastSentenceBreak + 1);
-    }
-    return slice;
-}
-function applyContentBudget(baseContent, maxCharacters) {
-    const totalCharacters = baseContent.length;
-    const truncated = totalCharacters > maxCharacters;
-    const clipped = truncated ? clipAtSentenceBoundary(baseContent, maxCharacters) : baseContent;
-    const content = clipped.trim();
-    const wordCount = content.length > 0 ? (0, es_toolkit_1.compact)(content.split(WORD_SPLIT_PATTERN)).length : 0;
-    return { content, truncated, totalCharacters, wordCount };
-}
-//# sourceMappingURL=cleaner.js.map

package/dist/cjs/content/link-preview/content/cleaner.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/cleaner.ts"],"names":[],"mappings":";;AAWA,gDAOC;AAED,kDAMC;AAED,gDAUC;AAED,gDAMC;AAED,wDAeC;AAED,gDAUC;AA3ED,2CAAoC;AAEpC,MAAM,kBAAkB,GAAG,MAAM,CAAA;AASjC,SAAgB,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,SAAgB,mBAAmB,CAAC,KAAa;IAC/C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC9B,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAAgC;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAA;IACb,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;AAC5C,CAAC;AAED,SAAgB,sBAAsB,CAAC,KAAa,EAAE,SAAiB;IACrE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAChC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAC1B,CAAA;IACD,IAAI,iBAAiB,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAgB,kBAAkB,CAChC,WAAmB,EACnB,aAAqB;IAErB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAA;IAC1C,MAAM,SAAS,GAAG,eAAe,GAAG,aAAa,CAAA;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAA;IAC5F,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAA,oBAAO,EAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5F,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,CAAA;AAC3D,CAAC"}

package/dist/cjs/content/link-preview/content/constants.js

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK = exports.READABILITY_RELATIVE_THRESHOLD = exports.MIN_METADATA_DESCRIPTION_CHARACTERS = exports.MIN_READABILITY_CONTENT_CHARACTERS = exports.MIN_HTML_CONTENT_CHARACTERS = exports.BLOCKED_HTML_HINT_PATTERN = void 0;
-exports.BLOCKED_HTML_HINT_PATTERN = /access denied|attention required|captcha|cloudflare|enable javascript|forbidden|please turn javascript on|verify you are human/i;
-exports.MIN_HTML_CONTENT_CHARACTERS = 200;
-exports.MIN_READABILITY_CONTENT_CHARACTERS = 200;
-exports.MIN_METADATA_DESCRIPTION_CHARACTERS = 120;
-exports.READABILITY_RELATIVE_THRESHOLD = 0.6;
-exports.MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK = 5000;
-//# sourceMappingURL=constants.js.map

package/dist/cjs/content/link-preview/content/constants.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,yBAAyB,GACpC,iIAAiI,CAAA;AACtH,QAAA,2BAA2B,GAAG,GAAG,CAAA;AACjC,QAAA,kCAAkC,GAAG,GAAG,CAAA;AACxC,QAAA,mCAAmC,GAAG,GAAG,CAAA;AACzC,QAAA,8BAA8B,GAAG,GAAG,CAAA;AACpC,QAAA,yCAAyC,GAAG,IAAI,CAAA"}

package/dist/cjs/content/link-preview/content/fetcher.js

@@ -1,128 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchHtmlDocument = fetchHtmlDocument;
-exports.fetchWithFirecrawl = fetchWithFirecrawl;
-const utils_js_1 = require("../../transcript/utils.js");
-const utils_js_2 = require("./utils.js");
-const REQUEST_HEADERS = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
-    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-    'Accept-Language': 'en-US,en;q=0.9',
-    'Cache-Control': 'no-cache',
-    Pragma: 'no-cache',
-};
-const DEFAULT_REQUEST_TIMEOUT_MS = 5000;
-async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress, } = {}) {
-    onProgress?.({ kind: 'fetch-html-start', url });
-    const controller = new AbortController();
-    const effectiveTimeoutMs = typeof timeoutMs === 'number' && Number.isFinite(timeoutMs)
-        ? timeoutMs
-        : DEFAULT_REQUEST_TIMEOUT_MS;
-    const timeout = setTimeout(() => {
-        controller.abort();
-    }, effectiveTimeoutMs);
-    try {
-        const response = await fetchImpl(url, {
-            headers: REQUEST_HEADERS,
-            redirect: 'follow',
-            signal: controller.signal,
-        });
-        if (!response.ok) {
-            throw new Error(`Failed to fetch HTML document (status ${response.status})`);
-        }
-        const contentType = response.headers.get('content-type')?.toLowerCase() ?? null;
-        if (contentType &&
-            !contentType.includes('text/html') &&
-            !contentType.includes('application/xhtml+xml') &&
-            !contentType.includes('application/xml') &&
-            !contentType.includes('text/xml') &&
-            !contentType.includes('application/rss+xml') &&
-            !contentType.includes('application/atom+xml') &&
-            !contentType.startsWith('text/')) {
-            throw new Error(`Unsupported content-type for HTML document fetch: ${contentType}`);
-        }
-        const totalBytes = (() => {
-            const raw = response.headers.get('content-length');
-            if (!raw)
-                return null;
-            const parsed = Number(raw);
-            return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : null;
-        })();
-        const body = response.body;
-        if (!body) {
-            const text = await response.text();
-            const bytes = new TextEncoder().encode(text).byteLength;
-            onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes: bytes, totalBytes });
-            return text;
-        }
-        const reader = body.getReader();
-        const decoder = new TextDecoder();
-        let downloadedBytes = 0;
-        let text = '';
-        onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes: 0, totalBytes });
-        while (true) {
-            const { value, done } = await reader.read();
-            if (done)
-                break;
-            if (!value)
-                continue;
-            downloadedBytes += value.byteLength;
-            text += decoder.decode(value, { stream: true });
-            onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes, totalBytes });
-        }
-        text += decoder.decode();
-        onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes, totalBytes });
-        return text;
-    }
-    catch (error) {
-        if (error instanceof DOMException && error.name === 'AbortError') {
-            throw new Error('Fetching HTML document timed out');
-        }
-        throw error;
-    }
-    finally {
-        clearTimeout(timeout);
-    }
-}
-async function fetchWithFirecrawl(url, scrapeWithFirecrawl, options = {}) {
-    const timeoutMs = options.timeoutMs;
-    const cacheMode = options.cacheMode ?? 'default';
-    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
-    const reason = typeof options.reason === 'string' ? options.reason : null;
-    const diagnostics = {
-        attempted: false,
-        used: false,
-        cacheMode,
-        cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
-        notes: null,
-    };
-    if ((0, utils_js_1.isYouTubeUrl)(url)) {
-        diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, 'Skipped Firecrawl for YouTube URL');
-        return { payload: null, diagnostics };
-    }
-    if (!scrapeWithFirecrawl) {
-        diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, 'Firecrawl is not configured');
-        return { payload: null, diagnostics };
-    }
-    diagnostics.attempted = true;
-    onProgress?.({ kind: 'firecrawl-start', url, reason: reason ?? 'firecrawl' });
-    try {
-        const payload = await scrapeWithFirecrawl(url, { timeoutMs, cacheMode });
-        if (!payload) {
-            diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, 'Firecrawl returned no content payload');
-            onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
-            return { payload: null, diagnostics };
-        }
-        const encoder = new TextEncoder();
-        const markdownBytes = typeof payload.markdown === 'string' ? encoder.encode(payload.markdown).byteLength : null;
-        const htmlBytes = typeof payload.html === 'string' ? encoder.encode(payload.html).byteLength : null;
-        onProgress?.({ kind: 'firecrawl-done', url, ok: true, markdownBytes, htmlBytes });
-        return { payload, diagnostics };
-    }
-    catch (error) {
-        diagnostics.notes = (0, utils_js_2.appendNote)(diagnostics.notes, `Firecrawl error: ${error instanceof Error ? error.message : 'unknown error'}`);
-        onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
-        return { payload: null, diagnostics };
-    }
-}
-//# sourceMappingURL=fetcher.js.map

package/dist/cjs/content/link-preview/content/fetcher.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/fetcher.ts"],"names":[],"mappings":";;AA2BA,8CAsFC;AAED,gDA2DC;AA9KD,wDAAwD;AAQxD,yCAAuC;AAEvC,MAAM,eAAe,GAA2B;IAC9C,YAAY,EACV,iHAAiH;IACnH,MAAM,EACJ,kGAAkG;IACpG,iBAAiB,EAAE,gBAAgB;IACnC,eAAe,EAAE,UAAU;IAC3B,MAAM,EAAE,UAAU;CACnB,CAAA;AAED,MAAM,0BAA0B,GAAG,IAAI,CAAA;AAOhC,KAAK,UAAU,iBAAiB,CACrC,SAAuB,EACvB,GAAW,EACX,EACE,SAAS,EACT,UAAU,MACiF,EAAE;IAE/F,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAA;IAE/C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,kBAAkB,GACtB,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;QACzD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,0BAA0B,CAAA;IAChC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC,EAAE,kBAAkB,CAAC,CAAA;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YACpC,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;QAC9E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,IAAI,IAAI,CAAA;QAC/E,IACE,WAAW;YACX,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;YAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAC9C,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YACxC,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC;YACjC,CAAC,WAAW,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YAC5C,CAAC,WAAW,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAC7C,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,EAChC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,WAAW,EAAE,CAAC,CAAA;QACrF,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE;YACvB,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;YAClD,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAA;YACrB,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;YAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAC1E,CAAC,CAAC,EAAE,CAAA;QAEJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YAClC,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAA;YACvD,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAA;YAClF,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;QAC/B,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,IAAI,eAAe,GAAG,CAAC,CAAA;QACvB,IAAI,IAAI,GAAG,EAAE,CAAA;QAEb,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAA;QAElF,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAC3C,IAAI,IAAI;gBAAE,MAAK;YACf,IAAI,CAAC,KAAK;gBAAE,SAAQ;YACpB,eAAe,IAAI,KAAK,CAAC,UAAU,CAAA;YACnC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/C,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAA;QACjF,CAAC;QAED,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAA;QACxB,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC,CAAA;QAC3E,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QACrD,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAA;IACvB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,GAAW,EACX,mBAA+C,EAC/C,UAKI,EAAE;IAEN,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;IACnC,MAAM,SAAS,GAAc,OAAO,CAAC,SAAS,IAAI,SAAS,CAAA;IAC3D,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IACvF,MAAM,MAAM,GAAG,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IACzE,MAAM,WAAW,GAAyB;QACxC,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,KAAK;QACX,SAAS;QACT,WAAW,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC5D,KAAK,EAAE,IAAI;KACZ,CAAA;IAED,IAAI,IAAA,uBAAY,EAAC,GAAG,CAAC,EAAE,CAAC;QACtB,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAAC,WAAW,CAAC,KAAK,EAAE,mCAAmC,CAAC,CAAA;QACtF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAAC,WAAW,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAA;QAChF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,WAAW,CAAC,SAAS,GAAG,IAAI,CAAA;IAC5B,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW,EAAE,CAAC,CAAA;IAE7E,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAA;QACxE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAAC,WAAW,CAAC,KAAK,EAAE,uCAAuC,CAAC,CAAA;YAC1F,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;QACvC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;QACjC,MAAM,aAAa,GACjB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;QAC3F,MAAM,SAAS,GACb,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;QACnF,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAA;QAEjF,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,KAAK,GAAG,IAAA,qBAAU,EAC5B,WAAW,CAAC,KAAK,EACjB,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAC/E,CAAA;QACD,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAA;IACvC,CAAC;AACH,CAAC"}

package/dist/cjs/content/link-preview/content/firecrawl.js

@@ -1,90 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.shouldFallbackToFirecrawl = shouldFallbackToFirecrawl;
-exports.buildResultFromFirecrawl = buildResultFromFirecrawl;
-const index_js_1 = require("../../transcript/index.js");
-const article_js_1 = require("./article.js");
-const cleaner_js_1 = require("./cleaner.js");
-const constants_js_1 = require("./constants.js");
-const jsonld_js_1 = require("./jsonld.js");
-const parsers_js_1 = require("./parsers.js");
-const podcast_utils_js_1 = require("./podcast-utils.js");
-const utils_js_1 = require("./utils.js");
-const video_js_1 = require("./video.js");
-function shouldFallbackToFirecrawl(html) {
-    const plainText = (0, cleaner_js_1.normalizeForPrompt)((0, article_js_1.extractPlainText)(html));
-    if (constants_js_1.BLOCKED_HTML_HINT_PATTERN.test(plainText))
-        return true;
-    const normalized = (0, cleaner_js_1.normalizeForPrompt)((0, article_js_1.extractArticleContent)(html));
-    if (normalized.length >= constants_js_1.MIN_HTML_CONTENT_CHARACTERS) {
-        return false;
-    }
-    // Avoid spending Firecrawl on truly small/simple pages where the extracted HTML content is short but
-    // likely complete (e.g. https://example.com). Only treat "thin" content as a Firecrawl signal when
-    // the HTML document itself is large (SSR/app-shell pages, blocked pages without a match, etc.).
-    return html.length >= constants_js_1.MIN_HTML_DOCUMENT_CHARACTERS_FOR_FALLBACK;
-}
-async function buildResultFromFirecrawl({ url, payload, cacheMode, maxCharacters, youtubeTranscriptMode, firecrawlDiagnostics, markdownRequested, deps, }) {
-    const normalizedMarkdown = (0, cleaner_js_1.normalizeForPrompt)(payload.markdown ?? '');
-    if (normalizedMarkdown.length === 0) {
-        firecrawlDiagnostics.notes = (0, utils_js_1.appendNote)(firecrawlDiagnostics.notes, 'Firecrawl markdown normalization yielded empty text');
-        return null;
-    }
-    const jsonLd = payload.html ? (0, jsonld_js_1.extractJsonLdContent)(payload.html) : null;
-    const isPodcastJsonLd = (0, podcast_utils_js_1.isPodcastLikeJsonLdType)(jsonLd?.type);
-    const transcriptResolution = await (0, index_js_1.resolveTranscriptForLink)(url, payload.html ?? null, deps, {
-        youtubeTranscriptMode,
-        cacheMode,
-    });
-    const htmlMetadata = payload.html
-        ? (0, parsers_js_1.extractMetadataFromHtml)(payload.html, url)
-        : { title: null, description: null, siteName: null };
-    const metadata = (0, parsers_js_1.extractMetadataFromFirecrawl)(payload.metadata ?? null);
-    const title = (0, utils_js_1.pickFirstText)([jsonLd?.title, metadata.title, htmlMetadata.title]);
-    const description = (0, utils_js_1.pickFirstText)([
-        jsonLd?.description,
-        metadata.description,
-        htmlMetadata.description,
-    ]);
-    const siteName = (0, utils_js_1.pickFirstText)([metadata.siteName, htmlMetadata.siteName, (0, utils_js_1.safeHostname)(url)]);
-    const descriptionCandidate = description ? (0, cleaner_js_1.normalizeForPrompt)(description) : '';
-    const preferDescription = descriptionCandidate.length >= constants_js_1.MIN_METADATA_DESCRIPTION_CHARACTERS &&
-        (isPodcastJsonLd ||
-            (0, podcast_utils_js_1.isPodcastHost)(url) ||
-            normalizedMarkdown.length < constants_js_1.MIN_HTML_CONTENT_CHARACTERS ||
-            descriptionCandidate.length >= normalizedMarkdown.length * constants_js_1.READABILITY_RELATIVE_THRESHOLD);
-    const baseCandidate = preferDescription ? descriptionCandidate : normalizedMarkdown;
-    const baseContent = (0, utils_js_1.selectBaseContent)(baseCandidate, transcriptResolution.text);
-    if (baseContent.length === 0) {
-        firecrawlDiagnostics.notes = (0, utils_js_1.appendNote)(firecrawlDiagnostics.notes, 'Firecrawl produced content that normalized to an empty string');
-        return null;
-    }
-    firecrawlDiagnostics.used = true;
-    const transcriptDiagnostics = (0, utils_js_1.ensureTranscriptDiagnostics)(transcriptResolution, cacheMode ?? 'default');
-    const video = payload.html ? (0, video_js_1.detectPrimaryVideoFromHtml)(payload.html, url) : null;
-    const isVideoOnly = !transcriptResolution.text &&
-        normalizedMarkdown.length < constants_js_1.MIN_HTML_CONTENT_CHARACTERS &&
-        video !== null;
-    return (0, utils_js_1.finalizeExtractedLinkContent)({
-        url,
-        baseContent,
-        maxCharacters,
-        title,
-        description,
-        siteName,
-        transcriptResolution,
-        video,
-        isVideoOnly,
-        diagnostics: {
-            strategy: 'firecrawl',
-            firecrawl: firecrawlDiagnostics,
-            markdown: {
-                requested: markdownRequested,
-                used: true,
-                provider: 'firecrawl',
-            },
-            transcript: transcriptDiagnostics,
-        },
-    });
-}
-//# sourceMappingURL=firecrawl.js.map

package/dist/cjs/content/link-preview/content/firecrawl.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/firecrawl.ts"],"names":[],"mappings":";;AA0BA,8DAYC;AAED,4DAmGC;AA3ID,wDAAoE;AAGpE,6CAAsE;AACtE,6CAAiD;AACjD,iDAMuB;AACvB,2CAAkD;AAClD,6CAAoF;AACpF,yDAA2E;AAE3E,yCAOmB;AACnB,yCAAuD;AAEvD,SAAgB,yBAAyB,CAAC,IAAY;IACpD,MAAM,SAAS,GAAG,IAAA,+BAAkB,EAAC,IAAA,6BAAgB,EAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,wCAAyB,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAA;IAC1D,MAAM,UAAU,GAAG,IAAA,+BAAkB,EAAC,IAAA,kCAAqB,EAAC,IAAI,CAAC,CAAC,CAAA;IAClE,IAAI,UAAU,CAAC,MAAM,IAAI,0CAA2B,EAAE,CAAC;QACrD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,qGAAqG;IACrG,mGAAmG;IACnG,gGAAgG;IAChG,OAAO,IAAI,CAAC,MAAM,IAAI,wDAAyC,CAAA;AACjE,CAAC;AAEM,KAAK,UAAU,wBAAwB,CAAC,EAC7C,GAAG,EACH,OAAO,EACP,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,IAAI,GAUL;IACC,MAAM,kBAAkB,GAAG,IAAA,+BAAkB,EAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACrE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,oBAAoB,CAAC,KAAK,GAAG,IAAA,qBAAU,EACrC,oBAAoB,CAAC,KAAK,EAC1B,qDAAqD,CACtD,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,gCAAoB,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACvE,MAAM,eAAe,GAAG,IAAA,0CAAuB,EAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAE7D,MAAM,oBAAoB,GAAG,MAAM,IAAA,mCAAwB,EAAC,GAAG,EAAE,OAAO,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,EAAE;QAC3F,qBAAqB;QACrB,SAAS;KACV,CAAC,CAAA;IACF,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI;QAC/B,CAAC,CAAC,IAAA,oCAAuB,EAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC;QAC5C,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;IACtD,MAAM,QAAQ,GAAG,IAAA,yCAA4B,EAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAA;IAEvE,MAAM,KAAK,GAAG,IAAA,wBAAa,EAAC,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAChF,MAAM,WAAW,GAAG,IAAA,wBAAa,EAAC;QAChC,MAAM,EAAE,WAAW;QACnB,QAAQ,CAAC,WAAW;QACpB,YAAY,CAAC,WAAW;KACzB,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,IAAA,wBAAa,EAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC,QAAQ,EAAE,IAAA,uBAAY,EAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAE7F,MAAM,oBAAoB,GAAG,WAAW,CAAC,CAAC,CAAC,IAAA,+BAAkB,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/E,MAAM,iBAAiB,GACrB,oBAAoB,CAAC,MAAM,IAAI,kDAAmC;QAClE,CAAC,eAAe;YACd,IAAA,gCAAa,EAAC,GAAG,CAAC;YAClB,kBAAkB,CAAC,MAAM,GAAG,0CAA2B;YACvD,oBAAoB,CAAC,MAAM,IAAI,kBAAkB,CAAC,MAAM,GAAG,6CAA8B,CAAC,CAAA;IAC9F,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,kBAAkB,CAAA;IACnF,MAAM,WAAW,GAAG,IAAA,4BAAiB,EAAC,aAAa,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAA;IAC/E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,oBAAoB,CAAC,KAAK,GAAG,IAAA,qBAAU,EACrC,oBAAoB,CAAC,KAAK,EAC1B,+DAA+D,CAChE,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,oBAAoB,CAAC,IAAI,GAAG,IAAI,CAAA;IAEhC,MAAM,qBAAqB,GAAG,IAAA,sCAA2B,EACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAA,qCAA0B,EAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACjF,MAAM,WAAW,GACf,CAAC,oBAAoB,CAAC,IAAI;QAC1B,kBAAkB,CAAC,MAAM,GAAG,0CAA2B;QACvD,KAAK,KAAK,IAAI,CAAA;IAEhB,OAAO,IAAA,uCAA4B,EAAC;QAClC,GAAG;QACH,WAAW;QACX,aAAa;QACb,KAAK;QACL,WAAW;QACX,QAAQ;QACR,oBAAoB;QACpB,KAAK;QACL,WAAW;QACX,WAAW,EAAE;YACX,QAAQ,EAAE,WAAW;YACrB,SAAS,EAAE,oBAAoB;YAC/B,QAAQ,EAAE;gBACR,SAAS,EAAE,iBAAiB;gBAC5B,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,WAAW;aACtB;YACD,UAAU,EAAE,qBAAqB;SAClC;KACF,CAAC,CAAA;AACJ,CAAC"}