@steipete/summarize-core 0.9.0 → 0.11.0

package/README.md

@@ -4,4 +4,3 @@ Core library for Summarize (content extraction + prompt builders).
 
 - CLI package: `@steipete/summarize`
 - Recommended imports (library use): `@steipete/summarize-core/content`, `@steipete/summarize-core/prompts`
-

package/dist/esm/content/index.js

@@ -1,6 +1,6 @@
-export { createLinkPreviewClient, } from './link-preview/client.js';
-export { DEFAULT_CACHE_MODE, DEFAULT_MAX_CONTENT_CHARACTERS, DEFAULT_TIMEOUT_MS, } from './link-preview/content/types.js';
-export { ProgressKind } from './link-preview/deps.js';
-export { CACHE_MODES, } from './link-preview/types.js';
-export { extractYouTubeVideoId, isDirectMediaUrl, isPodcastHost, isTwitterStatusUrl, isYouTubeUrl, isYouTubeVideoUrl, shouldPreferUrlMode, } from './url.js';
+export { createLinkPreviewClient, } from "./link-preview/client.js";
+export { DEFAULT_CACHE_MODE, DEFAULT_MAX_CONTENT_CHARACTERS, DEFAULT_TIMEOUT_MS, } from "./link-preview/content/types.js";
+export { ProgressKind } from "./link-preview/deps.js";
+export { CACHE_MODES, } from "./link-preview/types.js";
+export { DIRECT_MEDIA_EXTENSIONS, extractYouTubeVideoId, isDirectMediaExtension, isDirectMediaUrl, isPodcastHost, isTwitterBroadcastUrl, isTwitterStatusUrl, isYouTubeUrl, isYouTubeVideoUrl, shouldPreferUrlMode, } from "./url.js";
 //# sourceMappingURL=index.js.map

package/dist/esm/content/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/content/index.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,uBAAuB,GAGxB,MAAM,0BAA0B,CAAA;AACjC,OAAO,EACL,kBAAkB,EAClB,8BAA8B,EAC9B,kBAAkB,GAGnB,MAAM,iCAAiC,CAAA;AASxC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EACL,WAAW,GAIZ,MAAM,yBAAyB,CAAA;AAChC,OAAO,EACL,qBAAqB,EACrB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,UAAU,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/content/index.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,uBAAuB,GAGxB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,kBAAkB,EAClB,8BAA8B,EAC9B,kBAAkB,GAGnB,MAAM,iCAAiC,CAAC;AASzC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EACL,WAAW,GAIZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,uBAAuB,EACvB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,aAAa,EACb,qBAAqB,EACrB,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,UAAU,CAAC"}

package/dist/esm/content/link-preview/client.js

@@ -1,27 +1,42 @@
-import { fetchLinkContent } from './content/index.js';
+import { resolveTranscriptionConfig, } from "../transcript/transcription-config.js";
+import { fetchLinkContent } from "./content/index.js";
 /** Public factory for a link preview client with injectable dependencies. */
 export function createLinkPreviewClient(options = {}) {
     const fetchImpl = options.fetch ?? ((...args) => globalThis.fetch(...args));
+    const env = typeof options.env === "object" && options.env ? options.env : undefined;
     const scrape = options.scrapeWithFirecrawl ?? null;
-    const apifyApiToken = typeof options.apifyApiToken === 'string' ? options.apifyApiToken : null;
-    const ytDlpPath = typeof options.ytDlpPath === 'string' ? options.ytDlpPath : null;
-    const falApiKey = typeof options.falApiKey === 'string' ? options.falApiKey : null;
-    const openaiApiKey = typeof options.openaiApiKey === 'string' ? options.openaiApiKey : null;
+    const apifyApiToken = typeof options.apifyApiToken === "string" ? options.apifyApiToken : null;
+    const ytDlpPath = typeof options.ytDlpPath === "string" ? options.ytDlpPath : null;
+    const falApiKey = typeof options.falApiKey === "string" ? options.falApiKey : null;
+    const groqApiKey = typeof options.groqApiKey === "string" ? options.groqApiKey : null;
+    const openaiApiKey = typeof options.openaiApiKey === "string" ? options.openaiApiKey : null;
+    const transcription = resolveTranscriptionConfig({
+        env,
+        transcription: options.transcription ?? null,
+        falApiKey,
+        groqApiKey,
+        openaiApiKey,
+    });
     const convertHtmlToMarkdown = options.convertHtmlToMarkdown ?? null;
     const transcriptCache = options.transcriptCache ?? null;
-    const readTweetWithBird = typeof options.readTweetWithBird === 'function' ? options.readTweetWithBird : null;
-    const resolveTwitterCookies = typeof options.resolveTwitterCookies === 'function' ? options.resolveTwitterCookies : null;
-    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
+    const mediaCache = options.mediaCache ?? null;
+    const readTweetWithBird = typeof options.readTweetWithBird === "function" ? options.readTweetWithBird : null;
+    const resolveTwitterCookies = typeof options.resolveTwitterCookies === "function" ? options.resolveTwitterCookies : null;
+    const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
     return {
         fetchLinkContent: (url, contentOptions) => fetchLinkContent(url, contentOptions, {
             fetch: fetchImpl,
+            env,
             scrapeWithFirecrawl: scrape,
             apifyApiToken,
             ytDlpPath,
+            transcription,
             falApiKey,
+            groqApiKey,
             openaiApiKey,
             convertHtmlToMarkdown,
             transcriptCache,
+            mediaCache,
             readTweetWithBird,
             resolveTwitterCookies,
             onProgress,

package/dist/esm/content/link-preview/client.js.map

@@ -1 +1 @@
-{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../src/content/link-preview/client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AA8BrD,6EAA6E;AAC7E,MAAM,UAAU,uBAAuB,CAAC,UAAoC,EAAE;IAC5E,MAAM,SAAS,GACb,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;IACrF,MAAM,MAAM,GAA+B,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAA;IAC9E,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9F,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3F,MAAM,qBAAqB,GAAiC,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAA;IACjG,MAAM,eAAe,GAA2B,OAAO,CAAC,eAAe,IAAI,IAAI,CAAA;IAC/E,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAA;IACpF,MAAM,qBAAqB,GACzB,OAAO,OAAO,CAAC,qBAAqB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAA;IAC5F,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IAEvF,OAAO;QACL,gBAAgB,EAAE,CAAC,GAAW,EAAE,cAAwC,EAAE,EAAE,CAC1E,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE;YACpC,KAAK,EAAE,SAAS;YAChB,mBAAmB,EAAE,MAAM;YAC3B,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,qBAAqB;YACrB,eAAe;YACf,iBAAiB;YACjB,qBAAqB;YACrB,UAAU;SACX,CAAC;KACL,CAAA;AACH,CAAC"}
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../../src/content/link-preview/client.ts"],"names":[],"mappings":"AASA,OAAO,EACL,0BAA0B,GAE3B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AA0BtD,6EAA6E;AAC7E,MAAM,UAAU,uBAAuB,CAAC,UAAoC,EAAE;IAC5E,MAAM,SAAS,GACb,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IACtF,MAAM,GAAG,GAAG,OAAO,OAAO,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,MAAM,MAAM,GAA+B,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC;IAC/E,MAAM,aAAa,GAAG,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IACnF,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IACnF,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IACtF,MAAM,YAAY,GAAG,OAAO,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5F,MAAM,aAAa,GAAG,0BAA0B,CAAC;QAC/C,GAAG;QACH,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;QAC5C,SAAS;QACT,UAAU;QACV,YAAY;KACb,CAAC,CAAC;IACH,MAAM,qBAAqB,GAAiC,OAAO,CAAC,qBAAqB,IAAI,IAAI,CAAC;IAClG,MAAM,eAAe,GAA2B,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC;IAChF,MAAM,UAAU,GAAsB,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;IACjE,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,iBAAiB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC;IACrF,MAAM,qBAAqB,GACzB,OAAO,OAAO,CAAC,qBAAqB,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7F,MAAM,UAAU,GAAG,OAAO,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAExF,OAAO;QACL,gBAAgB,EAAE,CAAC,GAAW,EAAE,cAAwC,EAAE,EAAE,CAC1E,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE;YACpC,KAAK,EAAE,SAAS;YAChB,GAAG;YACH,mBAAmB,EAAE,MAAM;YAC3B,aAAa;YACb,SAAS;YACT,aAAa;YACb,SAAS;YACT,UAAU;YACV,YAAY;YACZ,qBAAqB;YACrB,eAAe;YACf,UAAU;YACV,iBAAiB;YACjB,qBAAqB;YACrB,UAAU;SACX,CAAC;KACL,CAAC;AACJ,CAAC"}

package/dist/esm/content/link-preview/content/article.js

@@ -1,45 +1,46 @@
-import { load } from 'cheerio';
-import sanitizeHtml from 'sanitize-html';
-import { decodeHtmlEntities, normalizeWhitespace } from './cleaner.js';
+import { load } from "cheerio";
+import sanitizeHtml from "sanitize-html";
+import { decodeHtmlEntities, normalizeWhitespace } from "./cleaner.js";
+import { stripHiddenHtml } from "./visibility.js";
 const MIN_SEGMENT_LENGTH = 30;
 export function sanitizeHtmlForMarkdownConversion(html) {
-    return sanitizeHtml(html, {
+    return sanitizeHtml(stripHiddenHtml(html), {
         allowedTags: [
-            'article',
-            'section',
-            'div',
-            'p',
-            'h1',
-            'h2',
-            'h3',
-            'h4',
-            'h5',
-            'h6',
-            'ol',
-            'ul',
-            'li',
-            'blockquote',
-            'pre',
-            'code',
-            'span',
-            'strong',
-            'em',
-            'br',
-            'a',
+            "article",
+            "section",
+            "div",
+            "p",
+            "h1",
+            "h2",
+            "h3",
+            "h4",
+            "h5",
+            "h6",
+            "ol",
+            "ul",
+            "li",
+            "blockquote",
+            "pre",
+            "code",
+            "span",
+            "strong",
+            "em",
+            "br",
+            "a",
         ],
         allowedAttributes: {
-            a: ['href'],
+            a: ["href"],
         },
         nonTextTags: [
-            'style',
-            'script',
-            'noscript',
-            'template',
-            'svg',
-            'canvas',
-            'iframe',
-            'object',
-            'embed',
+            "style",
+            "script",
+            "noscript",
+            "template",
+            "svg",
+            "canvas",
+            "iframe",
+            "object",
+            "embed",
         ],
         textFilter(text) {
             return decodeHtmlEntities(text);
@@ -49,46 +50,46 @@ export function sanitizeHtmlForMarkdownConversion(html) {
 export function extractArticleContent(html) {
     const segments = collectSegmentsFromHtml(html);
     if (segments.length > 0) {
-        return segments.join('\n');
+        return segments.join("\n");
     }
     const fallback = normalizeWhitespace(extractPlainText(html));
-    return fallback ?? '';
+    return fallback ?? "";
 }
 export function collectSegmentsFromHtml(html) {
-    const sanitized = sanitizeHtml(html, {
+    const sanitized = sanitizeHtml(stripHiddenHtml(html), {
         allowedTags: [
-            'article',
-            'section',
-            'div',
-            'p',
-            'h1',
-            'h2',
-            'h3',
-            'h4',
-            'h5',
-            'h6',
-            'ol',
-            'ul',
-            'li',
-            'blockquote',
-            'pre',
-            'code',
-            'span',
-            'strong',
-            'em',
-            'br',
+            "article",
+            "section",
+            "div",
+            "p",
+            "h1",
+            "h2",
+            "h3",
+            "h4",
+            "h5",
+            "h6",
+            "ol",
+            "ul",
+            "li",
+            "blockquote",
+            "pre",
+            "code",
+            "span",
+            "strong",
+            "em",
+            "br",
         ],
         allowedAttributes: {},
         nonTextTags: [
-            'style',
-            'script',
-            'noscript',
-            'template',
-            'svg',
-            'canvas',
-            'iframe',
-            'object',
-            'embed',
+            "style",
+            "script",
+            "noscript",
+            "template",
+            "svg",
+            "canvas",
+            "iframe",
+            "object",
+            "embed",
         ],
         textFilter(text) {
             return decodeHtmlEntities(text);
@@ -96,23 +97,23 @@ export function collectSegmentsFromHtml(html) {
     });
     const $ = load(sanitized);
     const segments = [];
-    $('h1,h2,h3,h4,h5,h6,li,p,blockquote,pre').each((_, element) => {
-        if (!('tagName' in element) || typeof element.tagName !== 'string') {
+    $("h1,h2,h3,h4,h5,h6,li,p,blockquote,pre").each((_, element) => {
+        if (!("tagName" in element) || typeof element.tagName !== "string") {
             return;
         }
         const tag = element.tagName.toLowerCase();
         const raw = $(element).text();
-        const text = normalizeWhitespace(raw).replaceAll(/\n+/g, ' ');
+        const text = normalizeWhitespace(raw).replaceAll(/\n+/g, " ");
         if (!text || text.length === 0) {
             return;
         }
-        if (tag.startsWith('h')) {
+        if (tag.startsWith("h")) {
             if (text.length >= 10) {
                 segments.push(text);
             }
             return;
         }
-        if (tag === 'li') {
+        if (tag === "li") {
             if (text.length >= 20) {
                 segments.push(`• ${text}`);
             }
@@ -124,25 +125,25 @@ export function collectSegmentsFromHtml(html) {
         segments.push(text);
     });
     if (segments.length === 0) {
-        const fallback = normalizeWhitespace($('body').text() || sanitized);
+        const fallback = normalizeWhitespace($("body").text() || sanitized);
         return fallback ? [fallback] : [];
     }
     return mergeConsecutiveSegments(segments);
 }
 export function extractPlainText(html) {
-    const stripped = sanitizeHtml(html, {
+    const stripped = sanitizeHtml(stripHiddenHtml(html), {
         allowedTags: [],
         allowedAttributes: {},
         nonTextTags: [
-            'style',
-            'script',
-            'noscript',
-            'template',
-            'svg',
-            'canvas',
-            'iframe',
-            'object',
-            'embed',
+            "style",
+            "script",
+            "noscript",
+            "template",
+            "svg",
+            "canvas",
+            "iframe",
+            "object",
+            "embed",
         ],
     });
     return decodeHtmlEntities(stripped);

package/dist/esm/content/link-preview/content/article.js.map

@@ -1 +1 @@
-{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/article.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,YAAY,MAAM,eAAe,CAAA;AAExC,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAEtE,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,MAAM,UAAU,iCAAiC,CAAC,IAAY;IAC5D,OAAO,YAAY,CAAC,IAAI,EAAE;QACxB,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;YACJ,GAAG;SACJ;QACD,iBAAiB,EAAE;YACjB,CAAC,EAAE,CAAC,MAAM,CAAC;SACZ;QACD,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IACD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,OAAO,QAAQ,IAAI,EAAE,CAAA;AACvB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE;QACnC,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;SACL;QACD,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAA;QACjC,CAAC;KACF,CAAC,CAAA;IAEF,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAA;IACzB,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,CAAC,CAAC,uCAAuC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnE,OAAM;QACR,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEzC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7B,MAAM,IAAI,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC7D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAM;QACR,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACrB,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;YAC5B,CAAC;YACD,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACrC,OAAM;QACR,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACrB,CAAC,CAAC,CAAA;IAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,CAAA;QACnE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACnC,CAAC;IAED,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAA;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE;QAClC,WAAW,EAAE,EAAE;QACf,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;KACF,CAAC,CAAA;IACF,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAA;AACrC,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAkB;IAClD,gGAAgG;IAChG,2FAA2F;IAC3F,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;AACjC,CAAC"}
+{"version":3,"file":"article.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/article.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,YAAY,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,MAAM,UAAU,iCAAiC,CAAC,IAAY;IAC5D,OAAO,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE;QACzC,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;YACJ,GAAG;SACJ;QACD,iBAAiB,EAAE;YACjB,CAAC,EAAE,CAAC,MAAM,CAAC;SACZ;QACD,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,OAAO,QAAQ,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,SAAS,GAAG,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE;QACpD,WAAW,EAAE;YACX,SAAS;YACT,SAAS;YACT,KAAK;YACL,GAAG;YACH,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,IAAI;YACJ,YAAY;YACZ,KAAK;YACL,MAAM;YACN,MAAM;YACN,QAAQ;YACR,IAAI;YACJ,IAAI;SACL;QACD,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;QACD,UAAU,CAAC,IAAY;YACrB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,CAAC,CAAC,uCAAuC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;QAC7D,IAAI,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACnE,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAE1C,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACT,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAC7B,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACrC,OAAO;QACT,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,CAAC;QACpE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,OAAO,wBAAwB,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,YAAY,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE;QACnD,WAAW,EAAE,EAAE;QACf,iBAAiB,EAAE,EAAE;QACrB,WAAW,EAAE;YACX,OAAO;YACP,QAAQ;YACR,UAAU;YACV,UAAU;YACV,KAAK;YACL,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,OAAO;SACR;KACF,CAAC,CAAC;IACH,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,wBAAwB,CAAC,QAAkB;IAClD,gGAAgG;IAChG,2FAA2F;IAC3F,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC"}

package/dist/esm/content/link-preview/content/cleaner.js

@@ -1,36 +1,39 @@
-import { compact } from 'es-toolkit';
+import { compact } from "es-toolkit";
 const WORD_SPLIT_PATTERN = /\s+/g;
 export function normalizeForPrompt(input) {
-    return input
-        .replaceAll('\u00A0', ' ')
-        .replaceAll(/[\t ]+/g, ' ')
-        .replaceAll(/\s*\n\s*/g, '\n')
-        .replaceAll(/\n{3,}/g, '\n\n')
+    return stripInvisibleUnicode(input)
+        .replaceAll("\u00A0", " ")
+        .replaceAll(/[\t ]+/g, " ")
+        .replaceAll(/\s*\n\s*/g, "\n")
+        .replaceAll(/\n{3,}/g, "\n\n")
         .trim();
 }
 export function normalizeWhitespace(input) {
-    return input
-        .replaceAll('\u00A0', ' ')
-        .replaceAll(/[\t ]+/g, ' ')
-        .replaceAll(/\s*\n\s*/g, '\n')
+    return stripInvisibleUnicode(input)
+        .replaceAll("\u00A0", " ")
+        .replaceAll(/[\t ]+/g, " ")
+        .replaceAll(/\s*\n\s*/g, "\n")
         .trim();
 }
 export function decodeHtmlEntities(input) {
     return input
-        .replaceAll('&', '&')
-        .replaceAll('&lt;', '<')
-        .replaceAll('&gt;', '>')
-        .replaceAll('&quot;', '"')
-        .replaceAll('&#39;', "'")
-        .replaceAll('&#x27;', "'")
-        .replaceAll('&#x2F;', '/')
-        .replaceAll('&nbsp;', ' ');
+        .replaceAll("&amp;", "&")
+        .replaceAll("&lt;", "<")
+        .replaceAll("&gt;", ">")
+        .replaceAll("&quot;", '"')
+        .replaceAll("&#39;", "'")
+        .replaceAll("&#x27;", "'")
+        .replaceAll("&#x2F;", "/")
+        .replaceAll("&nbsp;", " ");
+}
+export function stripInvisibleUnicode(input) {
+    return input.replaceAll(/[\u200B-\u200F\u202A-\u202E\u2060-\u2069\uFEFF\u{E0000}-\u{E007F}]/gu, "");
 }
 export function normalizeCandidate(value) {
     if (!value) {
         return null;
     }
-    const trimmed = value.replaceAll(/\s+/g, ' ').trim();
+    const trimmed = value.replaceAll(/\s+/g, " ").trim();
     return trimmed.length > 0 ? trimmed : null;
 }
 export function clipAtSentenceBoundary(input, maxLength) {
@@ -38,7 +41,7 @@ export function clipAtSentenceBoundary(input, maxLength) {
         return input;
     }
     const slice = input.slice(0, maxLength);
-    const lastSentenceBreak = Math.max(slice.lastIndexOf('. '), slice.lastIndexOf('! '), slice.lastIndexOf('? '), slice.lastIndexOf('\n\n'));
+    const lastSentenceBreak = Math.max(slice.lastIndexOf(". "), slice.lastIndexOf("! "), slice.lastIndexOf("? "), slice.lastIndexOf("\n\n"));
     if (lastSentenceBreak > maxLength * 0.5) {
         return slice.slice(0, lastSentenceBreak + 1);
     }

package/dist/esm/content/link-preview/content/cleaner.js.map

@@ -1 +1 @@
-{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/cleaner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAA;AAEpC,MAAM,kBAAkB,GAAG,MAAM,CAAA;AASjC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,OAAO,KAAK;SACT,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,IAAI,EAAE,CAAA;AACX,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAgC;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAA;IACb,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;AAC5C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa,EAAE,SAAiB;IACrE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;IACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAChC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAC1B,CAAA;IACD,IAAI,iBAAiB,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,WAAmB,EACnB,aAAqB;IAErB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAA;IAC1C,MAAM,SAAS,GAAG,eAAe,GAAG,aAAa,CAAA;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAA;IAC5F,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5F,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,CAAA;AAC3D,CAAC"}
+{"version":3,"file":"cleaner.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/cleaner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AASlC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,qBAAqB,CAAC,KAAK,CAAC;SAChC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC;SAC7B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,OAAO,qBAAqB,CAAC,KAAK,CAAC;SAChC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1B,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC;SAC7B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAa;IACjD,OAAO,KAAK,CAAC,UAAU,CACrB,sEAAsE,EACtE,EAAE,CACH,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAgC;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACrD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa,EAAE,SAAiB;IACrE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACxC,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAChC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,EACvB,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAC1B,CAAC;IACF,IAAI,iBAAiB,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,WAAmB,EACnB,aAAqB;IAErB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC;IAC3C,MAAM,SAAS,GAAG,eAAe,GAAG,aAAa,CAAC;IAClD,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAC7F,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC;AAC5D,CAAC"}

package/dist/esm/content/link-preview/content/constants.js.map

@@ -1 +1 @@
-{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,yBAAyB,GACpC,iIAAiI,CAAA;AACnI,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAA;AAC9C,MAAM,CAAC,MAAM,kCAAkC,GAAG,GAAG,CAAA;AACrD,MAAM,CAAC,MAAM,mCAAmC,GAAG,GAAG,CAAA;AACtD,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAA;AACjD,MAAM,CAAC,MAAM,yCAAyC,GAAG,IAAI,CAAA"}
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,yBAAyB,GACpC,iIAAiI,CAAC;AACpI,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC;AAC/C,MAAM,CAAC,MAAM,kCAAkC,GAAG,GAAG,CAAC;AACtD,MAAM,CAAC,MAAM,mCAAmC,GAAG,GAAG,CAAC;AACvD,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAC;AAClD,MAAM,CAAC,MAAM,yCAAyC,GAAG,IAAI,CAAC"}

package/dist/esm/content/link-preview/content/fetcher.js

@@ -1,17 +1,17 @@
-import { isYouTubeUrl } from '../../url.js';
-import { appendNote } from './utils.js';
+import { isYouTubeUrl } from "../../url.js";
+import { appendNote } from "./utils.js";
 const REQUEST_HEADERS = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
-    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-    'Accept-Language': 'en-US,en;q=0.9',
-    'Cache-Control': 'no-cache',
-    Pragma: 'no-cache',
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    Pragma: "no-cache",
 };
 const DEFAULT_REQUEST_TIMEOUT_MS = 5000;
 export async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress, } = {}) {
-    onProgress?.({ kind: 'fetch-html-start', url });
+    onProgress?.({ kind: "fetch-html-start", url });
     const controller = new AbortController();
-    const effectiveTimeoutMs = typeof timeoutMs === 'number' && Number.isFinite(timeoutMs)
+    const effectiveTimeoutMs = typeof timeoutMs === "number" && Number.isFinite(timeoutMs)
         ? timeoutMs
         : DEFAULT_REQUEST_TIMEOUT_MS;
     const timeout = setTimeout(() => {
@@ -20,26 +20,26 @@ export async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress,
     try {
         const response = await fetchImpl(url, {
             headers: REQUEST_HEADERS,
-            redirect: 'follow',
+            redirect: "follow",
             signal: controller.signal,
         });
         if (!response.ok) {
             throw new Error(`Failed to fetch HTML document (status ${response.status})`);
         }
         const finalUrl = response.url?.trim() || url;
-        const contentType = response.headers.get('content-type')?.toLowerCase() ?? null;
+        const contentType = response.headers.get("content-type")?.toLowerCase() ?? null;
         if (contentType &&
-            !contentType.includes('text/html') &&
-            !contentType.includes('application/xhtml+xml') &&
-            !contentType.includes('application/xml') &&
-            !contentType.includes('text/xml') &&
-            !contentType.includes('application/rss+xml') &&
-            !contentType.includes('application/atom+xml') &&
-            !contentType.startsWith('text/')) {
+            !contentType.includes("text/html") &&
+            !contentType.includes("application/xhtml+xml") &&
+            !contentType.includes("application/xml") &&
+            !contentType.includes("text/xml") &&
+            !contentType.includes("application/rss+xml") &&
+            !contentType.includes("application/atom+xml") &&
+            !contentType.startsWith("text/")) {
             throw new Error(`Unsupported content-type for HTML document fetch: ${contentType}`);
         }
         const totalBytes = (() => {
-            const raw = response.headers.get('content-length');
+            const raw = response.headers.get("content-length");
             if (!raw)
                 return null;
             const parsed = Number(raw);
@@ -49,14 +49,14 @@ export async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress,
         if (!body) {
             const text = await response.text();
             const bytes = new TextEncoder().encode(text).byteLength;
-            onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes: bytes, totalBytes });
+            onProgress?.({ kind: "fetch-html-done", url, downloadedBytes: bytes, totalBytes });
             return { html: text, finalUrl };
         }
         const reader = body.getReader();
         const decoder = new TextDecoder();
         let downloadedBytes = 0;
-        let text = '';
-        onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes: 0, totalBytes });
+        let text = "";
+        onProgress?.({ kind: "fetch-html-progress", url, downloadedBytes: 0, totalBytes });
         while (true) {
             const { value, done } = await reader.read();
             if (done)
@@ -65,15 +65,15 @@ export async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress,
                 continue;
             downloadedBytes += value.byteLength;
             text += decoder.decode(value, { stream: true });
-            onProgress?.({ kind: 'fetch-html-progress', url, downloadedBytes, totalBytes });
+            onProgress?.({ kind: "fetch-html-progress", url, downloadedBytes, totalBytes });
         }
         text += decoder.decode();
-        onProgress?.({ kind: 'fetch-html-done', url, downloadedBytes, totalBytes });
+        onProgress?.({ kind: "fetch-html-done", url, downloadedBytes, totalBytes });
         return { html: text, finalUrl };
     }
     catch (error) {
-        if (error instanceof DOMException && error.name === 'AbortError') {
-            throw new Error('Fetching HTML document timed out');
+        if (error instanceof DOMException && error.name === "AbortError") {
+            throw new Error("Fetching HTML document timed out");
         }
         throw error;
     }
@@ -83,42 +83,48 @@ export async function fetchHtmlDocument(fetchImpl, url, { timeoutMs, onProgress,
 }
 export async function fetchWithFirecrawl(url, scrapeWithFirecrawl, options = {}) {
     const timeoutMs = options.timeoutMs;
-    const cacheMode = options.cacheMode ?? 'default';
-    const onProgress = typeof options.onProgress === 'function' ? options.onProgress : null;
-    const reason = typeof options.reason === 'string' ? options.reason : null;
+    const cacheMode = options.cacheMode ?? "default";
+    const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
+    const reason = typeof options.reason === "string" ? options.reason : null;
     const diagnostics = {
         attempted: false,
         used: false,
         cacheMode,
-        cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
+        cacheStatus: cacheMode === "bypass" ? "bypassed" : "unknown",
         notes: null,
     };
     if (isYouTubeUrl(url)) {
-        diagnostics.notes = appendNote(diagnostics.notes, 'Skipped Firecrawl for YouTube URL');
+        diagnostics.notes = appendNote(diagnostics.notes, "Skipped Firecrawl for YouTube URL");
         return { payload: null, diagnostics };
     }
     if (!scrapeWithFirecrawl) {
-        diagnostics.notes = appendNote(diagnostics.notes, 'Firecrawl is not configured');
+        diagnostics.notes = appendNote(diagnostics.notes, "Firecrawl is not configured");
         return { payload: null, diagnostics };
     }
     diagnostics.attempted = true;
-    onProgress?.({ kind: 'firecrawl-start', url, reason: reason ?? 'firecrawl' });
+    onProgress?.({ kind: "firecrawl-start", url, reason: reason ?? "firecrawl" });
     try {
         const payload = await scrapeWithFirecrawl(url, { timeoutMs, cacheMode });
         if (!payload) {
-            diagnostics.notes = appendNote(diagnostics.notes, 'Firecrawl returned no content payload');
-            onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
+            diagnostics.notes = appendNote(diagnostics.notes, "Firecrawl returned no content payload");
+            onProgress?.({
+                kind: "firecrawl-done",
+                url,
+                ok: false,
+                markdownBytes: null,
+                htmlBytes: null,
+            });
             return { payload: null, diagnostics };
         }
         const encoder = new TextEncoder();
-        const markdownBytes = typeof payload.markdown === 'string' ? encoder.encode(payload.markdown).byteLength : null;
-        const htmlBytes = typeof payload.html === 'string' ? encoder.encode(payload.html).byteLength : null;
-        onProgress?.({ kind: 'firecrawl-done', url, ok: true, markdownBytes, htmlBytes });
+        const markdownBytes = typeof payload.markdown === "string" ? encoder.encode(payload.markdown).byteLength : null;
+        const htmlBytes = typeof payload.html === "string" ? encoder.encode(payload.html).byteLength : null;
+        onProgress?.({ kind: "firecrawl-done", url, ok: true, markdownBytes, htmlBytes });
         return { payload, diagnostics };
     }
     catch (error) {
-        diagnostics.notes = appendNote(diagnostics.notes, `Firecrawl error: ${error instanceof Error ? error.message : 'unknown error'}`);
-        onProgress?.({ kind: 'firecrawl-done', url, ok: false, markdownBytes: null, htmlBytes: null });
+        diagnostics.notes = appendNote(diagnostics.notes, `Firecrawl error: ${error instanceof Error ? error.message : "unknown error"}`);
+        onProgress?.({ kind: "firecrawl-done", url, ok: false, markdownBytes: null, htmlBytes: null });
         return { payload: null, diagnostics };
     }
 }