@steipete/summarize-core 0.7.0

package/dist/esm/content/link-preview/content/html.js

@@ -0,0 +1,162 @@
+import { resolveTranscriptForLink } from '../../transcript/index.js';
+import { extractYouTubeVideoId, isYouTubeUrl, isYouTubeVideoUrl } from '../../transcript/utils.js';
+import { extractArticleContent, sanitizeHtmlForMarkdownConversion } from './article.js';
+import { normalizeForPrompt } from './cleaner.js';
+import { MIN_HTML_CONTENT_CHARACTERS, MIN_METADATA_DESCRIPTION_CHARACTERS, MIN_READABILITY_CONTENT_CHARACTERS, READABILITY_RELATIVE_THRESHOLD, } from './constants.js';
+import { extractJsonLdContent } from './jsonld.js';
+import { extractMetadataFromHtml } from './parsers.js';
+import { isPodcastHost, isPodcastLikeJsonLdType } from './podcast-utils.js';
+import { extractReadabilityFromHtml, toReadabilityHtml } from './readability.js';
+import { ensureTranscriptDiagnostics, finalizeExtractedLinkContent, pickFirstText, selectBaseContent, } from './utils.js';
+import { detectPrimaryVideoFromHtml } from './video.js';
+import { extractYouTubeShortDescription } from './youtube.js';
+const LEADING_CONTROL_PATTERN = /^[\s\p{Cc}]+/u;
+function stripLeadingTitle(content, title) {
+    if (!(content && title)) {
+        return content;
+    }
+    const normalizedTitle = title.trim();
+    if (normalizedTitle.length === 0) {
+        return content;
+    }
+    const trimmedContent = content.trimStart();
+    if (!trimmedContent.toLowerCase().startsWith(normalizedTitle.toLowerCase())) {
+        return content;
+    }
+    const remainderOriginal = trimmedContent.slice(normalizedTitle.length);
+    const remainder = remainderOriginal.replace(LEADING_CONTROL_PATTERN, '');
+    return remainder;
+}
+export async function buildResultFromHtmlDocument({ url, html, cacheMode, maxCharacters, youtubeTranscriptMode, firecrawlDiagnostics, markdownRequested, markdownMode, timeoutMs, deps, readabilityCandidate, }) {
+    if (isYouTubeVideoUrl(url) && !extractYouTubeVideoId(url)) {
+        throw new Error('Invalid YouTube video id in URL');
+    }
+    const { title, description, siteName } = extractMetadataFromHtml(html, url);
+    const jsonLd = extractJsonLdContent(html);
+    const mergedTitle = pickFirstText([jsonLd?.title, title]);
+    const mergedDescription = pickFirstText([jsonLd?.description, description]);
+    const isPodcastJsonLd = isPodcastLikeJsonLdType(jsonLd?.type);
+    const readability = readabilityCandidate ?? (await extractReadabilityFromHtml(html, url));
+    const readabilityText = readability?.text ? normalizeForPrompt(readability.text) : '';
+    const readabilityHtml = toReadabilityHtml(readability);
+    const normalizedSegmentsFromHtml = normalizeForPrompt(extractArticleContent(html));
+    const normalizedSegmentsFromReadabilityHtml = readabilityHtml
+        ? normalizeForPrompt(extractArticleContent(readabilityHtml))
+        : '';
+    const preferReadabilityHtml = normalizedSegmentsFromReadabilityHtml.length >= MIN_READABILITY_CONTENT_CHARACTERS &&
+        (normalizedSegmentsFromHtml.length < MIN_HTML_CONTENT_CHARACTERS ||
+            normalizedSegmentsFromReadabilityHtml.length >=
+                normalizedSegmentsFromHtml.length * READABILITY_RELATIVE_THRESHOLD);
+    const normalizedSegments = preferReadabilityHtml
+        ? normalizedSegmentsFromReadabilityHtml
+        : normalizedSegmentsFromHtml;
+    const preferReadabilityText = !preferReadabilityHtml &&
+        readabilityText.length >= MIN_READABILITY_CONTENT_CHARACTERS &&
+        (normalizedSegmentsFromHtml.length < MIN_HTML_CONTENT_CHARACTERS ||
+            readabilityText.length >= normalizedSegmentsFromHtml.length * READABILITY_RELATIVE_THRESHOLD);
+    const preferReadability = preferReadabilityHtml || preferReadabilityText;
+    const effectiveNormalized = preferReadabilityText ? readabilityText : normalizedSegments;
+    const descriptionCandidate = mergedDescription ? normalizeForPrompt(mergedDescription) : '';
+    const preferDescription = descriptionCandidate.length >= MIN_METADATA_DESCRIPTION_CHARACTERS &&
+        (isPodcastJsonLd ||
+            isPodcastHost(url) ||
+            (!preferReadability &&
+                (effectiveNormalized.length < MIN_HTML_CONTENT_CHARACTERS ||
+                    descriptionCandidate.length >=
+                        effectiveNormalized.length * READABILITY_RELATIVE_THRESHOLD)));
+    const effectiveNormalizedWithDescription = preferDescription
+        ? descriptionCandidate
+        : effectiveNormalized;
+    const transcriptResolution = await resolveTranscriptForLink(url, html, deps, {
+        youtubeTranscriptMode,
+        cacheMode,
+    });
+    const youtubeDescription = transcriptResolution.text === null ? extractYouTubeShortDescription(html) : null;
+    const baseCandidate = youtubeDescription
+        ? normalizeForPrompt(youtubeDescription)
+        : effectiveNormalizedWithDescription;
+    let baseContent = selectBaseContent(baseCandidate, transcriptResolution.text);
+    if (baseContent === normalizedSegments) {
+        baseContent = stripLeadingTitle(baseContent, mergedTitle ?? title);
+    }
+    const transcriptDiagnostics = ensureTranscriptDiagnostics(transcriptResolution, cacheMode ?? 'default');
+    const markdownDiagnostics = await (async () => {
+        if (!markdownRequested) {
+            return { requested: false, used: false, provider: null, notes: null };
+        }
+        if (isYouTubeUrl(url)) {
+            return {
+                requested: true,
+                used: false,
+                provider: null,
+                notes: 'Skipping Markdown conversion for YouTube URLs',
+            };
+        }
+        if (!deps.convertHtmlToMarkdown) {
+            return {
+                requested: true,
+                used: false,
+                provider: null,
+                notes: 'No HTML→Markdown converter configured',
+            };
+        }
+        try {
+            const htmlForMarkdown = markdownMode === 'readability' && readabilityHtml ? readabilityHtml : html;
+            const sanitizedHtml = sanitizeHtmlForMarkdownConversion(htmlForMarkdown);
+            const markdown = await deps.convertHtmlToMarkdown({
+                url,
+                html: sanitizedHtml,
+                title: mergedTitle ?? title,
+                siteName,
+                timeoutMs,
+            });
+            const normalizedMarkdown = normalizeForPrompt(markdown);
+            if (normalizedMarkdown.length === 0) {
+                return {
+                    requested: true,
+                    used: false,
+                    provider: null,
+                    notes: 'HTML→Markdown conversion returned empty content',
+                };
+            }
+            baseContent = normalizedMarkdown;
+            return {
+                requested: true,
+                used: true,
+                provider: 'llm',
+                notes: markdownMode === 'readability' && readabilityHtml
+                    ? 'Readability HTML used for markdown input'
+                    : null,
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return {
+                requested: true,
+                used: false,
+                provider: null,
+                notes: `HTML→Markdown conversion failed: ${message}`,
+            };
+        }
+    })();
+    const video = detectPrimaryVideoFromHtml(html, url);
+    const isVideoOnly = !transcriptResolution.text && baseContent.length < MIN_HTML_CONTENT_CHARACTERS && video !== null;
+    return finalizeExtractedLinkContent({
+        url,
+        baseContent,
+        maxCharacters,
+        title: mergedTitle ?? title,
+        description: mergedDescription ?? description,
+        siteName,
+        transcriptResolution,
+        video,
+        isVideoOnly,
+        diagnostics: {
+            strategy: 'html',
+            firecrawl: firecrawlDiagnostics,
+            markdown: markdownDiagnostics,
+            transcript: transcriptDiagnostics,
+        },
+    });
+}
+//# sourceMappingURL=html.js.map

package/dist/esm/content/link-preview/content/html.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/html.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAA;AACpE,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAGlG,OAAO,EAAE,qBAAqB,EAAE,iCAAiC,EAAE,MAAM,cAAc,CAAA;AACvF,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,EACL,2BAA2B,EAC3B,mCAAmC,EACnC,kCAAkC,EAClC,8BAA8B,GAC/B,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAA;AAC3E,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAEhF,OAAO,EACL,2BAA2B,EAC3B,4BAA4B,EAC5B,aAAa,EACb,iBAAiB,GAClB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAA;AACvD,OAAO,EAAE,8BAA8B,EAAE,MAAM,cAAc,CAAA;AAE7D,MAAM,uBAAuB,GAAG,eAAe,CAAA;AAE/C,SAAS,iBAAiB,CAAC,OAAe,EAAE,KAAgC;IAC1E,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IACpC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAA;IAC1C,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;QAC5E,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,iBAAiB,GAAG,cAAc,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;IACtE,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAA;IACxE,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAChD,GAAG,EACH,IAAI,EACJ,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,YAAY,EACZ,SAAS,EACT,IAAI,EACJ,oBAAoB,GAarB;IACC,IAAI,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,uBAAuB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAC3E,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;IACzC,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAA;IACzD,MAAM,iBAAiB,GAAG,aAAa,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC,CAAA;IAC3E,MAAM,eAAe,GAAG,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAC7D,MAAM,WAAW,GAAG,oBAAoB,IAAI,CAAC,MAAM,0BAA0B,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAA;IACzF,MAAM,eAAe,GAAG,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACrF,MAAM,eAAe,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAA;IAEtD,MAAM,0BAA0B,GAAG,kBAAkB,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,MAAM,qCAAqC,GAAG,eAAe;QAC3D,CAAC,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;QAC5D,CAAC,CAAC,EAAE,CAAA;IACN,MAAM,qBAAqB,GACzB,qCAAqC,CAAC,MAAM,IAAI,kCAAkC;QAClF,CAAC,0BAA0B,CAAC,MAAM,GAAG,2BAA2B;YAC9D,qCAAqC,CAAC,MAAM;gBAC1C,0BAA0B,CAAC,MAAM,GAAG,8BAA8B,CAAC,CAAA;IACzE,MAAM,kBAAkB,GAAG,qBAAqB;QAC9C,CAAC,CAAC,qCAAqC;QACvC,CAAC,CAAC,0BAA0B,CAAA;IAE9B,MAAM,qBAAqB,GACzB,CAAC,qBAAqB;QACtB,eAAe,CAAC,MAAM,IAAI,kCAAkC;QAC5D,CAAC,0BAA0B,CAAC,MAAM,GAAG,2BAA2B;YAC9D,eAAe,CAAC,MAAM,IAAI,0BAA0B,CAAC,MAAM,GAAG,8BAA8B,CAAC,CAAA;IACjG,MAAM,iBAAiB,GAAG,qBAAqB,IAAI,qBAAqB,CAAA;IACxE,MAAM,mBAAmB,GAAG,qBAAqB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,kBAAkB,CAAA;IACxF,MAAM,oBAAoB,GAAG,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC3F,MAAM,iBAAiB,GACrB,oBAAoB,CAAC,MAAM,IAAI,mCAAmC;QAClE,CAAC,eAAe;YACd,aAAa,CAAC,GAAG,CAAC;YAClB,CAAC,CAAC,iBAAiB;gBACjB,CAAC,mBAAmB,CAAC,MAAM,GAAG,2BAA2B;oBACvD,oBAAoB,CAAC,MAAM;wBACzB,mBAAmB,CAAC,MAAM,GAAG,8BAA8B,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,kCAAkC,GAAG,iBAAiB;QAC1D,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,mBAAmB,CAAA;IACvB,MAAM,oBAAoB,GAAG,MAAM,wBAAwB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE;QAC3E,qBAAqB;QACrB,SAAS;KACV,CAAC,CAAA;IAEF,MAAM,kBAAkB,GACtB,oBAAoB,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAClF,MAAM,aAAa,GAAG,kBAAkB;QACtC,CAAC,CAAC,kBAAkB,CAAC,kBAAkB,CAAC;QACxC,CAAC,CAAC,kCAAkC,CAAA;IAEtC,IAAI,WAAW,GAAG,iBAAiB,CAAC,aAAa,EAAE,oBAAoB,CAAC,IAAI,CAAC,CAAA;IAC7E,IAAI,WAAW,KAAK,kBAAkB,EAAE,CAAC;QACvC,WAAW,GAAG,iBAAiB,CAAC,WAAW,EAAE,WAAW,IAAI,KAAK,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,qBAAqB,GAAG,2BAA2B,CACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;IAED,MAAM,mBAAmB,GAAwB,MAAM,CAAC,KAAK,IAAI,EAAE;QACjE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;QACvE,CAAC;QAED,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,+CAA+C;aACvD,CAAA;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAChC,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,uCAAuC;aAC/C,CAAA;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,eAAe,GACnB,YAAY,KAAK,aAAa,IAAI,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAA;YAC5E,MAAM,aAAa,GAAG,iCAAiC,CAAC,eAAe,CAAC,CAAA;YACxE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC;gBAChD,GAAG;gBACH,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,WAAW,IAAI,KAAK;gBAC3B,QAAQ;gBACR,SAAS;aACV,CAAC,CAAA;YACF,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAA;YACvD,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACpC,OAAO;oBACL,SAAS,EAAE,IAAI;oBACf,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,iDAAiD;iBACzD,CAAA;YACH,CAAC;YAED,WAAW,GAAG,kBAAkB,CAAA;YAChC,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,IAAI;gBACV,QAAQ,EAAE,KAAK;gBACf,KAAK,EACH,YAAY,KAAK,aAAa,IAAI,eAAe;oBAC/C,CAAC,CAAC,0CAA0C;oBAC5C,CAAC,CAAC,IAAI;aACX,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,OAAO;gBACL,SAAS,EAAE,IAAI;gBACf,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,IAAI;gBACd,KAAK,EAAE,oCAAoC,OAAO,EAAE;aACrD,CAAA;QACH,CAAC;IACH,CAAC,CAAC,EAAE,CAAA;IAEJ,MAAM,KAAK,GAAG,0BAA0B,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACnD,MAAM,WAAW,GACf,CAAC,oBAAoB,CAAC,IAAI,IAAI,WAAW,CAAC,MAAM,GAAG,2BAA2B,IAAI,KAAK,KAAK,IAAI,CAAA;IAElG,OAAO,4BAA4B,CAAC;QAClC,GAAG;QACH,WAAW;QACX,aAAa;QACb,KAAK,EAAE,WAAW,IAAI,KAAK;QAC3B,WAAW,EAAE,iBAAiB,IAAI,WAAW;QAC7C,QAAQ;QACR,oBAAoB;QACpB,KAAK;QACL,WAAW;QACX,WAAW,EAAE;YACX,QAAQ,EAAE,MAAM;YAChB,SAAS,EAAE,oBAAoB;YAC/B,QAAQ,EAAE,mBAAmB;YAC7B,UAAU,EAAE,qBAAqB;SAClC;KACF,CAAC,CAAA;AACJ,CAAC"}

package/dist/esm/content/link-preview/content/index.js

@@ -0,0 +1,345 @@
+import { resolveTranscriptForLink } from '../../transcript/index.js';
+import { isYouTubeUrl } from '../../transcript/utils.js';
+import { normalizeForPrompt } from './cleaner.js';
+import { MIN_READABILITY_CONTENT_CHARACTERS } from './constants.js';
+import { fetchHtmlDocument, fetchWithFirecrawl } from './fetcher.js';
+import { buildResultFromFirecrawl, shouldFallbackToFirecrawl } from './firecrawl.js';
+import { buildResultFromHtmlDocument } from './html.js';
+import { extractApplePodcastIds, extractSpotifyEpisodeId } from './podcast-utils.js';
+import { extractReadabilityFromHtml } from './readability.js';
+import { isAnubisHtml, isBlockedTwitterContent, isTwitterStatusUrl, toNitterUrls, } from './twitter-utils.js';
+import { appendNote, ensureTranscriptDiagnostics, finalizeExtractedLinkContent, resolveCacheMode, resolveFirecrawlMode, resolveMaxCharacters, resolveTimeoutMs, selectBaseContent, } from './utils.js';
+export async function fetchLinkContent(url, options, deps) {
+    const timeoutMs = resolveTimeoutMs(options);
+    const cacheMode = resolveCacheMode(options);
+    const maxCharacters = resolveMaxCharacters(options);
+    const youtubeTranscriptMode = options?.youtubeTranscript ?? 'auto';
+    const firecrawlMode = resolveFirecrawlMode(options);
+    const markdownRequested = (options?.format ?? 'text') === 'markdown';
+    const markdownMode = options?.markdownMode ?? 'auto';
+    const canUseFirecrawl = firecrawlMode !== 'off' && deps.scrapeWithFirecrawl !== null && !isYouTubeUrl(url);
+    const spotifyEpisodeId = extractSpotifyEpisodeId(url);
+    if (spotifyEpisodeId) {
+        if (!deps.openaiApiKey && !deps.falApiKey) {
+            throw new Error('Spotify episode transcription requires OPENAI_API_KEY or FAL_KEY (Whisper); otherwise you may only get a captcha/recaptcha HTML page.');
+        }
+        const transcriptResolution = await resolveTranscriptForLink(url, null, deps, {
+            youtubeTranscriptMode,
+            cacheMode,
+        });
+        if (!transcriptResolution.text) {
+            const notes = transcriptResolution.diagnostics?.notes;
+            const suffix = notes ? ` (${notes})` : '';
+            throw new Error(`Failed to transcribe Spotify episode${suffix}`);
+        }
+        const transcriptDiagnostics = ensureTranscriptDiagnostics(transcriptResolution, cacheMode ?? 'default');
+        transcriptDiagnostics.notes = appendNote(transcriptDiagnostics.notes, 'Spotify episode: skipped HTML fetch to avoid captcha pages');
+        return finalizeExtractedLinkContent({
+            url,
+            baseContent: selectBaseContent('', transcriptResolution.text),
+            maxCharacters,
+            title: null,
+            description: null,
+            siteName: 'Spotify',
+            transcriptResolution,
+            video: null,
+            isVideoOnly: false,
+            diagnostics: {
+                strategy: 'html',
+                firecrawl: {
+                    attempted: false,
+                    used: false,
+                    cacheMode,
+                    cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
+                    notes: 'Spotify short-circuit skipped HTML/Firecrawl',
+                },
+                markdown: {
+                    requested: markdownRequested,
+                    used: false,
+                    provider: null,
+                    notes: 'Spotify short-circuit uses transcript content',
+                },
+                transcript: transcriptDiagnostics,
+            },
+        });
+    }
+    const appleIds = extractApplePodcastIds(url);
+    if (appleIds) {
+        if (!deps.openaiApiKey && !deps.falApiKey) {
+            throw new Error('Apple Podcasts transcription requires OPENAI_API_KEY or FAL_KEY (Whisper); otherwise you may only get a slow/blocked HTML page.');
+        }
+        const transcriptResolution = await resolveTranscriptForLink(url, null, deps, {
+            youtubeTranscriptMode,
+            cacheMode,
+        });
+        if (!transcriptResolution.text) {
+            const notes = transcriptResolution.diagnostics?.notes;
+            const suffix = notes ? ` (${notes})` : '';
+            throw new Error(`Failed to transcribe Apple Podcasts episode${suffix}`);
+        }
+        const transcriptDiagnostics = ensureTranscriptDiagnostics(transcriptResolution, cacheMode ?? 'default');
+        transcriptDiagnostics.notes = appendNote(transcriptDiagnostics.notes, 'Apple Podcasts: skipped HTML fetch (prefer iTunes lookup / enclosures)');
+        return finalizeExtractedLinkContent({
+            url,
+            baseContent: selectBaseContent('', transcriptResolution.text),
+            maxCharacters,
+            title: null,
+            description: null,
+            siteName: 'Apple Podcasts',
+            transcriptResolution,
+            video: null,
+            isVideoOnly: false,
+            diagnostics: {
+                strategy: 'html',
+                firecrawl: {
+                    attempted: false,
+                    used: false,
+                    cacheMode,
+                    cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
+                    notes: 'Apple Podcasts short-circuit skipped HTML/Firecrawl',
+                },
+                markdown: {
+                    requested: markdownRequested,
+                    used: false,
+                    provider: null,
+                    notes: 'Apple Podcasts short-circuit uses transcript content',
+                },
+                transcript: transcriptDiagnostics,
+            },
+        });
+    }
+    let firecrawlAttempted = false;
+    let firecrawlPayload = null;
+    const firecrawlDiagnostics = {
+        attempted: false,
+        used: false,
+        cacheMode,
+        cacheStatus: cacheMode === 'bypass' ? 'bypassed' : 'unknown',
+        notes: null,
+    };
+    const twitterStatus = isTwitterStatusUrl(url);
+    const nitterUrls = twitterStatus ? toNitterUrls(url) : [];
+    let birdError = null;
+    let nitterError = null;
+    const attemptFirecrawl = async (reason) => {
+        if (!canUseFirecrawl) {
+            return null;
+        }
+        if (!firecrawlAttempted) {
+            const attempt = await fetchWithFirecrawl(url, deps.scrapeWithFirecrawl, {
+                timeoutMs,
+                cacheMode,
+                onProgress: deps.onProgress ?? null,
+                reason,
+            });
+            firecrawlAttempted = true;
+            firecrawlPayload = attempt.payload;
+            firecrawlDiagnostics.attempted = attempt.diagnostics.attempted;
+            firecrawlDiagnostics.used = attempt.diagnostics.used;
+            firecrawlDiagnostics.cacheMode = attempt.diagnostics.cacheMode;
+            firecrawlDiagnostics.cacheStatus = attempt.diagnostics.cacheStatus;
+            firecrawlDiagnostics.notes = attempt.diagnostics.notes ?? null;
+        }
+        firecrawlDiagnostics.notes = appendNote(firecrawlDiagnostics.notes, reason);
+        if (!firecrawlPayload) {
+            return null;
+        }
+        const firecrawlResult = await buildResultFromFirecrawl({
+            url,
+            payload: firecrawlPayload,
+            cacheMode,
+            maxCharacters,
+            youtubeTranscriptMode,
+            firecrawlDiagnostics,
+            markdownRequested,
+            deps,
+        });
+        if (firecrawlResult) {
+            return firecrawlResult;
+        }
+        firecrawlDiagnostics.notes = appendNote(firecrawlDiagnostics.notes, 'Firecrawl returned empty content');
+        return null;
+    };
+    const attemptBird = async () => {
+        if (!deps.readTweetWithBird || !twitterStatus) {
+            return null;
+        }
+        deps.onProgress?.({ kind: 'bird-start', url });
+        try {
+            const tweet = await deps.readTweetWithBird({ url, timeoutMs });
+            const text = tweet?.text?.trim() ?? '';
+            if (text.length === 0) {
+                deps.onProgress?.({ kind: 'bird-done', url, ok: false, textBytes: null });
+                return null;
+            }
+            const title = tweet?.author?.username ? `@${tweet.author.username}` : null;
+            const description = null;
+            const siteName = 'X';
+            const transcriptResolution = { text: null, source: null };
+            const transcriptDiagnostics = ensureTranscriptDiagnostics(transcriptResolution, cacheMode ?? 'default');
+            const result = finalizeExtractedLinkContent({
+                url,
+                baseContent: text,
+                maxCharacters,
+                title,
+                description,
+                siteName,
+                transcriptResolution,
+                video: null,
+                isVideoOnly: false,
+                diagnostics: {
+                    strategy: 'bird',
+                    firecrawl: firecrawlDiagnostics,
+                    markdown: {
+                        requested: markdownRequested,
+                        used: false,
+                        provider: null,
+                        notes: 'Bird tweet fetch provides plain text',
+                    },
+                    transcript: transcriptDiagnostics,
+                },
+            });
+            deps.onProgress?.({
+                kind: 'bird-done',
+                url,
+                ok: true,
+                textBytes: Buffer.byteLength(result.content, 'utf8'),
+            });
+            return result;
+        }
+        catch (error) {
+            birdError = error;
+            deps.onProgress?.({ kind: 'bird-done', url, ok: false, textBytes: null });
+            return null;
+        }
+    };
+    const birdResult = await attemptBird();
+    if (birdResult) {
+        return birdResult;
+    }
+    const attemptNitter = async () => {
+        if (nitterUrls.length === 0) {
+            return null;
+        }
+        for (const nitterUrl of nitterUrls) {
+            deps.onProgress?.({ kind: 'nitter-start', url: nitterUrl });
+            try {
+                const nitterHtml = await fetchHtmlDocument(deps.fetch, nitterUrl, { timeoutMs });
+                if (!nitterHtml.trim()) {
+                    nitterError = new Error(`Nitter returned empty body from ${new URL(nitterUrl).host}`);
+                    deps.onProgress?.({ kind: 'nitter-done', url: nitterUrl, ok: false, textBytes: null });
+                    continue;
+                }
+                if (isAnubisHtml(nitterHtml)) {
+                    nitterError = new Error(`Nitter returned Anubis challenge from ${new URL(nitterUrl).host}`);
+                    deps.onProgress?.({ kind: 'nitter-done', url: nitterUrl, ok: false, textBytes: null });
+                    continue;
+                }
+                deps.onProgress?.({
+                    kind: 'nitter-done',
+                    url: nitterUrl,
+                    ok: true,
+                    textBytes: Buffer.byteLength(nitterHtml, 'utf8'),
+                });
+                return nitterHtml;
+            }
+            catch (error) {
+                nitterError = error;
+                deps.onProgress?.({ kind: 'nitter-done', url: nitterUrl, ok: false, textBytes: null });
+            }
+        }
+        return null;
+    };
+    const nitterHtml = await attemptNitter();
+    if (nitterHtml) {
+        const nitterResult = await buildResultFromHtmlDocument({
+            url,
+            html: nitterHtml,
+            cacheMode,
+            maxCharacters,
+            youtubeTranscriptMode,
+            firecrawlDiagnostics,
+            markdownRequested,
+            markdownMode,
+            timeoutMs,
+            deps,
+            readabilityCandidate: null,
+        });
+        if (!isBlockedTwitterContent(nitterResult.content)) {
+            nitterResult.diagnostics.strategy = 'nitter';
+            return nitterResult;
+        }
+        nitterError = new Error('Nitter returned blocked or empty content');
+    }
+    if (firecrawlMode === 'always') {
+        const firecrawlResult = await attemptFirecrawl('Firecrawl forced via options');
+        if (firecrawlResult) {
+            return firecrawlResult;
+        }
+    }
+    let html = null;
+    let htmlError = null;
+    try {
+        html = await fetchHtmlDocument(deps.fetch, url, {
+            timeoutMs,
+            onProgress: deps.onProgress ?? null,
+        });
+    }
+    catch (error) {
+        htmlError = error;
+    }
+    if (!html) {
+        if (!canUseFirecrawl) {
+            throw htmlError instanceof Error ? htmlError : new Error('Failed to fetch HTML document');
+        }
+        const firecrawlResult = await attemptFirecrawl('HTML fetch failed; falling back to Firecrawl');
+        if (firecrawlResult) {
+            return firecrawlResult;
+        }
+        const firecrawlError = firecrawlDiagnostics.notes
+            ? `; Firecrawl notes: ${firecrawlDiagnostics.notes}`
+            : '';
+        throw new Error(`Failed to fetch HTML document${firecrawlError}${htmlError instanceof Error ? `; HTML error: ${htmlError.message}` : ''}`);
+    }
+    let readabilityCandidate = null;
+    if (firecrawlMode === 'auto' && shouldFallbackToFirecrawl(html)) {
+        readabilityCandidate = await extractReadabilityFromHtml(html, url);
+        const readabilityText = readabilityCandidate?.text
+            ? normalizeForPrompt(readabilityCandidate.text)
+            : '';
+        if (readabilityText.length < MIN_READABILITY_CONTENT_CHARACTERS) {
+            const firecrawlResult = await attemptFirecrawl('HTML content looked blocked/thin; falling back to Firecrawl');
+            if (firecrawlResult) {
+                return firecrawlResult;
+            }
+        }
+    }
+    const htmlResult = await buildResultFromHtmlDocument({
+        url,
+        html,
+        cacheMode,
+        maxCharacters,
+        youtubeTranscriptMode,
+        firecrawlDiagnostics,
+        markdownRequested,
+        markdownMode,
+        timeoutMs,
+        deps,
+        readabilityCandidate,
+    });
+    if (twitterStatus && isBlockedTwitterContent(htmlResult.content)) {
+        const birdNote = !deps.readTweetWithBird
+            ? 'Bird not available'
+            : birdError
+                ? `Bird failed: ${birdError instanceof Error ? birdError.message : String(birdError)}`
+                : 'Bird returned no text';
+        const nitterNote = nitterUrls.length > 0
+            ? nitterError
+                ? `Nitter failed: ${nitterError instanceof Error ? nitterError.message : String(nitterError)}`
+                : 'Nitter returned no text'
+            : 'Nitter not available';
+        throw new Error(`Unable to fetch tweet content from X. ${birdNote}. ${nitterNote}.`);
+    }
+    return htmlResult;
+}
+//# sourceMappingURL=index.js.map

package/dist/esm/content/link-preview/content/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAA;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAGxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,EAAE,kCAAkC,EAAE,MAAM,gBAAgB,CAAA;AACnE,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AACpE,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAA;AACpF,OAAO,EAAE,2BAA2B,EAAE,MAAM,WAAW,CAAA;AACvD,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAA;AACpF,OAAO,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAA;AAC7D,OAAO,EACL,YAAY,EACZ,uBAAuB,EACvB,kBAAkB,EAClB,YAAY,GACb,MAAM,oBAAoB,CAAA;AAE3B,OAAO,EACL,UAAU,EACV,2BAA2B,EAC3B,4BAA4B,EAC5B,gBAAgB,EAChB,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,YAAY,CAAA;AAEnB,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAW,EACX,OAA4C,EAC5C,IAAqB;IAErB,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;IAC3C,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAA;IAC3C,MAAM,aAAa,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAA;IACnD,MAAM,qBAAqB,GAAG,OAAO,EAAE,iBAAiB,IAAI,MAAM,CAAA;IAClE,MAAM,aAAa,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAA;IACnD,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,IAAI,MAAM,CAAC,KAAK,UAAU,CAAA;IACpE,MAAM,YAAY,GAAiB,OAAO,EAAE,YAAY,IAAI,MAAM,CAAA;IAElE,MAAM,eAAe,GACnB,aAAa,KAAK,KAAK,IAAI,IAAI,CAAC,mBAAmB,KAAK,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IAEpF,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAA;IACrD,IAAI,gBAAgB,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CACb,uIAAuI,CACxI,CAAA;QACH,CAAC;QAED,MAAM,oBAAoB,GAAG,MAAM,wBAAwB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE;YAC3E,qBAAqB;YACrB,SAAS;SACV,CAAC,CAAA;QACF,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,oBAAoB,CAAC,WAAW,EAAE,KAAK,CAAA;YACrD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;YACzC,MAAM,IAAI,KAAK,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAA;QAClE,CAAC;QAED,MAAM,qBAAqB,GAAG,2BAA2B,CACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;QACD,qBAAqB,CAAC,KAAK,GAAG,UAAU,CACtC,qBAAqB,CAAC,KAAK,EAC3B,4DAA4D,CAC7D,CAAA;QAED,OAAO,4BAA4B,CAAC;YAClC,GAAG;YACH,WAAW,EAAE,iBAAiB,CAAC,EAAE,EAAE,oBAAoB,CAAC,IAAI,CAAC;YAC7D,aAAa;YACb,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,SAAS;YACnB,oBAAoB;YACpB,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,KAAK;YAClB,WAAW,EAAE;gBACX,QAAQ,EAAE,MAAM;gBAChB,SAAS,EAAE;oBACT,SAAS,EAAE,KAAK;oBAChB,IAAI,EAAE,KAAK;oBACX,SAAS;oBACT,WAAW,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;oBAC5D,KAAK,EAAE,8CAA8C;iBACtD;gBACD,QAAQ,EAAE;oBACR,SAAS,EAAE,iBAAiB;oBAC5B,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,+CAA+C;iBACvD;gBACD,UAAU,EAAE,qBAAqB;aAClC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAA;IAC5C,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CACb,iIAAiI,CAClI,CAAA;QACH,CAAC;QAED,MAAM,oBAAoB,GAAG,MAAM,wBAAwB,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE;YAC3E,qBAAqB;YACrB,SAAS;SACV,CAAC,CAAA;QACF,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,oBAAoB,CAAC,WAAW,EAAE,KAAK,CAAA;YACrD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;YACzC,MAAM,IAAI,KAAK,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAA;QACzE,CAAC;QAED,MAAM,qBAAqB,GAAG,2BAA2B,CACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;QACD,qBAAqB,CAAC,KAAK,GAAG,UAAU,CACtC,qBAAqB,CAAC,KAAK,EAC3B,wEAAwE,CACzE,CAAA;QAED,OAAO,4BAA4B,CAAC;YAClC,GAAG;YACH,WAAW,EAAE,iBAAiB,CAAC,EAAE,EAAE,oBAAoB,CAAC,IAAI,CAAC;YAC7D,aAAa;YACb,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,IAAI;YACjB,QAAQ,EAAE,gBAAgB;YAC1B,oBAAoB;YACpB,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,KAAK;YAClB,WAAW,EAAE;gBACX,QAAQ,EAAE,MAAM;gBAChB,SAAS,EAAE;oBACT,SAAS,EAAE,KAAK;oBAChB,IAAI,EAAE,KAAK;oBACX,SAAS;oBACT,WAAW,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;oBAC5D,KAAK,EAAE,qDAAqD;iBAC7D;gBACD,QAAQ,EAAE;oBACR,SAAS,EAAE,iBAAiB;oBAC5B,IAAI,EAAE,KAAK;oBACX,QAAQ,EAAE,IAAI;oBACd,KAAK,EAAE,sDAAsD;iBAC9D;gBACD,UAAU,EAAE,qBAAqB;aAClC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,IAAI,kBAAkB,GAAG,KAAK,CAAA;IAC9B,IAAI,gBAAgB,GAAiC,IAAI,CAAA;IACzD,MAAM,oBAAoB,GAAyB;QACjD,SAAS,EAAE,KAAK;QAChB,IAAI,EAAE,KAAK;QACX,SAAS;QACT,WAAW,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QAC5D,KAAK,EAAE,IAAI;KACZ,CAAA;IAED,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;IAC7C,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IACzD,IAAI,SAAS,GAAY,IAAI,CAAA;IAC7B,IAAI,WAAW,GAAY,IAAI,CAAA;IAE/B,MAAM,gBAAgB,GAAG,KAAK,EAAE,MAAc,EAAwC,EAAE;QACtF,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,OAAO,IAAI,CAAA;QACb,CAAC;QAED,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,mBAAmB,EAAE;gBACtE,SAAS;gBACT,SAAS;gBACT,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;gBACnC,MAAM;aACP,CAAC,CAAA;YACF,kBAAkB,GAAG,IAAI,CAAA;YACzB,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAA;YAClC,oBAAoB,CAAC,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAA;YAC9D,oBAAoB,CAAC,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,IAAI,CAAA;YACpD,oBAAoB,CAAC,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAA;YAC9D,oBAAoB,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,CAAA;YAClE,oBAAoB,CAAC,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI,IAAI,CAAA;QAChE,CAAC;QAED,oBAAoB,CAAC,KAAK,GAAG,UAAU,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QAE3E,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,IAAI,CAAA;QACb,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,wBAAwB,CAAC;YACrD,GAAG;YACH,OAAO,EAAE,gBAAgB;YACzB,SAAS;YACT,aAAa;YACb,qBAAqB;YACrB,oBAAoB;YACpB,iBAAiB;YACjB,IAAI;SACL,CAAC,CAAA;QACF,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,eAAe,CAAA;QACxB,CAAC;QAED,oBAAoB,CAAC,KAAK,GAAG,UAAU,CACrC,oBAAoB,CAAC,KAAK,EAC1B,kCAAkC,CACnC,CAAA;QACD,OAAO,IAAI,CAAA;IACb,CAAC,CAAA;IAED,MAAM,WAAW,GAAG,KAAK,IAA0C,EAAE;QACnE,IAAI,CAAC,IAAI,CAAC,iBAAiB,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAA;QACb,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,CAAA;QAC9C,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAA;YAC9D,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;YACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;gBACzE,OAAO,IAAI,CAAA;YACb,CAAC;YAED,MAAM,KAAK,GAAG,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;YAC1E,MAAM,WAAW,GAAG,IAAI,CAAA;YACxB,MAAM,QAAQ,GAAG,GAAG,CAAA;YACpB,MAAM,oBAAoB,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;YACzD,MAAM,qBAAqB,GAAG,2BAA2B,CACvD,oBAAoB,EACpB,SAAS,IAAI,SAAS,CACvB,CAAA;YACD,MAAM,MAAM,GAAG,4BAA4B,CAAC;gBAC1C,GAAG;gBACH,WAAW,EAAE,IAAI;gBACjB,aAAa;gBACb,KAAK;gBACL,WAAW;gBACX,QAAQ;gBACR,oBAAoB;gBACpB,KAAK,EAAE,IAAI;gBACX,WAAW,EAAE,KAAK;gBAClB,WAAW,EAAE;oBACX,QAAQ,EAAE,MAAM;oBAChB,SAAS,EAAE,oBAAoB;oBAC/B,QAAQ,EAAE;wBACR,SAAS,EAAE,iBAAiB;wBAC5B,IAAI,EAAE,KAAK;wBACX,QAAQ,EAAE,IAAI;wBACd,KAAK,EAAE,sCAAsC;qBAC9C;oBACD,UAAU,EAAE,qBAAqB;iBAClC;aACF,CAAC,CAAA;YACF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,IAAI,EAAE,WAAW;gBACjB,GAAG;gBACH,EAAE,EAAE,IAAI;gBACR,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC;aACrD,CAAC,CAAA;YACF,OAAO,MAAM,CAAA;QACf,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAA;YACjB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YACzE,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,MAAM,WAAW,EAAE,CAAA;IACtC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,MAAM,aAAa,GAAG,KAAK,IAA4B,EAAE;QACvD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QACD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAA;YAC3D,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;gBAChF,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;oBACvB,WAAW,GAAG,IAAI,KAAK,CAAC,mCAAmC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;oBACrF,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBACtF,SAAQ;gBACV,CAAC;gBACD,IAAI,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7B,WAAW,GAAG,IAAI,KAAK,CACrB,yCAAyC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CACnE,CAAA;oBACD,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBACtF,SAAQ;gBACV,CAAC;gBACD,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,IAAI,EAAE,aAAa;oBACnB,GAAG,EAAE,SAAS;oBACd,EAAE,EAAE,IAAI;oBACR,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC;iBACjD,CAAC,CAAA;gBACF,OAAO,UAAU,CAAA;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,WAAW,GAAG,KAAK,CAAA;gBACnB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YACxF,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAA;IACxC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,MAAM,2BAA2B,CAAC;YACrD,GAAG;YACH,IAAI,EAAE,UAAU;YAChB,SAAS;YACT,aAAa;YACb,qBAAqB;YACrB,oBAAoB;YACpB,iBAAiB;YACjB,YAAY;YACZ,SAAS;YACT,IAAI;YACJ,oBAAoB,EAAE,IAAI;SAC3B,CAAC,CAAA;QACF,IAAI,CAAC,uBAAuB,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,YAAY,CAAC,WAAW,CAAC,QAAQ,GAAG,QAAQ,CAAA;YAC5C,OAAO,YAAY,CAAA;QACrB,CAAC;QACD,WAAW,GAAG,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;IACrE,CAAC;IAED,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAAC,8BAA8B,CAAC,CAAA;QAC9E,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,eAAe,CAAA;QACxB,CAAC;IACH,CAAC;IAED,IAAI,IAAI,GAAkB,IAAI,CAAA;IAC9B,IAAI,SAAS,GAAY,IAAI,CAAA;IAE7B,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE;YAC9C,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;SACpC,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,GAAG,KAAK,CAAA;IACnB,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAA;QAC3F,CAAC;QAED,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAAC,8CAA8C,CAAC,CAAA;QAC9F,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,eAAe,CAAA;QACxB,CAAC;QAED,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK;YAC/C,CAAC,CAAC,sBAAsB,oBAAoB,CAAC,KAAK,EAAE;YACpD,CAAC,CAAC,EAAE,CAAA;QACN,MAAM,IAAI,KAAK,CACb,gCAAgC,cAAc,GAC5C,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,iBAAiB,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EACtE,EAAE,CACH,CAAA;IACH,CAAC;IAED,IAAI,oBAAoB,GAAkE,IAAI,CAAA;IAE9F,IAAI,aAAa,KAAK,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,EAAE,CAAC;QAChE,oBAAoB,GAAG,MAAM,0BAA0B,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QAClE,MAAM,eAAe,GAAG,oBAAoB,EAAE,IAAI;YAChD,CAAC,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,IAAI,CAAC;YAC/C,CAAC,CAAC,EAAE,CAAA;QACN,IAAI,eAAe,CAAC,MAAM,GAAG,kCAAkC,EAAE,CAAC;YAChE,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAC5C,6DAA6D,CAC9D,CAAA;YACD,IAAI,eAAe,EAAE,CAAC;gBACpB,OAAO,eAAe,CAAA;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,2BAA2B,CAAC;QACnD,GAAG;QACH,IAAI;QACJ,SAAS;QACT,aAAa;QACb,qBAAqB;QACrB,oBAAoB;QACpB,iBAAiB;QACjB,YAAY;QACZ,SAAS;QACT,IAAI;QACJ,oBAAoB;KACrB,CAAC,CAAA;IACF,IAAI,aAAa,IAAI,uBAAuB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACjE,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,iBAAiB;YACtC,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,SAAS;gBACT,CAAC,CAAC,gBAAgB,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE;gBACtF,CAAC,CAAC,uBAAuB,CAAA;QAC7B,MAAM,UAAU,GACd,UAAU,CAAC,MAAM,GAAG,CAAC;YACnB,CAAC,CAAC,WAAW;gBACX,CAAC,CAAC,kBAAkB,WAAW,YAAY,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE;gBAC9F,CAAC,CAAC,yBAAyB;YAC7B,CAAC,CAAC,sBAAsB,CAAA;QAC5B,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,KAAK,UAAU,GAAG,CAAC,CAAA;IACtF,CAAC;IACD,OAAO,UAAU,CAAA;AACnB,CAAC"}

package/dist/esm/content/link-preview/content/jsonld.js

@@ -0,0 +1,77 @@
+import { load } from 'cheerio';
+import { normalizeCandidate } from './cleaner.js';
+export function extractJsonLdContent(html) {
+    try {
+        const $ = load(html);
+        const scripts = $('script[type="application/ld+json"]').toArray();
+        const candidates = [];
+        for (const script of scripts) {
+            const raw = $(script).text();
+            if (!raw)
+                continue;
+            try {
+                const data = JSON.parse(raw);
+                collectCandidates(data, candidates);
+            }
+            catch {
+                // ignore malformed jsonld
+            }
+        }
+        if (candidates.length === 0)
+            return null;
+        const sorted = candidates
+            .map((c) => ({
+            title: c.title ? normalizeCandidate(c.title) : null,
+            description: c.description ? normalizeCandidate(c.description) : null,
+            type: c.type ? normalizeCandidate(c.type) : null,
+        }))
+            .filter((c) => c.title || c.description)
+            .sort((a, b) => (b.description?.length ?? 0) - (a.description?.length ?? 0));
+        return sorted[0] ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+function collectCandidates(input, out) {
+    if (!input)
+        return;
+    if (Array.isArray(input)) {
+        for (const item of input)
+            collectCandidates(item, out);
+        return;
+    }
+    if (typeof input !== 'object')
+        return;
+    const record = input;
+    if (record['@graph'] && Array.isArray(record['@graph'])) {
+        collectCandidates(record['@graph'], out);
+    }
+    const type = extractType(record);
+    if (type) {
+        const title = firstString(record, ['name', 'headline', 'title']);
+        const description = firstString(record, ['description', 'summary']);
+        if (title || description) {
+            out.push({ title: title ?? null, description: description ?? null, type });
+        }
+    }
+}
+function extractType(record) {
+    const raw = record['@type'];
+    if (typeof raw === 'string')
+        return raw.toLowerCase();
+    if (Array.isArray(raw)) {
+        const found = raw.find((entry) => typeof entry === 'string');
+        return typeof found === 'string' ? found.toLowerCase() : null;
+    }
+    return null;
+}
+function firstString(record, keys) {
+    for (const key of keys) {
+        const value = record[key];
+        if (typeof value === 'string' && value.trim())
+            return value.trim();
+    }
+    return null;
+}
+//# sourceMappingURL=jsonld.js.map

package/dist/esm/content/link-preview/content/jsonld.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonld.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/content/jsonld.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAE9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAQjD,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAA;QACpB,MAAM,OAAO,GAAG,CAAC,CAAC,oCAAoC,CAAC,CAAC,OAAO,EAAE,CAAA;QACjE,MAAM,UAAU,GAAoB,EAAE,CAAA;QAEtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5B,IAAI,CAAC,GAAG;gBAAE,SAAQ;YAClB,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;gBAC5B,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YACrC,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAExC,MAAM,MAAM,GAAG,UAAU;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;YACnD,WAAW,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI;YACrE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;SACjD,CAAC,CAAC;aACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,WAAW,CAAC;aACvC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9E,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,GAAoB;IAC7D,IAAI,CAAC,KAAK;QAAE,OAAM;IAElB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QACtD,OAAM;IACR,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAM;IAErC,MAAM,MAAM,GAAG,KAAgC,CAAA;IAC/C,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACxD,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAA;IAC1C,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;IAChC,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;QAChE,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAA;QACnE,IAAI,KAAK,IAAI,WAAW,EAAE,CAAC;YACzB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,IAAI,IAAI,EAAE,WAAW,EAAE,WAAW,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5E,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IAClD,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAA;IAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC,WAAW,EAAE,CAAA;IACrD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAA;QAC5D,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;IAC/D,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B,EAAE,IAAc;IAClE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;QACzB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;YAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;IACpE,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC"}