@llm-newsletter-kit/core 1.1.6 → 1.3.0

package/README.md

@@ -168,6 +168,49 @@ This kit prioritizes flexibility over rigid tooling. Instead of locking you into
 - **Asynchronous Injection:** Parsing logic is injected asynchronously, allowing you to integrate third-party APIs or AI-based parsers effortlessly.
 - **Recommendation:** While the kit supports LLM-based parsing (HTML-to-JSON), we generally recommend **rule-based parsing** (e.g., CSS selectors) for production environments to ensure speed, cost-efficiency, and stability.
 
+## Playground
+
+Playground scripts let you run individual LLM query classes in isolation — no full pipeline needed. Useful for prompt tuning, testing new options, or debugging output quality.
+
+### Setup
+
+1. Install playground dependencies:
+   ```bash
+   npm install -D tsx @ai-sdk/openai
+   ```
+
+2. Copy example data files and customize:
+   ```bash
+   mkdir -p playground/data
+   cp playground/data-examples/config.example.json playground/data/config.json
+   cp playground/data-examples/articles.example.json playground/data/articles.json
+   cp playground/data-examples/template.example.html playground/data/template.html
+   ```
+
+3. Edit `playground/data/config.json` with your OpenAI API key and options.
+4. Edit `playground/data/articles.json` with your target articles.
+5. (Optional) Replace `playground/data/template.html` with your actual email template.
+
+### Run
+
+```bash
+npm run playground:generate-newsletter
+```
+
+### Output
+
+Results are saved to `playground/output/` (git-ignored):
+- `newsletter.md` — Generated markdown with title in frontmatter
+- `newsletter.html` — Rendered HTML with CSS inlined (juice)
+
+### Data Management
+
+| Directory | Git | Purpose |
+|---|---|---|
+| `playground/data-examples/` | Tracked | Format reference files (`.example.*`) |
+| `playground/data/` | Ignored | Your actual config, articles, templates |
+| `playground/output/` | Ignored | Generated results |
+
 ## Development / Build / Test / CI
 
 For the full developer guide (environment, scripts, testing/coverage, and CI), see [CONTRIBUTING.md](./CONTRIBUTING.md).

package/dist/index.cjs

@@ -4,10 +4,9 @@ var runnables = require('@langchain/core/runnables');
 var esToolkit = require('es-toolkit');
 var ai = require('ai');
 var zod = require('zod');
-var juice = require('juice');
-var DOMPurify = require('dompurify');
 var jsdom = require('jsdom');
-var marked = require('marked');
+var juice = require('juice');
+var safeMarkdown2Html = require('safe-markdown2html');
 var node_crypto = require('node:crypto');
 
 /**
@@ -120,6 +119,17 @@ class LLMQuery extends BaseLLMQuery {
     }
 }
 
+const ZERO_USAGE = {
+    inputTokens: undefined,
+    inputTokenDetails: {
+        noCacheTokens: undefined,
+        cacheReadTokens: undefined,
+        cacheWriteTokens: undefined,
+    },
+    outputTokens: undefined,
+    outputTokenDetails: { textTokens: undefined, reasoningTokens: undefined },
+    totalTokens: undefined,
+};
 class AnalyzeImages extends LLMQuery {
     schema = zod.z.object({
         imageContext: zod.z
@@ -132,12 +142,12 @@ class AnalyzeImages extends LLMQuery {
     async execute() {
         if (!this.targetArticle.hasAttachedImage ||
             !this.targetArticle.detailContent) {
-            return null;
+            return { result: null, usage: ZERO_USAGE };
         }
         if (this.imageMessages.length === 0) {
-            return null;
+            return { result: null, usage: ZERO_USAGE };
         }
-        const { output } = await ai.generateText({
+        const { output, usage } = await ai.generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             output: ai.Output.object({
@@ -151,7 +161,7 @@ class AnalyzeImages extends LLMQuery {
                 },
             ],
         });
-        return output.imageContext;
+        return { result: output.imageContext, usage };
     }
     get systemPrompt() {
         return `# Image Analysis Expert System
@@ -258,7 +268,7 @@ class ClassifyTags extends LLMQuery {
     }
     async execute({ existTags }) {
         this.existTags = existTags;
-        const { output } = await ai.generateText({
+        const { output, usage } = await ai.generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             output: ai.Output.object({
@@ -267,7 +277,7 @@ class ClassifyTags extends LLMQuery {
             system: this.systemPrompt,
             prompt: this.userPrompt,
         });
-        return output;
+        return { result: output, usage };
     }
     get systemPrompt() {
         return `You are an AI specializing in analyzing and categorizing articles for professionals in ${this.expertFields.join(', ')}.
@@ -340,7 +350,7 @@ class DetermineArticleImportance extends LLMQuery {
         this.dateService = config.dateService;
     }
     async execute() {
-        const { output } = await ai.generateText({
+        const { output, usage } = await ai.generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             output: ai.Output.object({
@@ -349,7 +359,7 @@ class DetermineArticleImportance extends LLMQuery {
             system: this.systemPrompt,
             prompt: this.userPrompt,
         });
-        return output.importanceScore;
+        return { result: output.importanceScore, usage };
     }
     get minPoint() {
         const targetRule = this.minimumImportanceScoreRules.find(({ targetUrl }) => targetUrl === this.targetArticle.targetUrl);
@@ -523,7 +533,9 @@ class ArticleInsightsChain extends PrivateChain {
                 });
                 try {
                     const classifyTags = new ClassifyTags(this.getLlmQueryConfig(this.provider.classifyTagOptions.model, article));
-                    const generatedTags = await classifyTags.execute({ existTags });
+                    const { result: generatedTags } = await classifyTags.execute({
+                        existTags,
+                    });
                     pushTag(generatedTags.tag1);
                     pushTag(generatedTags.tag2);
                     pushTag(generatedTags.tag3);
@@ -596,7 +608,7 @@ class ArticleInsightsChain extends PrivateChain {
                 });
                 try {
                     const analyzeImages = new AnalyzeImages(this.getLlmQueryConfig(this.provider.analyzeImagesOptions.model, article));
-                    const imageContextByLlm = await analyzeImages.execute();
+                    const { result: imageContextByLlm } = await analyzeImages.execute();
                     if (imageContextByLlm) {
                         articlesWithImageContext.push({
                             id: article.id,
@@ -695,7 +707,7 @@ class ArticleInsightsChain extends PrivateChain {
                         minimumImportanceScoreRules: this.provider.determineScoreOptions.minimumImportanceScoreRules,
                         dateService: this.dateService,
                     });
-                    const importanceScore = await determineArticleImportance.execute();
+                    const { result: importanceScore } = await determineArticleImportance.execute();
                     const processedArticle = {
                         ...article,
                         importanceScore,
@@ -838,55 +850,6 @@ class AnalysisChain extends Chain {
     }
 }
 
-function preprocessBoldSyntax(markdown) {
-    // Convert **text** to <strong>text</strong> before marked parsing
-    // This fixes issues where marked doesn't properly handle bold syntax with parentheses
-    return markdown.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
-}
-function markdownToHtml(markdown) {
-    const preprocessed = preprocessBoldSyntax(markdown);
-    const html = marked.marked.parse(preprocessed);
-    const window = new jsdom.JSDOM('').window;
-    const purify = DOMPurify(window);
-    const sanitized = purify.sanitize(html);
-    const withCorrectedUrls = correctMalformedUrls(sanitized);
-    const withTargetBlank = addTargetBlankToAnchors(withCorrectedUrls);
-    const withDelReplaced = replaceDelTagsWithTilde(withTargetBlank);
-    return correctUnconvertedBoldSyntax(withDelReplaced);
-}
-function addTargetBlankToAnchors(htmlString) {
-    // DOMPurify removes target attributes, so we can safely add target="_blank" to all anchors
-    return htmlString.replace(/<a\s+([^>]*)>/gi, (_match, attributes) => {
-        return `<a ${attributes} target="_blank">`;
-    });
-}
-function replaceDelTagsWithTilde(htmlString) {
-    // Replace opening and closing del tags with tilde (~)
-    return htmlString.replace(/<del>/gi, '~').replace(/<\/del>/gi, '~');
-}
-function correctUnconvertedBoldSyntax(htmlString) {
-    // Replace unconverted "**text**" markdown syntax with <b> tags
-    // Matches "**" followed by one or more non-asterisk characters, followed by "**"
-    return htmlString.replace(/\*\*([^*]+)\*\*/g, '<b>$1</b>');
-}
-function correctMalformedUrls(htmlString) {
-    // Pattern matches anchors with `)` followed by optional closing markup (</b> or **) and URL-encoded characters
-    // Capture groups:
-    // 1: attributes before href
-    // 2: URL base (before `)`)
-    // 3: closing markup (</b> or ** or empty)
-    // 4: URL-encoded part (starts with %)
-    // 5: attributes after href
-    // 6: link text base (before `)`)
-    // 7: text after `)` in link text (may include **)
-    const regex = /<a\s+([^>]*?)href="([^"]*?)\)((?:<\/b>|\*\*)?)(%[0-9A-Fa-f]{2}[^"]*?)"([^>]*?)>([^<]*?)\)((?:\*\*)?[^<]*?)<\/a>/g;
-    return htmlString.replace(regex, (_match, beforeHref, urlBase, closingMarkup, _encodedPart, afterHref, textBase, textAfterClosingParen) => {
-        // Remove leading ** from textAfterClosingParen since it's already captured as closingMarkup (</b>)
-        const cleanedText = textAfterClosingParen.replace(/^\*\*/, '');
-        return `<a ${beforeHref}href="${urlBase}"${afterHref}>${textBase}</a>${closingMarkup})${cleanedText}`;
-    });
-}
-
 let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
     maxOutputTokens;
     temperature;
@@ -929,7 +892,7 @@ let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
         this.newsletterBrandName = config.newsletterBrandName;
     }
     async execute() {
-        const { output } = await ai.generateText({
+        const { output, usage } = await ai.generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             maxOutputTokens: this.maxOutputTokens,
@@ -944,16 +907,19 @@ let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
             system: this.systemPrompt,
             prompt: this.userPrompt,
         });
-        if (!output.isWrittenInOutputLanguage) {
-            return this.execute();
-        }
-        if (!output.copyrightVerified) {
-            return this.execute();
-        }
-        if (!output.factAccuracy) {
-            return this.execute();
+        const needsRetry = !output.isWrittenInOutputLanguage ||
+            !output.copyrightVerified ||
+            !output.factAccuracy ||
+            (this.options.content.titleContext &&
+                !output.title.includes(this.options.content.titleContext));
+        if (needsRetry) {
+            const retryResult = await this.execute();
+            return {
+                result: retryResult.result,
+                usage: addUsage(usage, retryResult.usage),
+            };
         }
-        return esToolkit.pick(output, ['title', 'content']);
+        return { result: esToolkit.pick(output, ['title', 'content']), usage };
     }
     get systemPrompt() {
         return `You are a newsletter production expert for "${this.newsletterBrandName}" who analyzes and delivers trends in the fields of ${this.expertFields.join(', ')}. Your goal is to provide in-depth analysis that helps industry professionals easily understand complex information and make informed decisions.
@@ -1007,14 +973,14 @@ Copyright Protection & Fact-Checking Principles:
 Output Format & Requirements:
 1. Language: ${this.options.content.outputLanguage}
 
-2. Start: Specify date (${this.dateService.getDisplayDateString()}) and begin with neutral, objective greeting. Briefly introduce key factual information to be covered in today's newsletter.
+2. Start: ${this.options.content.freeFormIntro ? 'Begin directly with the Overall Briefing section (no separate opening heading or greeting).' : `Specify date (${this.dateService.getDisplayDateString()}) and begin with neutral, objective greeting. Briefly introduce key factual information to be covered in today's newsletter.`}
 
-3. Overall Briefing: Before the main listing, create a briefing section conveying objective facts about today's news in these aspects:
+3. Overall Briefing: Before the main listing, create a briefing section conveying objective facts about today's news${this.options.content.freeFormIntro ? `. Structure: Start with a Heading 2 (##) briefing section heading in the format "## 📮 ${this.dateService.getDisplayDateString()} [Briefing/Summary word in output language]" (e.g., "## 📮 2월 6일 브리핑" for Korean, "## 📮 Feb 6 Briefing" for English) — do NOT include domain or field names in the heading. Immediately follow with a brief paragraph introducing key factual information to be covered in today's newsletter, then include the following bullet points:` : ' in these aspects:'}
    - Key Trends: Explain major patterns or trends found in this news based on data. Ex: 'Over 00% of today's news relates to 00'.
    - Immediate Impact: Emphasize most important changes or decisions affecting industry immediately, specifically mentioning which fields will be most impacted.
 
 4. Category Classification & Content Organization:
-   - Group news by logical categories based on related tags and content (e.g., Policy/Regulation, Budget/Support, Research/Development, Products/Services, Operations/Process, Recruitment/Events) rather than just listing by importance.
+   - Group news by logical categories based on related tags and content (e.g., Policy/Regulation, Budget/Support, Research/Development, Products/Services, Operations/Process, Recruitment/Events) rather than just listing by importance.${this.options.content.freeFormIntro ? '\n   - Use Heading 2 (##) for each category heading (same level as the briefing heading). Do NOT use Heading 3 (###) for categories.' : ''}
    - Use appropriate emoticons for each category for visual distinction.
    - Sort by importance within categories, making high-importance items more prominent.
    - Add short paragraph at category start summarizing overall trends or changes in that area, specifying important points and areas to focus on.
@@ -1054,18 +1020,22 @@ Output Format & Requirements:
    - Do not write preview or anticipatory messages about next newsletter.
    - Do not include contact information for inquiries.
 
-7. Title Writing Guidelines:
+7. Title Writing Guidelines:${this.options.content.titleContext
+            ? `\n   - **Required title keyword**: "${this.options.content.titleContext}". This phrase MUST appear in the title. Combine it with key context from today's newsletter content to form a natural, complete title.
+   - Keep title length 20-100 characters and can include 1-2 relevant emoticons.
+   - Use neutral and objective terms in title (e.g., 'announced', 'implementing', 'deadline approaching').
+   - Write title clearly and factually to maintain professionalism and credibility.`
+            : `
    - Title should objectively convey core facts of 1-2 most important news items today.
    - Write with key facts rather than simple "Newsletter", more effective with specific figures or schedules.
    - Use neutral and objective terms in title (e.g., 'announced', 'implementing', 'deadline approaching').
    - Keep title length 20-50 characters and can include 1-2 relevant emoticons.
    - Place most important key facts at beginning of title.
-   - Write title clearly and factually to maintain professionalism and credibility.
+   - Write title clearly and factually to maintain professionalism and credibility.`}
 
 8. Additional Requirements:
    - Comprehensively analyze posts to create email containing most important information for ${this.expertFields.join(', ')} field experts.
-   - Naturally include date at beginning in the format: "${this.dateService.getDisplayDateString()} ${this.expertFields.join(', ')} [News Term]". Replace [News Term] with the word for "News" appropriate for the output language (e.g., "News" for English, "소식" for Korean). Declare this part as \`Heading 1\`(#).
-   - Write body in markdown format, effectively using headings(#, ##, ###), bold(**), italics(_), bullet points(-, *) etc. to improve readability.
+   ${this.options.content.freeFormIntro ? '' : `- Naturally include date at beginning in the format: "${this.dateService.getDisplayDateString()} ${this.expertFields.join(', ')} [News Term]". Replace [News Term] with the word for "News" appropriate for the output language (e.g., "News" for English, "소식" for Korean). Declare this part as \`Heading 1\`(#).\n   `}- Write body in markdown format, effectively using headings(#, ##, ###), bold(**), italics(_), bullet points(-, *) etc. to improve readability.
    - Group related news to provide broader context, and mention development status if there's continuity with content covered in previous issues.
    - **Source citation is most important for ensuring credibility.** Must provide links in [original title](URL) format using source's title. Do not write as "View", "Article", "[Post3](URL)" format.
    - Specify source whenever article titles or content are quoted in newsletter, ensure all information is provided with links.
@@ -1116,6 +1086,27 @@ Based on all post information provided above, please generate a ${this.expertFie
 Please follow the roles and output format defined in the system prompt (friendly introduction, overall briefing, category classification, in-depth analysis, polite closing, etc.).`;
     }
 };
+function addNum(a, b) {
+    if (a == null && b == null)
+        return undefined;
+    return (a ?? 0) + (b ?? 0);
+}
+function addUsage(a, b) {
+    return {
+        inputTokens: addNum(a.inputTokens, b.inputTokens),
+        inputTokenDetails: {
+            noCacheTokens: addNum(a.inputTokenDetails?.noCacheTokens, b.inputTokenDetails?.noCacheTokens),
+            cacheReadTokens: addNum(a.inputTokenDetails?.cacheReadTokens, b.inputTokenDetails?.cacheReadTokens),
+            cacheWriteTokens: addNum(a.inputTokenDetails?.cacheWriteTokens, b.inputTokenDetails?.cacheWriteTokens),
+        },
+        outputTokens: addNum(a.outputTokens, b.outputTokens),
+        outputTokenDetails: {
+            textTokens: addNum(a.outputTokenDetails?.textTokens, b.outputTokenDetails?.textTokens),
+            reasoningTokens: addNum(a.outputTokenDetails?.reasoningTokens, b.outputTokenDetails?.reasoningTokens),
+        },
+        totalTokens: addNum(a.totalTokens, b.totalTokens),
+    };
+}
 
 class ContentGenerateChain extends Chain {
     dateService;
@@ -1208,7 +1199,8 @@ class ContentGenerateChain extends Chain {
                 newsletterBrandName: this.provider.newsletterBrandName,
                 dateService: this.dateService,
             });
-            return await generateNewsletter.execute();
+            const { result } = await generateNewsletter.execute();
+            return result;
         });
     }
     async renderHtml(coreContent) {
@@ -1223,7 +1215,13 @@ class ContentGenerateChain extends Chain {
             }
             return this.htmlTemplate.html
                 .replaceAll(`{{${this.htmlTemplate.markers.title}}}`, coreContent.title)
-                .replaceAll(`{{${this.htmlTemplate.markers.content}}}`, markdownToHtml(coreContent.content));
+                .replaceAll(`{{${this.htmlTemplate.markers.content}}}`, safeMarkdown2Html(coreContent.content, {
+                window: new jsdom.JSDOM('').window,
+                linkTargetBlank: true,
+                fixMalformedUrls: true,
+                fixBoldSyntax: true,
+                convertStrikethrough: true,
+            }));
         });
     }
     async createNewsletter(html, coreContent, candidateArticles) {
@@ -1304,7 +1302,7 @@ function shouldRetry(status, error) {
     }
     return false;
 }
-async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/') {
+async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/', customFetch) {
     const maxRetries = 5;
     const baseTimeoutMs = 10_000; // Base 10s, increases per attempt
     let lastError = null;
@@ -1314,7 +1312,7 @@ async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/')
         const timeout = setTimeout(() => controller.abort(`timeout after ${timeoutMs}ms`), timeoutMs);
         try {
             const startedAt = Date.now();
-            const response = await fetch(url, {
+            const response = await (customFetch ?? fetch)(url, {
                 // mode: 'cors' // Not applicable in Node, left here for behavioral parity with browsers
                 redirect: 'follow',
                 // @ts-expect-error Undici/Fetch in Node may allow duplex; safe to ignore
@@ -1458,7 +1456,7 @@ class CrawlingChain extends Chain {
             startFields: { target: this.describeTarget(target) },
         }, async () => {
             try {
-                return await getHtmlFromUrl(this.logger, target.url);
+                return await getHtmlFromUrl(this.logger, target.url, undefined, this.provider.customFetch);
             }
             catch (error) {
                 this.logger.error({
@@ -1537,7 +1535,7 @@ class CrawlingChain extends Chain {
                 failedCount: result.failedCount,
             }),
         }, async () => {
-            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl)));
+            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl, undefined, this.provider.customFetch)));
             const detailPagesHtmlWithPipelineId = [];
             const successList = [];
             let failedCount = 0;

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}
+{"version":3,"file":"index.cjs","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/dist/index.d.ts

@@ -254,6 +254,17 @@ type ContentOptions = {
      * @example ["AI", "Cloud"]
      */
     expertField: string | string[];
+    /**
+     * When true, removes the fixed date+field heading directive
+     * and moves the brief introduction into the briefing section.
+     */
+    freeFormIntro?: boolean;
+    /**
+     * Context string to prioritize when generating the newsletter title.
+     * When provided, the LLM will consider this value as the top priority
+     * along with the generated newsletter content for title creation.
+     */
+    titleContext?: string;
 };
 type LLMQueryOptions = {
     /**
@@ -515,6 +526,11 @@ interface CrawlingProvider {
      * @default 5
      */
     maxConcurrency?: number;
+    /**
+     * Optional custom fetch function (e.g., proxy-based fetch).
+     * When provided, this function is used instead of the global `fetch` for HTTP requests.
+     */
+    customFetch?: typeof fetch;
     /**
      * Crawling target groups.
      */

package/dist/index.js

@@ -2,10 +2,9 @@ import { RunnablePassthrough, RunnableSequence } from '@langchain/core/runnables
 import { pick, omit } from 'es-toolkit';
 import { generateText, Output } from 'ai';
 import { z } from 'zod';
-import juice from 'juice';
-import DOMPurify from 'dompurify';
 import { JSDOM } from 'jsdom';
-import { marked } from 'marked';
+import juice from 'juice';
+import safeMarkdown2Html from 'safe-markdown2html';
 import { randomUUID } from 'node:crypto';
 
 /**
@@ -118,6 +117,17 @@ class LLMQuery extends BaseLLMQuery {
     }
 }
 
+const ZERO_USAGE = {
+    inputTokens: undefined,
+    inputTokenDetails: {
+        noCacheTokens: undefined,
+        cacheReadTokens: undefined,
+        cacheWriteTokens: undefined,
+    },
+    outputTokens: undefined,
+    outputTokenDetails: { textTokens: undefined, reasoningTokens: undefined },
+    totalTokens: undefined,
+};
 class AnalyzeImages extends LLMQuery {
     schema = z.object({
         imageContext: z
@@ -130,12 +140,12 @@ class AnalyzeImages extends LLMQuery {
     async execute() {
         if (!this.targetArticle.hasAttachedImage ||
             !this.targetArticle.detailContent) {
-            return null;
+            return { result: null, usage: ZERO_USAGE };
         }
         if (this.imageMessages.length === 0) {
-            return null;
+            return { result: null, usage: ZERO_USAGE };
         }
-        const { output } = await generateText({
+        const { output, usage } = await generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             output: Output.object({
@@ -149,7 +159,7 @@ class AnalyzeImages extends LLMQuery {
                 },
             ],
         });
-        return output.imageContext;
+        return { result: output.imageContext, usage };
     }
     get systemPrompt() {
         return `# Image Analysis Expert System
@@ -256,7 +266,7 @@ class ClassifyTags extends LLMQuery {
     }
     async execute({ existTags }) {
         this.existTags = existTags;
-        const { output } = await generateText({
+        const { output, usage } = await generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             output: Output.object({
@@ -265,7 +275,7 @@ class ClassifyTags extends LLMQuery {
             system: this.systemPrompt,
             prompt: this.userPrompt,
         });
-        return output;
+        return { result: output, usage };
     }
     get systemPrompt() {
         return `You are an AI specializing in analyzing and categorizing articles for professionals in ${this.expertFields.join(', ')}.
@@ -338,7 +348,7 @@ class DetermineArticleImportance extends LLMQuery {
         this.dateService = config.dateService;
     }
     async execute() {
-        const { output } = await generateText({
+        const { output, usage } = await generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             output: Output.object({
@@ -347,7 +357,7 @@ class DetermineArticleImportance extends LLMQuery {
             system: this.systemPrompt,
             prompt: this.userPrompt,
         });
-        return output.importanceScore;
+        return { result: output.importanceScore, usage };
     }
     get minPoint() {
         const targetRule = this.minimumImportanceScoreRules.find(({ targetUrl }) => targetUrl === this.targetArticle.targetUrl);
@@ -521,7 +531,9 @@ class ArticleInsightsChain extends PrivateChain {
                 });
                 try {
                     const classifyTags = new ClassifyTags(this.getLlmQueryConfig(this.provider.classifyTagOptions.model, article));
-                    const generatedTags = await classifyTags.execute({ existTags });
+                    const { result: generatedTags } = await classifyTags.execute({
+                        existTags,
+                    });
                     pushTag(generatedTags.tag1);
                     pushTag(generatedTags.tag2);
                     pushTag(generatedTags.tag3);
@@ -594,7 +606,7 @@ class ArticleInsightsChain extends PrivateChain {
                 });
                 try {
                     const analyzeImages = new AnalyzeImages(this.getLlmQueryConfig(this.provider.analyzeImagesOptions.model, article));
-                    const imageContextByLlm = await analyzeImages.execute();
+                    const { result: imageContextByLlm } = await analyzeImages.execute();
                     if (imageContextByLlm) {
                         articlesWithImageContext.push({
                             id: article.id,
@@ -693,7 +705,7 @@ class ArticleInsightsChain extends PrivateChain {
                         minimumImportanceScoreRules: this.provider.determineScoreOptions.minimumImportanceScoreRules,
                         dateService: this.dateService,
                     });
-                    const importanceScore = await determineArticleImportance.execute();
+                    const { result: importanceScore } = await determineArticleImportance.execute();
                     const processedArticle = {
                         ...article,
                         importanceScore,
@@ -836,55 +848,6 @@ class AnalysisChain extends Chain {
     }
 }
 
-function preprocessBoldSyntax(markdown) {
-    // Convert **text** to <strong>text</strong> before marked parsing
-    // This fixes issues where marked doesn't properly handle bold syntax with parentheses
-    return markdown.replace(/\*\*(.+?)\*\*/g, '<strong>$1</strong>');
-}
-function markdownToHtml(markdown) {
-    const preprocessed = preprocessBoldSyntax(markdown);
-    const html = marked.parse(preprocessed);
-    const window = new JSDOM('').window;
-    const purify = DOMPurify(window);
-    const sanitized = purify.sanitize(html);
-    const withCorrectedUrls = correctMalformedUrls(sanitized);
-    const withTargetBlank = addTargetBlankToAnchors(withCorrectedUrls);
-    const withDelReplaced = replaceDelTagsWithTilde(withTargetBlank);
-    return correctUnconvertedBoldSyntax(withDelReplaced);
-}
-function addTargetBlankToAnchors(htmlString) {
-    // DOMPurify removes target attributes, so we can safely add target="_blank" to all anchors
-    return htmlString.replace(/<a\s+([^>]*)>/gi, (_match, attributes) => {
-        return `<a ${attributes} target="_blank">`;
-    });
-}
-function replaceDelTagsWithTilde(htmlString) {
-    // Replace opening and closing del tags with tilde (~)
-    return htmlString.replace(/<del>/gi, '~').replace(/<\/del>/gi, '~');
-}
-function correctUnconvertedBoldSyntax(htmlString) {
-    // Replace unconverted "**text**" markdown syntax with <b> tags
-    // Matches "**" followed by one or more non-asterisk characters, followed by "**"
-    return htmlString.replace(/\*\*([^*]+)\*\*/g, '<b>$1</b>');
-}
-function correctMalformedUrls(htmlString) {
-    // Pattern matches anchors with `)` followed by optional closing markup (</b> or **) and URL-encoded characters
-    // Capture groups:
-    // 1: attributes before href
-    // 2: URL base (before `)`)
-    // 3: closing markup (</b> or ** or empty)
-    // 4: URL-encoded part (starts with %)
-    // 5: attributes after href
-    // 6: link text base (before `)`)
-    // 7: text after `)` in link text (may include **)
-    const regex = /<a\s+([^>]*?)href="([^"]*?)\)((?:<\/b>|\*\*)?)(%[0-9A-Fa-f]{2}[^"]*?)"([^>]*?)>([^<]*?)\)((?:\*\*)?[^<]*?)<\/a>/g;
-    return htmlString.replace(regex, (_match, beforeHref, urlBase, closingMarkup, _encodedPart, afterHref, textBase, textAfterClosingParen) => {
-        // Remove leading ** from textAfterClosingParen since it's already captured as closingMarkup (</b>)
-        const cleanedText = textAfterClosingParen.replace(/^\*\*/, '');
-        return `<a ${beforeHref}href="${urlBase}"${afterHref}>${textBase}</a>${closingMarkup})${cleanedText}`;
-    });
-}
-
 let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
     maxOutputTokens;
     temperature;
@@ -927,7 +890,7 @@ let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
         this.newsletterBrandName = config.newsletterBrandName;
     }
     async execute() {
-        const { output } = await generateText({
+        const { output, usage } = await generateText({
             model: this.model,
             maxRetries: this.options.llm.maxRetries,
             maxOutputTokens: this.maxOutputTokens,
@@ -942,16 +905,19 @@ let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
             system: this.systemPrompt,
             prompt: this.userPrompt,
         });
-        if (!output.isWrittenInOutputLanguage) {
-            return this.execute();
-        }
-        if (!output.copyrightVerified) {
-            return this.execute();
-        }
-        if (!output.factAccuracy) {
-            return this.execute();
+        const needsRetry = !output.isWrittenInOutputLanguage ||
+            !output.copyrightVerified ||
+            !output.factAccuracy ||
+            (this.options.content.titleContext &&
+                !output.title.includes(this.options.content.titleContext));
+        if (needsRetry) {
+            const retryResult = await this.execute();
+            return {
+                result: retryResult.result,
+                usage: addUsage(usage, retryResult.usage),
+            };
         }
-        return pick(output, ['title', 'content']);
+        return { result: pick(output, ['title', 'content']), usage };
     }
     get systemPrompt() {
         return `You are a newsletter production expert for "${this.newsletterBrandName}" who analyzes and delivers trends in the fields of ${this.expertFields.join(', ')}. Your goal is to provide in-depth analysis that helps industry professionals easily understand complex information and make informed decisions.
@@ -1005,14 +971,14 @@ Copyright Protection & Fact-Checking Principles:
 Output Format & Requirements:
 1. Language: ${this.options.content.outputLanguage}
 
-2. Start: Specify date (${this.dateService.getDisplayDateString()}) and begin with neutral, objective greeting. Briefly introduce key factual information to be covered in today's newsletter.
+2. Start: ${this.options.content.freeFormIntro ? 'Begin directly with the Overall Briefing section (no separate opening heading or greeting).' : `Specify date (${this.dateService.getDisplayDateString()}) and begin with neutral, objective greeting. Briefly introduce key factual information to be covered in today's newsletter.`}
 
-3. Overall Briefing: Before the main listing, create a briefing section conveying objective facts about today's news in these aspects:
+3. Overall Briefing: Before the main listing, create a briefing section conveying objective facts about today's news${this.options.content.freeFormIntro ? `. Structure: Start with a Heading 2 (##) briefing section heading in the format "## 📮 ${this.dateService.getDisplayDateString()} [Briefing/Summary word in output language]" (e.g., "## 📮 2월 6일 브리핑" for Korean, "## 📮 Feb 6 Briefing" for English) — do NOT include domain or field names in the heading. Immediately follow with a brief paragraph introducing key factual information to be covered in today's newsletter, then include the following bullet points:` : ' in these aspects:'}
    - Key Trends: Explain major patterns or trends found in this news based on data. Ex: 'Over 00% of today's news relates to 00'.
    - Immediate Impact: Emphasize most important changes or decisions affecting industry immediately, specifically mentioning which fields will be most impacted.
 
 4. Category Classification & Content Organization:
-   - Group news by logical categories based on related tags and content (e.g., Policy/Regulation, Budget/Support, Research/Development, Products/Services, Operations/Process, Recruitment/Events) rather than just listing by importance.
+   - Group news by logical categories based on related tags and content (e.g., Policy/Regulation, Budget/Support, Research/Development, Products/Services, Operations/Process, Recruitment/Events) rather than just listing by importance.${this.options.content.freeFormIntro ? '\n   - Use Heading 2 (##) for each category heading (same level as the briefing heading). Do NOT use Heading 3 (###) for categories.' : ''}
    - Use appropriate emoticons for each category for visual distinction.
    - Sort by importance within categories, making high-importance items more prominent.
    - Add short paragraph at category start summarizing overall trends or changes in that area, specifying important points and areas to focus on.
@@ -1052,18 +1018,22 @@ Output Format & Requirements:
    - Do not write preview or anticipatory messages about next newsletter.
    - Do not include contact information for inquiries.
 
-7. Title Writing Guidelines:
+7. Title Writing Guidelines:${this.options.content.titleContext
+            ? `\n   - **Required title keyword**: "${this.options.content.titleContext}". This phrase MUST appear in the title. Combine it with key context from today's newsletter content to form a natural, complete title.
+   - Keep title length 20-100 characters and can include 1-2 relevant emoticons.
+   - Use neutral and objective terms in title (e.g., 'announced', 'implementing', 'deadline approaching').
+   - Write title clearly and factually to maintain professionalism and credibility.`
+            : `
    - Title should objectively convey core facts of 1-2 most important news items today.
    - Write with key facts rather than simple "Newsletter", more effective with specific figures or schedules.
    - Use neutral and objective terms in title (e.g., 'announced', 'implementing', 'deadline approaching').
    - Keep title length 20-50 characters and can include 1-2 relevant emoticons.
    - Place most important key facts at beginning of title.
-   - Write title clearly and factually to maintain professionalism and credibility.
+   - Write title clearly and factually to maintain professionalism and credibility.`}
 
 8. Additional Requirements:
    - Comprehensively analyze posts to create email containing most important information for ${this.expertFields.join(', ')} field experts.
-   - Naturally include date at beginning in the format: "${this.dateService.getDisplayDateString()} ${this.expertFields.join(', ')} [News Term]". Replace [News Term] with the word for "News" appropriate for the output language (e.g., "News" for English, "소식" for Korean). Declare this part as \`Heading 1\`(#).
-   - Write body in markdown format, effectively using headings(#, ##, ###), bold(**), italics(_), bullet points(-, *) etc. to improve readability.
+   ${this.options.content.freeFormIntro ? '' : `- Naturally include date at beginning in the format: "${this.dateService.getDisplayDateString()} ${this.expertFields.join(', ')} [News Term]". Replace [News Term] with the word for "News" appropriate for the output language (e.g., "News" for English, "소식" for Korean). Declare this part as \`Heading 1\`(#).\n   `}- Write body in markdown format, effectively using headings(#, ##, ###), bold(**), italics(_), bullet points(-, *) etc. to improve readability.
    - Group related news to provide broader context, and mention development status if there's continuity with content covered in previous issues.
    - **Source citation is most important for ensuring credibility.** Must provide links in [original title](URL) format using source's title. Do not write as "View", "Article", "[Post3](URL)" format.
    - Specify source whenever article titles or content are quoted in newsletter, ensure all information is provided with links.
@@ -1114,6 +1084,27 @@ Based on all post information provided above, please generate a ${this.expertFie
 Please follow the roles and output format defined in the system prompt (friendly introduction, overall briefing, category classification, in-depth analysis, polite closing, etc.).`;
     }
 };
+function addNum(a, b) {
+    if (a == null && b == null)
+        return undefined;
+    return (a ?? 0) + (b ?? 0);
+}
+function addUsage(a, b) {
+    return {
+        inputTokens: addNum(a.inputTokens, b.inputTokens),
+        inputTokenDetails: {
+            noCacheTokens: addNum(a.inputTokenDetails?.noCacheTokens, b.inputTokenDetails?.noCacheTokens),
+            cacheReadTokens: addNum(a.inputTokenDetails?.cacheReadTokens, b.inputTokenDetails?.cacheReadTokens),
+            cacheWriteTokens: addNum(a.inputTokenDetails?.cacheWriteTokens, b.inputTokenDetails?.cacheWriteTokens),
+        },
+        outputTokens: addNum(a.outputTokens, b.outputTokens),
+        outputTokenDetails: {
+            textTokens: addNum(a.outputTokenDetails?.textTokens, b.outputTokenDetails?.textTokens),
+            reasoningTokens: addNum(a.outputTokenDetails?.reasoningTokens, b.outputTokenDetails?.reasoningTokens),
+        },
+        totalTokens: addNum(a.totalTokens, b.totalTokens),
+    };
+}
 
 class ContentGenerateChain extends Chain {
     dateService;
@@ -1206,7 +1197,8 @@ class ContentGenerateChain extends Chain {
                 newsletterBrandName: this.provider.newsletterBrandName,
                 dateService: this.dateService,
             });
-            return await generateNewsletter.execute();
+            const { result } = await generateNewsletter.execute();
+            return result;
         });
     }
     async renderHtml(coreContent) {
@@ -1221,7 +1213,13 @@ class ContentGenerateChain extends Chain {
             }
             return this.htmlTemplate.html
                 .replaceAll(`{{${this.htmlTemplate.markers.title}}}`, coreContent.title)
-                .replaceAll(`{{${this.htmlTemplate.markers.content}}}`, markdownToHtml(coreContent.content));
+                .replaceAll(`{{${this.htmlTemplate.markers.content}}}`, safeMarkdown2Html(coreContent.content, {
+                window: new JSDOM('').window,
+                linkTargetBlank: true,
+                fixMalformedUrls: true,
+                fixBoldSyntax: true,
+                convertStrikethrough: true,
+            }));
         });
     }
     async createNewsletter(html, coreContent, candidateArticles) {
@@ -1302,7 +1300,7 @@ function shouldRetry(status, error) {
     }
     return false;
 }
-async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/') {
+async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/', customFetch) {
     const maxRetries = 5;
     const baseTimeoutMs = 10_000; // Base 10s, increases per attempt
     let lastError = null;
@@ -1312,7 +1310,7 @@ async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/')
         const timeout = setTimeout(() => controller.abort(`timeout after ${timeoutMs}ms`), timeoutMs);
         try {
             const startedAt = Date.now();
-            const response = await fetch(url, {
+            const response = await (customFetch ?? fetch)(url, {
                 // mode: 'cors' // Not applicable in Node, left here for behavioral parity with browsers
                 redirect: 'follow',
                 // @ts-expect-error Undici/Fetch in Node may allow duplex; safe to ignore
@@ -1456,7 +1454,7 @@ class CrawlingChain extends Chain {
             startFields: { target: this.describeTarget(target) },
         }, async () => {
             try {
-                return await getHtmlFromUrl(this.logger, target.url);
+                return await getHtmlFromUrl(this.logger, target.url, undefined, this.provider.customFetch);
             }
             catch (error) {
                 this.logger.error({
@@ -1535,7 +1533,7 @@ class CrawlingChain extends Chain {
                 failedCount: result.failedCount,
             }),
         }, async () => {
-            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl)));
+            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl, undefined, this.provider.customFetch)));
             const detailPagesHtmlWithPipelineId = [];
             const successList = [];
             let failedCount = 0;

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}
+{"version":3,"file":"index.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@llm-newsletter-kit/core",
   "private": false,
   "type": "module",
-  "version": "1.1.6",
+  "version": "1.3.0",
   "description": "An extensible framework to automate your entire newsletter workflow. Handles data collection, LLM-based content analysis, and email generation, letting you focus on your unique domain logic.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",
@@ -41,25 +41,30 @@
     "lint:fix": "eslint --fix ./src",
     "lint:ci": "eslint --quiet ./src",
     "typecheck": "tsc --noEmit",
-    "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx,json,css,md,mdx}\""
+    "format": "prettier --write \"src/**/*.{ts,tsx,js,jsx,json,css,md,mdx}\"",
+    "playground": "tsx",
+    "playground:generate-newsletter": "tsx playground/generate-newsletter.ts"
   },
   "author": "kimhongyeon",
   "license": "Apache-2.0",
   "dependencies": {
     "@langchain/core": "^1.1.19",
-    "ai": "^6.0.69",
-    "dompurify": "^3.3.1",
+    "ai": "^6.0.77",
     "es-toolkit": "^1.44.0",
-    "jsdom": "^27.4.0",
-    "juice": "^11.1.0",
-    "marked": "^17.0.1",
+    "jsdom": "^28.0.0",
+    "juice": "^11.1.1",
+    "safe-markdown2html": "^1.0.0",
     "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@ai-sdk/anthropic": "^3.0.38",
+    "@ai-sdk/google": "^3.0.22",
+    "@ai-sdk/openai": "^3.0.26",
+    "@ai-sdk/togetherai": "^2.0.31",
     "@eslint/js": "^9.39.2",
     "@trivago/prettier-plugin-sort-imports": "^6.0.2",
     "@types/jsdom": "^27.0.0",
-    "@types/node": "^25.2.0",
+    "@types/node": "^25.2.1",
     "@vitest/coverage-v8": "^3.2.4",
     "@vitest/expect": "^3.2.4",
     "eslint": "^9.39.2",
@@ -69,6 +74,7 @@
     "rollup": "^4.57.1",
     "rollup-plugin-dts": "^6.3.0",
     "rollup-plugin-typescript2": "^0.36.0",
+    "tsx": "^4.21.0",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.54.0",
     "vitest": "^3.2.4"