@llm-newsletter-kit/core 1.0.0

package/dist/index.js

@@ -0,0 +1,1755 @@
+import { RunnablePassthrough, RunnableSequence } from '@langchain/core/runnables';
+import { pick, omit } from 'es-toolkit';
+import { generateObject } from 'ai';
+import { z } from 'zod';
+import juice from 'juice';
+import DOMPurify from 'dompurify';
+import { JSDOM } from 'jsdom';
+import { marked } from 'marked';
+import { randomUUID } from 'node:crypto';
+
+/**
+ * Common type aliases.
+ *
+ * - Provides explicit alias types for date/URL/Markdown/HTML, etc.
+ * - All comments are written in English JSDoc style.
+ */
+/**
+ * Type for date identifiers.
+ *
+ * The DateType enum is used to distinguish date-related values.
+ * It can be used to differentiate between registered dates and ranges.
+ *
+ * Enum members:
+ * - REGISTERED: indicates a registered date.
+ * - DURATION: indicates a duration or time range.
+ *
+ * @example
+ * ```ts
+ * const type: DateType = DateType.REGISTERED;
+ * ```
+ */
+var DateType;
+(function (DateType) {
+    DateType["REGISTERED"] = "registered";
+    DateType["DURATION"] = "duration";
+})(DateType || (DateType = {}));
+
+/**
+ * Executor that provides a standardized start/done/error logging pattern.
+ * - Uses the injected logger and taskId to attach common fields to every log.
+ * - Pass config.event as a prefix like "crawl.group";
+ *   ".start"/".done"/".error" are appended automatically.
+ */
+class LoggingExecutor {
+    logger;
+    taskId;
+    constructor(logger, taskId) {
+        this.logger = logger;
+        this.taskId = taskId;
+    }
+    async executeWithLogging(config, fn) {
+        const level = config.level ?? 'debug';
+        const startedAt = Date.now();
+        const startMsg = {
+            event: `${config.event}.start`,
+            level,
+            taskId: this.taskId,
+            data: config.startFields ?? {},
+        };
+        this.logger[level](startMsg);
+        try {
+            const result = await fn();
+            const durationMs = Date.now() - startedAt;
+            const doneExtra = config.doneFields
+                ? (config.doneFields(result) ?? {})
+                : {};
+            const doneMsg = {
+                event: `${config.event}.done`,
+                level,
+                taskId: this.taskId,
+                durationMs,
+                data: { ...(config.startFields ?? {}), ...doneExtra },
+            };
+            this.logger[level](doneMsg);
+            return result;
+        }
+        catch (err) {
+            const durationMs = Date.now() - startedAt;
+            const errorMsg = {
+                event: `${config.event}.error`,
+                level,
+                taskId: this.taskId,
+                durationMs,
+                data: { ...(config.startFields ?? {}) },
+            };
+            this.logger[level](errorMsg);
+            this.logger.error(err);
+            throw err;
+        }
+    }
+}
+
+const ensureStringArray = (value) => {
+    return typeof value === 'string' ? [value] : value;
+};
+
+class BaseLLMQuery {
+    model;
+    expertFields;
+    logger;
+    taskId;
+    options;
+    executeWithLogging;
+    constructor(config) {
+        this.model = config.model;
+        this.expertFields = ensureStringArray(config.options.content.expertField);
+        this.logger = config.logger;
+        this.taskId = config.taskId;
+        this.options = config.options;
+        this.executeWithLogging = config.loggingExecutor.executeWithLogging.bind(config.loggingExecutor);
+    }
+}
+class LLMQuery extends BaseLLMQuery {
+    targetArticle;
+    constructor(config) {
+        super(config);
+        this.targetArticle = config.targetArticle;
+    }
+}
+
+class AnalyzeImages extends LLMQuery {
+    schema = z.object({
+        imageContext: z
+            .string()
+            .describe('A comprehensive description of all information extracted from the images'),
+    });
+    constructor(config) {
+        super(config);
+    }
+    async execute() {
+        if (!this.targetArticle.hasAttachedImage ||
+            !this.targetArticle.detailContent) {
+            return null;
+        }
+        if (this.imageMessages.length === 0) {
+            return null;
+        }
+        const { object } = await generateObject({
+            model: this.model,
+            maxRetries: this.options.llm.maxRetries,
+            schema: this.schema,
+            system: this.systemPrompt,
+            messages: [
+                {
+                    role: 'user',
+                    content: [this.textMessage, ...this.imageMessages],
+                },
+            ],
+        });
+        return object.imageContext;
+    }
+    get systemPrompt() {
+        return `# Image Analysis Expert System
+
+## Identity & Expertise
+You are a specialized image analysis expert in: ${this.expertFields.join(', ')}
+
+## Core Responsibilities
+1. Extract visual information unavailable from text alone
+2. Identify industry-specific elements, facilities, and stakeholders
+3. Accurately read and transcribe text, charts, and data visualizations
+4. Synthesize visual information with article context
+
+## Analysis Framework
+
+### Information Categories to Extract
+- Industry-relevant visual elements
+- Text and numerical data within images
+- Key subjects (people, places, objects, infrastructure)
+- Contextual relationships to ${this.expertFields.join(', ')}
+- Information gaps filled by visual analysis
+
+### Quality Standards
+- Accuracy and specificity in descriptions
+- Professional relevance for industry practitioners
+- Integration with accompanying text content
+- Completeness in covering all visual information
+
+## Output Specifications
+- Language: ${this.options.content.outputLanguage}
+- Format: Single cohesive explanation (not numbered list)
+- Focus: Practical insights for industry professionals
+- Integration: Seamlessly merge all extracted information`;
+    }
+    get imageUrls() {
+        // Markdown image pattern: ![alt text](url) or ![](url)
+        // Includes http, https, relative paths, and data URIs
+        const imageRegex = /!\[.*?\]\(([^)]+)\)/g;
+        const urls = [];
+        let match;
+        while ((match = imageRegex.exec(this.targetArticle.detailContent)) !== null) {
+            const url = match[1].trim();
+            // Validate URL format (http, https, relative path, data URI)
+            if (url &&
+                (url.startsWith('http://') ||
+                    url.startsWith('https://') ||
+                    url.startsWith('//') || // Protocol-relative URL
+                    url.startsWith('/') || // Absolute path
+                    url.startsWith('./') || // Relative path
+                    url.startsWith('../') || // Parent directory relative path
+                    url.startsWith('data:image/')) // Data URI
+            ) {
+                urls.push(url);
+            }
+        }
+        // Process max 5 images only (to save cost)
+        return urls.slice(0, 5);
+    }
+    get imageMessages() {
+        return this.imageUrls.map((url) => ({
+            type: 'image',
+            image: url,
+        }));
+    }
+    get textMessage() {
+        return {
+            type: 'text',
+            text: `## Analysis Task
+
+**Document Context:**
+- Title: ${this.targetArticle.title}
+- Content: ${this.targetArticle.detailContent}
+
+## Instructions
+
+Analyze the provided images and synthesize your findings into a single comprehensive explanation that:
+
+1. **Identifies Visual Content**: Extract industry-specific elements, infrastructure, and stakeholders relevant to ${this.expertFields.join(', ')}
+
+2. **Captures Text & Data**: Accurately read and include all visible text, numerical data, charts, and graphs
+
+3. **Describes Visual Elements**: Detail important subjects (people, places, objects) and their significance
+
+4. **Establishes Connections**: Link visual information to ${this.expertFields.join(', ')} context and article content
+
+5. **Provides Context**: Explain what industry professionals should understand from these images
+
+6. **Complements Text**: Add visual insights not covered in the article text
+
+**Format**: Present all findings as one flowing narrative without enumeration.`,
+        };
+    }
+}
+
+class ClassifyTags extends LLMQuery {
+    schema = z.object({
+        tag1: z.string(),
+        tag2: z.string(),
+        tag3: z.string(),
+    });
+    existTags = [];
+    constructor(config) {
+        super(config);
+    }
+    async execute({ existTags }) {
+        this.existTags = existTags;
+        const { object } = await generateObject({
+            model: this.model,
+            maxRetries: this.options.llm.maxRetries,
+            schema: this.schema,
+            system: this.systemPrompt,
+            prompt: this.userPrompt,
+        });
+        return object;
+    }
+    get systemPrompt() {
+        return `You are an AI specializing in analyzing and categorizing articles for professionals in ${this.expertFields.join(', ')}.
+
+## Core Responsibility
+Analyze article titles and content to generate 3 optimal, detailed classifications by evaluating compatibility with existing tags and determining when new tags are justified.
+
+## Output Language
+All classifications must be written in ${this.options.content.outputLanguage}.
+
+## Classification Rules
+1. **Reuse Threshold**: Use existing classifications if compatibility is 80% or higher
+2. **New Tag Criteria**: Create new classifications only when:
+   - Best existing match scores below 80% compatibility
+   - New tag demonstrates versatility across 10+ similar articles
+3. **Naming Standards**: 
+   - Length: 3-15 characters
+   - Style: Clear, intuitive ${this.options.content.outputLanguage} terms
+   - Balance industry precision with general reader comprehension
+4. **Scope Exclusion**: Avoid broad, general tags like ${this.expertFields.map((v) => `"${v}"`).join(', ')} (too generic for this expert audience)
+
+## Decision Framework
+Prioritize in order:
+- Content accuracy and relevance
+- Classification system consistency
+- User intuitiveness and searchability
+- Long-term scalability and maintainability`;
+    }
+    get userPrompt() {
+        return `**Task**: Classify this article with 3 optimal detailed tags.
+
+**Article Information**
+- Title: ${this.targetArticle.title}
+- Content: ${this.targetArticle.detailContent}
+
+**Available Existing Tags**
+\`\`\`
+${JSON.stringify(this.existTags, null, 2)}
+\`\`\`
+
+**Analysis Steps**
+1. Extract core concepts, industry sectors, and specific topics from the article
+2. Score each existing tag for compatibility (0-100%)
+3. Identify which existing tags meet the 80%+ threshold
+4. For tags below 80%, determine if a new tag would better serve the content
+5. Validate new tag names for clarity, length, and future applicability
+
+**Output Requirements**
+Return exactly 3 classifications following the system rules. Each classification should:
+- Match the article's core content
+- Fit logically within the overall classification system
+- Be immediately understandable to your target audience
+- Support future similar articles`;
+    }
+}
+
+class DetermineArticleImportance extends LLMQuery {
+    minimumImportanceScoreRules;
+    schema = z.object({
+        importanceScore: z
+            .number()
+            .min(1)
+            .max(10)
+            .describe('Article importance score (1-10, 10 is most important)'),
+    });
+    dateService;
+    constructor(config) {
+        super(config);
+        this.minimumImportanceScoreRules = config.minimumImportanceScoreRules ?? [];
+        this.dateService = config.dateService;
+    }
+    async execute() {
+        const { object } = await generateObject({
+            model: this.model,
+            maxRetries: this.options.llm.maxRetries,
+            schema: this.schema,
+            system: this.systemPrompt,
+            prompt: this.userPrompt,
+        });
+        return object.importanceScore;
+    }
+    get minPoint() {
+        const targetRule = this.minimumImportanceScoreRules.find(({ targetUrl }) => targetUrl === this.targetArticle.targetUrl);
+        return targetRule?.minScore ?? 1;
+    }
+    get hasHigherMinimumScore() {
+        return this.minPoint > 1;
+    }
+    get systemPrompt() {
+        return `You are an expert in importance evaluation in the field of ${this.expertFields.join(', ')}.
+
+Role:
+- Analyze titles and content in depth to objectively evaluate the importance of news and announcements.
+- Extract the most important insights for industry professionals. Main readers are practitioners from research institutions, local/public officials, graduate students, and field experts in ${this.expertFields.join(', ')}.
+- Score based on urgency, impact, and scarcity of information.
+
+Importance Score Criteria (${this.minPoint}-10):
+10: Information with immediate and significant impact on entire industry (e.g., major legislation passed, large budget allocation, critical discoveries/events that transform the field)
+8-9: Information with important impact on many stakeholders (e.g., major policy changes, major findings/achievements released, large project announcements)
+7-8: Very important academic/professional achievements or information in specific fields (e.g., journal publication/release, major research results announcement, professional report publication, important academic events, research database construction/release, designation of important field resources/assets, medium-scale bid information)
+5-6: General important information limited to specific fields or regions (e.g., small project permits, general event notices, small-scale bids)
+4-5: General industry news or small/medium-scale event information
+2-3: Simple information sharing or repetitive daily news
+${this.hasHigherMinimumScore ? '' : `1: Information without current significance - Expired support programs, past events, invalid bid notices or recruitment information, notices that have lost practical value, or administrative/simple notices like "membership fee status", "meeting minutes", "internal schedule notices"`}
+
+Evaluation Criteria:
+- Academic Value: Journal publications, research reports, academic seminars/symposiums, research output presentations etc. minimum 7 points (knowledge base expansion and long-term reference value)
+- Practical Impact: Information requiring immediate response like policies, regulations, bids, recruitment
+- Impact Range: How many stakeholders are affected
+- Scarcity: How rare and exclusive the information is
+- Temporal Context: Practical value at current time considering deadlines, event schedules${this.hasHigherMinimumScore ? '' : ' (However, recent academic achievements maintain high scores)'}
+
+Important Notes:
+- Evaluate considering characteristics and context of ${this.expertFields.join(', ')} fields.
+- Be sensitive to core keywords, events, policies considered important in the field.`;
+    }
+    get userPrompt() {
+        return `Please rate the importance of this article from ${this.minPoint} to 10.
+
+**Current Date:** ${this.dateService.getCurrentISODateString()}
+
+**Title:** ${this.targetArticle.title || 'No Title'}
+
+**Content:** ${this.targetArticle.detailContent || 'No Content'}
+
+**Tags:** ${this.targetArticle.tag1 || ''}, ${this.targetArticle.tag2 || ''}, ${this.targetArticle.tag3 || ''}
+${this.targetArticle.imageContextByLlm
+            ? `
+**Image Analysis:** ${this.targetArticle.imageContextByLlm}`
+            : ''}`;
+    }
+}
+
+class BaseChain {
+    logger;
+    taskId;
+    provider;
+    options;
+    executeWithLogging;
+    constructor(config) {
+        this.logger = config.logger;
+        this.taskId = config.taskId;
+        this.provider = config.provider;
+        this.options = config.options;
+        this.executeWithLogging = config.loggingExecutor.executeWithLogging.bind(config.loggingExecutor);
+    }
+}
+class Chain extends BaseChain {
+}
+class PrivateChain extends BaseChain {
+}
+
+class ArticleInsightsChain extends PrivateChain {
+    dateService;
+    constructor(config) {
+        super(config);
+        this.dateService = config.dateService;
+    }
+    /* istanbul ignore next - pipeline arrow functions are exercised via higher-level tests */
+    get chain() {
+        return RunnablePassthrough.assign({
+            generatedTags: () => this.classifyArticles(),
+            generatedImageContextList: () => this.extractImageContext(),
+        })
+            .pipe({
+            mergedArticles: ({ generatedTags, generatedImageContextList }) => this.mergeTagsAndImageContext(generatedTags, generatedImageContextList),
+        })
+            .pipe({
+            determinedArticles: ({ mergedArticles }) => this.determineImportance(mergedArticles),
+        });
+    }
+    async generateInsights() {
+        const { determinedArticles: initial } = await this.chain.invoke({});
+        const maxIterations = 5; // Maximum number of iterations to prevent infinite loop
+        const reprocess = async (current, iteration) => {
+            if (iteration >= maxIterations) {
+                this.logger.debug({
+                    event: 'insights.warning.maxIterationsReached',
+                    taskId: this.taskId,
+                    data: {
+                        iterationCount: iteration,
+                        maxIterationCount: maxIterations,
+                    },
+                });
+                return current;
+            }
+            // Filter incomplete posts (where any of tag1, tag2, tag3, or importance_score is null)
+            const incompleteArticles = current.filter((article) => !article.tag1 ||
+                !article.tag2 ||
+                !article.tag3 ||
+                !article.importanceScore);
+            if (incompleteArticles.length === 0) {
+                this.logger.debug({
+                    event: 'insights.complete',
+                    taskId: this.taskId,
+                });
+                return current; // Exit when all posts have been fully processed
+            }
+            this.logger.debug({
+                event: 'insights.incomplete.restart',
+                taskId: this.taskId,
+                data: {
+                    incompleteArticleCount: incompleteArticles.length,
+                    iterationCount: iteration,
+                },
+            });
+            // Reprocess incomplete posts only by reusing an insight object
+            const { determinedArticles: reprocessedArticles } = await this.chain.invoke({});
+            // Update original determinedPosts with reprocessed articles
+            const updated = current.map((article) => {
+                const reprocessedArticle = reprocessedArticles.find((reArticle) => reArticle.id === article.id);
+                return reprocessedArticle || article;
+            });
+            this.logger.debug({
+                event: 'insights.incomplete.restart.done',
+                taskId: this.taskId,
+                data: {
+                    iterationCount: iteration,
+                },
+            });
+            return reprocess(updated, iteration + 1);
+        };
+        return reprocess(initial, 0);
+    }
+    async classifyArticles() {
+        return this.executeWithLogging({
+            event: 'insights.articles.classify',
+            level: 'debug',
+            doneFields: (articles) => ({ count: articles }),
+        }, async () => {
+            const pushTag = (tag) => {
+                if (tag && !this.provider.tags.includes(tag)) {
+                    this.provider.tags.push(tag);
+                }
+            };
+            const articlesWithTags = [];
+            for (const [i, article] of this.provider.unscoredArticles.entries()) {
+                const existTags = this.provider.tags;
+                if (article.tag1 && article.tag2 && article.tag3) {
+                    continue;
+                }
+                this.logger.debug({
+                    event: 'insights.articles.classify.start',
+                    taskId: this.taskId,
+                    data: {
+                        count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                        articleId: article.id,
+                        title: article.title?.substring(0, 50) + '...',
+                        existingTags: existTags.length,
+                    },
+                });
+                try {
+                    const classifyTags = new ClassifyTags(this.getLlmQueryConfig(this.provider.classifyTagOptions.model, article));
+                    const generatedTags = await classifyTags.execute({ existTags });
+                    pushTag(generatedTags.tag1);
+                    pushTag(generatedTags.tag2);
+                    pushTag(generatedTags.tag3);
+                    this.logger.debug({
+                        event: 'insights.articles.classify.end',
+                        taskId: this.taskId,
+                        data: {
+                            count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                            articleId: article.id,
+                            result: `tag1: ${generatedTags.tag1}, tag2: ${generatedTags.tag2}, tag3: ${generatedTags.tag3}`,
+                        },
+                    });
+                    articlesWithTags.push({
+                        id: article.id,
+                        ...generatedTags,
+                    });
+                }
+                catch (error) {
+                    this.logger.debug({
+                        event: 'insights.articles.classify.end.error',
+                        taskId: this.taskId,
+                        data: {
+                            count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                            articleId: article.id,
+                            error: error instanceof Error ? error.message : String(error),
+                            title: article.title?.substring(0, 50) + '...',
+                        },
+                    });
+                    // NOTE: Despite the error, it does not significantly hinder newsletter generation, so we proceed. Tagging helps produce a better newsletter, but it is not strictly required.
+                }
+            }
+            return articlesWithTags;
+        });
+    }
+    async extractImageContext() {
+        return this.executeWithLogging({
+            event: 'insights.images.extract',
+            level: 'debug',
+            doneFields: (articles) => ({ articles }),
+        }, async () => {
+            const articlesWithImageContext = [];
+            for (const [i, article] of this.provider.unscoredArticles.entries()) {
+                if (!article.hasAttachedImage) {
+                    this.logger.debug({
+                        event: 'insights.images.extract.pass.noimage',
+                        taskId: this.taskId,
+                        data: {
+                            articleId: article.id,
+                        },
+                    });
+                    continue;
+                }
+                if (article.imageContextByLlm) {
+                    this.logger.debug({
+                        event: 'insights.images.extract.pass.exist',
+                        taskId: this.taskId,
+                        data: {
+                            articleId: article.id,
+                        },
+                    });
+                    continue;
+                }
+                this.logger.debug({
+                    event: 'insights.images.extract.start',
+                    taskId: this.taskId,
+                    data: {
+                        count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                        articleId: article.id,
+                    },
+                });
+                try {
+                    const analyzeImages = new AnalyzeImages(this.getLlmQueryConfig(this.provider.analyzeImagesOptions.model, article));
+                    const imageContextByLlm = await analyzeImages.execute();
+                    if (imageContextByLlm) {
+                        articlesWithImageContext.push({
+                            id: article.id,
+                            imageContextByLlm,
+                        });
+                        this.logger.debug({
+                            event: 'insights.images.extract.end',
+                            taskId: this.taskId,
+                            data: {
+                                count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                                articleId: article.id,
+                            },
+                        });
+                    }
+                    else {
+                        this.logger.debug({
+                            event: 'insights.images.extract.end.noimage',
+                            taskId: this.taskId,
+                            data: {
+                                count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                                articleId: article.id,
+                            },
+                        });
+                    }
+                }
+                catch (error) {
+                    this.logger.debug({
+                        event: 'insights.images.extract.end.error',
+                        taskId: this.taskId,
+                        data: {
+                            count: `${i + 1} / ${this.provider.unscoredArticles.length}`,
+                            articleId: article.id,
+                            error: error instanceof Error ? error.message : String(error),
+                        },
+                    });
+                    // NOTE: Image analysis failure should not interrupt the overall process
+                }
+            }
+            return articlesWithImageContext;
+        });
+    }
+    async mergeTagsAndImageContext(generatedTags, generatedImageContextList) {
+        return this.executeWithLogging({
+            event: 'insights.context.merge',
+            level: 'debug',
+            startFields: {
+                generatedTags,
+                generatedImageContextList,
+            },
+            doneFields: (count) => ({ count }),
+        }, async () => {
+            return this.provider.unscoredArticles.map((article) => {
+                const articleWithTags = generatedTags.find(({ id }) => id === article.id);
+                const articleWithImageContext = generatedImageContextList.find(({ id }) => id === article.id);
+                if (articleWithTags) {
+                    article = {
+                        ...article,
+                        tag1: articleWithTags.tag1,
+                        tag2: articleWithTags.tag2,
+                        tag3: articleWithTags.tag3,
+                    };
+                }
+                if (articleWithImageContext) {
+                    article = {
+                        ...article,
+                        imageContextByLlm: articleWithImageContext.imageContextByLlm,
+                    };
+                }
+                return article;
+            });
+        });
+    }
+    async determineImportance(mergedArticles) {
+        return this.executeWithLogging({
+            event: 'insights.importance.determine',
+            level: 'debug',
+            startFields: {
+                mergedArticles,
+            },
+            doneFields: (articles) => ({ articles }),
+        }, async () => {
+            const determinedArticles = [];
+            for (const [i, article] of mergedArticles.entries()) {
+                this.logger.debug({
+                    event: 'insights.importance.determine.start',
+                    taskId: this.taskId,
+                    data: {
+                        count: `${i + 1} / ${mergedArticles.length}`,
+                        articleId: article.id,
+                        title: article.title?.substring(0, 50) + '...',
+                    },
+                });
+                try {
+                    const determineArticleImportance = new DetermineArticleImportance({
+                        ...this.getLlmQueryConfig(this.provider.determineScoreOptions.model, article),
+                        minimumImportanceScoreRules: this.provider.determineScoreOptions.minimumImportanceScoreRules,
+                        dateService: this.dateService,
+                    });
+                    const importanceScore = await determineArticleImportance.execute();
+                    const processedArticle = {
+                        ...article,
+                        importanceScore,
+                    };
+                    // Push result first to avoid losing it if logging fails
+                    determinedArticles.push(processedArticle);
+                    // Best-effort logging that won't affect the result
+                    try {
+                        this.logger.debug({
+                            event: 'insights.importance.determine.end',
+                            taskId: this.taskId,
+                            data: {
+                                count: `${i + 1} / ${mergedArticles.length}`,
+                                articleId: article.id,
+                                importanceScore: importanceScore,
+                            },
+                        });
+                    }
+                    catch {
+                        // ignore logging errors
+                    }
+                }
+                catch (error) {
+                    // Log error but ensure we still return a fallback score
+                    try {
+                        this.logger.debug({
+                            event: 'insights.importance.determine.end.error',
+                            taskId: this.taskId,
+                            data: {
+                                count: `${i + 1} / ${mergedArticles.length}`,
+                                articleId: article.id,
+                                error: error instanceof Error ? error.message : String(error),
+                            },
+                        });
+                    }
+                    catch {
+                        // ignore logging errors
+                    }
+                    // NOTE: Importance analysis failure should not stop the pipeline; use a fallback score instead
+                    determinedArticles.push({
+                        ...article,
+                        importanceScore: 1, // Set to minimum importance as a sane default
+                    });
+                }
+            }
+            return determinedArticles;
+        });
+    }
+    getLlmQueryConfig(model, targetArticle) {
+        return {
+            model,
+            logger: this.logger,
+            taskId: this.taskId,
+            targetArticle: targetArticle,
+            options: pick(this.options, ['content', 'llm']),
+            loggingExecutor: new LoggingExecutor(this.logger, this.taskId),
+        };
+    }
+}
+
+class AnalysisChain extends Chain {
+    dateService;
+    constructor(config) {
+        super(config);
+        this.dateService = config.dateService;
+    }
+    get chain() {
+        return RunnablePassthrough.assign({
+            unscoredArticles: () => this.fetchUnscoredArticles(),
+            tags: () => this.fetchTags(),
+        })
+            .pipe(RunnablePassthrough.assign({
+            determinedArticles: ({ unscoredArticles, tags }) => this.analyzeArticles(unscoredArticles, tags),
+        }))
+            .pipe(RunnablePassthrough.assign({
+            processedCount: ({ determinedArticles }) => this.updateAnalysisContext(determinedArticles),
+        }))
+            .withRetry({ stopAfterAttempt: this.options.chain.stopAfterAttempt });
+    }
+    async fetchUnscoredArticles() {
+        return this.executeWithLogging({
+            event: 'analysis.articles.fetch',
+            level: 'debug',
+            doneFields: (items) => ({ count: items.length }),
+        }, async () => {
+            return await this.provider.fetchUnscoredArticles();
+        });
+    }
+    async fetchTags() {
+        return this.executeWithLogging({
+            event: 'analysis.tags.fetch',
+            level: 'debug',
+            doneFields: (items) => ({ count: items.length }),
+        }, async () => {
+            return await this.provider.fetchTags();
+        });
+    }
+    async analyzeArticles(unscoredArticles, tags) {
+        return this.executeWithLogging({
+            event: 'analysis.articles.analyze',
+            level: 'debug',
+            startFields: {
+                unscoredArticles,
+                tags,
+            },
+            doneFields: (items) => ({ count: items.length }),
+        }, async () => {
+            const articleInsightsChain = new ArticleInsightsChain({
+                logger: this.logger,
+                taskId: this.taskId,
+                provider: {
+                    unscoredArticles,
+                    tags,
+                    classifyTagOptions: this.provider.classifyTagOptions,
+                    analyzeImagesOptions: this.provider.analyzeImagesOptions,
+                    determineScoreOptions: this.provider.determineScoreOptions,
+                },
+                options: this.options,
+                loggingExecutor: new LoggingExecutor(this.logger, this.taskId),
+                dateService: this.dateService,
+            });
+            return await articleInsightsChain.generateInsights();
+        });
+    }
+    async updateAnalysisContext(determinedArticles) {
+        return this.executeWithLogging({
+            event: 'analysis.articles.update',
+            level: 'debug',
+            startFields: {
+                determinedArticles,
+            },
+            doneFields: (count) => ({ count }),
+        }, async () => {
+            for (let i = 0; i < determinedArticles.length; i++) {
+                const article = determinedArticles[i];
+                await this.provider.update(article);
+            }
+            return determinedArticles.length;
+        });
+    }
+}
+
+function markdownToHtml(markdown) {
+    const html = marked.parse(markdown);
+    const window = new JSDOM('').window;
+    const purify = DOMPurify(window);
+    const sanitized = purify.sanitize(html);
+    const withTargetBlank = addTargetBlankToAnchors(sanitized);
+    const withDelReplaced = replaceDelTagsWithTilde(withTargetBlank);
+    return correctUnconvertedBoldSyntax(withDelReplaced);
+}
+function addTargetBlankToAnchors(htmlString) {
+    // Regular expression to find '<a>' tags
+    // This regex matches '<a>' tags that contain 'href' attribute and optionally other attributes
+    // Excludes 'target="[^"]*"' to check if target attribute already exists
+    const regex = /<a(\s+[^>]*?)?(?<!target="[^"]*")>/gi;
+    // Use regex to find '<a>' tags and add 'target="_blank"'
+    return htmlString.replace(regex, (_match, attributes) => {
+        // Handle undefined attributes as empty string
+        const currentAttributes = attributes || '';
+        // Double check if target attribute exists (safety check for regex limitations)
+        if (currentAttributes.includes('target=')) {
+            return `<a${currentAttributes}>`; // If target attribute exists, return without modification
+        }
+        else {
+            // Add target="_blank" attribute
+            return `<a${currentAttributes} target="_blank">`;
+        }
+    });
+}
+function replaceDelTagsWithTilde(htmlString) {
+    // Replace opening and closing del tags with tilde (~)
+    return htmlString.replace(/<del>/gi, '~').replace(/<\/del>/gi, '~');
+}
+function correctUnconvertedBoldSyntax(htmlString) {
+    // Replace unconverted "**text**" markdown syntax with <b> tags
+    // Matches "**" followed by one or more non-asterisk characters, followed by "**"
+    return htmlString.replace(/\*\*([^*]+)\*\*/g, '<b>$1</b>');
+}
+
+let GenerateNewsletter$1 = class GenerateNewsletter extends BaseLLMQuery {
+    maxOutputTokens;
+    temperature;
+    topP;
+    topK;
+    presencePenalty;
+    frequencyPenalty;
+    targetArticles;
+    dateService;
+    subscribePageUrl;
+    newsletterBrandName;
+    schema = z.object({
+        title: z
+            .string()
+            .max(100)
+            .min(20)
+            .describe('Title of the newsletter email'),
+        content: z.string().describe('Email content in markdown format'),
+        isWrittenInOutputLanguage: z
+            .boolean()
+            .describe(`Whether the content is written in ${this.options.content.outputLanguage}`),
+        copyrightVerified: z
+            .boolean()
+            .describe('Verification status of copyright compliance (true: verified, false: potential violation)'),
+        factAccuracy: z
+            .boolean()
+            .describe('Verification of fact-based content from provided data (true: facts only, false: contains unsupported content)'),
+    });
+    constructor(config) {
+        super(config);
+        this.maxOutputTokens = config.maxOutputTokens;
+        this.temperature = config.temperature ?? 0.3;
+        this.topP = config.topP;
+        this.topK = config.topK;
+        this.presencePenalty = config.presencePenalty;
+        this.frequencyPenalty = config.frequencyPenalty;
+        this.targetArticles = config.targetArticles;
+        this.dateService = config.dateService;
+        this.subscribePageUrl = config.subscribePageUrl;
+        this.newsletterBrandName = config.newsletterBrandName;
+    }
+    async execute() {
+        const { object } = await generateObject({
+            model: this.model,
+            maxRetries: this.options.llm.maxRetries,
+            maxOutputTokens: this.maxOutputTokens,
+            temperature: this.temperature,
+            topP: this.topP,
+            topK: this.topK,
+            presencePenalty: this.presencePenalty,
+            frequencyPenalty: this.frequencyPenalty,
+            schema: this.schema,
+            system: this.systemPrompt,
+            prompt: this.userPrompt,
+        });
+        if (!object.isWrittenInOutputLanguage) {
+            return this.execute();
+        }
+        if (!object.copyrightVerified) {
+            return this.execute();
+        }
+        if (!object.factAccuracy) {
+            return this.execute();
+        }
+        return pick(object, ['title', 'content']);
+    }
+    get systemPrompt() {
+        return `You are a newsletter production expert for "${this.newsletterBrandName}" who analyzes and delivers trends in the fields of ${this.expertFields.join(', ')}. Your goal is to provide in-depth analysis that helps industry professionals easily understand complex information and make informed decisions.
+
+Important rule for displaying date ranges: When displaying date ranges, you must use a hyphen (-) instead of a tilde (~). For example, use 'June 1-2, 2025' instead of 'June 1~2, 2025'. The tilde (~) can be rendered as strikethrough in markdown.
+
+**Key Principles for Preventing Hallucination:**
+1. **Fact-Based Writing**: Use only content explicitly stated in the provided sources, do not expand through inference or speculation.
+2. **Accurate Citation**: Use expressions directly from the sources without arbitrarily interpreting or elaborating on meanings.
+3. **Conservative Approach**: Do not mention uncertain or ambiguous content, or express it very cautiously.
+4. **Verifiable Information**: All information must be directly verifiable from the provided sources.
+5. **No Speculation**: Do not use speculative expressions like "appears to be" or "is expected to".
+6. **No Fictional Standards/Policies**: Do not mention non-existent standards/policies or systems incorrectly reported as planned for future implementation.
+
+Roles:
+- Friendly Guide: Deliver information like a trusted colleague rather than a rigid expert. Use appropriate emoticons in titles and section headings to improve readability.
+- Information Integrator: Group similar topics or related news to show broader context and trends. Focus on connections between news items rather than individual stories, and explain patterns based on data.
+- Credibility Builder: All information must be provided with sources. Whenever specific content or titles are mentioned in the body, links must be provided in [original title](URL) format. Understand that source citation is not just formal but a key element in enhancing newsletter credibility and accessibility.
+- Fact Checker: Use only facts from provided source materials. Do not make unsubstantiated claims or speculate beyond the materials.
+
+**Important Prohibitions:**
+- Do not bundle or omit structured list items (permits/reports/notices etc.) with "... and n more" etc. (tables must list all items in individual rows).
+- Do not describe policies or plans of governments/organizations/companies not explicitly mentioned in sources as facts.
+- Do not mention unconfirmed future plans or non-existent standards/policies.
+- Do not add details not present in source materials.
+
+**Content Organization Principles:** 
+- Use only accurate content from provided sources
+- No additional details or specific interpretations beyond source materials
+- All information must be verifiable and traceable
+- Focus on clear facts rather than inference or speculation  
+- Exclude uncertain content and include only confirmed information
+- For tables, include as much identifying information as possible, but mark "—" if not in source
+- No arbitrary estimation of domain-specific procedures or schedules (state only confirmed facts)
+
+Importance Score Criteria (1-10 points, expressed as stars):
+- ★★★★★ (9-10 points): [Very Important] Laws/regulations that can change industry landscape, large budgets/investments, groundbreaking research/technology announcements that all professionals must know and prepare immediate responses for. These require immediate action and direct changes to business strategy.
+- ★★★★☆ (7-8 points): [Important] Major policy changes, large projects/programs, important research/product announcements affecting specific fields or multiple organizations that should be referenced for key decisions. These need action soon and affect mid-term planning.
+- ★★★☆☆ (5-6 points): [Reference] Medium-scale projects, services, approvals, major events/campaigns affecting specific regions or organizations. These are changes that professionals in the field should know about.
+- ★★☆☆☆ (3-4 points): General industry trends, small events, routine permits/reports that are good to know. No direct action needed but helpful for trend awareness.
+- ★☆☆☆☆ (1-2 points): Simple information sharing or repetitive news. Just for reference.
+
+Copyright Protection & Fact-Checking Principles:
+- Extract only factual information from sources, completely exclude creative expressions
+- When constructing new sentences from extracted facts, do not follow source structure
+- Review for remaining source expressions after writing and modify to dry style
+- Do not present content not specified in provided materials as fact
+- Analysis and insights must be data-based; avoid baseless predictions or claims
+- If information is uncertain or requires speculation, clearly use phrases like "is estimated" or "may be possible"
+
+Output Format & Requirements:
+1. Language: ${this.options.content.outputLanguage}
+
+2. Start: Specify date (${this.dateService.getDisplayDateString()}) and begin with neutral, objective greeting. Briefly introduce key factual information to be covered in today's newsletter.
+
+3. Overall Briefing: Before the main listing, create a briefing section conveying objective facts about today's news in these aspects:
+   - Key Trends: Explain major patterns or trends found in this news based on data. Ex: 'Over 00% of today's news relates to 00'.
+   - Immediate Impact: Emphasize most important changes or decisions affecting industry immediately, specifically mentioning which fields will be most impacted.
+
+4. Category Classification & Content Organization:
+   - Group news by logical categories based on related tags and content (e.g., Policy/Regulation, Budget/Support, Research/Development, Products/Services, Operations/Process, Recruitment/Events) rather than just listing by importance.
+   - Use appropriate emoticons for each category for visual distinction.
+   - Sort by importance within categories, making high-importance items more prominent.
+   - Add short paragraph at category start summarizing overall trends or changes in that area, specifying important points and areas to focus on.
+   - Group similar news together for joint analysis when multiple related items exist.
+   - When content is essentially identical (e.g., same job posting, event notice, announcement) from different sources, integrate around most detailed and accurate information without duplication.
+   - Use tables when helpful to show commonalities and differences between multiple items at a glance.
+   - Always provide links in [original title](URL) format whenever article titles or content are mentioned. Do not write as general text like "View" or "Article" or numbered references like [Post3](URL).
+
+5. Detailed Content Writing Guidelines (Importance-Based Length Control):
+   **ABSOLUTE RULE: The writing length limits below are MAXIMUM constraints. DO NOT EXCEED these limits under any circumstances.**
+
+   - **Tier 1 (9-10 points) - Full Detail Allowed:**
+     - Key Facts: 1-2 sentences in **bold** with source link [original title](URL).
+     - Related Targets & Scope: Bullet points for different target groups.
+     - Important Dates & Procedures: Deadlines, methods, required documents by step.
+     - Related Facts: Budget/scale/participants/scope as factual data.
+     - Use tables when comparing multiple items.
+
+   - **Tier 2 (6-8 points) - ABSOLUTE MAXIMUM 3 SENTENCES:**
+     - Format: ONE sentence with **bold** key fact + [original title](URL) link. OPTIONALLY add ONE more sentence with critical detail (deadline/budget/target). NEVER EXCEED 3 SENTENCES TOTAL.
+     - DO NOT write bullet points, DO NOT write multiple paragraphs, DO NOT add subsections.
+
+   - **Tier 3 (1-5 points) - ABSOLUTE MAXIMUM 1 SENTENCE:**
+     - Format: ONE single sentence with core fact + [original title](URL) link. PERIOD. NO ADDITIONAL SENTENCES.
+     - Multiple low-priority items can be grouped into a single bullet list.
+
+   **VIOLATION WARNING: If you write more than the maximum sentences allowed for Tier 2 or Tier 3, the output will be rejected and you must regenerate.**
+
+   - **Structured Lists (Permits/Reports/Notices):** Create tables listing every item in individual rows without abbreviation, regardless of importance score.
+   - Use professional but friendly tone that's easy to understand. (Ex: Use "is notable" instead of "is", "is recommended" instead of "must", "needs to" etc.)
+   - Can use blockquotes to highlight expert comments or particularly emphasized insights.
+
+6. Closing: Write objective closing including:
+   - Brief summary of key factual information covered today.
+   - Objectively list ongoing important schedules or imminent deadlines.
+   - Maintain neutral and objective tone.
+   - Do not write preview or anticipatory messages about next newsletter.
+   - Do not include contact information for inquiries.
+
+7. Title Writing Guidelines:
+   - Title should objectively convey core facts of 1-2 most important news items today.
+   - Write with key facts rather than simple "Newsletter", more effective with specific figures or schedules.
+   - Use neutral and objective terms in title (e.g., 'announced', 'implementing', 'deadline approaching').
+   - Keep title length 20-50 characters and can include 1-2 relevant emoticons.
+   - Place most important key facts at beginning of title.
+   - Write title clearly and factually to maintain professionalism and credibility.
+
+8. Additional Requirements:
+   - Comprehensively analyze posts to create email containing most important information for ${this.expertFields.join(', ')} field experts.
+   - Naturally include date at beginning in the format: "${this.dateService.getDisplayDateString()} ${this.expertFields.join(', ')} [News Term]". Replace [News Term] with the word for "News" appropriate for the output language (e.g., "News" for English, "소식" for Korean). Declare this part as \`Heading 1\`(#).
+   - Write body in markdown format, effectively using headings(#, ##, ###), bold(**), italics(_), bullet points(-, *) etc. to improve readability.
+   - Group related news to provide broader context, and mention development status if there's continuity with content covered in previous issues.
+   - **Source citation is most important for ensuring credibility.** Must provide links in [original title](URL) format using source's title. Do not write as "View", "Article", "[Post3](URL)" format.
+   - Specify source whenever article titles or content are quoted in newsletter, ensure all information is provided with links.
+   - Discover connections and patterns between news items to provide integrated insights rather than simple listing, and provide data-based insightful analysis.
+   - Structure entire content so experts can quickly scan and grasp key information, design so busy experts can understand most important content within 2-3 minutes.
+   - Including simple small data analysis (e.g., "00% of this news budget-related", "30% increase in 00-related news vs last week") adds more valuable insight where possible.
+   ${this.subscribePageUrl ? `-   Add \`${this.subscribePageUrl}\`(Subscribe to ${this.newsletterBrandName}) page access link button at appropriate attention-worthy spot for natural recommendation to others.` : ''}`;
+    }
+    get userPrompt() {
+        return `Below is the complete list of newly collected ${this.expertFields.join(', ')} related news:
+
+${this.targetArticles
+            .map((post, index) => `## Post ${index + 1}
+**Title:** ${post.title}
+**Content:** ${post.detailContent}
+**Importance:** ${post.importanceScore}/10
+**Tags:** ${[post.tag1, post.tag2, post.tag3].filter(Boolean).join(', ')}
+**Content Type:** ${post.contentType}
+**URL:** ${post.url}
+${post.imageContextByLlm ? `**Image Analysis:** ${post.imageContextByLlm}` : ''}
+`)
+            .join('\n\n')}
+
+
+---
+**Comprehensive Analysis and Daily Newsletter Generation Request:**
+Based on all post information provided above, please generate a ${this.expertFields.join(', ')} trends newsletter for ${this.dateService.getDisplayDateString()}. Please note the following:
+
+1. **STRICT LENGTH CONTROL BY IMPORTANCE SCORE:**
+   - 9-10 points: Full detailed coverage allowed (Key Facts + Targets + Dates + Related Facts)
+   - 6-8 points: MAXIMUM 3 SENTENCES ONLY. Do not write detailed analysis, subsections, or bullet points.
+   - 1-5 points: MAXIMUM 1 SENTENCE ONLY. Just core fact + link.
+
+2. Prioritize high importance items (9-10 points) first and structure information by importance and topic.
+3. Instead of simply listing news items, group similar topics to strengthen information connectivity. For duplicate content from different sources (e.g., same job posting, same event notice), minimize redundancy by mentioning once or grouping around the most detailed content.
+4. Sort by highest importance within categories and analyze category trends and patterns to assess industry-wide impact.
+5. Include 1-2 most important news items in the title and use appropriate emoticons to enhance visual readability.
+6. Structure the final output so experts can quickly grasp key information and provide insights that aid practical decision-making.
+7. Source citation is crucial for credibility. Whenever mentioning any news, always provide links in [original title](URL) format. Do not use formats like "View Details" or "Post3". Content must always be accompanied by source links.
+8. Never create content not present in the provided materials. All analysis and insights must be based strictly on the provided post materials, without adding arbitrary information or presenting it as fact.
+9. Structured List (Permits/Reports/Notices etc.) Table Formatting Principles (No Omissions/Abbreviations):
+    - Do not bundle any items in this category; never abbreviate with "... and n more", "others", etc.
+    - List all items in tables with one item per row. Do not shorten tables regardless of length.
+    - Each row must include [original title](URL) and when available, list the following fields in order: Organization (or Publisher) | Region/Basin | Number/ID (Permit/Receipt/Report number etc.) | Date (Post/Issue/Permit date as shown in source).
+    - Mark missing values as "—" rather than leaving blank or omitting columns.
+    - Maintain table format without using bullet/number lists.
+
+Please follow the roles and output format defined in the system prompt (friendly introduction, overall briefing, category classification, in-depth analysis, polite closing, etc.).`;
+    }
+};
+
+class ContentGenerateChain extends Chain {
+    dateService;
+    minimumArticleCountForIssue;
+    priorityArticleScoreThreshold;
+    htmlTemplate;
+    constructor(config) {
+        super(config);
+        this.dateService = config.dateService;
+        this.minimumArticleCountForIssue =
+            config.provider.publicationCriteria?.minimumArticleCountForIssue ?? 5;
+        this.priorityArticleScoreThreshold =
+            config.provider.publicationCriteria?.priorityArticleScoreThreshold ?? 8;
+        this.htmlTemplate = {
+            html: config.provider.htmlTemplate.html,
+            markers: {
+                title: config.provider.htmlTemplate.markers?.title ?? 'title',
+                content: config.provider.htmlTemplate.markers?.content ?? 'content',
+            },
+        };
+    }
+    get chain() {
+        return RunnablePassthrough.assign({
+            candidateArticles: () => this.fetchArticleCandidates(),
+        })
+            .pipe(RunnablePassthrough.assign({
+            generatedCoreContent: ({ candidateArticles }) => this.generateCoreContent(candidateArticles),
+            candidateArticles: ({ candidateArticles }) => candidateArticles,
+        }))
+            .pipe(RunnablePassthrough.assign({
+            html: ({ generatedCoreContent }) => this.renderHtml(generatedCoreContent),
+            generatedCoreContent: ({ generatedCoreContent }) => generatedCoreContent,
+            candidateArticles: ({ candidateArticles }) => candidateArticles,
+        }))
+            .pipe(RunnablePassthrough.assign({
+            newsletterId: ({ html, generatedCoreContent, candidateArticles }) => this.createNewsletter(html, generatedCoreContent, candidateArticles),
+        }))
+            .withRetry({ stopAfterAttempt: this.options.chain.stopAfterAttempt });
+    }
+    async fetchArticleCandidates() {
+        return this.executeWithLogging({
+            event: 'generate.content.articles.fetch',
+            level: 'debug',
+            doneFields: (items) => ({ count: items.length }),
+        }, async () => {
+            return await this.provider.fetchArticleCandidates();
+        });
+    }
+    async generateCoreContent(candidateArticles) {
+        return this.executeWithLogging({
+            event: 'generate.content.core.generate',
+            level: 'info',
+            startFields: { count: candidateArticles.length },
+            doneFields: (result) => ({ title: result?.title }),
+        }, async () => {
+            if (candidateArticles.length === 0) {
+                this.logger.debug({
+                    event: 'generate.content.core.generate.noarticle',
+                    taskId: this.taskId,
+                });
+                return null;
+            }
+            const hasHighImportancePost = candidateArticles.some(({ importanceScore }) => importanceScore >= this.priorityArticleScoreThreshold);
+            if (candidateArticles.length <= this.minimumArticleCountForIssue &&
+                !hasHighImportancePost) {
+                this.logger.debug({
+                    event: 'generate.content.core.generate.criteria',
+                    taskId: this.taskId,
+                    data: {
+                        count: candidateArticles.length,
+                        hasHighImportancePost,
+                    },
+                });
+                return null;
+            }
+            const generateNewsletter = new GenerateNewsletter$1({
+                model: this.provider.model,
+                maxOutputTokens: this.provider.maxOutputTokens,
+                temperature: this.provider.temperature,
+                topP: this.provider.topP,
+                topK: this.provider.topK,
+                presencePenalty: this.provider.presencePenalty,
+                frequencyPenalty: this.provider.frequencyPenalty,
+                logger: this.logger,
+                taskId: this.taskId,
+                targetArticles: candidateArticles,
+                options: pick(this.options, ['content', 'llm']),
+                loggingExecutor: new LoggingExecutor(this.logger, this.taskId),
+                subscribePageUrl: this.provider.subscribePageUrl,
+                newsletterBrandName: this.provider.newsletterBrandName,
+                dateService: this.dateService,
+            });
+            return await generateNewsletter.execute();
+        });
+    }
+    async renderHtml(coreContent) {
+        return this.executeWithLogging({
+            event: 'generate.content.html.render',
+            level: 'debug',
+            startFields: { coreContent },
+            doneFields: (html) => ({ html }),
+        }, async () => {
+            if (!coreContent) {
+                return null;
+            }
+            return this.htmlTemplate.html
+                .replaceAll(`{{${this.htmlTemplate.markers.title}}}`, coreContent.title)
+                .replaceAll(`{{${this.htmlTemplate.markers.content}}}`, markdownToHtml(coreContent.content));
+        });
+    }
+    async createNewsletter(html, coreContent, candidateArticles) {
+        return this.executeWithLogging({
+            event: 'generate.content.newsletter.create',
+            level: 'debug',
+            startFields: { html, count: candidateArticles.length },
+            doneFields: (id) => ({ id }),
+        }, async () => {
+            if (!html || !coreContent) {
+                return null;
+            }
+            const { id } = await this.provider.saveNewsletter({
+                newsletter: {
+                    ...coreContent,
+                    htmlBody: juice(html),
+                    issueOrder: this.provider.issueOrder,
+                    date: this.dateService.getCurrentISODateString(),
+                },
+                usedArticles: candidateArticles,
+            });
+            return id;
+        });
+    }
+}
+
+// User-Agent list used by real browsers
+const USER_AGENTS = [
+    // Windows - Chrome, Edge, Firefox
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0',
+    // macOS - Chrome, Safari, Firefox
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0',
+    // Linux - Chrome, Firefox
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:126.0) Gecko/20100101 Firefox/126.0',
+    // Additional common combinations
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+];
+// Pick a random User-Agent
+const getRandomUserAgent = () => USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const clamp = (n, min, max) => Math.max(min, Math.min(max, n));
+function parseRetryAfter(header) {
+    if (!header)
+        return null;
+    // Seconds value
+    const seconds = Number(header);
+    if (!Number.isNaN(seconds))
+        return clamp(seconds * 1000, 0, 60_000);
+    // HTTP date format
+    const date = new Date(header);
+    const diff = date.getTime() - Date.now();
+    if (Number.isFinite(diff) && diff > 0)
+        return clamp(diff, 0, 60_000);
+    return null;
+}
+function shouldRetry(status, error) {
+    if (status === 429)
+        return true; // Too Many Requests (429)
+    if (status && status >= 500)
+        return true; // 5xx server error
+    if (status && status >= 400 && status < 500)
+        return false; // Fatal client error
+    // Network error or aborted
+    if (error instanceof Error) {
+        const msg = error.message.toLowerCase();
+        if (msg.includes('aborted') ||
+            msg.includes('timeout') ||
+            msg.includes('network') ||
+            msg.includes('fetch')) {
+            return true;
+        }
+    }
+    return false;
+}
+async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/') {
+    const maxRetries = 5;
+    const baseTimeoutMs = 10_000; // Base 10s, increases per attempt
+    let lastError = null;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        const controller = new AbortController();
+        const timeoutMs = clamp(baseTimeoutMs * Math.pow(1.3, attempt - 1), 5_000, 30_000);
+        const timeout = setTimeout(() => controller.abort(`timeout after ${timeoutMs}ms`), timeoutMs);
+        try {
+            const startedAt = Date.now();
+            const response = await fetch(url, {
+                // mode: 'cors' // Not applicable in Node, left here for behavioral parity with browsers
+                redirect: 'follow',
+                // @ts-expect-error Undici/Fetch in Node may allow duplex; safe to ignore
+                duplex: 'half',
+                signal: controller.signal,
+                headers: {
+                    'User-Agent': getRandomUserAgent(), // Randomize User-Agent
+                    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+                    'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
+                    Referer: referer, // Include previous page information
+                    Connection: 'keep-alive',
+                    // Compression is handled automatically by undici/node-fetch, no need to set Accept-Encoding explicitly
+                },
+            });
+            clearTimeout(timeout);
+            const duration = Date.now() - startedAt;
+            const status = response.status;
+            if (!response.ok) {
+                const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
+                const canRetry = shouldRetry(status, null);
+                logger.debug({
+                    event: 'fetch.error',
+                    data: { url, status, attempt, canRetry, duration, retryAfterMs },
+                });
+                if (!canRetry || attempt === maxRetries) {
+                    const msg = `Request failed (status=${status}) - ${url}`;
+                    lastError = new Error(msg);
+                    break;
+                }
+                const backoff = Math.pow(2, attempt - 1) * 1000 + Math.random() * 1000;
+                const delay = retryAfterMs != null ? Math.max(retryAfterMs, backoff) : backoff;
+                await sleep(delay);
+                continue;
+            }
+            const contentType = response.headers.get('content-type') || '';
+            if (!contentType.toLowerCase().includes('text/html')) {
+                // If not HTML, log a warning and continue (keep I/O compatibility)
+                logger.debug({
+                    event: 'fetch.nonHtml',
+                    data: { url, contentType, attempt },
+                });
+            }
+            const html = await response.text();
+            logger.debug({
+                event: 'fetch.success',
+                data: { url, status, attempt, duration, size: html.length },
+            });
+            // Short randomized sleep after success to reduce server load during crawling
+            const sleepTime = Math.random() * 500 + 250;
+            await sleep(sleepTime);
+            return html;
+        }
+        catch (error) {
+            clearTimeout(timeout);
+            const canRetry = shouldRetry(null, error);
+            lastError = error instanceof Error ? error : new Error(String(error));
+            logger.debug({
+                event: 'fetch.catch',
+                data: { url, attempt, canRetry, error: lastError.message },
+            });
+            if (!canRetry || attempt === maxRetries) {
+                logger.error({
+                    event: 'fetch.failed',
+                    data: { url, attempt, error: lastError.message },
+                });
+                throw lastError;
+            }
+            // Wait before the next attempt (exponential backoff + jitter)
+            const retryDelay = Math.pow(2, attempt - 1) * 1000 + Math.random() * 1000;
+            await sleep(retryDelay);
+        }
+    }
+    // Should not reach here; keep for type safety
+    throw lastError;
+}
+
+class CrawlingChain extends Chain {
+    constructor(config) {
+        const provider = config.provider;
+        provider.maxConcurrency ??= 5;
+        super({ ...config, provider });
+    }
+    get chain() {
+        const mapping = this.provider.crawlingTargetGroups.reduce((result, group) => {
+            return {
+                ...result,
+                [group.name]: () => this.executeGroupPipeline(group),
+            };
+        }, {});
+        return RunnablePassthrough.assign(mapping);
+    }
+    async executeGroupPipeline(group) {
+        const groupLabel = group.name;
+        const chain = RunnablePassthrough.assign({
+            listPageHtml: ({ target }) => this.fetchListPageHtml(target),
+        })
+            .pipe({
+            parsedList: ({ target, listPageHtml }) => this.parseListPageHtml(target, listPageHtml),
+            target: ({ target }) => target,
+        })
+            .pipe({
+            list: ({ target, parsedList }) => this.dedupeListItems(target, parsedList),
+            target: ({ target }) => target,
+        })
+            .pipe({
+            detailPagesHtmlWithPipelineId: ({ target, list }) => this.fetchDetailPagesHtml(target, list),
+            target: ({ target }) => target,
+            list: ({ list }) => list,
+        })
+            .pipe({
+            parsedDetails: ({ target, detailPagesHtmlWithPipelineId }) => this.parseDetailPagesHtml(target, detailPagesHtmlWithPipelineId),
+            target: ({ target }) => target,
+            list: ({ list }) => list,
+        })
+            .pipe({
+            processedArticles: ({ target, list, parsedDetails }) => this.mergeParsedArticles(target, list, parsedDetails),
+            target: ({ target }) => target,
+        })
+            .pipe({
+            count: ({ target, processedArticles }) => this.saveArticles(group, target, processedArticles),
+        })
+            .withRetry({ stopAfterAttempt: this.options.chain.stopAfterAttempt });
+        return this.executeWithLogging({
+            event: 'crawl.group',
+            level: 'debug',
+            startFields: {
+                group: groupLabel,
+                targets: group.targets.length,
+            },
+            doneFields: (total) => ({ totalSaved: total }),
+        }, async () => {
+            const results = await chain.batch(group.targets.map((target) => ({ target })), {
+                maxConcurrency: this.provider.maxConcurrency,
+            });
+            return results.reduce((sum, result) => sum + result.count, 0);
+        });
+    }
+    async fetchListPageHtml(target) {
+        return this.executeWithLogging({
+            event: 'crawl.list.fetch',
+            level: 'debug',
+            startFields: { target: this.describeTarget(target) },
+        }, async () => {
+            return await getHtmlFromUrl(this.logger, target.url);
+        });
+    }
+    async parseListPageHtml(target, listPageHtml) {
+        return this.executeWithLogging({
+            event: 'crawl.list.parse',
+            level: 'debug',
+            startFields: {
+                target: this.describeTarget(target),
+                htmlLength: listPageHtml.length,
+            },
+            doneFields: (items) => ({ count: items.length }),
+        }, async () => {
+            return (await target.parseList(listPageHtml)).map((item) => ({
+                ...item,
+                pipelineId: randomUUID(),
+            }));
+        });
+    }
+    async dedupeListItems(target, parsedList) {
+        return this.executeWithLogging({
+            event: 'crawl.list.dedupe',
+            level: 'debug',
+            startFields: {
+                target: this.describeTarget(target),
+                inCount: parsedList.length,
+            },
+            doneFields: (deduped) => ({
+                outCount: deduped.length,
+                filtered: parsedList.length - deduped.length,
+            }),
+        }, async () => {
+            const existingArticles = await this.provider.fetchExistingArticlesByUrls(parsedList.map(({ detailUrl }) => detailUrl));
+            const existingUrlSet = new Set(existingArticles.map(({ detailUrl }) => detailUrl));
+            return parsedList.filter((item) => !existingUrlSet.has(item.detailUrl));
+        });
+    }
+    async fetchDetailPagesHtml(target, list) {
+        return this.executeWithLogging({
+            event: 'crawl.detail.fetch',
+            level: 'debug',
+            startFields: {
+                target: this.describeTarget(target),
+                count: list.length,
+            },
+            doneFields: (htmlList) => ({ count: htmlList.length }),
+        }, async () => {
+            const htmlList = await Promise.all(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl)));
+            return htmlList.map((html, index) => ({
+                pipelineId: list[index].pipelineId,
+                html,
+            }));
+        });
+    }
+    async parseDetailPagesHtml(target, detailPagesHtmlWithPipelineId) {
+        return this.executeWithLogging({
+            event: 'crawl.detail.parse',
+            level: 'debug',
+            startFields: {
+                target: this.describeTarget(target),
+                count: detailPagesHtmlWithPipelineId.length,
+            },
+            doneFields: (details) => ({ count: details.length }),
+        }, async () => {
+            const detail = await Promise.all(detailPagesHtmlWithPipelineId.map(({ html }) => target.parseDetail(html)));
+            return detail.map((detail, index) => ({
+                pipelineId: detailPagesHtmlWithPipelineId[index].pipelineId,
+                ...detail,
+            }));
+        });
+    }
+    // Although this is a synchronous method, using async wrapping to maintain consistency with the executeWithLogging interface
+    async mergeParsedArticles(target, list, parsedDetails) {
+        return this.executeWithLogging({
+            event: 'crawl.merge',
+            level: 'debug',
+            startFields: {
+                target: this.describeTarget(target),
+                listCount: list.length,
+                detailCount: parsedDetails.length,
+            },
+            doneFields: (merged) => ({ count: merged.length }),
+        }, async () => {
+            const listItemMap = new Map(list.map((item) => [item.pipelineId, item]));
+            const merged = parsedDetails.map((detail) => {
+                const listItem = listItemMap.get(detail.pipelineId);
+                if (!listItem) {
+                    throw new Error(`No matching list item for detail with pipelineId: ${detail.pipelineId}`);
+                }
+                return {
+                    ...omit(listItem, ['pipelineId']),
+                    ...omit(detail, ['pipelineId']),
+                };
+            });
+            return merged;
+        });
+    }
+    async saveArticles(group, target, processedArticles) {
+        const omittedGroup = omit(group, ['targets']);
+        return this.executeWithLogging({
+            event: 'crawl.save',
+            level: 'debug',
+            startFields: {
+                group: omittedGroup,
+                target: this.describeTarget(target),
+                count: processedArticles.length,
+            },
+            doneFields: (saved) => ({ saved }),
+        }, async () => {
+            return await this.provider.saveCrawledArticles(processedArticles, {
+                taskId: this.taskId,
+                targetGroup: omittedGroup,
+                target,
+            });
+        });
+    }
+    describeTarget(target) {
+        return {
+            name: target.name || 'unknown',
+            listUrl: target.url,
+        };
+    }
+}
+
+/**
+ * Core class that orchestrates LLM-based newsletter generation.
+ * - Responsible for the flow: Crawling → Analysis → Content Generation → Save; external dependencies are injected via DI.
+ */
+class GenerateNewsletter {
+    /** Internal fields provided via dependency injection */
+    dateService;
+    taskService;
+    crawlingProvider;
+    analysisProvider;
+    contentGenerateProvider;
+    logger;
+    options;
+    previewNewsletterOptions;
+    /** Independent internal field **/
+    taskId = null;
+    /**
+     * Constructor
+     *
+     * @param config
+     * @example
+     * const generator = new GenerateNewsletter({
+     *   outputLanguage: 'English',
+     *   expertField: ['AI', 'Cloud'],
+     *   dateService,
+     *   taskService,
+     *   tagProvider,
+     *   crawlingProvider,
+     *   analysisProvider,
+     *   contentGenerateProvider,
+     *   options: { llm: { maxRetries: 5 } },
+     * });
+     */
+    constructor(config) {
+        const defaultOptions = {
+            content: config.contentOptions,
+            llm: { maxRetries: 5 },
+            chain: { stopAfterAttempt: 3 },
+        };
+        this.dateService = config.dateService;
+        this.taskService = config.taskService;
+        this.crawlingProvider = config.crawlingProvider;
+        this.analysisProvider = config.analysisProvider;
+        this.contentGenerateProvider = config.contentGenerateProvider;
+        this.options = {
+            ...defaultOptions,
+            ...config.options,
+            llm: {
+                ...defaultOptions.llm,
+                ...config.options?.llm,
+            },
+            chain: {
+                ...defaultOptions.chain,
+                ...config.options?.chain,
+            },
+        };
+        // Default logger (no-op)
+        this.logger = config.options?.logger ?? {
+            info: (_msg) => { },
+            debug: (_msg) => { },
+            error: (_msg) => { },
+        };
+        // Store preview newsletter options
+        this.previewNewsletterOptions = config.options?.previewNewsletter;
+    }
+    /**
+     * Execute the full newsletter generation pipeline.
+     */
+    async generate() {
+        const { newsletterId } = await this.executeWithTaskManagement(async () => {
+            const loggingExecutor = new LoggingExecutor(this.logger, this.taskId);
+            const crawlingChain = new CrawlingChain({
+                logger: this.logger,
+                taskId: this.taskId,
+                provider: this.crawlingProvider,
+                options: this.options,
+                loggingExecutor,
+            });
+            const analysisChain = new AnalysisChain({
+                logger: this.logger,
+                taskId: this.taskId,
+                provider: this.analysisProvider,
+                options: this.options,
+                loggingExecutor,
+                dateService: this.dateService,
+            });
+            const contentGenerateChain = new ContentGenerateChain({
+                logger: this.logger,
+                taskId: this.taskId,
+                provider: this.contentGenerateProvider,
+                options: this.options,
+                loggingExecutor,
+                dateService: this.dateService,
+            });
+            const taskChain = RunnableSequence.from([
+                crawlingChain.chain,
+                analysisChain.chain,
+                contentGenerateChain.chain,
+            ]);
+            return await taskChain.invoke({});
+        });
+        this.logNewsletterResult(newsletterId);
+        await this.sendPreviewNewsletterIfConfigured(newsletterId);
+        return newsletterId;
+    }
+    /**
+     * Run the pipeline while managing the task lifecycle.
+     */
+    async executeWithTaskManagement(pipeline) {
+        await this.startTask();
+        const executor = new LoggingExecutor(this.logger, this.taskId);
+        try {
+            return await executor.executeWithLogging({
+                event: 'task',
+                level: 'info',
+            }, async () => {
+                return await pipeline();
+            });
+        }
+        finally {
+            await this.endTask();
+        }
+    }
+    logNewsletterResult(newsletterId) {
+        if (newsletterId === null) {
+            this.logger.info({
+                event: 'generate.result.skipped',
+                taskId: this.taskId,
+                data: { reason: 'publicationCriteria.notMet' },
+            });
+            return;
+        }
+        this.logger.info({
+            event: 'generate.result.created',
+            taskId: this.taskId,
+            data: { newsletterId },
+        });
+    }
+    async sendPreviewNewsletterIfConfigured(newsletterId) {
+        const preview = this.previewNewsletterOptions;
+        if (!preview) {
+            return;
+        }
+        if (newsletterId === null) {
+            this.logger.info({
+                event: 'generate.preview.skip',
+                taskId: this.taskId,
+                data: { reason: 'noNewsletterCreated' },
+            });
+            return;
+        }
+        try {
+            // Fetch newsletter entity for preview
+            const newsletter = await preview.fetchNewsletterForPreview();
+            // Compose email subject/html/text
+            const subject = `[Preview] ${newsletter.title}`;
+            const html = newsletter.htmlBody;
+            const text = `${newsletter.title}\n\nIssue #${newsletter.issueOrder} - ${newsletter.date}`;
+            await preview.emailService.send({
+                ...preview.emailMessage,
+                subject,
+                html,
+                text,
+            });
+            this.logger.info({
+                event: 'generate.preview.sent',
+                taskId: this.taskId,
+                data: {
+                    newsletterId,
+                    to: preview.emailMessage.to,
+                },
+            });
+        }
+        catch (err) {
+            this.logger.error({
+                event: 'generate.preview.error',
+                taskId: this.taskId,
+                data: { newsletterId },
+            });
+            this.logger.error(err);
+        }
+    }
+    async startTask() {
+        this.taskId = await this.taskService.start();
+    }
+    async endTask() {
+        await this.taskService.end();
+    }
+}
+
+export { DateType, GenerateNewsletter };
+//# sourceMappingURL=index.js.map