@llm-newsletter-kit/core 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,828 @@
+import { LanguageModel } from 'ai';
+
+/**
+ * Common type aliases.
+ *
+ * - Provides explicit alias types for date/URL/Markdown/HTML, etc.
+ * - All comments are written in English JSDoc style.
+ */
+/**
+ * ISO 8601 date string. Use YYYY-MM-DD without time.
+ * @example "2025-10-15"
+ */
+type IsoDateString = string;
+/**
+ * URL string pointing to external or internal resources.
+ * @example "https://example.com/news/123"
+ */
+type UrlString = string;
+/**
+ * Markdown-formatted text.
+ * @example "# Title\n\nBody content."
+ */
+type MarkdownString = string;
+/**
+ * HTML-formatted string.
+ * @example "<h1>Title</h1><p>Body</p>"
+ */
+type HtmlString = string;
+/**
+ * String-based unique identifier. Used for DOM ids, data record ids, etc.
+ * @example "item-42"
+ */
+type UniqueIdentifier = string;
+/**
+ * Type for date identifiers.
+ *
+ * The DateType enum is used to distinguish date-related values.
+ * It can be used to differentiate between registered dates and ranges.
+ *
+ * Enum members:
+ * - REGISTERED: indicates a registered date.
+ * - DURATION: indicates a duration or time range.
+ *
+ * @example
+ * ```ts
+ * const type: DateType = DateType.REGISTERED;
+ * ```
+ */
+declare enum DateType {
+    REGISTERED = "registered",
+    DURATION = "duration"
+}
+
+/**
+ * Structure of an article that has not yet been processed (no importance score, image analysis, or tagging).
+ */
+type UnscoredArticle = {
+    /**
+     * Identifier. Supports both string and number to accommodate various DB schemas.
+     */
+    id: string | number;
+    /**
+     * Article title.
+     * @example "Weekly Tech Newsletter"
+     */
+    title: string;
+    /**
+     * Article body in Markdown format.
+     * @example "### Highlights\n- New framework announced"
+     */
+    detailContent: MarkdownString;
+    /**
+     * Whether the article has an attached image.
+     * @example true
+     */
+    hasAttachedImage: boolean;
+    /**
+     * Image analysis result. Null if no LLM analysis result exists.
+     */
+    imageContextByLlm: string | null;
+    /**
+     * First classification tag.
+     * @example "News"
+     */
+    tag1: string | null;
+    /**
+     * Second classification tag.
+     * @example "Jobs"
+     */
+    tag2: string | null;
+    /**
+     * Third classification tag.
+     * @example "Announcement"
+     */
+    tag3: string | null;
+    /**
+     * URL for board collection. Not the original article detail URL. Typically matches CrawlingTarget.url.
+     * @example "https://example.com/board/notice"
+     */
+    targetUrl: UrlString;
+};
+/**
+ * Article type used after the analysis phase (image/tagging/score) for updates.
+ */
+type ArticleForUpdateByAnalysis = UnscoredArticle & {
+    /**
+     * Importance score. Range 1–10.
+     * @example 8
+     */
+    importanceScore: number;
+};
+/**
+ * Article type used in the newsletter content generation phase.
+ */
+type ArticleForGenerateContent = ArticleForUpdateByAnalysis & {
+    /**
+     * Content type of the article. Typically a group name from CrawlingTargetGroup or similar grouping.
+     * @example "News"
+     */
+    contentType: string;
+    /**
+     * Original detail page URL of the article.
+     * @example "https://example.com/news/123"
+     */
+    url: UrlString;
+};
+
+/**
+ * Structure returned after parsing a list page (HTML).
+ */
+type ParsedTargetListItem = {
+    /**
+     * Unique id present in the HTML. If none exists, you may omit it.
+     * @example "post-2025-0001"
+     */
+    uniqId?: UniqueIdentifier;
+    /**
+     * Article title.
+     * @example "AI Industry Trends Report Released"
+     */
+    title: string;
+    /**
+     * Article date in ISO format (YYYY-MM-DD), no time included.
+     * @example "2025-10-15"
+     */
+    date: IsoDateString;
+    /**
+     * Whether it is a registered date or a duration.
+     * @example DateType.REGISTERED
+     */
+    dateType: DateType;
+    /**
+     * URL to the linked detail page.
+     * @example "https://example.com/board/notice/1234"
+     */
+    detailUrl: UrlString;
+};
+/**
+ * Structure returned after parsing a detail page (HTML).
+ */
+type ParsedTargetDetail = {
+    /**
+     * Parsed detail page content in Markdown.
+     * Convert HTML to Markdown using libraries such as turndown.
+     * @example "## Notice\n\n- Application period: 2025-10-15 ~ 2025-10-31"
+     */
+    detailContent: MarkdownString;
+    /**
+     * Whether there is any file attachment.
+     * @example true
+     */
+    hasAttachedFile: boolean;
+    /**
+     * Whether an image is included.
+     * @example false
+     */
+    hasAttachedImage: boolean;
+};
+/**
+ * Fully structured crawling target object combining list and detail parsing results.
+ */
+type ParsedTarget = ParsedTargetListItem & ParsedTargetDetail;
+/**
+ * Target to crawl.
+ * For example, a board/list page of a website. Parsing methods must be defined.
+ */
+type CrawlingTarget = {
+    /**
+     * Identifier for the crawling target. Any unique id works; uuid is recommended.
+     * @example "crawling-target-001"
+     */
+    id: UniqueIdentifier;
+    /**
+     * Name of the crawling target.
+     * @example "Notice Board"
+     */
+    name: string;
+    /**
+     * URL of the crawling target. Should point to a specific board (list) page URL.
+     * @example "https://example.com/board/notice"
+     */
+    url: UrlString;
+    /**
+     * Method to structurally parse data from a list page (HTML).
+     * Synchronous parsing with clear rules is recommended, but async is supported to allow LLM/external backends.
+     *
+     * @param html Original HTML string of the list page
+     * @returns Parsed list items
+     * @example
+     * ```ts
+     * const items = target.parseList(html);
+     * items[0].title; // "Notice Title"
+     * ```
+     */
+    parseList: (html: string) => Promise<ParsedTargetListItem[]> | ParsedTargetListItem[];
+    /**
+     * Method to structurally parse data from a detail page (HTML).
+     * Synchronous parsing with clear rules is recommended, but async is supported to allow LLM/external backends.
+     *
+     * @param html Original HTML string of the detail page
+     * @returns Parsed detail information
+     */
+    parseDetail: (html: string) => Promise<ParsedTargetDetail> | ParsedTargetDetail;
+};
+/**
+ * Grouped type for crawling targets.
+ * For example, groups like News, Jobs, Programs/Bids, etc.
+ */
+type CrawlingTargetGroup = {
+    /**
+     * Identifier for a group. Any unique id works; uuid is recommended.
+     * @example "group-news"
+     */
+    id: UniqueIdentifier;
+    /**
+     * Group name.
+     * @example "News"
+     */
+    name: string;
+    /**
+     * Targets included in this group.
+     */
+    targets: CrawlingTarget[];
+};
+
+type ContentOptions = {
+    /**
+     * Output language for the newsletter. e.g., "English", "Spanish"
+     * @example "English"
+     */
+    outputLanguage: string;
+    /**
+     * Target domain(s) for the newsletter (one or many)
+     * @example ["AI", "Cloud"]
+     */
+    expertField: string | string[];
+};
+type LLMQueryOptions = {
+    /**
+     * Number of retries when LLM calls fail.
+     * @default 5
+     */
+    maxRetries?: number;
+};
+type ChainOptions = {
+    /**
+     * Maximum retry attempts when the chain fails while running.
+     * @default 3
+     */
+    stopAfterAttempt?: number;
+};
+
+/**
+ * Token markers in template.
+ */
+type HtmlTemplateMarkers = {
+    /**
+     * Title token
+     *
+     * When the title is set to "NEWSLETTER_TITLE", it replaces the "{{NEWSLETTER_TITLE}}" pattern in the template string.
+     *
+     * @default "NEWSLETTER_TITLE"
+     */
+    title?: string;
+    /**
+     * Content HTML token
+     *
+     * When content is set to "NEWSLETTER_CONTENT", it replaces the "{{NEWSLETTER_CONTENT}}" pattern in the template string.
+     *
+     * @default "NEWSLETTER_CONTENT"
+     */
+    content?: string;
+};
+/**
+ * String template and marker set
+ */
+type HtmlTemplate = {
+    /**
+     * Original template string
+     */
+    html: string;
+    /**
+     * Uses default markers when not specified
+     */
+    markers?: HtmlTemplateMarkers;
+};
+type RequiredHtmlTemplate = Pick<HtmlTemplate, 'html'> & {
+    markers: Required<HtmlTemplateMarkers>;
+};
+
+/**
+ * Type representing an email address.
+ * @example "user@example.com"
+ */
+type EmailAddress = string;
+/**
+ * Information about an email attachment.
+ */
+type EmailAttachment = {
+    /**
+     * Attachment file name
+     */
+    filename: string;
+    /**
+     * Attachment content (Buffer or string)
+     */
+    content: Buffer | string;
+    /**
+     * Attachment MIME type (optional)
+     * @example 'application/pdf', 'image/png'
+     */
+    contentType?: string;
+    /**
+     * Content-ID for inline images (optional)
+     * Can be referenced in HTML as <img src="cid:contentId">
+     */
+    contentId?: string;
+};
+/**
+ * Email message information.
+ */
+type EmailMessage = {
+    /**
+     * Sender email address
+     */
+    from: EmailAddress;
+    /**
+     * Recipient email address(es)
+     */
+    to: EmailAddress | EmailAddress[];
+    /**
+     * Email subject
+     */
+    subject: string;
+    /**
+     * HTML body
+     */
+    html: HtmlString;
+    /**
+     * Plain text body (optional)
+     */
+    text?: string;
+    /**
+     * CC address(es) (optional)
+     */
+    cc?: EmailAddress | EmailAddress[];
+    /**
+     * BCC address(es) (optional)
+     */
+    bcc?: EmailAddress | EmailAddress[];
+    /**
+     * Reply-to address (optional)
+     * Address for recipients to reply to
+     */
+    replyTo?: EmailAddress;
+    /**
+     * Additional headers (optional)
+     * @example { 'X-Priority': '1', 'X-Mailer': 'MyApp' }
+     */
+    headers?: Record<string, string>;
+    /**
+     * Attachments (optional)
+     * Supports both regular attachments and inline images.
+     */
+    attachments?: EmailAttachment[];
+};
+
+/**
+ * Global logging level type used across the project.
+ */
+type LogLevel = 'debug' | 'info' | 'error';
+/**
+ * Structured log message format.
+ * - event: Recommended "domain.action[.state]" style, e.g., "crawl.group.start" | "fetch.success" | "task.error"
+ * - level: Usually implied by the called method, but can be explicit (info/debug).
+ * - taskId: Identifier to correlate logs for the same job.
+ * - durationMs: Include elapsed time (ms) on ".done"/".error" logs.
+ * - data: Additional context. Prefer JSON‑serializable values only.
+ * Examples:
+ * logger.info({ event: 'task.start', taskId })
+ * logger.debug({ event: 'crawl.list.fetch.start', data: { url } })
+ */
+type LogMessage<TaskId = unknown, Extra extends Record<string, unknown> = Record<string, unknown>> = {
+    /** Event name, e.g., "crawl.group.start", "fetch.success" */
+    event: string;
+    /** Log level (optional; implied by the method if omitted) */
+    level?: LogLevel;
+    /** Associated task identifier */
+    taskId?: TaskId;
+    /** Elapsed time in milliseconds (typically for done/error) */
+    durationMs?: number;
+    /** Additional data container */
+    data?: Extra;
+};
+
+/**
+ * Logger interface used across the application.
+ * - Accepts structured LogMessage, implement to integrate with systems like Logstash, CloudWatch, Datadog, etc.
+ * - Typically, info is for operational events, debug for detailed tracing, and error for exceptions/critical failures.
+ * - error accepts either a structured LogMessage or an arbitrary error object (Error, unknown).
+ *
+ * Usage examples:
+ * logger.info({ event: 'task.start', taskId })
+ * logger.debug({ event: 'crawl.list.fetch.start', data: { url } })
+ * logger.error({ event: 'fetch.failed', data: { url, attempt, error: err.message } })
+ */
+interface AppLogger {
+    /** Info-level logs for operational events/state. */
+    info: (message: LogMessage) => void;
+    /** Debug-level logs for detailed debugging/tracing. */
+    debug: (message: LogMessage) => void;
+    /** Error-level logs. Accepts structured messages or arbitrary errors (Error/unknown). */
+    error: (message: LogMessage | unknown) => void;
+}
+/**
+ * Email sending service interface.
+ */
+interface EmailService {
+    /**
+     * Send an email.
+     * @param message Email message to send
+     * @throws May throw on delivery failures
+     */
+    send: (message: EmailMessage) => Promise<void>;
+}
+/**
+ * Service that provides dates for internal use or insertion into the newsletter.
+ * Clients can consider locale/language/timezone themselves.
+ */
+interface DateService {
+    /**
+     * Return current date in ISO format (YYYY-MM-DD).
+     * @returns ISO date string
+     * @example "2024-10-15"
+     */
+    getCurrentISODateString: () => IsoDateString;
+    /**
+     * Return a localized display date string for use in newsletter content.
+     */
+    getDisplayDateString: () => string;
+}
+
+/**
+ * Publishable final Newsletter entity.
+ */
+type Newsletter = {
+    /**
+     * Newsletter title.
+     * @example "LLM Newsletter #12"
+     */
+    title: string;
+    /**
+     * Newsletter content in Markdown. Will be applied to a template and converted to HTML.
+     */
+    content: MarkdownString;
+    /**
+     * Final HTML body of the newsletter, ready to send.
+     */
+    htmlBody: HtmlString;
+    /**
+     * Issue number of the newsletter.
+     * @example 12
+     */
+    issueOrder: number;
+    /**
+     * Publication date of the newsletter in ISO format (YYYY-MM-DD). Time is not included.
+     * @example "2025-10-15"
+     */
+    date: IsoDateString;
+};
+
+/**
+ * Task module managed by the client for newsletter generation.
+ * e.g., can ensure single execution (prevent duplicates) like "run once a day".
+ */
+interface TaskService<TaskId> {
+    /**
+     * Start a task. May return an error if a task is already running.
+     * @returns taskId
+     */
+    start: () => Promise<TaskId>;
+    /**
+     * End a task.
+     * @returns void
+     */
+    end: () => Promise<void>;
+}
+/**
+ * Values and methods required for crawling.
+ */
+interface CrawlingProvider {
+    /**
+     * Maximum number of concurrent jobs.
+     * Used to control parallelism for resource management and to prevent overload.
+     * @default 5
+     */
+    maxConcurrency?: number;
+    /**
+     * Crawling target groups.
+     */
+    crawlingTargetGroups: CrawlingTargetGroup[];
+    /**
+     * Look up existing stored articles for the given URLs to compare with newly collected ones.
+     * @param articleUrls Original article URLs to query
+     * @returns Previously stored (parsed) articles
+     */
+    fetchExistingArticlesByUrls: (articleUrls: UrlString[]) => Promise<ParsedTarget[]>;
+    /**
+     * Persist structured results collected for a specific crawling group (batch-save recommended).
+     * @param articles Parsed articles
+     * @param context Crawling execution context
+     * @returns Number of saved articles
+     */
+    saveCrawledArticles: <TaskId>(articles: ParsedTarget[], context: {
+        taskId: TaskId;
+        targetGroup: Omit<CrawlingTargetGroup, 'targets'>;
+        target: CrawlingTarget;
+    }) => Promise<number>;
+}
+/**
+ * Minimum importance score policy for a specific crawling target.
+ */
+type MinimumImportanceScoreRule = {
+    /**
+     * Target URL the minimum score applies to. Same as CrawlingTarget.url.
+     */
+    targetUrl: UrlString;
+    /**
+     * Minimum importance score to apply.
+     * @example 5
+     */
+    minScore: number;
+};
+/**
+ * Values and methods required for LLM analysis.
+ */
+interface AnalysisProvider {
+    /**
+     * Options for classification and tag generation.
+     */
+    classifyTagOptions: {
+        /**
+         * Model to use. A relatively light model is acceptable.
+         */
+        model: LanguageModel;
+    };
+    /**
+     * Options for image analysis.
+     */
+    analyzeImagesOptions: {
+        /**
+         * Model to use. Must be a multimodal model.
+         */
+        model: LanguageModel;
+    };
+    /**
+     * Options for importance score generation.
+     */
+    determineScoreOptions: {
+        /**
+         * Model to use. A relatively light model is acceptable.
+         */
+        model: LanguageModel;
+        /**
+         * Minimum score policies per crawling target.
+         * @example
+         * ```ts
+         * minimumImportanceScoreRules: [
+         *   { targetUrl: 'https://example.com/board/notice', minScore: 5 }
+         * ]
+         * ```
+         */
+        minimumImportanceScoreRules?: MinimumImportanceScoreRule[];
+    };
+    /**
+     * Fetch articles without an importance score from the DB.
+     * @returns List of unscored articles
+     */
+    fetchUnscoredArticles: () => Promise<UnscoredArticle[]>;
+    /**
+     * Fetch existing tag list to classify collected articles before newsletter generation.
+     * @returns Array of tag strings
+     */
+    fetchTags: () => Promise<string[]>;
+    /**
+     * Update analysis results and importance scores after all work is done.
+     * @param article Article data to update
+     */
+    update: (article: ArticleForUpdateByAnalysis) => Promise<void>;
+}
+/**
+ * Values and methods required to generate a newsletter.
+ */
+interface ContentGenerateProvider {
+    /**
+     * Language model to use. A high‑performance model is recommended.
+     */
+    model: LanguageModel;
+    /**
+     * Maximum tokens allowed for generation.
+     * Used to prevent excessively long outputs and control token usage.
+     */
+    maxOutputTokens?: number;
+    /**
+     * Temperature controlling randomness (0.0–1.0).
+     * Higher values can be more creative but less consistent.
+     * @default 0.3
+     */
+    temperature?: number;
+    /**
+     * Controls nucleus sampling (top‑p, 0.0–1.0).
+     * Sample from tokens whose cumulative probability exceeds the threshold.
+     * @default 0.95
+     */
+    topP?: number;
+    /**
+     * Restrict sampling to top‑K tokens.
+     * Helps balance diversity and quality.
+     */
+    topK?: number;
+    /**
+     * Controls penalty for repeating tokens (−2.0–2.0).
+     * Higher values discourage repetition.
+     */
+    presencePenalty?: number;
+    /**
+     * Controls penalty based on token frequency (−2.0–2.0).
+     * Higher values discourage frequent tokens.
+     */
+    frequencyPenalty?: number;
+    /**
+     * Issue number of the newsletter.
+     */
+    issueOrder: number;
+    /**
+     * Publication criteria for issuing a newsletter.
+     * @example
+     * ```ts
+     * publicationCriteria: {
+     *   minimumArticleCountForIssue: 5,
+     *   priorityArticleScoreThreshold: 8,
+     * }
+     * ```
+     */
+    publicationCriteria?: {
+        /**
+         * Minimum number of articles required to issue a newsletter
+         * @default 5
+         */
+        minimumArticleCountForIssue: number;
+        /**
+         * If there exists an article with importance ≥ this score, issue regardless of count
+         * @default 8
+         */
+        priorityArticleScoreThreshold: number;
+    };
+    /**
+     * Subscription page URL. Can be inserted as a CTA link in the newsletter.
+     */
+    subscribePageUrl?: UrlString;
+    /**
+     * Brand name of the newsletter.
+     * @example "Dev Insight"
+     */
+    newsletterBrandName: string;
+    /**
+     * Fetch candidate articles from the DB for newsletter generation.
+     */
+    fetchArticleCandidates: () => Promise<ArticleForGenerateContent[]>;
+    /**
+     * HTML template for the newsletter.
+     * Generated content is applied to this template to produce the final HTML.
+     */
+    htmlTemplate: HtmlTemplate;
+    /**
+     * Persist the newsletter (recommend saving relationships together).
+     * - Receives `usedArticles` so relations can be handled transactionally.
+     */
+    saveNewsletter: (input: {
+        newsletter: Newsletter;
+        usedArticles: ArticleForGenerateContent[];
+    }) => Promise<{
+        id: string | number;
+    }>;
+}
+/**
+ * Options for newsletter generation.
+ * - Controls LLM retry counts, logger injection, etc.
+ */
+type GenerateNewsletterOptions = {
+    /**
+     * Logger implementation. If not provided, a no‑op logger is used.
+     */
+    logger?: AppLogger;
+    /**
+     * LLM behavior configuration.
+     */
+    llm?: LLMQueryOptions;
+    /**
+     * Internal chain behavior configuration.
+     */
+    chain?: ChainOptions;
+    /**
+     * Preview newsletter delivery configuration.
+     * When present, a preview email is sent to reviewers.
+     */
+    previewNewsletter?: {
+        /**
+         * Fetch a newsletter entity to use for preview.
+         */
+        fetchNewsletterForPreview: () => Promise<Newsletter>;
+        /**
+         * Email delivery service implementation.
+         */
+        emailService: EmailService;
+        /**
+         * Base configuration for the preview email.
+         * subject/html/text are generated automatically and omitted here.
+         */
+        emailMessage: Omit<EmailMessage, 'subject' | 'html' | 'text'>;
+    };
+};
+/**
+ * Configuration object passed to the GenerateNewsletter constructor.
+ */
+type GenerateNewsletterConfig<TaskId> = {
+    /**
+     * Content generation settings.
+     * Defines the output language and target domains.
+     */
+    contentOptions: ContentOptions;
+    /**
+     * Service that supplies date values.
+     * Manages publication date and display strings.
+     */
+    dateService: DateService;
+    /**
+     * Task service used to ensure single execution and avoid duplicates.
+     */
+    taskService: TaskService<TaskId>;
+    /**
+     * Provider for crawling (targets, persistence, queries, etc.).
+     */
+    crawlingProvider: CrawlingProvider;
+    /**
+     * Provider for analysis (image analysis, tagging, scoring, etc.).
+     */
+    analysisProvider: AnalysisProvider;
+    /**
+     * Provider for content generation (LLM, template, save/publish, etc.).
+     */
+    contentGenerateProvider: ContentGenerateProvider;
+    /**
+     * Optional behavior/settings.
+     */
+    options?: GenerateNewsletterOptions;
+};
+
+/**
+ * Core class that orchestrates LLM-based newsletter generation.
+ * - Responsible for the flow: Crawling → Analysis → Content Generation → Save; external dependencies are injected via DI.
+ */
+declare class GenerateNewsletter<TaskId> {
+    /** Internal fields provided via dependency injection */
+    private readonly dateService;
+    private readonly taskService;
+    private readonly crawlingProvider;
+    private readonly analysisProvider;
+    private readonly contentGenerateProvider;
+    private readonly logger;
+    private readonly options;
+    private readonly previewNewsletterOptions?;
+    /** Independent internal field **/
+    private taskId;
+    /**
+     * Constructor
+     *
+     * @param config
+     * @example
+     * const generator = new GenerateNewsletter({
+     *   outputLanguage: 'English',
+     *   expertField: ['AI', 'Cloud'],
+     *   dateService,
+     *   taskService,
+     *   tagProvider,
+     *   crawlingProvider,
+     *   analysisProvider,
+     *   contentGenerateProvider,
+     *   options: { llm: { maxRetries: 5 } },
+     * });
+     */
+    constructor(config: GenerateNewsletterConfig<TaskId>);
+    /**
+     * Execute the full newsletter generation pipeline.
+     */
+    generate(): Promise<string | number | null>;
+    /**
+     * Run the pipeline while managing the task lifecycle.
+     */
+    private executeWithTaskManagement;
+    private logNewsletterResult;
+    private sendPreviewNewsletterIfConfigured;
+    private startTask;
+    private endTask;
+}
+
+export { DateType, GenerateNewsletter };
+export type { AnalysisProvider, AppLogger, ArticleForGenerateContent, ArticleForUpdateByAnalysis, ContentGenerateProvider, CrawlingProvider, CrawlingTarget, CrawlingTargetGroup, DateService, EmailAddress, EmailAttachment, EmailMessage, EmailService, GenerateNewsletterConfig, GenerateNewsletterOptions, HtmlString, HtmlTemplate, HtmlTemplateMarkers, IsoDateString, LogLevel, LogMessage, MarkdownString, MinimumImportanceScoreRule, Newsletter, ParsedTarget, ParsedTargetDetail, ParsedTargetListItem, RequiredHtmlTemplate, TaskService, UniqueIdentifier, UnscoredArticle, UrlString };