@llm-newsletter-kit/core 1.2.0 → 1.3.1

package/dist/index.cjs

@@ -1302,7 +1302,7 @@ function shouldRetry(status, error) {
     }
     return false;
 }
-async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/') {
+async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/', customFetch) {
     const maxRetries = 5;
     const baseTimeoutMs = 10_000; // Base 10s, increases per attempt
     let lastError = null;
@@ -1312,7 +1312,7 @@ async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/')
         const timeout = setTimeout(() => controller.abort(`timeout after ${timeoutMs}ms`), timeoutMs);
         try {
             const startedAt = Date.now();
-            const response = await fetch(url, {
+            const response = await (customFetch ?? fetch)(url, {
                 // mode: 'cors' // Not applicable in Node, left here for behavioral parity with browsers
                 redirect: 'follow',
                 // @ts-expect-error Undici/Fetch in Node may allow duplex; safe to ignore
@@ -1456,7 +1456,7 @@ class CrawlingChain extends Chain {
             startFields: { target: this.describeTarget(target) },
         }, async () => {
             try {
-                return await getHtmlFromUrl(this.logger, target.url);
+                return await getHtmlFromUrl(this.logger, target.url, undefined, this.provider.customFetch);
             }
             catch (error) {
                 this.logger.error({
@@ -1535,7 +1535,7 @@ class CrawlingChain extends Chain {
                 failedCount: result.failedCount,
             }),
         }, async () => {
-            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl)));
+            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl, undefined, this.provider.customFetch)));
             const detailPagesHtmlWithPipelineId = [];
             const successList = [];
             let failedCount = 0;

package/dist/index.d.ts

@@ -526,6 +526,11 @@ interface CrawlingProvider {
      * @default 5
      */
     maxConcurrency?: number;
+    /**
+     * Optional custom fetch function (e.g., proxy-based fetch).
+     * When provided, this function is used instead of the global `fetch` for HTTP requests.
+     */
+    customFetch?: typeof fetch;
     /**
      * Crawling target groups.
      */

package/dist/index.js

@@ -1300,7 +1300,7 @@ function shouldRetry(status, error) {
     }
     return false;
 }
-async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/') {
+async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/', customFetch) {
     const maxRetries = 5;
     const baseTimeoutMs = 10_000; // Base 10s, increases per attempt
     let lastError = null;
@@ -1310,7 +1310,7 @@ async function getHtmlFromUrl(logger, url, referer = 'https://www.google.com/')
         const timeout = setTimeout(() => controller.abort(`timeout after ${timeoutMs}ms`), timeoutMs);
         try {
             const startedAt = Date.now();
-            const response = await fetch(url, {
+            const response = await (customFetch ?? fetch)(url, {
                 // mode: 'cors' // Not applicable in Node, left here for behavioral parity with browsers
                 redirect: 'follow',
                 // @ts-expect-error Undici/Fetch in Node may allow duplex; safe to ignore
@@ -1454,7 +1454,7 @@ class CrawlingChain extends Chain {
             startFields: { target: this.describeTarget(target) },
         }, async () => {
             try {
-                return await getHtmlFromUrl(this.logger, target.url);
+                return await getHtmlFromUrl(this.logger, target.url, undefined, this.provider.customFetch);
             }
             catch (error) {
                 this.logger.error({
@@ -1533,7 +1533,7 @@ class CrawlingChain extends Chain {
                 failedCount: result.failedCount,
             }),
         }, async () => {
-            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl)));
+            const settled = await Promise.allSettled(list.map((data) => getHtmlFromUrl(this.logger, data.detailUrl, undefined, this.provider.customFetch)));
             const detailPagesHtmlWithPipelineId = [];
             const successList = [];
             let failedCount = 0;

package/package.json

@@ -2,7 +2,7 @@
   "name": "@llm-newsletter-kit/core",
   "private": false,
   "type": "module",
-  "version": "1.2.0",
+  "version": "1.3.1",
   "description": "An extensible framework to automate your entire newsletter workflow. Handles data collection, LLM-based content analysis, and email generation, letting you focus on your unique domain logic.",
   "main": "dist/index.cjs",
   "module": "dist/index.js",
@@ -48,8 +48,8 @@
   "author": "kimhongyeon",
   "license": "Apache-2.0",
   "dependencies": {
-    "@langchain/core": "^1.1.19",
-    "ai": "^6.0.77",
+    "@langchain/core": "^1.1.24",
+    "ai": "^6.0.84",
     "es-toolkit": "^1.44.0",
     "jsdom": "^28.0.0",
     "juice": "^11.1.1",
@@ -57,18 +57,18 @@
     "zod": "^4.3.6"
   },
   "devDependencies": {
-    "@ai-sdk/anthropic": "^3.0.38",
-    "@ai-sdk/google": "^3.0.22",
-    "@ai-sdk/openai": "^3.0.26",
-    "@ai-sdk/togetherai": "^2.0.31",
+    "@ai-sdk/anthropic": "^3.0.43",
+    "@ai-sdk/google": "^3.0.29",
+    "@ai-sdk/openai": "^3.0.28",
+    "@ai-sdk/togetherai": "^2.0.33",
     "@eslint/js": "^9.39.2",
     "@trivago/prettier-plugin-sort-imports": "^6.0.2",
     "@types/jsdom": "^27.0.0",
-    "@types/node": "^25.2.1",
+    "@types/node": "^25.2.3",
     "@vitest/coverage-v8": "^3.2.4",
     "@vitest/expect": "^3.2.4",
     "eslint": "^9.39.2",
-    "eslint-plugin-unused-imports": "^4.3.0",
+    "eslint-plugin-unused-imports": "^4.4.1",
     "prettier": "^3.8.1",
     "rimraf": "^6.1.2",
     "rollup": "^4.57.1",
@@ -76,7 +76,7 @@
     "rollup-plugin-typescript2": "^0.36.0",
     "tsx": "^4.21.0",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.54.0",
+    "typescript-eslint": "^8.55.0",
     "vitest": "^3.2.4"
   },
   "repository": {