@howuse/electron-crawler 0.1.0 → 0.3.0

package/dist/index.js

@@ -1,12 +1,12 @@
-"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const k=require("electron-store"),x=require("electron"),U=require("turndown");function v(){const e=x.BrowserWindow.getAllWindows().find(r=>r.title==="crawler-hidden-window");return e||new x.BrowserWindow({show:!1,webPreferences:{sandbox:!1},title:"crawler-hidden-window"})}const S=new U({headingStyle:"atx",codeBlockStyle:"fenced",bulletListMarker:"-",emDelimiter:"*",strongDelimiter:"**",linkStyle:"inlined",linkReferenceStyle:"full",preformattedCode:!1,blankReplacement:(e,t)=>t.nodeName==="BR"?`
-`:""});S.addRule("preserveLineBreaks",{filter:["br"],replacement:()=>`
-`});S.addRule("images",{filter:"img",replacement:(e,t)=>{const r=t.alt||"",s=t.src||t.getAttribute("src")||"",a=t.title||"";return a?`![${r}](${s} "${a}")`:`![${r}](${s})`}});function A(e){return e.replace(/<script[\s\S]*?<\/script>/gi,"").replace(/<style[\s\S]*?<\/style>/gi,"").replace(/<noscript[\s\S]*?<\/noscript>/gi,"").replace(/<iframe[\s\S]*?<\/iframe>/gi,"").replace(/on\w+="[^"]*"/gi,"").replace(/on\w+='[^']*'/gi,"").trim()}function C(e){if(!e||!e.trim())return"";try{const t=A(e);if(!t)return"";let r=S.turndown(t);return r=r.replace(/\n{3,}/g,`
+"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const x=require("electron-store"),A=require("electron"),O=require("turndown");function L(){const e=A.BrowserWindow.getAllWindows().find(r=>r.title==="crawler-hidden-window");return e||new A.BrowserWindow({show:!1,webPreferences:{sandbox:!1},title:"crawler-hidden-window"})}const k=new O({headingStyle:"atx",codeBlockStyle:"fenced",bulletListMarker:"-",emDelimiter:"*",strongDelimiter:"**",linkStyle:"inlined",linkReferenceStyle:"full",preformattedCode:!1,blankReplacement:(e,t)=>t.nodeName==="BR"?`
+`:""});k.addRule("preserveLineBreaks",{filter:["br"],replacement:()=>`
+`});k.addRule("images",{filter:"img",replacement:(e,t)=>{const r=t.alt||"",s=t.src||t.getAttribute("src")||"",o=t.title||"";return o?`![${r}](${s} "${o}")`:`![${r}](${s})`}});function b(e){return e.replace(/<script[\s\S]*?<\/script>/gi,"").replace(/<style[\s\S]*?<\/style>/gi,"").replace(/<noscript[\s\S]*?<\/noscript>/gi,"").replace(/<iframe[\s\S]*?<\/iframe>/gi,"").replace(/on\w+="[^"]*"/gi,"").replace(/on\w+='[^']*'/gi,"").trim()}function v(e){if(!e||!e.trim())return"";try{const t=b(e);if(!t)return"";let r=k.turndown(t);return r=r.replace(/\n{3,}/g,`
 
 `),r=r.split(`
 `).map(s=>s.trimEnd()).join(`
-`),r.trim()}catch(t){return console.error("[normalizeMarkdown] 转换失败:",t),A(e).replace(/<[^>]+>/g,"").replace(/\n{3,}/g,`
+`),r.trim()}catch(t){return console.error("[normalizeMarkdown] 转换失败:",t),b(e).replace(/<[^>]+>/g,"").replace(/\n{3,}/g,`
 
-`).trim()}}function D(e){const t=new Date,r=t.getFullYear(),s=t.getMonth()+1,a=t.getDate(),o=t.getHours(),i=t.getMinutes();if(!e||!e.trim())return t.toISOString();const c=e.trim(),w=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]+(\d{1,2}):(\d{1,2})\s+(?:星期|周)[一二三四五六日天]/,y=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/,T=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日]?/,_=/(\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/,R=/(\d{1,2})[月\-/](\d{1,2})[日]?/,O=/(\d{1,2})[:时](\d{1,2})[分]?/;let g=r,u=s,f=a,m=o,d=i,n=c.match(w);if(n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10),m=parseInt(n[4],10),d=parseInt(n[5],10);else if(n=c.match(y),n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10),m=parseInt(n[4],10),d=parseInt(n[5],10);else if(n=c.match(T),n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10);else if(n=c.match(_),n)u=parseInt(n[1],10),f=parseInt(n[2],10),m=parseInt(n[3],10),d=parseInt(n[4],10);else if(n=c.match(R),n)u=parseInt(n[1],10),f=parseInt(n[2],10);else if(n=c.match(O),n)m=parseInt(n[1],10),d=parseInt(n[2],10);else{const I=new Date(c);return Number.isNaN(I.getTime())?t.toISOString():I.toISOString()}(u<1||u>12)&&(u=s),(f<1||f>31)&&(f=a),(m<0||m>23)&&(m=o),(d<0||d>59)&&(d=i);const h=new Date(g,u-1,f,m,d,0,0);return h.getFullYear()!==g||h.getMonth()!==u-1||h.getDate()!==f?t.toISOString():L(h)}function L(e){const t=y=>y.toString().padStart(2,"0"),r=e.getFullYear(),s=t(e.getMonth()+1),a=t(e.getDate()),o=t(e.getHours()),i=t(e.getMinutes()),c=t(e.getSeconds()),w=e.getMilliseconds().toString().padStart(3,"0");return`${r}-${s}-${a}T${o}:${i}:${c}.${w}`}const M=k.default||k,b=new M;async function P(){if(l.rules&&l.rules.length>0)return l.rules;const e=b.get("news.rules")||[];if(e.length>0)return e;const t=await H();return t.length>0?(b.set("news.rules",t),t):[]}const p={running:!1,interval_ms:1800*1e3};let l={rules:[],rulesApiUrl:void 0,pushApiUrl:void 0,ruleTransformer:e=>e,newsItemFieldMap:void 0},$=!1;function E(e){l={...l,...e,ruleTransformer:e.ruleTransformer||(t=>t&&typeof t=="object"&&"data"in t?t.data:t)},j()}async function H(){if(!l.rulesApiUrl)return[];try{const e=await fetch(l.rulesApiUrl);if(!e.ok)throw new Error(`Failed to fetch rules from API: ${e.status} ${e.statusText}`);const t=await e.json(),r=l.ruleTransformer(t);return Array.isArray(r)?r:(console.warn("[crawler] Rules API returned non-array data, using empty array instead"),[])}catch(e){return console.error("[crawler] Failed to fetch rules from API:",e),[]}}async function N(e){if(l.pushApiUrl)try{const t=q(e),r=await fetch(l.pushApiUrl,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok)throw new Error(`Failed to push results to API: ${r.status} ${r.statusText}`);console.log("[crawler] Results pushed to API successfully")}catch(t){console.error("[crawler] Failed to push results to API:",t)}}function q(e){const t=l.newsItemFieldMap;if(!t||Object.keys(t).length===0)return e;const r={},s=Object.entries(e);for(const[a,o]of s){const i=t[a];if(i==="-")continue;const c=typeof i=="string"?i:a;r[c]=o}return r}async function F(e,t){return await e.webContents.executeJavaScript(`
+`).trim()}}function C(e){const t=new Date,r=t.getFullYear(),s=t.getMonth()+1,o=t.getDate(),a=t.getHours(),c=t.getMinutes();if(!e||!e.trim())return t.toISOString();const l=e.trim(),y=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]+(\d{1,2}):(\d{1,2})\s+(?:星期|周)[一二三四五六日天]/,S=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/,T=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日]?/,U=/(\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/,_=/(\d{1,2})[月\-/](\d{1,2})[日]?/,R=/(\d{1,2})[:时](\d{1,2})[分]?/;let g=r,u=s,f=o,d=a,m=c,n=l.match(y);if(n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10),d=parseInt(n[4],10),m=parseInt(n[5],10);else if(n=l.match(S),n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10),d=parseInt(n[4],10),m=parseInt(n[5],10);else if(n=l.match(T),n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10);else if(n=l.match(U),n)u=parseInt(n[1],10),f=parseInt(n[2],10),d=parseInt(n[3],10),m=parseInt(n[4],10);else if(n=l.match(_),n)u=parseInt(n[1],10),f=parseInt(n[2],10);else if(n=l.match(R),n)d=parseInt(n[1],10),m=parseInt(n[2],10);else{const I=new Date(l);return Number.isNaN(I.getTime())?t.toISOString():I.toISOString()}(u<1||u>12)&&(u=s),(f<1||f>31)&&(f=o),(d<0||d>23)&&(d=a),(m<0||m>59)&&(m=c);const h=new Date(g,u-1,f,d,m,0,0);return h.getFullYear()!==g||h.getMonth()!==u-1||h.getDate()!==f?t.toISOString():D(h)}function D(e){const t=S=>S.toString().padStart(2,"0"),r=e.getFullYear(),s=t(e.getMonth()+1),o=t(e.getDate()),a=t(e.getHours()),c=t(e.getMinutes()),l=t(e.getSeconds()),y=e.getMilliseconds().toString().padStart(3,"0");return`${r}-${s}-${o}T${a}:${c}:${l}.${y}`}const M=x.default||x,w=new M;async function P(){if(i.rules&&i.rules.length>0)return i.rules;const e=w.get("news.rules")||[];if(e.length>0)return e;const t=await H();return t.length>0?(w.set("news.rules",t),t):[]}const p={running:!1,interval_ms:1800*1e3};let i={rules:[],rulesApiUrl:void 0,pushApiUrl:void 0,ruleTransformer:e=>e,newsItemFieldMap:void 0},$=!1;function E(e){i={...i,...e,ruleTransformer:e.ruleTransformer||(t=>t&&typeof t=="object"&&"data"in t?t.data:t)},j()}async function H(){if(!i.rulesApiUrl)return[];try{const e=await fetch(i.rulesApiUrl);if(!e.ok)throw new Error(`Failed to fetch rules from API: ${e.status} ${e.statusText}`);const t=await e.json(),r=i.ruleTransformer(t);return Array.isArray(r)?r:(console.warn("[crawler] Rules API returned non-array data, using empty array instead"),[])}catch(e){return console.error("[crawler] Failed to fetch rules from API:",e),[]}}async function N(e){if(!i.pushApiUrl)return;const t=w.get("news.pushedUrls")||[];if(t.includes(e.url)){console.log(`[crawler] URL already pushed, skipping: ${e.url}`);return}try{const r=q(e),s=await fetch(i.pushApiUrl,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(r)});if(!s.ok)throw new Error(`Failed to push results to API: ${s.status} ${s.statusText}`);const o=[...t,e.url];w.set("news.pushedUrls",o),console.log("[crawler] Results pushed to API successfully")}catch(r){console.error("[crawler] Failed to push results to API:",r)}}function q(e){const t=i.newsItemFieldMap;if(!t||Object.keys(t).length===0)return e;const r={},s=Object.entries(e);for(const[o,a]of s){const c=t[o];if(c==="-")continue;const l=typeof c=="string"?c:o;r[l]=a}return r}async function F(e,t){return await e.webContents.executeJavaScript(`
       (() => {
         const links = []
         // 在指定范围内查找所有链接
@@ -83,4 +83,4 @@
             url: location.href
           }
         })()
-      `)}catch(s){return console.warn("[crawler] failed to extract page content",r,s),null}}async function J(e){const t=v();try{await t.loadURL(e.base_url,{httpReferrer:e.base_url});const r=await F(t,e);console.log(`[crawler] found ${r.length} links from ${e.remark||e.base_url}`);const s=[];for(const a of r){const o=await W(t,e,a);if(!o||!o.title||!o.contentHtml){console.log(`[crawler] skip empty result for ${a}`);continue}const i={url:o.url||a,title:o.title,content_html:o.contentHtml,content_markdown:C(o.contentHtml),published_at:D(o.timeText)};s.push(i);try{await N(i)}catch(c){console.warn("[crawler] push single news item failed",c)}}return console.log(`[crawler] processed ${s.length} items from ${e.remark||e.base_url}`),{success:!0,data:s}}catch(r){return console.warn("[crawler] rule failed",e.remark||e.base_url,r),{success:!1,error:r instanceof Error?r.message:String(r)}}}function j(){if($)return;$=!0;const e=p.interval_ms,t=async()=>{const r=await P();console.log(`[crawler] scheduled run, rules=${r.length}`),p.running=!0,p.running_source=void 0;try{for(const s of r)await J(s)}finally{p.running=!1,p.running_source=void 0,p.next_run_at=new Date(Date.now()+e).toISOString()}};p.next_run_at=new Date(Date.now()+5e3).toISOString(),setTimeout(t,5e3),setInterval(t,e)}exports.initCrawler=E;
+      `)}catch(s){return console.warn("[crawler] failed to extract page content",r,s),null}}async function J(e){const t=L();try{await t.loadURL(e.base_url,{httpReferrer:e.base_url});const r=await F(t,e);console.log(`[crawler] found ${r.length} links from ${e.remark||e.base_url}`);const s=[];for(const o of r){const a=await W(t,e,o);if(!a||!a.title||!a.contentHtml){console.log(`[crawler] skip empty result for ${o}`);continue}const c={url:a.url||o,title:a.title,content_html:a.contentHtml,content_markdown:v(a.contentHtml),published_at:C(a.timeText)};s.push(c);try{await N(c)}catch(l){console.warn("[crawler] push single news item failed",l)}}return console.log(`[crawler] processed ${s.length} items from ${e.remark||e.base_url}`),{success:!0,data:s}}catch(r){return console.warn("[crawler] rule failed",e.remark||e.base_url,r),{success:!1,error:r instanceof Error?r.message:String(r)}}}function j(){if($)return;$=!0;const e=p.interval_ms,t=async()=>{const r=await P();console.log(`[crawler] scheduled run, rules=${r.length}`),p.running=!0,p.running_source=void 0;try{for(const s of r)await J(s)}finally{p.running=!1,p.running_source=void 0,p.next_run_at=new Date(Date.now()+e).toISOString()}};p.next_run_at=new Date(Date.now()+5e3).toISOString(),setTimeout(t,5e3),setInterval(t,e)}exports.initCrawler=E;

package/dist/index.mjs

@@ -1,9 +1,9 @@
-import k from "electron-store";
-import { BrowserWindow as x } from "electron";
-import U from "turndown";
-function D() {
-  const e = x.getAllWindows().find((r) => r.title === "crawler-hidden-window");
-  return e || new x({
+import x from "electron-store";
+import { BrowserWindow as A } from "electron";
+import L from "turndown";
+function O() {
+  const e = A.getAllWindows().find((r) => r.title === "crawler-hidden-window");
+  return e || new A({
     show: !1,
     webPreferences: {
       sandbox: !1
@@ -11,7 +11,7 @@ function D() {
     title: "crawler-hidden-window"
   });
 }
-const S = new U({
+const k = new L({
   headingStyle: "atx",
   // 使用 # 格式的标题
   codeBlockStyle: "fenced",
@@ -31,79 +31,79 @@ const S = new U({
   blankReplacement: (e, t) => t.nodeName === "BR" ? `
 ` : ""
 });
-S.addRule("preserveLineBreaks", {
+k.addRule("preserveLineBreaks", {
   filter: ["br"],
   replacement: () => `
 `
 });
-S.addRule("images", {
+k.addRule("images", {
   filter: "img",
   replacement: (e, t) => {
-    const r = t.alt || "", s = t.src || t.getAttribute("src") || "", a = t.title || "";
-    return a ? `![${r}](${s} "${a}")` : `![${r}](${s})`;
+    const r = t.alt || "", s = t.src || t.getAttribute("src") || "", o = t.title || "";
+    return o ? `![${r}](${s} "${o}")` : `![${r}](${s})`;
   }
 });
-function A(e) {
+function $(e) {
   return e.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<noscript[\s\S]*?<\/noscript>/gi, "").replace(/<iframe[\s\S]*?<\/iframe>/gi, "").replace(/on\w+="[^"]*"/gi, "").replace(/on\w+='[^']*'/gi, "").trim();
 }
-function L(e) {
+function D(e) {
   if (!e || !e.trim())
     return "";
   try {
-    const t = A(e);
+    const t = $(e);
     if (!t)
       return "";
-    let r = S.turndown(t);
+    let r = k.turndown(t);
     return r = r.replace(/\n{3,}/g, `
 
 `), r = r.split(`
 `).map((s) => s.trimEnd()).join(`
 `), r.trim();
   } catch (t) {
-    return console.error("[normalizeMarkdown] 转换失败:", t), A(e).replace(/<[^>]+>/g, "").replace(/\n{3,}/g, `
+    return console.error("[normalizeMarkdown] 转换失败:", t), $(e).replace(/<[^>]+>/g, "").replace(/\n{3,}/g, `
 
 `).trim();
   }
 }
 function v(e) {
-  const t = /* @__PURE__ */ new Date(), r = t.getFullYear(), s = t.getMonth() + 1, a = t.getDate(), o = t.getHours(), l = t.getMinutes();
+  const t = /* @__PURE__ */ new Date(), r = t.getFullYear(), s = t.getMonth() + 1, o = t.getDate(), a = t.getHours(), c = t.getMinutes();
   if (!e || !e.trim())
     return t.toISOString();
-  const c = e.trim(), w = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]+(\d{1,2}):(\d{1,2})\s+(?:星期|周)[一二三四五六日天]/, y = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/, _ = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日]?/, T = /(\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/, R = /(\d{1,2})[月\-/](\d{1,2})[日]?/, O = /(\d{1,2})[:时](\d{1,2})[分]?/;
-  let g = r, u = s, f = a, m = o, p = l, n = c.match(w);
+  const l = e.trim(), y = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]+(\d{1,2}):(\d{1,2})\s+(?:星期|周)[一二三四五六日天]/, S = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/, U = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日]?/, _ = /(\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/, T = /(\d{1,2})[月\-/](\d{1,2})[日]?/, R = /(\d{1,2})[:时](\d{1,2})[分]?/;
+  let g = r, u = s, f = o, m = a, p = c, n = l.match(y);
   if (n)
     g = parseInt(n[1], 10), u = parseInt(n[2], 10), f = parseInt(n[3], 10), m = parseInt(n[4], 10), p = parseInt(n[5], 10);
-  else if (n = c.match(y), n)
+  else if (n = l.match(S), n)
     g = parseInt(n[1], 10), u = parseInt(n[2], 10), f = parseInt(n[3], 10), m = parseInt(n[4], 10), p = parseInt(n[5], 10);
-  else if (n = c.match(_), n)
+  else if (n = l.match(U), n)
     g = parseInt(n[1], 10), u = parseInt(n[2], 10), f = parseInt(n[3], 10);
-  else if (n = c.match(T), n)
+  else if (n = l.match(_), n)
     u = parseInt(n[1], 10), f = parseInt(n[2], 10), m = parseInt(n[3], 10), p = parseInt(n[4], 10);
-  else if (n = c.match(R), n)
+  else if (n = l.match(T), n)
     u = parseInt(n[1], 10), f = parseInt(n[2], 10);
-  else if (n = c.match(O), n)
+  else if (n = l.match(R), n)
     m = parseInt(n[1], 10), p = parseInt(n[2], 10);
   else {
-    const I = new Date(c);
+    const I = new Date(l);
     return Number.isNaN(I.getTime()) ? t.toISOString() : I.toISOString();
   }
-  (u < 1 || u > 12) && (u = s), (f < 1 || f > 31) && (f = a), (m < 0 || m > 23) && (m = o), (p < 0 || p > 59) && (p = l);
+  (u < 1 || u > 12) && (u = s), (f < 1 || f > 31) && (f = o), (m < 0 || m > 23) && (m = a), (p < 0 || p > 59) && (p = c);
   const h = new Date(g, u - 1, f, m, p, 0, 0);
   return h.getFullYear() !== g || h.getMonth() !== u - 1 || h.getDate() !== f ? t.toISOString() : C(h);
 }
 function C(e) {
-  const t = (y) => y.toString().padStart(2, "0"), r = e.getFullYear(), s = t(e.getMonth() + 1), a = t(e.getDate()), o = t(e.getHours()), l = t(e.getMinutes()), c = t(e.getSeconds()), w = e.getMilliseconds().toString().padStart(3, "0");
-  return `${r}-${s}-${a}T${o}:${l}:${c}.${w}`;
+  const t = (S) => S.toString().padStart(2, "0"), r = e.getFullYear(), s = t(e.getMonth() + 1), o = t(e.getDate()), a = t(e.getHours()), c = t(e.getMinutes()), l = t(e.getSeconds()), y = e.getMilliseconds().toString().padStart(3, "0");
+  return `${r}-${s}-${o}T${a}:${c}:${l}.${y}`;
 }
-const M = k.default || k, b = new M();
+const M = x.default || x, w = new M();
 async function P() {
   if (i.rules && i.rules.length > 0)
     return i.rules;
-  const e = b.get("news.rules") || [];
+  const e = w.get("news.rules") || [];
   if (e.length > 0)
     return e;
   const t = await E();
-  return t.length > 0 ? (b.set("news.rules", t), t) : [];
+  return t.length > 0 ? (w.set("news.rules", t), t) : [];
 }
 const d = {
   running: !1,
@@ -115,7 +115,7 @@ let i = {
   pushApiUrl: void 0,
   ruleTransformer: (e) => e,
   newsItemFieldMap: void 0
-}, $ = !1;
+}, b = !1;
 function z(e) {
   i = {
     ...i,
@@ -138,32 +138,39 @@ async function E() {
   }
 }
 async function H(e) {
-  if (i.pushApiUrl)
-    try {
-      const t = N(e), r = await fetch(i.pushApiUrl, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json"
-        },
-        body: JSON.stringify(t)
-      });
-      if (!r.ok)
-        throw new Error(`Failed to push results to API: ${r.status} ${r.statusText}`);
-      console.log("[crawler] Results pushed to API successfully");
-    } catch (t) {
-      console.error("[crawler] Failed to push results to API:", t);
-    }
+  if (!i.pushApiUrl)
+    return;
+  const t = w.get("news.pushedUrls") || [];
+  if (t.includes(e.url)) {
+    console.log(`[crawler] URL already pushed, skipping: ${e.url}`);
+    return;
+  }
+  try {
+    const r = N(e), s = await fetch(i.pushApiUrl, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(r)
+    });
+    if (!s.ok)
+      throw new Error(`Failed to push results to API: ${s.status} ${s.statusText}`);
+    const o = [...t, e.url];
+    w.set("news.pushedUrls", o), console.log("[crawler] Results pushed to API successfully");
+  } catch (r) {
+    console.error("[crawler] Failed to push results to API:", r);
+  }
 }
 function N(e) {
   const t = i.newsItemFieldMap;
   if (!t || Object.keys(t).length === 0)
     return e;
   const r = {}, s = Object.entries(e);
-  for (const [a, o] of s) {
-    const l = t[a];
-    if (l === "-") continue;
-    const c = typeof l == "string" ? l : a;
-    r[c] = o;
+  for (const [o, a] of s) {
+    const c = t[o];
+    if (c === "-") continue;
+    const l = typeof c == "string" ? c : o;
+    r[l] = a;
   }
   return r;
 }
@@ -261,30 +268,30 @@ async function q(e, t, r) {
   }
 }
 async function J(e) {
-  const t = D();
+  const t = O();
   try {
     await t.loadURL(e.base_url, { httpReferrer: e.base_url });
     const r = await F(t, e);
     console.log(`[crawler] found ${r.length} links from ${e.remark || e.base_url}`);
     const s = [];
-    for (const a of r) {
-      const o = await q(t, e, a);
-      if (!o || !o.title || !o.contentHtml) {
-        console.log(`[crawler] skip empty result for ${a}`);
+    for (const o of r) {
+      const a = await q(t, e, o);
+      if (!a || !a.title || !a.contentHtml) {
+        console.log(`[crawler] skip empty result for ${o}`);
         continue;
       }
-      const l = {
-        url: o.url || a,
-        title: o.title,
-        content_html: o.contentHtml,
-        content_markdown: L(o.contentHtml),
-        published_at: v(o.timeText)
+      const c = {
+        url: a.url || o,
+        title: a.title,
+        content_html: a.contentHtml,
+        content_markdown: D(a.contentHtml),
+        published_at: v(a.timeText)
       };
-      s.push(l);
+      s.push(c);
       try {
-        await H(l);
-      } catch (c) {
-        console.warn("[crawler] push single news item failed", c);
+        await H(c);
+      } catch (l) {
+        console.warn("[crawler] push single news item failed", l);
       }
     }
     return console.log(`[crawler] processed ${s.length} items from ${e.remark || e.base_url}`), {
@@ -299,8 +306,8 @@ async function J(e) {
   }
 }
 function W() {
-  if ($) return;
-  $ = !0;
+  if (b) return;
+  b = !0;
   const e = d.interval_ms, t = async () => {
     const r = await P();
     console.log(`[crawler] scheduled run, rules=${r.length}`), d.running = !0, d.running_source = void 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@howuse/electron-crawler",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "基于Electron的爬虫工具包，用于爬取新闻和股票详情",
   "keywords": [
     "electron",
@@ -18,8 +18,8 @@
     "url": "https://github.com/your-username/your-repo/issues"
   },
   "homepage": "https://github.com/your-username/your-repo#readme",
-  "main": "dist/index.cjs",
-  "module": "dist/index.js",
+  "main": "dist/index.js",
+  "module": "dist/index.mjs",
   "types": "dist/index.d.ts",
   "files": [
     "dist/**/*",
@@ -29,8 +29,8 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "require": "./dist/index.cjs"
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
     }
   },
   "scripts": {