@howuse/electron-crawler 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Your Name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,102 @@
+# @howuse/electron-crawler
+
+基于 Electron 的爬虫工具包，用于爬取新闻和股票详情。
+
+## 特性
+
+- 🚀 基于 Electron 构建，支持现代网页渲染
+- 📰 专为新闻网站设计的爬虫功能
+- 📊 股票详情数据抓取
+- ⚙️ 可配置的爬取规则
+- 🔄 自动定时任务
+- 📦 TypeScript 支持，提供完整的类型定义
+
+## 安装
+
+```bash
+npm install @howuse/electron-crawler
+```
+
+## 使用方法
+
+### 基本用法
+
+```javascript
+import { initCrawler } from '@howuse/electron-crawler'
+
+// 初始化爬虫配置（调用后自动启动定时任务，默认每30分钟执行一次）
+initCrawler({
+  rules: [
+    {
+      remark: '示例新闻站点',
+      base_url: 'https://example-news-site.com',
+      home_range_selector: '.news-list',
+      title_selector: 'h1',
+      content_selector: '.article-content',
+      time_selector: '.publish-time',
+      exclude_selectors: ['.ad', '.related-links'],
+      enabled: true
+    }
+  ],
+  pushApiUrl: 'https://api.example.com/news/push',
+  // 可选：字段映射，'-' 表示忽略该字段
+  newsItemFieldMap: {
+    url: 'link',               // 重命名链接
+    title: 'title',            // 保持不变
+    content_html: 'content',   // 重命名 HTML 内容
+    content_markdown: '-',     // 不推送 markdown
+    published_at: 'publishTime'// 重命名发布时间
+  }
+})
+```
+
+### 从接口拉取规则
+
+```javascript
+import { initCrawler } from '@howuse/electron-crawler'
+
+initCrawler({
+  // 优先使用内存 rules，其次本地 store，最后从接口获取并写入 store
+  rulesApiUrl: 'https://example-news-site.com/rules.json',
+  ruleTransformer: (data) => data.rules ?? data, // 若返回 { rules: [...] }，可在此转换
+  pushApiUrl: 'https://api.example.com/news/push'
+})
+```
+
+### 配置选项
+
+| 选项 | 类型 | 描述 |
+|------|------|------|
+| `rules` | `NewsRule[]` | 直接提供的规则数组 |
+| `rulesApiUrl` | `string` | 规则API接口URL |
+| `pushApiUrl` | `string` | 结果推送API接口URL（必填以启用推送） |
+| `ruleTransformer` | `(data: any) => any` | 规则转换函数 |
+| `newsItemFieldMap` | `Partial<Record<keyof NewsItem, string \| '-'>>` | 推送字段映射，值为 '-' 表示忽略该字段 |
+
+### NewsRule 结构
+
+| 属性 | 类型 | 必需 | 描述 |
+|------|------|------|------|
+| `remark` | `string` | 是 | 备注 |
+| `base_url` | `string` | 是 | 网站URL |
+| `home_range_selector` | `string` | 是 | 首页范围选择器 |
+| `title_selector` | `string` | 是 | 标题选择器 |
+| `content_selector` | `string` | 是 | 内容选择器 |
+| `time_selector` | `string` | 是 | 时间选择器 |
+| `exclude_selectors` | `string[]` | 否 | 排除元素选择器数组 |
+| `enabled` | `boolean` | 否 | 是否启用，默认为 true |
+
+## API
+
+### `initCrawler(config: CrawlerConfig)`
+初始化并启动定时爬虫。会按配置自动拉取规则（内存 > 本地 store > API），并在每次抓取时为每条新闻立即推送到 `pushApiUrl`。
+
+## 注意事项
+
+1. 此包需要在 Electron 环境中运行
+2. 请遵守目标网站的 robots.txt 和使用条款
+3. 建议设置合理的请求间隔，避免对目标服务器造成压力
+
+## 许可证
+
+MIT

package/dist/common.d.ts

@@ -0,0 +1,25 @@
+import { BrowserWindow } from 'electron';
+/**
+ * 确保隐藏窗口存在，用于爬虫操作
+ * 所有爬虫模块共享同一个隐藏窗口
+ */
+export declare function ensureHiddenWindow(): BrowserWindow;
+/**
+ * 清理所有爬虫隐藏窗口
+ */
+export declare function cleanupHiddenWindows(): void;
+/**
+ * 将 HTML 转换为 Markdown 格式
+ * 使用专业的 turndown 库进行转换，保留原始HTML结构
+ */
+export declare function normalizeMarkdown(html: string): string;
+/**
+ * 解析和补全时间文本
+ * 支持多种时间格式：年月日、年月日时分、时分等
+ * 规则：
+ * 1. 如果时间不存在，返回当前时间的年月日时分
+ * 2. 如果缺少年月日，取今年的年月日
+ * 3. 如果缺少时分，取当前系统时间的时分
+ * 4. 使用正则表达式匹配时间
+ */
+export declare function parseDate(text?: string): string;

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { initCrawler } from './newsCrawler';

package/dist/index.js

@@ -0,0 +1,86 @@
+"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const k=require("electron-store"),x=require("electron"),U=require("turndown");function v(){const e=x.BrowserWindow.getAllWindows().find(r=>r.title==="crawler-hidden-window");return e||new x.BrowserWindow({show:!1,webPreferences:{sandbox:!1},title:"crawler-hidden-window"})}const S=new U({headingStyle:"atx",codeBlockStyle:"fenced",bulletListMarker:"-",emDelimiter:"*",strongDelimiter:"**",linkStyle:"inlined",linkReferenceStyle:"full",preformattedCode:!1,blankReplacement:(e,t)=>t.nodeName==="BR"?`
+`:""});S.addRule("preserveLineBreaks",{filter:["br"],replacement:()=>`
+`});S.addRule("images",{filter:"img",replacement:(e,t)=>{const r=t.alt||"",s=t.src||t.getAttribute("src")||"",a=t.title||"";return a?`![${r}](${s} "${a}")`:`![${r}](${s})`}});function A(e){return e.replace(/<script[\s\S]*?<\/script>/gi,"").replace(/<style[\s\S]*?<\/style>/gi,"").replace(/<noscript[\s\S]*?<\/noscript>/gi,"").replace(/<iframe[\s\S]*?<\/iframe>/gi,"").replace(/on\w+="[^"]*"/gi,"").replace(/on\w+='[^']*'/gi,"").trim()}function C(e){if(!e||!e.trim())return"";try{const t=A(e);if(!t)return"";let r=S.turndown(t);return r=r.replace(/\n{3,}/g,`
+
+`),r=r.split(`
+`).map(s=>s.trimEnd()).join(`
+`),r.trim()}catch(t){return console.error("[normalizeMarkdown] 转换失败:",t),A(e).replace(/<[^>]+>/g,"").replace(/\n{3,}/g,`
+
+`).trim()}}function D(e){const t=new Date,r=t.getFullYear(),s=t.getMonth()+1,a=t.getDate(),o=t.getHours(),i=t.getMinutes();if(!e||!e.trim())return t.toISOString();const c=e.trim(),w=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]+(\d{1,2}):(\d{1,2})\s+(?:星期|周)[一二三四五六日天]/,y=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/,T=/(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日]?/,_=/(\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/,R=/(\d{1,2})[月\-/](\d{1,2})[日]?/,O=/(\d{1,2})[:时](\d{1,2})[分]?/;let g=r,u=s,f=a,m=o,d=i,n=c.match(w);if(n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10),m=parseInt(n[4],10),d=parseInt(n[5],10);else if(n=c.match(y),n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10),m=parseInt(n[4],10),d=parseInt(n[5],10);else if(n=c.match(T),n)g=parseInt(n[1],10),u=parseInt(n[2],10),f=parseInt(n[3],10);else if(n=c.match(_),n)u=parseInt(n[1],10),f=parseInt(n[2],10),m=parseInt(n[3],10),d=parseInt(n[4],10);else if(n=c.match(R),n)u=parseInt(n[1],10),f=parseInt(n[2],10);else if(n=c.match(O),n)m=parseInt(n[1],10),d=parseInt(n[2],10);else{const I=new Date(c);return Number.isNaN(I.getTime())?t.toISOString():I.toISOString()}(u<1||u>12)&&(u=s),(f<1||f>31)&&(f=a),(m<0||m>23)&&(m=o),(d<0||d>59)&&(d=i);const h=new Date(g,u-1,f,m,d,0,0);return h.getFullYear()!==g||h.getMonth()!==u-1||h.getDate()!==f?t.toISOString():L(h)}function L(e){const t=y=>y.toString().padStart(2,"0"),r=e.getFullYear(),s=t(e.getMonth()+1),a=t(e.getDate()),o=t(e.getHours()),i=t(e.getMinutes()),c=t(e.getSeconds()),w=e.getMilliseconds().toString().padStart(3,"0");return`${r}-${s}-${a}T${o}:${i}:${c}.${w}`}const M=k.default||k,b=new M;async function P(){if(l.rules&&l.rules.length>0)return l.rules;const e=b.get("news.rules")||[];if(e.length>0)return e;const t=await H();return t.length>0?(b.set("news.rules",t),t):[]}const p={running:!1,interval_ms:1800*1e3};let l={rules:[],rulesApiUrl:void 0,pushApiUrl:void 0,ruleTransformer:e=>e,newsItemFieldMap:void 0},$=!1;function E(e){l={...l,...e,ruleTransformer:e.ruleTransformer||(t=>t&&typeof t=="object"&&"data"in t?t.data:t)},j()}async function H(){if(!l.rulesApiUrl)return[];try{const e=await fetch(l.rulesApiUrl);if(!e.ok)throw new Error(`Failed to fetch rules from API: ${e.status} ${e.statusText}`);const t=await e.json(),r=l.ruleTransformer(t);return Array.isArray(r)?r:(console.warn("[crawler] Rules API returned non-array data, using empty array instead"),[])}catch(e){return console.error("[crawler] Failed to fetch rules from API:",e),[]}}async function N(e){if(l.pushApiUrl)try{const t=q(e),r=await fetch(l.pushApiUrl,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)});if(!r.ok)throw new Error(`Failed to push results to API: ${r.status} ${r.statusText}`);console.log("[crawler] Results pushed to API successfully")}catch(t){console.error("[crawler] Failed to push results to API:",t)}}function q(e){const t=l.newsItemFieldMap;if(!t||Object.keys(t).length===0)return e;const r={},s=Object.entries(e);for(const[a,o]of s){const i=t[a];if(i==="-")continue;const c=typeof i=="string"?i:a;r[c]=o}return r}async function F(e,t){return await e.webContents.executeJavaScript(`
+      (() => {
+        const links = []
+        // 在指定范围内查找所有链接
+        const rangeElements = document.querySelectorAll(${JSON.stringify(t.home_range_selector)})
+        if (rangeElements.length === 0) {
+          // 如果没有找到范围，则在整个文档中查找
+          const allLinks = document.querySelectorAll('a')
+          allLinks.forEach((a) => {
+            if (a.href && a.textContent && a.textContent.trim()) {
+              links.push({
+                href: a.href,
+                text: a.textContent.trim()
+              })
+            }
+          })
+        } else {
+          rangeElements.forEach((range) => {
+            const rangeLinks = range.querySelectorAll('a')
+            rangeLinks.forEach((a) => {
+              if (a.href && a.textContent && a.textContent.trim()) {
+                links.push({
+                  href: a.href,
+                  text: a.textContent.trim()
+                })
+              }
+            })
+          })
+        }
+        // 去重并返回
+        const uniqueLinks = [...new Map(links.map(l => [l.href, l])).values()]
+        return uniqueLinks.map(l => l.href)
+      })()
+    `)}async function W(e,t,r){try{return await e.loadURL(r,{httpReferrer:t.base_url}),await e.webContents.executeJavaScript(`
+        (() => {
+          const pickText = (sel) => {
+            const el = document.querySelector(sel)
+            return el ? (el.textContent || '').trim() : ''
+          }
+          const pickContentHtml = (sel, excludes = []) => {
+            const el = document.querySelector(sel)
+            if (!el) return ''
+            const clone = el.cloneNode(true)
+            excludes.forEach((ex) => clone.querySelectorAll(ex).forEach((n) => n.remove()))
+            
+            // 将相对路径的图片 URL 转换为绝对路径
+            const images = clone.querySelectorAll('img')
+            images.forEach((img) => {
+              if (img.src) {
+                try {
+                  // 如果 src 是相对路径，转换为绝对路径
+                  const imgUrl = new URL(img.src, location.href)
+                  img.src = imgUrl.href
+                } catch {
+                  // 如果 URL 解析失败，尝试使用 getAttribute('src') 并手动拼接
+                  const srcAttr = img.getAttribute('src')
+                  if (srcAttr && !srcAttr.startsWith('http://') && !srcAttr.startsWith('https://') && !srcAttr.startsWith('data:')) {
+                    try {
+                      const baseUrl = new URL(location.href)
+                      img.src = new URL(srcAttr, baseUrl.origin + baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1)).href
+                    } catch {
+                      // 如果还是失败，保持原样
+                    }
+                  }
+                }
+              }
+            })
+            
+            return clone.innerHTML || ''
+          }
+          return {
+            title: pickText(${JSON.stringify(t.title_selector)}),
+            contentHtml: pickContentHtml(${JSON.stringify(t.content_selector)}, ${JSON.stringify(t.exclude_selectors||[])}),
+            timeText: pickText(${JSON.stringify(t.time_selector)}),
+            url: location.href
+          }
+        })()
+      `)}catch(s){return console.warn("[crawler] failed to extract page content",r,s),null}}async function J(e){const t=v();try{await t.loadURL(e.base_url,{httpReferrer:e.base_url});const r=await F(t,e);console.log(`[crawler] found ${r.length} links from ${e.remark||e.base_url}`);const s=[];for(const a of r){const o=await W(t,e,a);if(!o||!o.title||!o.contentHtml){console.log(`[crawler] skip empty result for ${a}`);continue}const i={url:o.url||a,title:o.title,content_html:o.contentHtml,content_markdown:C(o.contentHtml),published_at:D(o.timeText)};s.push(i);try{await N(i)}catch(c){console.warn("[crawler] push single news item failed",c)}}return console.log(`[crawler] processed ${s.length} items from ${e.remark||e.base_url}`),{success:!0,data:s}}catch(r){return console.warn("[crawler] rule failed",e.remark||e.base_url,r),{success:!1,error:r instanceof Error?r.message:String(r)}}}function j(){if($)return;$=!0;const e=p.interval_ms,t=async()=>{const r=await P();console.log(`[crawler] scheduled run, rules=${r.length}`),p.running=!0,p.running_source=void 0;try{for(const s of r)await J(s)}finally{p.running=!1,p.running_source=void 0,p.next_run_at=new Date(Date.now()+e).toISOString()}};p.next_run_at=new Date(Date.now()+5e3).toISOString(),setTimeout(t,5e3),setInterval(t,e)}exports.initCrawler=E;

package/dist/index.mjs

@@ -0,0 +1,318 @@
+import k from "electron-store";
+import { BrowserWindow as x } from "electron";
+import U from "turndown";
+function D() {
+  const e = x.getAllWindows().find((r) => r.title === "crawler-hidden-window");
+  return e || new x({
+    show: !1,
+    webPreferences: {
+      sandbox: !1
+    },
+    title: "crawler-hidden-window"
+  });
+}
+const S = new U({
+  headingStyle: "atx",
+  // 使用 # 格式的标题
+  codeBlockStyle: "fenced",
+  // 使用 ``` 格式的代码块
+  bulletListMarker: "-",
+  // 使用 - 作为列表标记
+  emDelimiter: "*",
+  // 使用 * 作为强调标记
+  strongDelimiter: "**",
+  // 使用 ** 作为粗体标记
+  linkStyle: "inlined",
+  // 内联链接格式 [text](url)
+  linkReferenceStyle: "full",
+  // 完整的链接引用格式
+  preformattedCode: !1,
+  // 不使用预格式化代码
+  blankReplacement: (e, t) => t.nodeName === "BR" ? `
+` : ""
+});
+S.addRule("preserveLineBreaks", {
+  filter: ["br"],
+  replacement: () => `
+`
+});
+S.addRule("images", {
+  filter: "img",
+  replacement: (e, t) => {
+    const r = t.alt || "", s = t.src || t.getAttribute("src") || "", a = t.title || "";
+    return a ? `![${r}](${s} "${a}")` : `![${r}](${s})`;
+  }
+});
+function A(e) {
+  return e.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<noscript[\s\S]*?<\/noscript>/gi, "").replace(/<iframe[\s\S]*?<\/iframe>/gi, "").replace(/on\w+="[^"]*"/gi, "").replace(/on\w+='[^']*'/gi, "").trim();
+}
+function L(e) {
+  if (!e || !e.trim())
+    return "";
+  try {
+    const t = A(e);
+    if (!t)
+      return "";
+    let r = S.turndown(t);
+    return r = r.replace(/\n{3,}/g, `
+
+`), r = r.split(`
+`).map((s) => s.trimEnd()).join(`
+`), r.trim();
+  } catch (t) {
+    return console.error("[normalizeMarkdown] 转换失败:", t), A(e).replace(/<[^>]+>/g, "").replace(/\n{3,}/g, `
+
+`).trim();
+  }
+}
+function v(e) {
+  const t = /* @__PURE__ */ new Date(), r = t.getFullYear(), s = t.getMonth() + 1, a = t.getDate(), o = t.getHours(), l = t.getMinutes();
+  if (!e || !e.trim())
+    return t.toISOString();
+  const c = e.trim(), w = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]+(\d{1,2}):(\d{1,2})\s+(?:星期|周)[一二三四五六日天]/, y = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/, _ = /(\d{4})[年\-/](\d{1,2})[月\-/](\d{1,2})[日]?/, T = /(\d{1,2})[月\-/](\d{1,2})[日\s]*(\d{1,2}):(\d{1,2})/, R = /(\d{1,2})[月\-/](\d{1,2})[日]?/, O = /(\d{1,2})[:时](\d{1,2})[分]?/;
+  let g = r, u = s, f = a, m = o, p = l, n = c.match(w);
+  if (n)
+    g = parseInt(n[1], 10), u = parseInt(n[2], 10), f = parseInt(n[3], 10), m = parseInt(n[4], 10), p = parseInt(n[5], 10);
+  else if (n = c.match(y), n)
+    g = parseInt(n[1], 10), u = parseInt(n[2], 10), f = parseInt(n[3], 10), m = parseInt(n[4], 10), p = parseInt(n[5], 10);
+  else if (n = c.match(_), n)
+    g = parseInt(n[1], 10), u = parseInt(n[2], 10), f = parseInt(n[3], 10);
+  else if (n = c.match(T), n)
+    u = parseInt(n[1], 10), f = parseInt(n[2], 10), m = parseInt(n[3], 10), p = parseInt(n[4], 10);
+  else if (n = c.match(R), n)
+    u = parseInt(n[1], 10), f = parseInt(n[2], 10);
+  else if (n = c.match(O), n)
+    m = parseInt(n[1], 10), p = parseInt(n[2], 10);
+  else {
+    const I = new Date(c);
+    return Number.isNaN(I.getTime()) ? t.toISOString() : I.toISOString();
+  }
+  (u < 1 || u > 12) && (u = s), (f < 1 || f > 31) && (f = a), (m < 0 || m > 23) && (m = o), (p < 0 || p > 59) && (p = l);
+  const h = new Date(g, u - 1, f, m, p, 0, 0);
+  return h.getFullYear() !== g || h.getMonth() !== u - 1 || h.getDate() !== f ? t.toISOString() : C(h);
+}
+function C(e) {
+  const t = (y) => y.toString().padStart(2, "0"), r = e.getFullYear(), s = t(e.getMonth() + 1), a = t(e.getDate()), o = t(e.getHours()), l = t(e.getMinutes()), c = t(e.getSeconds()), w = e.getMilliseconds().toString().padStart(3, "0");
+  return `${r}-${s}-${a}T${o}:${l}:${c}.${w}`;
+}
+const M = k.default || k, b = new M();
+async function P() {
+  if (i.rules && i.rules.length > 0)
+    return i.rules;
+  const e = b.get("news.rules") || [];
+  if (e.length > 0)
+    return e;
+  const t = await E();
+  return t.length > 0 ? (b.set("news.rules", t), t) : [];
+}
+const d = {
+  running: !1,
+  interval_ms: 1800 * 1e3
+};
+let i = {
+  rules: [],
+  rulesApiUrl: void 0,
+  pushApiUrl: void 0,
+  ruleTransformer: (e) => e,
+  newsItemFieldMap: void 0
+}, $ = !1;
+function z(e) {
+  i = {
+    ...i,
+    ...e,
+    // 确保ruleTransformer始终存在
+    ruleTransformer: e.ruleTransformer || ((t) => t && typeof t == "object" && "data" in t ? t.data : t)
+  }, W();
+}
+async function E() {
+  if (!i.rulesApiUrl)
+    return [];
+  try {
+    const e = await fetch(i.rulesApiUrl);
+    if (!e.ok)
+      throw new Error(`Failed to fetch rules from API: ${e.status} ${e.statusText}`);
+    const t = await e.json(), r = i.ruleTransformer(t);
+    return Array.isArray(r) ? r : (console.warn("[crawler] Rules API returned non-array data, using empty array instead"), []);
+  } catch (e) {
+    return console.error("[crawler] Failed to fetch rules from API:", e), [];
+  }
+}
+async function H(e) {
+  if (i.pushApiUrl)
+    try {
+      const t = N(e), r = await fetch(i.pushApiUrl, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify(t)
+      });
+      if (!r.ok)
+        throw new Error(`Failed to push results to API: ${r.status} ${r.statusText}`);
+      console.log("[crawler] Results pushed to API successfully");
+    } catch (t) {
+      console.error("[crawler] Failed to push results to API:", t);
+    }
+}
+function N(e) {
+  const t = i.newsItemFieldMap;
+  if (!t || Object.keys(t).length === 0)
+    return e;
+  const r = {}, s = Object.entries(e);
+  for (const [a, o] of s) {
+    const l = t[a];
+    if (l === "-") continue;
+    const c = typeof l == "string" ? l : a;
+    r[c] = o;
+  }
+  return r;
+}
+async function F(e, t) {
+  return await e.webContents.executeJavaScript(
+    `
+      (() => {
+        const links = []
+        // 在指定范围内查找所有链接
+        const rangeElements = document.querySelectorAll(${JSON.stringify(t.home_range_selector)})
+        if (rangeElements.length === 0) {
+          // 如果没有找到范围，则在整个文档中查找
+          const allLinks = document.querySelectorAll('a')
+          allLinks.forEach((a) => {
+            if (a.href && a.textContent && a.textContent.trim()) {
+              links.push({
+                href: a.href,
+                text: a.textContent.trim()
+              })
+            }
+          })
+        } else {
+          rangeElements.forEach((range) => {
+            const rangeLinks = range.querySelectorAll('a')
+            rangeLinks.forEach((a) => {
+              if (a.href && a.textContent && a.textContent.trim()) {
+                links.push({
+                  href: a.href,
+                  text: a.textContent.trim()
+                })
+              }
+            })
+          })
+        }
+        // 去重并返回
+        const uniqueLinks = [...new Map(links.map(l => [l.href, l])).values()]
+        return uniqueLinks.map(l => l.href)
+      })()
+    `
+  );
+}
+async function q(e, t, r) {
+  try {
+    return await e.loadURL(r, { httpReferrer: t.base_url }), await e.webContents.executeJavaScript(
+      `
+        (() => {
+          const pickText = (sel) => {
+            const el = document.querySelector(sel)
+            return el ? (el.textContent || '').trim() : ''
+          }
+          const pickContentHtml = (sel, excludes = []) => {
+            const el = document.querySelector(sel)
+            if (!el) return ''
+            const clone = el.cloneNode(true)
+            excludes.forEach((ex) => clone.querySelectorAll(ex).forEach((n) => n.remove()))
+            
+            // 将相对路径的图片 URL 转换为绝对路径
+            const images = clone.querySelectorAll('img')
+            images.forEach((img) => {
+              if (img.src) {
+                try {
+                  // 如果 src 是相对路径，转换为绝对路径
+                  const imgUrl = new URL(img.src, location.href)
+                  img.src = imgUrl.href
+                } catch {
+                  // 如果 URL 解析失败，尝试使用 getAttribute('src') 并手动拼接
+                  const srcAttr = img.getAttribute('src')
+                  if (srcAttr && !srcAttr.startsWith('http://') && !srcAttr.startsWith('https://') && !srcAttr.startsWith('data:')) {
+                    try {
+                      const baseUrl = new URL(location.href)
+                      img.src = new URL(srcAttr, baseUrl.origin + baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1)).href
+                    } catch {
+                      // 如果还是失败，保持原样
+                    }
+                  }
+                }
+              }
+            })
+            
+            return clone.innerHTML || ''
+          }
+          return {
+            title: pickText(${JSON.stringify(t.title_selector)}),
+            contentHtml: pickContentHtml(${JSON.stringify(t.content_selector)}, ${JSON.stringify(
+        t.exclude_selectors || []
+      )}),
+            timeText: pickText(${JSON.stringify(t.time_selector)}),
+            url: location.href
+          }
+        })()
+      `
+    );
+  } catch (s) {
+    return console.warn("[crawler] failed to extract page content", r, s), null;
+  }
+}
+async function J(e) {
+  const t = D();
+  try {
+    await t.loadURL(e.base_url, { httpReferrer: e.base_url });
+    const r = await F(t, e);
+    console.log(`[crawler] found ${r.length} links from ${e.remark || e.base_url}`);
+    const s = [];
+    for (const a of r) {
+      const o = await q(t, e, a);
+      if (!o || !o.title || !o.contentHtml) {
+        console.log(`[crawler] skip empty result for ${a}`);
+        continue;
+      }
+      const l = {
+        url: o.url || a,
+        title: o.title,
+        content_html: o.contentHtml,
+        content_markdown: L(o.contentHtml),
+        published_at: v(o.timeText)
+      };
+      s.push(l);
+      try {
+        await H(l);
+      } catch (c) {
+        console.warn("[crawler] push single news item failed", c);
+      }
+    }
+    return console.log(`[crawler] processed ${s.length} items from ${e.remark || e.base_url}`), {
+      success: !0,
+      data: s
+    };
+  } catch (r) {
+    return console.warn("[crawler] rule failed", e.remark || e.base_url, r), {
+      success: !1,
+      error: r instanceof Error ? r.message : String(r)
+    };
+  }
+}
+function W() {
+  if ($) return;
+  $ = !0;
+  const e = d.interval_ms, t = async () => {
+    const r = await P();
+    console.log(`[crawler] scheduled run, rules=${r.length}`), d.running = !0, d.running_source = void 0;
+    try {
+      for (const s of r)
+        await J(s);
+    } finally {
+      d.running = !1, d.running_source = void 0, d.next_run_at = new Date(Date.now() + e).toISOString();
+    }
+  };
+  d.next_run_at = new Date(Date.now() + 5e3).toISOString(), setTimeout(t, 5e3), setInterval(t, e);
+}
+export {
+  z as initCrawler
+};

package/dist/newsCrawler.d.ts

@@ -0,0 +1,53 @@
+export type CrawlerStatus = {
+    running: boolean;
+    running_source?: string;
+    next_run_at?: string;
+    interval_ms: number;
+};
+export type NewsRule = {
+    remark: string;
+    base_url: string;
+    home_range_selector: string;
+    title_selector: string;
+    content_selector: string;
+    time_selector: string;
+    exclude_selectors?: string[];
+    enabled?: boolean;
+};
+export type CrawlerConfig = {
+    rules?: NewsRule[];
+    rulesApiUrl?: string;
+    pushApiUrl?: string;
+    ruleTransformer?: (data: any) => any;
+    newsItemFieldMap?: Partial<Record<keyof NewsItem, string | '-'>>;
+};
+export type NewsItem = {
+    url: string;
+    title: string;
+    content_html: string;
+    content_markdown: string;
+    published_at: string;
+};
+export type CrawlResult = {
+    success: boolean;
+    data?: NewsItem[];
+    error?: string;
+};
+/**
+ * 获取爬虫规则
+ * 优先使用配置中的rules数组，然后使用API获取，最后使用本地存储的规则
+ * @returns 爬虫规则数组
+ */
+export declare function getNewsRules(): Promise<NewsRule[]>;
+/**
+ * 初始化爬虫配置
+ * @param config 爬虫配置
+ */
+export declare function initCrawler(config: CrawlerConfig): void;
+export declare function getCrawlerStatus(): CrawlerStatus;
+/**
+ * 爬取单个规则的新闻
+ * @param rule 爬虫规则
+ * @returns 爬取结果
+ */
+export declare function crawlRule(rule: NewsRule): Promise<CrawlResult>;

package/dist/vite.config.d.ts

@@ -0,0 +1,2 @@
+declare const _default: import('vite').UserConfig;
+export default _default;

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@howuse/electron-crawler",
+  "version": "0.1.0",
+  "description": "基于Electron的爬虫工具包，用于爬取新闻和股票详情",
+  "keywords": [
+    "electron",
+    "crawler",
+    "news",
+    "spider"
+  ],
+  "author": "无名氏",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/your-username/your-repo.git"
+  },
+  "bugs": {
+    "url": "https://github.com/your-username/your-repo/issues"
+  },
+  "homepage": "https://github.com/your-username/your-repo#readme",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist/**/*",
+    "README.md",
+    "LICENSE"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "build": "vite build",
+    "dev": "vite build --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "electron": "^39.2.6",
+    "electron-store": "^9.0.0",
+    "turndown": "^7.2.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.1",
+    "typescript": "^5.9.3",
+    "vite": "^5.0.0",
+    "vite-plugin-dts": "^4.0.0"
+  },
+  "peerDependencies": {
+    "electron": "^39.2.6"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}