koishi-plugin-nitter 0.0.7 → 0.0.9

package/lib/download.d.ts

@@ -0,0 +1,2 @@
+export declare function downloadImagesToBase64(imageUrls: any): Promise<any[]>;
+export declare function downloadVideosToBase64(videoUrls: any, maxSize?: number): Promise<any[]>;

package/lib/fixedSet.d.ts

@@ -0,0 +1,7 @@
+export declare class FixedSet<T> {
+    private maxSize;
+    private set;
+    constructor(maxSize: number);
+    add(element: T): void;
+    has(element: T): boolean;
+}

package/lib/index.d.ts

@@ -16,8 +16,16 @@ export interface Config {
     app: string;
     enableReTweet: boolean;
     sendPic: boolean;
+    cronString: string;
     maxSize: number;
 }
+declare module 'koishi' {
+    interface Tables {
+        nitter_records: {
+            id: string;
+            createdAt: Date;
+        };
+    }
+}
 export declare const Config: Schema<Config, Dict>;
 export declare function apply(ctx: Context, config: Config): void;
-export declare function downloadVideosToBase64(videoUrls: any, maxSize?: number): Promise<any[]>;

package/lib/index.js

@@ -32,7 +32,6 @@ var src_exports = {};
 __export(src_exports, {
   Config: () => Config,
   apply: () => apply,
-  downloadVideosToBase64: () => downloadVideosToBase64,
   inject: () => inject,
   name: () => name
 });
@@ -42,18 +41,21 @@ var import_koishi = require("koishi");
 // src/translate.ts
 var import_axios = __toESM(require("axios"));
 var import_https_proxy_agent = require("https-proxy-agent");
+
+// src/retry.ts
 async function retry(retries, fn, delay = 500) {
   try {
-    await fn();
-    return Promise.resolve();
+    return await fn();
   } catch (err) {
     console.log(`剩余${retries}次尝试`, err);
-    await new Promise((resolve) => setTimeout(resolve, delay));
-    return retries > 1 ? retry(retries - 1, fn, delay * 2) : Promise.reject(err);
+    if (retries <= 1) throw err;
+    await new Promise((r) => setTimeout(r, delay));
+    return retry(retries - 1, fn, delay * 2);
   }
-  ;
 }
 __name(retry, "retry");
+
+// src/translate.ts
 var translate;
 function setGoogleTranslate(key, proxy) {
   translate = /* @__PURE__ */ __name((texts) => {
@@ -109,6 +111,7 @@ json输出格式：["翻译结果1", "翻译结果2", ...]`
       });
       if (response.data && response.data.choices && response.data.choices[0]) {
         const content = response.data.choices[0].message.content;
+        console.log(content);
         data = JSON.parse(content);
         if (!Array.isArray(data)) {
           if (typeof data === "object" && Object.keys(data).length === 1) {
@@ -122,7 +125,7 @@ json输出格式：["翻译结果1", "翻译结果2", ...]`
         throw new Error("API返回数据格式异常");
       }
     });
-    return data;
+    return data.map((content) => content.replace(/\\n/g, "\n"));
   }, "translate");
 }
 __name(setOpenAiTranslate, "setOpenAiTranslate");
@@ -142,8 +145,8 @@ async function addTranslate(page, className) {
         tempDiv.innerHTML = translatedHTMLs2[index];
         const newElement = tempDiv.firstElementChild;
         if (newElement) {
-          element.insertAdjacentElement("afterend", newElement);
-          element.insertAdjacentHTML("afterend", '<div style="height: 1px;background-color: #ccc; margin-top: 10px; margin-bottom: 10px;"></div>');
+          element.insertAdjacentElement("beforebegin", newElement);
+          element.insertAdjacentHTML("beforebegin", '<div style="height: 1px;background-color: #ccc; margin-top: 10px; margin-bottom: 10px;"></div>');
         }
       }
     });
@@ -153,14 +156,87 @@ __name(addTranslate, "addTranslate");
 
 // src/index.tsx
 var import_rettiwt_api = require("rettiwt-api");
-var import_node_cron = require("node-cron");
+
+// src/download.ts
 var import_fluent_ffmpeg = __toESM(require("fluent-ffmpeg"));
 var import_path = __toESM(require("path"));
 var import_promises = __toESM(require("fs/promises"));
+var import_axios2 = __toESM(require("axios"));
+var tempDir = import_path.default.join(process.cwd(), "tmp");
+async function downloadVideosToBase64(videoUrls, maxSize = 20) {
+  await import_promises.default.mkdir(tempDir, { recursive: true });
+  const results = [];
+  for (const url of videoUrls) {
+    let outputPath = null;
+    try {
+      outputPath = import_path.default.join(tempDir, `video_${Date.now()}_${Math.random().toString(36).substr(2, 9)}.mp4`);
+      await new Promise((resolve, reject) => {
+        (0, import_fluent_ffmpeg.default)(url).inputOptions([
+          "-protocol_whitelist",
+          "file,http,https,tcp,tls,crypto"
+        ]).outputOptions([
+          "-c",
+          "copy"
+        ]).output(outputPath).on("end", () => resolve(outputPath)).on("error", reject).run();
+      });
+      const stats = await import_promises.default.stat(outputPath);
+      const fileSizeInMB = stats.size / (1024 * 1024);
+      if (fileSizeInMB > maxSize) {
+        console.log(`视频大小 ${fileSizeInMB.toFixed(2)}MB 超过${maxSize}MB限制，跳过转换`);
+      } else {
+        const fileBuffer = await import_promises.default.readFile(outputPath);
+        const base64String = fileBuffer.toString("base64");
+        const dataUrl = `data:video/mp4;base64,${base64String}`;
+        results.push(dataUrl);
+      }
+    } catch (error) {
+      console.error(`处理失败 (${url}):`, error.message);
+      results.push(null);
+    } finally {
+      if (outputPath) {
+        try {
+          await import_promises.default.unlink(outputPath);
+        } catch (deleteError) {
+        }
+      }
+    }
+  }
+  return results;
+}
+__name(downloadVideosToBase64, "downloadVideosToBase64");
+
+// src/taskqueue.ts
+var taskQueue = class {
+  static {
+    __name(this, "taskQueue");
+  }
+  queue = [];
+  consumer;
+  processing = false;
+  push(task) {
+    this.queue.push(task);
+    if (!this.processing) {
+      this.process();
+    }
+  }
+  onProcess(handler) {
+    this.consumer = handler;
+  }
+  async process() {
+    if (this.processing || !this.consumer) return;
+    this.processing = true;
+    while (this.queue.length > 0) {
+      const task = this.queue.shift();
+      await this.consumer(task);
+    }
+    this.processing = false;
+  }
+};
+
+// src/index.tsx
 var import_jsx_runtime = require("@satorijs/element/jsx-runtime");
 var name = "nitter";
-var logger = new import_koishi.Logger(name);
-var inject = ["puppeteer", "subscription"];
+var inject = ["puppeteer", "subscription", "database"];
 var Config = import_koishi.Schema.intersect([
   import_koishi.Schema.object({
     apiKey: import_koishi.Schema.string().required().description("Twitter API Key"),
@@ -195,23 +271,32 @@ var Config = import_koishi.Schema.intersect([
   }).description("订阅配置"),
   import_koishi.Schema.object({
     enableReTweet: import_koishi.Schema.boolean().default(false).description("是否发送转推"),
-    sendPic: import_koishi.Schema.boolean().default(false).description("是否单独发送推文中的图片")
+    sendPic: import_koishi.Schema.boolean().default(false).description("是否单独发送推文中的图片"),
+    cronString: import_koishi.Schema.string().default("15 */5 * * * *").description("使用cron表达式描述检查更新的时间，默认为每隔5分钟检查一次")
   }).description("推送设置"),
   import_koishi.Schema.union([
     import_koishi.Schema.object({
       sendPic: import_koishi.Schema.const(true).required(),
-      maxSize: import_koishi.Schema.number().default(10).description("发送视频的最大大小，单位为mb")
+      maxSize: import_koishi.Schema.number().default(20).description("发送视频的最大大小，单位为mb")
     })
   ])
 ]);
 function apply(ctx, config) {
+  ctx.model.extend("nitter_records", {
+    id: "string",
+    createdAt: "timestamp"
+  }, {
+    primary: "id",
+    autoInc: false
+  });
   config.nitterUrl = config.nitterUrl.replace(/\/+$/, "");
   const twitterClient = new import_rettiwt_api.Rettiwt({
     apiKey: config.apiKey,
-    proxyUrl: config.proxy ? new URL(config.proxy) : void 0
+    proxyUrl: config.proxy ? new URL(config.proxy) : void 0,
+    logging: true
   });
-  let latestTweetId;
   let cronJob;
+  const queue = new taskQueue();
   (async () => {
     if (config.enableTranslate == "google") {
       setGoogleTranslate(config.googleApiKey, config.proxy);
@@ -219,16 +304,13 @@ function apply(ctx, config) {
       setOpenAiTranslate(config.baseurl, config.openaiApiKey, config.model, config.prompt, config.temperature, config.timeout);
     }
     const tweetList = await getFollowedFeed();
-    for (const data of tweetList) {
-      latestTweetId ||= "";
-      if (latestTweetId < data.id)
-        latestTweetId = data.id;
-    }
-    cronJob = (0, import_node_cron.schedule)("15 * * * * *", checkForUpdates);
-    ctx.on("dispose", () => {
-      cronJob.stop();
-    });
-    ctx.logger("nitter").info("开始监听推特动态");
+    await ctx.database.upsert("nitter_records", tweetList.map((data) => {
+      return {
+        id: data.id,
+        createdAt: new Date(data.createdAt)
+      };
+    }));
+    queue.onProcess(broadcast);
   })();
   ctx.command("nitter.follow", "按照subscription中的订阅配置，使用登录的推特账号关注所有需要订阅的账号", { authority: 3 }).action(async () => {
     const whiteList = ctx.subscription.getAvailableAccounts(config.app);
@@ -236,9 +318,14 @@ function apply(ctx, config) {
     const followingIdList = followingList.map((user) => user.userName);
     for (const id of whiteList) {
       if (!followingIdList.includes(id)) {
-        const user = await twitterClient.user.details(id);
+        let user;
+        await retry(3, async () => {
+          user = await twitterClient.user.details(id);
+        });
         await new Promise((resolve) => setTimeout(resolve, 3 * 1e3));
-        await twitterClient.user.follow(user.id);
+        await retry(3, async () => {
+          await twitterClient.user.follow(user.id);
+        }, 30 * 1e3);
         console.log(`关注${id}成功`);
         await new Promise((resolve) => setTimeout(resolve, 30 * 1e3));
       }
@@ -250,7 +337,7 @@ function apply(ctx, config) {
       return "请输入推文ID";
     }
     try {
-      const [screenshot, imageUrls, hlsUrls] = await renderTweetScreenshot(ctx, tweetId, config);
+      const [screenshot, imageUrls, hlsUrls] = await renderTweetScreenshot(tweetId);
       let msg = /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: "data:image/png;base64," + screenshot.toString("base64") });
       let forwardMsg = [];
       const videoUrls = await downloadVideosToBase64(hlsUrls, config.maxSize);
@@ -270,14 +357,15 @@ function apply(ctx, config) {
     if (!tweetId) {
       return "请输入推文ID";
     }
-    broadcast(account, tweetId);
+    queue.push({ account, tweetId });
   });
-  async function broadcast(account, tweetId) {
+  async function broadcast(task) {
+    const { account, tweetId } = task;
     try {
-      const [screenshot, imageUrls, hlsUrls] = await renderTweetScreenshot(ctx, tweetId, config);
+      const [screenshot, imageUrls, hlsUrls] = await renderTweetScreenshot(tweetId);
       const screenshotMsg = /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: "data:image/png;base64," + screenshot.toString("base64") });
-      ctx.subscription.broadcast(config.app, account, screenshotMsg);
-      let forwardMsg = [];
+      await ctx.subscription.broadcast(config.app, account, screenshotMsg);
+      let forwardMsg = [`${ctx.subscription.getName(config.app, account)} ${ctx.subscription.getAccount(config.app, account)}`];
       const videoUrls = await downloadVideosToBase64(hlsUrls, config.maxSize);
       if (imageUrls.length > 0) {
         forwardMsg.push(...imageUrls.map((url) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: url }) })));
@@ -306,117 +394,106 @@ function apply(ctx, config) {
     for (const data of tweetList) {
       if (!config.enableReTweet && data.retweetedTweet)
         continue;
-      if (data.id <= latestTweetId)
+      if ((/* @__PURE__ */ new Date()).getTime() - new Date(data.createdAt).getTime() > 2 * 60 * 60 * 1e3)
+        continue;
+      if ((await ctx.database.get("nitter_records", { id: data.id })).length > 0)
         continue;
+      await ctx.database.upsert("nitter_records", [{
+        id: data.id,
+        createdAt: new Date(data.createdAt)
+      }]);
       if (!ctx.subscription.getAvailableAccounts(config.app).includes(data.tweetBy.userName))
         continue;
-      latestTweetId = data.id;
       ctx.logger("nitter").info(`检测到推文id:${data.id}，开始推送`);
-      await broadcast(data.tweetBy.userName, data.id);
+      queue.push({ account: data.tweetBy.userName, tweetId: data.id });
     }
   }
   __name(checkForUpdates, "checkForUpdates");
-}
-__name(apply, "apply");
-var tempDir = import_path.default.join(process.cwd(), "tmp");
-async function downloadVideosToBase64(videoUrls, maxSize = 20) {
-  await import_promises.default.mkdir(tempDir, { recursive: true });
-  const results = [];
-  for (const url of videoUrls) {
-    let outputPath = null;
+  async function renderTweetScreenshot(tweetId) {
+    const puppeteer = ctx.puppeteer;
+    if (!puppeteer) {
+      throw new Error("Puppeteer 服务未找到，请安装 koishi-plugin-puppeteer");
+    }
+    const page = await puppeteer.page();
     try {
-      outputPath = import_path.default.join(tempDir, `video_${Date.now()}_${Math.random().toString(36).substr(2, 9)}.mp4`);
-      await new Promise((resolve, reject) => {
-        (0, import_fluent_ffmpeg.default)(url).inputOptions([
-          "-protocol_whitelist",
-          "file,http,https,tcp,tls,crypto"
-        ]).outputOptions([
-          "-c",
-          "copy"
-        ]).output(outputPath).on("end", () => resolve(outputPath)).on("error", reject).run();
-      });
-      const stats = await import_promises.default.stat(outputPath);
-      const fileSizeInMB = stats.size / (1024 * 1024);
-      if (fileSizeInMB > maxSize) {
-        logger.info(`视频大小 ${fileSizeInMB.toFixed(2)}MB 超过${maxSize}MB限制，跳过转换`);
-      } else {
-        const fileBuffer = await import_promises.default.readFile(outputPath);
-        const base64String = fileBuffer.toString("base64");
-        const dataUrl = `data:video/mp4;base64,${base64String}`;
-        results.push(dataUrl);
-      }
-    } catch (error) {
-      console.error(`处理失败 (${url}):`, error.message);
-      results.push(null);
-    } finally {
-      if (outputPath) {
+      await page.setCacheEnabled(false);
+      const tweetUrl = `${config.nitterUrl}/i/status/${tweetId}`;
+      await retry(3, async () => {
+        await page.goto(tweetUrl);
+        const element2 = await page.$(".main-thread");
+        if (!element2) throw new Error("Rate Limited");
+      }, 2e3);
+      if (config.enableTranslate) {
         try {
-          await import_promises.default.unlink(outputPath);
-        } catch (deleteError) {
+          await addTranslate(page, ".main-thread .tweet-content, .main-thread .quote-text");
+        } catch (e) {
+          ctx.logger("nitter").info("翻译失败", e);
         }
       }
-    }
-  }
-  return results;
-}
-__name(downloadVideosToBase64, "downloadVideosToBase64");
-async function renderTweetScreenshot(ctx, tweetId, config) {
-  const puppeteer = ctx.puppeteer;
-  if (!puppeteer) {
-    throw new Error("Puppeteer 服务未找到，请安装 koishi-plugin-puppeteer");
-  }
-  const page = await puppeteer.page();
-  try {
-    const tweetUrl = `${config.nitterUrl}/i/status/${tweetId}`;
-    await retry(3, async () => {
-      await page.goto(tweetUrl);
-      const element2 = await page.$(".main-thread");
-      if (!element2) throw new Error("Rate Limited");
-    }, 2e3);
-    if (config.enableTranslate) {
-      try {
-        await addTranslate(page, ".main-thread .tweet-content, .main-thread .quote-text");
-      } catch (e) {
-        ctx.logger("nitter").info("翻译失败", e);
-      }
-    }
-    await new Promise((resolve) => setTimeout(resolve, 200));
-    await page.evaluate(() => {
-      const nav = document.querySelector("nav");
-      if (nav) {
-        nav.style.visibility = "hidden";
+      await page.evaluate(() => {
+        const nav = document.querySelector("nav");
+        if (nav) {
+          nav.style.visibility = "hidden";
+        }
+      });
+      await new Promise((resolve) => setTimeout(resolve, 1e3));
+      await page.evaluate(() => {
+        const element2 = document.querySelector(".main-thread");
+        if (!element2) return;
+        Object.assign(element2.style, {
+          border: "1px solid #1DA1F2",
+          borderRadius: "8px",
+          boxShadow: "0px 1px 9px 12px rgba(29, 161, 242, 0.2)",
+          margin: "20px",
+          boxSizing: "border-box",
+          overflow: "hidden",
+          width: "100%",
+          padding: "10px 10px 0 10px",
+          backgroundColor: "#fff"
+        });
+      });
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      const element = await page.$(".main-thread");
+      const boundingBox = await page.evaluate((el) => {
+        const rect = el.getBoundingClientRect();
+        const margin = 20;
+        return {
+          x: rect.left - margin,
+          y: rect.top - margin,
+          width: rect.width + margin * 2,
+          height: rect.height + margin * 2
+        };
+      }, element);
+      const buffer = await page.screenshot({
+        type: "png",
+        omitBackground: false,
+        clip: boundingBox
+      });
+      if (config.sendPic) {
+        const originalImages = await page.$$eval(
+          ".main-tweet a.still-image",
+          (links, baseUrl) => links.map((link) => link.getAttribute("href")).filter((href) => href).map((href) => `${baseUrl}${href}`),
+          config.nitterUrl
+        );
+        const hlsUrls = await page.$$eval(".main-tweet video", (videos, baseUrl) => {
+          return videos.map((video) => video.getAttribute("data-url")).filter((dataUrl) => dataUrl).map((dataUrl) => `${baseUrl}${dataUrl}`);
+        }, config.nitterUrl);
+        return [buffer, originalImages, hlsUrls];
       }
-    });
-    const element = await page.$(".main-thread");
-    const buffer = await element.screenshot({
-      // 不指定path参数，直接返回buffer
-      type: "png",
-      omitBackground: false
-    });
-    if (config.sendPic) {
-      const originalImages = await page.$$eval(
-        ".main-tweet a.still-image",
-        (links, baseUrl) => links.map((link) => link.getAttribute("href")).filter((href) => href).map((href) => `${baseUrl}${href}`),
-        config.nitterUrl
-      );
-      const hlsUrls = await page.$$eval(".main-tweet video", (videos, baseUrl) => {
-        return videos.map((video) => video.getAttribute("data-url")).filter((dataUrl) => dataUrl).map((dataUrl) => `${baseUrl}${dataUrl}`);
-      }, config.nitterUrl);
-      return [buffer, originalImages, hlsUrls];
+      return [buffer, [], []];
+    } catch (e) {
+      throw e;
+    } finally {
+      await page.close();
     }
-    return [buffer, [], []];
-  } catch (e) {
-    throw e;
-  } finally {
-    await page.close();
   }
+  __name(renderTweetScreenshot, "renderTweetScreenshot");
 }
-__name(renderTweetScreenshot, "renderTweetScreenshot");
+__name(apply, "apply");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   Config,
   apply,
-  downloadVideosToBase64,
   inject,
   name
 });

package/lib/retry.d.ts

@@ -0,0 +1 @@
+export declare function retry<T>(retries: number, fn: () => Promise<T> | T, delay?: number): Promise<T>;

package/lib/taskqueue.d.ts

@@ -0,0 +1,8 @@
+export declare class taskQueue<T> {
+    private queue;
+    private consumer?;
+    private processing;
+    push(task: T): void;
+    onProcess(handler: (task: T) => Promise<void> | void): void;
+    private process;
+}

package/lib/translate.d.ts

@@ -1,6 +1,4 @@
 import { Page } from 'puppeteer-core';
-export declare function retry(retries: number, fn: () => any, delay?: number): any;
 export declare function setGoogleTranslate(key: string, proxy: string): void;
-export declare function setSiliconTranslate(key: string, model: string, prompt: string, timeout?: number): void;
 export declare function setOpenAiTranslate(url: string, key: string, model: string, prompt: string, temperature: number, timeout?: number): void;
 export declare function addTranslate(page: Page, className: string): Promise<void>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-nitter",
   "description": "使用Rettiwt-API订阅推文，并使用nitter渲染",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
   "files": [
@@ -19,11 +19,11 @@
     "axios": "^1.12.2",
     "https-proxy-agent": "^7.0.6",
     "koishi": "^4.18.9",
-    "koishi-plugin-subscription": "^0.0.5"
+    "koishi-plugin-subscription": "^0.0.6"
   },
   "dependencies": {
+    "fluent-ffmpeg": "^2.1.3",
     "node-cron": "^4.2.1",
-    "rettiwt-api": "^6.0.6",
-    "fluent-ffmpeg": "^2.1.3"
+    "rettiwt-api": "^6.1.7"
   }
 }