koishi-plugin-nitter 0.0.9 → 0.0.11

package/lib/download.d.ts

@@ -1,2 +1,3 @@
 export declare function downloadImagesToBase64(imageUrls: any): Promise<any[]>;
-export declare function downloadVideosToBase64(videoUrls: any, maxSize?: number): Promise<any[]>;
+export declare function downloadVideosToTempFiles(videoUrls: string[], tempDir: string, maxSize?: number): Promise<string[]>;
+export declare function cleanupTempFiles(files: Array<string | null>): Promise<void>;

package/lib/index.d.ts

@@ -14,18 +14,12 @@ export interface Config {
     temperature: number;
     timeout?: number;
     app: string;
+    enablePollingOnStart: boolean;
     enableReTweet: boolean;
     sendPic: boolean;
     cronString: string;
-    maxSize: number;
-}
-declare module 'koishi' {
-    interface Tables {
-        nitter_records: {
-            id: string;
-            createdAt: Date;
-        };
-    }
+    maxSize?: number;
+    tmpDir?: string;
 }
 export declare const Config: Schema<Config, Dict>;
 export declare function apply(ctx: Context, config: Config): void;

package/lib/index.js

@@ -43,14 +43,16 @@ var import_axios = __toESM(require("axios"));
 var import_https_proxy_agent = require("https-proxy-agent");
 
 // src/retry.ts
-async function retry(retries, fn, delay = 500) {
+async function retry(retries, fn, error_fn = (err) => {
+  throw err;
+}, delay = 500) {
   try {
     return await fn();
   } catch (err) {
     console.log(`剩余${retries}次尝试`, err);
-    if (retries <= 1) throw err;
+    if (retries <= 1) error_fn(err);
     await new Promise((r) => setTimeout(r, delay));
-    return retry(retries - 1, fn, delay * 2);
+    return retry(retries - 1, fn, error_fn, delay * 2);
   }
 }
 __name(retry, "retry");
@@ -137,6 +139,13 @@ async function addTranslate(page, className) {
   }, className);
   if (!elementsHTML || !elementsHTML.length) return;
   const translatedHTMLs = await translate(elementsHTML);
+  const translatedTexts = await page.evaluate((htmlArr) => {
+    return htmlArr.map((html) => {
+      const div = document.createElement("div");
+      div.innerHTML = html;
+      return (div.innerText || "").replace(/\n{3,}/g, "\n\n").trim();
+    });
+  }, translatedHTMLs);
   await page.evaluate((className2, originalHTMLs, translatedHTMLs2) => {
     const elements = document.querySelectorAll(className2);
     elements.forEach((element, index) => {
@@ -151,59 +160,58 @@ async function addTranslate(page, className) {
       }
     });
   }, className, elementsHTML, translatedHTMLs);
+  return translatedTexts;
 }
 __name(addTranslate, "addTranslate");
 
 // src/index.tsx
 var import_rettiwt_api = require("rettiwt-api");
+var import_node_cron = require("node-cron");
 
 // src/download.ts
 var import_fluent_ffmpeg = __toESM(require("fluent-ffmpeg"));
 var import_path = __toESM(require("path"));
 var import_promises = __toESM(require("fs/promises"));
 var import_axios2 = __toESM(require("axios"));
-var tempDir = import_path.default.join(process.cwd(), "tmp");
-async function downloadVideosToBase64(videoUrls, maxSize = 20) {
+async function downloadVideosToTempFiles(videoUrls, tempDir, maxSize = 20) {
   await import_promises.default.mkdir(tempDir, { recursive: true });
   const results = [];
   for (const url of videoUrls) {
     let outputPath = null;
     try {
-      outputPath = import_path.default.join(tempDir, `video_${Date.now()}_${Math.random().toString(36).substr(2, 9)}.mp4`);
+      outputPath = import_path.default.join(
+        tempDir,
+        `video_${Date.now()}_${crypto.randomUUID()}.mp4`
+      );
       await new Promise((resolve, reject) => {
-        (0, import_fluent_ffmpeg.default)(url).inputOptions([
-          "-protocol_whitelist",
-          "file,http,https,tcp,tls,crypto"
-        ]).outputOptions([
-          "-c",
-          "copy"
-        ]).output(outputPath).on("end", () => resolve(outputPath)).on("error", reject).run();
+        (0, import_fluent_ffmpeg.default)(url).inputOptions(["-protocol_whitelist", "file,http,https,tcp,tls,crypto"]).outputOptions(["-c", "copy"]).output(outputPath).on("end", () => resolve(outputPath)).on("error", reject).run();
       });
       const stats = await import_promises.default.stat(outputPath);
       const fileSizeInMB = stats.size / (1024 * 1024);
       if (fileSizeInMB > maxSize) {
-        console.log(`视频大小 ${fileSizeInMB.toFixed(2)}MB 超过${maxSize}MB限制，跳过转换`);
+        console.log(`视频大小 ${fileSizeInMB.toFixed(2)}MB 超过 ${maxSize}MB，跳过发送`);
+        await import_promises.default.unlink(outputPath).catch(() => {
+        });
+        results.push(null);
       } else {
-        const fileBuffer = await import_promises.default.readFile(outputPath);
-        const base64String = fileBuffer.toString("base64");
-        const dataUrl = `data:video/mp4;base64,${base64String}`;
-        results.push(dataUrl);
+        results.push(outputPath);
       }
     } catch (error) {
-      console.error(`处理失败 (${url}):`, error.message);
+      console.error(`下载失败 (${url}):`, error?.message || error);
+      if (outputPath) await import_promises.default.unlink(outputPath).catch(() => {
+      });
       results.push(null);
-    } finally {
-      if (outputPath) {
-        try {
-          await import_promises.default.unlink(outputPath);
-        } catch (deleteError) {
-        }
-      }
     }
   }
   return results;
 }
-__name(downloadVideosToBase64, "downloadVideosToBase64");
+__name(downloadVideosToTempFiles, "downloadVideosToTempFiles");
+async function cleanupTempFiles(files) {
+  await Promise.allSettled(
+    files.filter((f) => !!f).map((f) => import_promises.default.unlink(f))
+  );
+}
+__name(cleanupTempFiles, "cleanupTempFiles");
 
 // src/taskqueue.ts
 var taskQueue = class {
@@ -236,7 +244,7 @@ var taskQueue = class {
 // src/index.tsx
 var import_jsx_runtime = require("@satorijs/element/jsx-runtime");
 var name = "nitter";
-var inject = ["puppeteer", "subscription", "database"];
+var inject = ["puppeteer", "subscription"];
 var Config = import_koishi.Schema.intersect([
   import_koishi.Schema.object({
     apiKey: import_koishi.Schema.string().required().description("Twitter API Key"),
@@ -270,6 +278,7 @@ var Config = import_koishi.Schema.intersect([
     app: import_koishi.Schema.string().description("subscription配置中应用名")
   }).description("订阅配置"),
   import_koishi.Schema.object({
+    enablePollingOnStart: import_koishi.Schema.boolean().default(true).description("启动后是否立刻开始轮询"),
     enableReTweet: import_koishi.Schema.boolean().default(false).description("是否发送转推"),
     sendPic: import_koishi.Schema.boolean().default(false).description("是否单独发送推文中的图片"),
     cronString: import_koishi.Schema.string().default("15 */5 * * * *").description("使用cron表达式描述检查更新的时间，默认为每隔5分钟检查一次")
@@ -277,18 +286,12 @@ var Config = import_koishi.Schema.intersect([
   import_koishi.Schema.union([
     import_koishi.Schema.object({
       sendPic: import_koishi.Schema.const(true).required(),
-      maxSize: import_koishi.Schema.number().default(20).description("发送视频的最大大小，单位为mb")
+      maxSize: import_koishi.Schema.number().default(20).description("发送视频的最大大小，单位为mb"),
+      tmpDir: import_koishi.Schema.string().default("/shared/tmp").description("临时存放视频的目录，需要koishi和对接的应用都能访问")
     })
   ])
 ]);
 function apply(ctx, config) {
-  ctx.model.extend("nitter_records", {
-    id: "string",
-    createdAt: "timestamp"
-  }, {
-    primary: "id",
-    autoInc: false
-  });
   config.nitterUrl = config.nitterUrl.replace(/\/+$/, "");
   const twitterClient = new import_rettiwt_api.Rettiwt({
     apiKey: config.apiKey,
@@ -303,14 +306,30 @@ function apply(ctx, config) {
     } else if (config.enableTranslate == "openai") {
       setOpenAiTranslate(config.baseurl, config.openaiApiKey, config.model, config.prompt, config.temperature, config.timeout);
     }
-    const tweetList = await getFollowedFeed();
-    await ctx.database.upsert("nitter_records", tweetList.map((data) => {
-      return {
-        id: data.id,
-        createdAt: new Date(data.createdAt)
-      };
-    }));
-    queue.onProcess(broadcast);
+    queue.onProcess(
+      (task) => {
+        return work(
+          task.tweetId,
+          async (msg) => await ctx.subscription.broadcast(config.app, task.account, msg),
+          async (forwardMsg) => await ctx.subscription.broadcastForward(config.app, task.account, forwardMsg)
+        );
+      }
+    );
+    if (config.enablePollingOnStart) {
+      const tweetList = await getFollowedFeed();
+      if (tweetList.length === 0) {
+        ctx.logger("nitter").info("初始化失败，请重试");
+        return;
+      }
+      await Promise.all(tweetList.map((data) => {
+        return ctx.subscription.checkExist(config.app, data.id);
+      }));
+      cronJob = (0, import_node_cron.schedule)(config.cronString, checkForUpdates);
+      ctx.on("dispose", () => {
+        cronJob.stop();
+      });
+      ctx.logger("nitter").info("开始监听推特动态");
+    }
   })();
   ctx.command("nitter.follow", "按照subscription中的订阅配置，使用登录的推特账号关注所有需要订阅的账号", { authority: 3 }).action(async () => {
     const whiteList = ctx.subscription.getAvailableAccounts(config.app);
@@ -321,12 +340,13 @@ function apply(ctx, config) {
         let user;
         await retry(3, async () => {
           user = await twitterClient.user.details(id);
-        });
+        }, (err) => ctx.logger("nitter").error(`获取用户${id}信息失败: ${err}`));
+        if (!user) continue;
         await new Promise((resolve) => setTimeout(resolve, 3 * 1e3));
         await retry(3, async () => {
           await twitterClient.user.follow(user.id);
-        }, 30 * 1e3);
-        console.log(`关注${id}成功`);
+          ctx.logger("nitter").info(`关注${id}成功`);
+        }, (err) => ctx.logger("nitter").error(`关注${id}失败: ${err}`));
         await new Promise((resolve) => setTimeout(resolve, 30 * 1e3));
       }
     }
@@ -337,17 +357,11 @@ function apply(ctx, config) {
       return "请输入推文ID";
     }
     try {
-      const [screenshot, imageUrls, hlsUrls] = await renderTweetScreenshot(tweetId);
-      let msg = /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: "data:image/png;base64," + screenshot.toString("base64") });
-      let forwardMsg = [];
-      const videoUrls = await downloadVideosToBase64(hlsUrls, config.maxSize);
-      if (imageUrls.length > 0) {
-        forwardMsg.push(...imageUrls.map((url) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: url }) })));
-      }
-      if (videoUrls.length > 0) {
-        forwardMsg.push(...videoUrls.map((url) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("video", { src: url }) })));
-      }
-      await session.send(msg + /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { forward: true, children: forwardMsg }));
+      await work(
+        tweetId,
+        async (msg) => await session.send(msg),
+        async (forwardMsg) => await session.send(/* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { forward: true, children: forwardMsg }))
+      );
     } catch (error) {
       ctx.logger("nitter").error("获取推文失败:", error);
       return "获取推文失败";
@@ -359,53 +373,79 @@ function apply(ctx, config) {
     }
     queue.push({ account, tweetId });
   });
-  async function broadcast(task) {
-    const { account, tweetId } = task;
+  async function work(tweetId, sendFunc, sendForwardFunc) {
+    let videoFiles = [];
     try {
-      const [screenshot, imageUrls, hlsUrls] = await renderTweetScreenshot(tweetId);
+      const { screenshot, pieces } = await renderTweetScreenshot(tweetId);
       const screenshotMsg = /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: "data:image/png;base64," + screenshot.toString("base64") });
-      await ctx.subscription.broadcast(config.app, account, screenshotMsg);
-      let forwardMsg = [`${ctx.subscription.getName(config.app, account)} ${ctx.subscription.getAccount(config.app, account)}`];
-      const videoUrls = await downloadVideosToBase64(hlsUrls, config.maxSize);
-      if (imageUrls.length > 0) {
-        forwardMsg.push(...imageUrls.map((url) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: url }) })));
+      await sendFunc(screenshotMsg);
+      const forwardMsg = [];
+      const videoUrls = pieces.filter((p) => p.type === "video").map((p) => p.url);
+      const tmpDir = config.tmpDir ?? "/shared/tmp";
+      if (videoUrls.length) {
+        videoFiles = await downloadVideosToTempFiles(videoUrls, tmpDir, config.maxSize ?? 20);
       }
-      if (videoUrls.length > 0) {
-        forwardMsg.push(...videoUrls.map((url) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("video", { src: url }) })));
+      let vIdx = 0;
+      for (const p of pieces) {
+        if (p.type === "account") {
+          forwardMsg.push(/* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: p.text }));
+        } else if (p.type === "text") {
+          forwardMsg.push(/* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: p.text }));
+        } else if (p.type === "image") {
+          forwardMsg.push(
+            /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("img", { src: p.url }) })
+          );
+        } else if (p.type === "video") {
+          const file = videoFiles[vIdx++];
+          if (file) {
+            forwardMsg.push(
+              /* @__PURE__ */ (0, import_jsx_runtime.jsx)("message", { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)("video", { src: "file://" + file }) })
+            );
+          }
+        }
       }
       if (forwardMsg.length > 0) {
-        await ctx.subscription.broadcastForward(config.app, account, forwardMsg);
+        await sendForwardFunc(forwardMsg);
       }
     } catch (error) {
       ctx.logger("nitter").error("获取推文失败:", error);
+    } finally {
+      await cleanupTempFiles(videoFiles);
     }
   }
-  __name(broadcast, "broadcast");
+  __name(work, "work");
   async function getFollowedFeed() {
     let tweetList;
-    await retry(3, async () => {
+    await retry(5, async () => {
       ({ list: tweetList } = await twitterClient.user.followed());
-    });
+    }, (err) => ctx.logger("nitter").error(`获取推文列表失败: ${err}`));
+    if (!tweetList) return [];
     return tweetList.reverse();
   }
   __name(getFollowedFeed, "getFollowedFeed");
+  let checking = false;
   async function checkForUpdates() {
-    const tweetList = await getFollowedFeed();
-    for (const data of tweetList) {
-      if (!config.enableReTweet && data.retweetedTweet)
-        continue;
-      if ((/* @__PURE__ */ new Date()).getTime() - new Date(data.createdAt).getTime() > 2 * 60 * 60 * 1e3)
-        continue;
-      if ((await ctx.database.get("nitter_records", { id: data.id })).length > 0)
-        continue;
-      await ctx.database.upsert("nitter_records", [{
-        id: data.id,
-        createdAt: new Date(data.createdAt)
-      }]);
-      if (!ctx.subscription.getAvailableAccounts(config.app).includes(data.tweetBy.userName))
-        continue;
-      ctx.logger("nitter").info(`检测到推文id:${data.id}，开始推送`);
-      queue.push({ account: data.tweetBy.userName, tweetId: data.id });
+    if (checking) return;
+    checking = true;
+    try {
+      const tweetList = await getFollowedFeed();
+      if (tweetList.length === 0) {
+        return;
+      }
+      for (const data of tweetList) {
+        if (!config.enableReTweet && data.retweetedTweet)
+          continue;
+        if ((/* @__PURE__ */ new Date()).getTime() - new Date(data.createdAt).getTime() > 2 * 60 * 60 * 1e3)
+          continue;
+        if (await ctx.subscription.checkExist(config.app, data.id))
+          continue;
+        if (!ctx.subscription.getAvailableAccounts(config.app).includes(data.tweetBy.userName))
+          continue;
+        ctx.logger("nitter").info(`检测到推文id:${data.id}，开始推送`);
+        queue.push({ account: data.tweetBy.userName, tweetId: data.id });
+      }
+    } finally {
+      checking = false;
     }
   }
   __name(checkForUpdates, "checkForUpdates");
@@ -419,13 +459,48 @@ function apply(ctx, config) {
       await page.setCacheEnabled(false);
       const tweetUrl = `${config.nitterUrl}/i/status/${tweetId}`;
       await retry(3, async () => {
-        await page.goto(tweetUrl);
+        await page.goto(tweetUrl, { timeout: 12e4 });
         const element2 = await page.$(".main-thread");
         if (!element2) throw new Error("Rate Limited");
+      }, (err) => {
+        throw new Error(`获取推文${tweetId}失败: ${err}`);
       }, 2e3);
-      if (config.enableTranslate) {
+      const captured = await page.evaluate((baseUrl) => {
+        const root = document.querySelector(".main-thread");
+        if (!root) return { accountText: "unknown", flow: [] };
+        const fullName = (root.querySelector(".fullname")?.textContent || "").trim();
+        const username = (root.querySelector(".username")?.textContent || "").trim();
+        const accountText = fullName && username ? `${fullName} ${username}` : username || fullName || "unknown";
+        const selector = ".tweet-content, .quote-text, a.still-image[href], video[data-url]";
+        const nodes = Array.from(root.querySelectorAll(selector));
+        const flow = [];
+        let textIndex = 0;
+        for (const n of nodes) {
+          const el = n;
+          if (el.classList.contains("tweet-content") || el.classList.contains("quote-text")) {
+            const text = (el.innerText || "").replace(/\n{3,}/g, "\n\n").trim();
+            flow.push({ type: "text", textIndex, fallbackText: text });
+            textIndex++;
+            continue;
+          }
+          if (el.tagName.toLowerCase() === "a" && el.classList.contains("still-image")) {
+            const href = el.getAttribute("href");
+            if (href) flow.push({ type: "image", url: `${baseUrl}${href}` });
+            continue;
+          }
+          if (el.tagName.toLowerCase() === "video") {
+            const dataUrl = el.getAttribute("data-url");
+            if (dataUrl) flow.push({ type: "video", url: `${baseUrl}${dataUrl}` });
+            continue;
+          }
+        }
+        return { accountText, flow };
+      }, config.nitterUrl);
+      let translatedTexts = [];
+      if (config.enableTranslate != "disable") {
         try {
-          await addTranslate(page, ".main-thread .tweet-content, .main-thread .quote-text");
+          const result = await addTranslate(page, ".main-thread .tweet-content, .main-thread .quote-text");
+          if (result?.length) translatedTexts = result;
         } catch (e) {
           ctx.logger("nitter").info("翻译失败", e);
         }
@@ -469,18 +544,19 @@ function apply(ctx, config) {
         omitBackground: false,
         clip: boundingBox
       });
-      if (config.sendPic) {
-        const originalImages = await page.$$eval(
-          ".main-tweet a.still-image",
-          (links, baseUrl) => links.map((link) => link.getAttribute("href")).filter((href) => href).map((href) => `${baseUrl}${href}`),
-          config.nitterUrl
-        );
-        const hlsUrls = await page.$$eval(".main-tweet video", (videos, baseUrl) => {
-          return videos.map((video) => video.getAttribute("data-url")).filter((dataUrl) => dataUrl).map((dataUrl) => `${baseUrl}${dataUrl}`);
-        }, config.nitterUrl);
-        return [buffer, originalImages, hlsUrls];
+      const pieces = [{ type: "account", text: `${captured.accountText}
+https://x.com/i/status/${tweetId}` }];
+      for (const item of captured.flow) {
+        if (item.type === "text") {
+          const t = translatedTexts[item.textIndex] || item.fallbackText || "";
+          if (t.trim()) pieces.push({ type: "text", text: t });
+        } else if (item.type === "image" && config.sendPic) {
+          pieces.push({ type: "image", url: item.url });
+        } else if (item.type === "video" && config.sendPic) {
+          pieces.push({ type: "video", url: item.url });
+        }
       }
-      return [buffer, [], []];
+      return { screenshot: buffer, pieces };
     } catch (e) {
       throw e;
     } finally {

package/lib/retry.d.ts

@@ -1 +1,2 @@
-export declare function retry<T>(retries: number, fn: () => Promise<T> | T, delay?: number): Promise<T>;
+export declare function retry<T>(retries: number, fn: () => Promise<T> | T, error_fn?: (err: any) => void, delay?: number): Promise<T>;
+export declare function withTimeout(promise: any, ms: any, fallbackValue?: any): Promise<any>;

package/lib/translate.d.ts

@@ -1,4 +1,4 @@
 import { Page } from 'puppeteer-core';
 export declare function setGoogleTranslate(key: string, proxy: string): void;
 export declare function setOpenAiTranslate(url: string, key: string, model: string, prompt: string, temperature: number, timeout?: number): void;
-export declare function addTranslate(page: Page, className: string): Promise<void>;
+export declare function addTranslate(page: Page, className: string): Promise<any>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "koishi-plugin-nitter",
   "description": "使用Rettiwt-API订阅推文，并使用nitter渲染",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
   "files": [
@@ -19,7 +19,7 @@
     "axios": "^1.12.2",
     "https-proxy-agent": "^7.0.6",
     "koishi": "^4.18.9",
-    "koishi-plugin-subscription": "^0.0.6"
+    "koishi-plugin-subscription": "^0.0.7"
   },
   "dependencies": {
     "fluent-ffmpeg": "^2.1.3",