comfy-pr 0.2.24 → 0.2.26

package/src/readTemplateTitle.ts

@@ -1,11 +1,11 @@
-import { readFile } from "fs/promises";
-import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
-
-export async function readTemplateTitle(filename: string) {
-  return await readTemplate(filename).then((e) => e.title);
-}
-export async function readTemplate(filename: string) {
-  return readFile("./templates/" + filename, "utf8").then(
-    parseTitleBodyOfMarkdown,
-  );
-}
+import { readFile } from "fs/promises";
+import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
+
+export async function readTemplateTitle(filename: string) {
+  return await readTemplate(filename).then((e) => e.title);
+}
+export async function readTemplate(filename: string) {
+  return readFile("./templates/" + filename, "utf8").then(
+    parseTitleBodyOfMarkdown,
+  );
+}

package/src/sendEmailAction.ts

@@ -0,0 +1,56 @@
+"use server";
+import { $elemMatch } from "@/packages/mongodb-pipeline-ts/$elemMatch";
+import { TaskDataOrNull, type Task } from "@/packages/mongodb-pipeline-ts/Task";
+import DIE from "@snomiao/die";
+import pMap from "p-map";
+import { snoflow } from "snoflow";
+import type { z } from "zod";
+import type { PullStatusShown } from "./analyzePullsStatus";
+import { CNRepos } from "./CNRepos";
+import { $filaten } from "./db";
+import { enqueueEmailTask } from "./EmailTasks";
+import { zSendEmailAction } from "./followRuleSchema";
+import { yaml } from "./utils/yaml";
+
+export async function sendEmailAction({
+  matched,
+  action,
+  runAction,
+  rule,
+}: {
+  matched: Task<PullStatusShown[]>;
+  action: z.infer<typeof zSendEmailAction>;
+  runAction: boolean;
+  rule: { name: string };
+}) {
+  return await pMap(
+    TaskDataOrNull(matched) ?? DIE("NO-PAYLOAD-AVAILABLE"),
+    async (payload) => {
+      if (action.provider !== "google") DIE("Currently we only support gmail sender");
+      const loadedAction = await snoflow([action])
+        .map((e) => yaml.stringify(e))
+        .map((e) =>
+          // replace {{var}} s
+          e.replace(
+            /{{\$([_A-Za-z0-9]+)}}/g,
+            (_, key: string) =>
+              (payload as any)[key] || DIE("Missing key: " + key + " in payload: " + JSON.stringify(payload)),
+          ),
+        )
+        .map((e) => yaml.parse(e))
+        .map((y) => zSendEmailAction.parse(y))
+        .map((a) => ({ ...a, action: "send-email" }))
+        .toLast();
+
+      if (runAction) {
+        const task = await enqueueEmailTask(loadedAction);
+        console.log(rule.name + " email enqueued :" + yaml.stringify(loadedAction));
+        await CNRepos.updateOne($filaten({ crPulls: { data: $elemMatch({ pull: { html_url: payload.url } }) } }), {
+          $set: { "crPulls.data.$.emailTask_id": task._id },
+        });
+      }
+      return loadedAction;
+    },
+    { concurrency: 1 },
+  );
+}

package/src/sendGmail.ts

@@ -0,0 +1,105 @@
+import DIE from "@snomiao/die";
+// import type { Credentials } from "google-auth-library";
+import type { Credentials } from "google-auth-library";
+import { GoogleApis } from "googleapis";
+import type { OAuth2Client } from "googleapis-common";
+import { createMimeMessage } from "mail-mime-builder";
+import markdownIt from "markdown-it";
+import {
+  GCloudOAuth2Credentials,
+  getGCloudOAuth2Client,
+  handleGCloudOAuth2Callback,
+} from "./gcloud/GCloudOAuth2Credentials";
+if (import.meta.main) {
+  // send test email
+  const name = "snomiao";
+  const from = "snomiao@gmail.com";
+  const to = ".snomiao@gmail.com";
+  const subject = "Hello! snomiao";
+  const markdown = `
+# hello from sno
+  
+You've <b>just</b> received an *email* from snomiao.
+
+Thank you for receiving this email!!!!
+  `;
+
+  const html = markdownIt().render(markdown);
+  console.log(html);
+
+  const sent = await sendGmail({
+    name,
+    from,
+    to,
+    subject,
+    html,
+    auth: await getGCloudOAuth2Client({
+      email: from,
+      scope: ["https://www.googleapis.com/auth/gmail.compose"],
+      authorize: async (url) => {
+        // check saved credential in db
+        const getCred = async () =>
+          (
+            await GCloudOAuth2Credentials.findOne({
+              scopes: "https://www.googleapis.com/auth/gmail.compose",
+              email: from,
+              credentials: { $exists: true },
+            })
+          )?.credentials;
+        const cred = await getCred();
+        if (cred) return cred;
+
+        // otherwise wait for user approve
+        (await (await import("open")).default(url)).unref();
+        // setup one time server to receive ?code={{....}}
+        return await new Promise<Credentials>((r) => {
+          const server = Bun.serve({
+            fetch: async (req) => {
+              // wait for code, and then save to db
+              const res = await handleGCloudOAuth2Callback(req);
+              const cred = await getCred();
+              if (cred) r(cred);
+              server.stop();
+              return res;
+            },
+          });
+        });
+      },
+    }),
+  });
+  console.log(sent);
+}
+
+export async function sendGmail({
+  name,
+  from,
+  to,
+  subject,
+  text,
+  html,
+  auth,
+}: {
+  name: string;
+  from: string;
+  to: string;
+  subject: string;
+  text?: string;
+  html?: string;
+  auth: OAuth2Client;
+}) {
+  const msg = createMimeMessage();
+  msg.setSender({ name: name, addr: from });
+  msg.setRecipient(to);
+  msg.setSubject(subject);
+  text && msg.addMessage({ contentType: "text/plain", data: text });
+  html && msg.addMessage({ contentType: "text/html", data: html });
+  text || html || DIE("Missing msg body");
+
+  const result = await new GoogleApis().gmail("v1").users.messages.send({
+    auth,
+    userId: from,
+    requestBody: { raw: btoa(msg.asRaw()) },
+  });
+  const sent = result.data;
+  return sent;
+}

package/src/showFollowRuleSet.ts

@@ -1,12 +1,12 @@
-"use server";
-import DIE from "@snomiao/die";
-import { FollowRuleSets } from "./FollowRules";
-import { updateFollowRuleSet } from "./updateFollowRuleSet";
-
-export async function showFollowRuleSet({ name = "default" } = {}) {
-  const ruleset = (await FollowRuleSets.findOne({ name })) ?? DIE("default ruleset not found");
-  return await updateFollowRuleSet({
-    name: ruleset.name,
-    yaml: ruleset.yamlWhenEnabled ?? DIE("Rule not enabled"),
-  });
-}
+"use server";
+import DIE from "@snomiao/die";
+import { FollowRuleSets } from "./FollowRules";
+import { updateFollowRuleSet } from "./updateFollowRuleSet";
+
+export async function showFollowRuleSet({ name = "default" } = {}) {
+  const ruleset = (await FollowRuleSets.findOne({ name })) ?? DIE("default ruleset not found");
+  return await updateFollowRuleSet({
+    name: ruleset.name,
+    yaml: ruleset.yamlWhenEnabled ?? DIE("Rule not enabled"),
+  });
+}

package/src/summaryLastPullComment.ts

@@ -1,8 +1,8 @@
-import { type GithubIssueComment } from "./GithubIssueComments";
-
-export function summaryLastPullComment(comments: GithubIssueComment[]) {
-  // assume ascending order
-  const last = comments.toReversed()[0];
-  const lastText = (last ?? "") && "@" + last.user!.name + ":" + last.body!.replace(/\s+/gim, " ");
-  return lastText;
-}
+import { type GithubIssueComment } from "./GithubIssueComments";
+
+export function summaryLastPullComment(comments: GithubIssueComment[]) {
+  // assume ascending order
+  const last = comments.toReversed()[0];
+  const lastText = (last ?? "") && "@" + last.user!.name + ":" + last.body!.replace(/\s+/gim, " ");
+  return lastText;
+}

package/src/tomlFillDescription.ts

@@ -1,17 +1,17 @@
-import { writeFile, readFile } from "fs/promises";
-import DIE from "@snomiao/die";
-import toml from "toml";
-import { fetchRepoDescriptionMap } from "./fetchRepoDescriptionMap";
-
-export async function tomlFillDescription(referenceUrl: string, pyprojectToml: string) {
-  const repoDescriptionMap = await fetchRepoDescriptionMap();
-  const matchedDescription = repoDescriptionMap[referenceUrl]?.toString() ||
-    DIE("Warn: missing description for " + referenceUrl);
-  const replaced = (await readFile(pyprojectToml, "utf8")).replace(
-    `description = ""`,
-    `description = ${JSON.stringify(matchedDescription)}`
-  );
-  // check validity
-  toml.parse(replaced);
-  await writeFile(pyprojectToml, replaced);
-}
+import { writeFile, readFile } from "fs/promises";
+import DIE from "@snomiao/die";
+import toml from "toml";
+import { fetchRepoDescriptionMap } from "./fetchRepoDescriptionMap";
+
+export async function tomlFillDescription(referenceUrl: string, pyprojectToml: string) {
+  const repoDescriptionMap = await fetchRepoDescriptionMap();
+  const matchedDescription = repoDescriptionMap[referenceUrl]?.toString() ||
+    DIE("Warn: missing description for " + referenceUrl);
+  const replaced = (await readFile(pyprojectToml, "utf8")).replace(
+    `description = ""`,
+    `description = ${JSON.stringify(matchedDescription)}`
+  );
+  // check validity
+  toml.parse(replaced);
+  await writeFile(pyprojectToml, replaced);
+}

package/src/updateAuthors.ts

@@ -1,7 +1,7 @@
-import { updateAuthorsForGithub } from "./updateAuthorsForGithub";
-import { updateAuthorsFromCNRepo } from "./updateAuthorsFromCNRepo";
-
-export async function updateAuthors() {
-  await updateAuthorsFromCNRepo();
-  await updateAuthorsForGithub();
-}
+import { updateAuthorsForGithub } from "./updateAuthorsForGithub";
+import { updateAuthorsFromCNRepo } from "./updateAuthorsFromCNRepo";
+
+export async function updateAuthors() {
+  await updateAuthorsFromCNRepo();
+  await updateAuthorsForGithub();
+}

package/src/updateAuthorsForGithub.ts

@@ -1,50 +1,54 @@
-import { $OK, TaskDataOrNull, TaskError, TaskOK } from "@/packages/mongodb-pipeline-ts/Task";
-import DIE from "@snomiao/die";
-import console from "console";
-import { peekYaml } from "peek-log";
-import { snoflow } from "snoflow";
-import { Authors, GithubUsers } from "./Authors";
-import { $stale } from "./db";
-import { gh } from "./gh";
-
-export async function updateAuthorsForGithub() {
-  await snoflow(Authors.find({ githubMtime: $stale("7d") }))
-    .map((e) => e.githubId)
-    .filter()
-    .pMap(2, async (username) => {
-      const cached = await GithubUsers.findOne({ username, mtime: $stale("1d"), ...$OK });
-      if (cached) return cached;
-      console.log("Fetching github user " + username);
-      const result = await gh.users
-        .getByUsername({ username })
-        .then((e) => e.data)
-        .then(TaskOK)
-        .catch(TaskError);
-      return (
-        (await GithubUsers.findOneAndUpdate(
-          { username },
-          { $set: result },
-          { returnDocument: "after", upsert: true }
-        )) ?? DIE(`fail to udpate gh users`)
-      );
-    })
-    .map((e) => TaskDataOrNull(e))
-    .filter()
-    .map(({ email, avatar_url, blog, updated_at, location, company, hireable, bio, login }) => Authors.findOneAndUpdate(
-      { githubId: login },
-      {
-        $set: { githubMtime: new Date(), ...(email && { email }), ...(null != hireable && { hireable }) },
-        $addToSet: {
-          ...(bio && { bios: bio }),
-          avatars: avatar_url,
-          ...(location && { locations: location }),
-          ...(blog && { blogs: blog }),
-          ...(company && { companies: company }),
-        },
-      },
-      { upsert: true, returnDocument: "after" }
-    )
-    )
-    .peek(peekYaml)
-    .done();
-}
+import { $OK, TaskDataOrNull, TaskError, TaskOK } from "@/packages/mongodb-pipeline-ts/Task";
+import DIE from "@snomiao/die";
+import console from "console";
+import { peekYaml } from "peek-log";
+import { snoflow } from "snoflow";
+import { Authors, GithubUsers } from "./Authors";
+import { $stale } from "./db";
+import { gh } from "./gh";
+
+if (import.meta.main) {
+  await updateAuthorsForGithub();
+}
+export async function updateAuthorsForGithub() {
+  await snoflow(Authors.find({ githubMtime: $stale("7d") }))
+    .map((e) => e.githubId)
+    .filter()
+    .pMap(2, async (username) => {
+      const cached = await GithubUsers.findOne({ username, mtime: $stale("1d"), ...$OK });
+      if (cached) return cached;
+      console.log("Fetching github user " + username);
+      const result = await gh.users
+        .getByUsername({ username })
+        .then((e) => e.data)
+        .then(TaskOK)
+        .catch(TaskError);
+      return (
+        (await GithubUsers.findOneAndUpdate(
+          { username },
+          { $set: result },
+          { returnDocument: "after", upsert: true },
+        )) ?? DIE(`fail to udpate gh users`)
+      );
+    })
+    .map((e) => TaskDataOrNull(e))
+    .filter()
+    .map(({ email, avatar_url, blog, updated_at, location, company, hireable, bio, login }) =>
+      Authors.findOneAndUpdate(
+        { githubId: login },
+        {
+          $set: { githubMtime: new Date(), ...(email && { email }), ...(null != hireable && { hireable }) },
+          $addToSet: {
+            ...(bio && { bios: bio }),
+            avatars: avatar_url,
+            ...(location && { locations: location }),
+            ...(blog && { blogs: blog }),
+            ...(company && { companies: company }),
+          },
+        },
+        { upsert: true, returnDocument: "after" },
+      ),
+    )
+    .peek(peekYaml)
+    .done();
+}

package/src/updateAuthorsFromCNRepo.ts

@@ -1,30 +1,30 @@
-import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
-import { snoflow } from "snoflow";
-import { Authors } from "./Authors";
-import { CNRepos } from "./CNRepos";
-import { $filaten } from "./db";
-
-/** Update authors for gh users, collecting emails/username/hireable */
-export async function updateAuthorsFromCNRepo() {
-  return await snoflow(
-    $pipeline(CNRepos)
-      .match($filaten({ info: { data: { owner: { login: { $exists: true } } } } }))
-      .group({
-        _id: "$info.data.owner.login",
-        // author: "$info.data.owner.login",
-        cm: { $sum: { $cond: [{ $eq: [{ $type: "$cm" }, "missing"] }, 0, 1] } },
-        cr: { $sum: { $cond: [{ $eq: [{ $type: "$cr" }, "missing"] }, 0, 1] } },
-
-        // TODO: get totals open/closed/merged
-        // pulls:{
-        //   OPEN: {"$crPulls.data.pull.prState", "open"}
-        // }
-        All: { $sum: 1 },
-      })
-      .set({ login: "$_id" })
-      .project({ _id: 0 })
-      .aggregate()
-  )
-    .peek(({ login, ...$set }) => Authors.updateOne({ login }, { $set }, { upsert: true }))
-    .done();
-}
+import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
+import { snoflow } from "snoflow";
+import { Authors } from "./Authors";
+import { CNRepos } from "./CNRepos";
+import { $filaten } from "./db";
+
+/** Update authors for gh users, collecting emails/username/hireable */
+export async function updateAuthorsFromCNRepo() {
+  return await snoflow(
+    $pipeline(CNRepos)
+      .match($filaten({ info: { data: { owner: { login: { $exists: true } } } } }))
+      .group({
+        _id: "$info.data.owner.login",
+        // author: "$info.data.owner.login",
+        cm: { $sum: { $cond: [{ $eq: [{ $type: "$cm" }, "missing"] }, 0, 1] } },
+        cr: { $sum: { $cond: [{ $eq: [{ $type: "$cr" }, "missing"] }, 0, 1] } },
+
+        // TODO: get totals open/closed/merged
+        // pulls:{
+        //   OPEN: {"$crPulls.data.pull.prState", "open"}
+        // }
+        All: { $sum: 1 },
+      })
+      .set({ githubId: "$_id" })
+      .project({ _id: 0 })
+      .aggregate(),
+  )
+    .map(({ githubId, ...$set }) => Authors.updateOne({ githubId }, { $set }, { upsert: true }))
+    .done();
+}

package/src/updateCMNodesDuplicationWarnings.ts

@@ -1,56 +1,56 @@
-import { unary } from "lodash-es";
-import pMap from "p-map";
-import { dissoc, filter, groupBy, map, prop, toPairs } from "rambda";
-import YAML from "yaml";
-import { CMNodes, type CMNode } from "./CMNodes";
-import { notifySlack } from "./slack/notifySlack";
-
-export async function updateCMNodesDuplicationWarnings(nodes: CMNode[]) {
-  console.log("CMNodes checking duplicates");
-  // prettier-ignore
-  const dups = {
-    ID: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.id, nodes)),
-    TITLE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.title, nodes)),
-    REFERENCE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.reference, nodes)),
-  };
-  const dupsSummary = JSON.stringify(map((x) => map((x) => x.length, x), dups));
-  await notifySlack(
-    `[WARN] CMNodes duplicates: ${dupsSummary}\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json`,
-  );
-
-  await pMap(
-    toPairs(dups),
-    async ([topic, nodes]) =>
-      await pMap(
-        toPairs(nodes),
-        async ([key, nodesRaw]) => {
-          const nodes = nodesRaw.map(unary(dissoc("hash")));
-          const hashes = nodesRaw.map(prop("hash"));
-          // check sent
-          const someDuplicateSent = await CMNodes.findOne({
-            hash: { $in: hashes },
-            [`duplicated.${topic}`]: { $exists: true },
-          });
-          if (someDuplicateSent) return;
-          // send slack notification
-          const slackNotification = await notifySlack(
-            `[ACTION NEEDED WARNING]: please resolve duplicated node in ${topic}: ${key}\n` +
-              "```\n" +
-              YAML.stringify(nodes) +
-              "```" +
-              "\n\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json",
-            { unique: true },
-          );
-          // mark duplicates
-          await CMNodes.updateMany(
-            { hash: { $in: hashes } },
-            {
-              $set: { [`duplicated.${topic}`]: { hashes, slackNotification } },
-            },
-          );
-        },
-        { concurrency: 2 },
-      ),
-    { concurrency: 2 },
-  );
-}
+import { unary } from "lodash-es";
+import pMap from "p-map";
+import { dissoc, filter, groupBy, map, prop, toPairs } from "rambda";
+import YAML from "yaml";
+import { CMNodes, type CMNode } from "./CMNodes";
+import { notifySlack } from "./slack/notifySlack";
+
+export async function updateCMNodesDuplicationWarnings(nodes: CMNode[]) {
+  console.log("CMNodes checking duplicates");
+  // prettier-ignore
+  const dups = {
+    ID: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.id, nodes)),
+    TITLE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.title, nodes)),
+    REFERENCE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.reference, nodes)),
+  };
+  const dupsSummary = JSON.stringify(map((x) => map((x) => x.length, x), dups));
+  await notifySlack(
+    `[WARN] CMNodes duplicates: ${dupsSummary}\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json`,
+  );
+
+  await pMap(
+    toPairs(dups),
+    async ([topic, nodes]) =>
+      await pMap(
+        toPairs(nodes),
+        async ([key, nodesRaw]) => {
+          const nodes = nodesRaw.map(unary(dissoc("hash")));
+          const hashes = nodesRaw.map(prop("hash"));
+          // check sent
+          const someDuplicateSent = await CMNodes.findOne({
+            hash: { $in: hashes },
+            [`duplicated.${topic}`]: { $exists: true },
+          });
+          if (someDuplicateSent) return;
+          // send slack notification
+          const slackNotification = await notifySlack(
+            `[ACTION NEEDED WARNING]: please resolve duplicated node in ${topic}: ${key}\n` +
+              "```\n" +
+              YAML.stringify(nodes) +
+              "```" +
+              "\n\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json",
+            { unique: true },
+          );
+          // mark duplicates
+          await CMNodes.updateMany(
+            { hash: { $in: hashes } },
+            {
+              $set: { [`duplicated.${topic}`]: { hashes, slackNotification } },
+            },
+          );
+        },
+        { concurrency: 2 },
+      ),
+    { concurrency: 2 },
+  );
+}

package/src/updateCMRepos.ts

@@ -1,27 +1,27 @@
-import pMap from "p-map";
-import { CMNodes } from "./CMNodes";
-import { CNRepos } from "./CNRepos";
-import { tLog } from "./utils/tLog";
-if (import.meta.main) {
-  await tLog("Update Repos from ComfyUI Manager", updateCMRepos);
-  console.log(await CMNodes.estimatedDocumentCount());
-}
-export async function updateCMRepos() {
-  await CMNodes.createIndex({ repo_id: 1 });
-  return await pMap(CMNodes.find({ repo_id: { $exists: false } }), async (cm, i) => {
-    const { reference: repository, _id } = cm;
-    return await CNRepos.updateOne(
-      { repository },
-      {
-        $set: {
-          cm: (await CMNodes.findOneAndUpdate(
-            { _id },
-            { $set: { repo_id: cm._id } },
-            { upsert: true, returnDocument: "after" },
-          ))!,
-        },
-      },
-      { upsert: true },
-    );
-  });
-}
+import pMap from "p-map";
+import { CMNodes } from "./CMNodes";
+import { CNRepos } from "./CNRepos";
+import { tLog } from "./utils/tLog";
+if (import.meta.main) {
+  await tLog("Update Repos from ComfyUI Manager", updateCMRepos);
+  console.log(await CMNodes.estimatedDocumentCount());
+}
+export async function updateCMRepos() {
+  await CMNodes.createIndex({ repo_id: 1 });
+  return await pMap(CMNodes.find({ repo_id: { $exists: false } }), async (cm, i) => {
+    const { reference: repository, _id } = cm;
+    return await CNRepos.updateOne(
+      { repository },
+      {
+        $set: {
+          cm: (await CMNodes.findOneAndUpdate(
+            { _id },
+            { $set: { repo_id: cm._id } },
+            { upsert: true, returnDocument: "after" },
+          ))!,
+        },
+      },
+      { upsert: true },
+    );
+  });
+}