comfy-pr 0.2.20 → 0.2.25

package/src/analyzePullsStatus.ts

@@ -1,11 +1,15 @@
 import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
-import pMap from "p-map";
+import type { ObjectId } from "mongodb";
 import prettyMs from "pretty-ms";
+import { snoflow } from "snoflow";
 import type { z } from "zod";
 import type { Task } from "../packages/mongodb-pipeline-ts/Task";
+import type { Author } from "./Authors";
 import { CNRepos, type CRPull } from "./CNRepos";
+import type { EmailTask } from "./EmailTasks";
 import type { GithubIssueComment } from "./GithubIssueComments";
 import { db } from "./db";
+import { yaml } from "./utils/yaml";
 import type { zPullStatus } from "./zod/zPullsStatus";
 // import { $pipeline } from "./db/$pipeline";
 // in case of dump production in local environment:
@@ -22,12 +26,19 @@ if (import.meta.main) {
   // await writeFile("src/zPullsStatus.ts", jsonToZod(await analyzePullsStatus({ limit: 1 }), "zPullsStatus", true));
 
   // analyzePullsStatusPipeline
-
-  const base = await pMap(analyzePullsStatusPipeline().group({ _id: "$ownername" }).aggregate(), (e) => e);
+  await snoflow(analyzePullsStatusPipeline().aggregate())
+    .filter((e) => e.email)
+    .limit(2)
+    .map((e) => yaml.stringify({ e }))
+    .log()
+    .chunk()
+    .map((e) => e.length)
+    .toLog();
 }
 
 export type PullStatus = z.infer<typeof zPullStatus>;
 export type PullsStatus = PullStatus[];
+export type PullStatusShown = Awaited<ReturnType<typeof analyzePullsStatus>>[number];
 export async function analyzePullsStatus({ skip = 0, limit = 0, pipeline = analyzePullsStatusPipeline() } = {}) {
   "use server";
   return await pipeline
@@ -47,18 +58,25 @@ export async function analyzePullsStatus({ skip = 0, limit = 0, pipeline = analy
     })
     .toArray();
 }
-export function analyzePullsStatusPipeline() {
+export function baseCRPullStatusPipeline() {
   return (
     $pipeline(CNRepos)
+      // get latest pr comments time
       .set({ "crPulls.data.pull.latest_comment_at": { $max: { $max: "$crPulls.data.comments.data.updated_at" } } })
-      .unwind("$crPulls.data")
+      // unwind
+      .stage({ $unwind: "$crPulls.data" })
       .match({ "crPulls.data.comments.data": { $exists: true } })
+      // repo infos
       .set({ "crPulls.data.pull.actived_at": { $toDate: "$info.data.updated_at" } })
       .set({ "crPulls.data.pull.repo": "$repository" })
+      .set({ "crPulls.data.pull.email": "$email" })
       .set({ "crPulls.data.pull.on_registry": "$on_registry" })
+      // pull info
       .set({ "crPulls.data.pull.type": "$crPulls.data.type" })
-      .set({ "crPulls.data.pull.comments": "$crPulls.data.comments.data" })
       .set({ "crPulls.data.pull": "$crPulls.data.pull" })
+      .set({ "crPulls.data.pull.comments": "$crPulls.data.comments.data" })
+      .set({ "crPulls.data.pull.emailTask_id": "$crPulls.data.emailTask_id" })
+      // replace root as pull
       .replaceRoot({ newRoot: "$crPulls.data.pull" })
       .as<
         CRPull & {
@@ -66,17 +84,59 @@ export function analyzePullsStatusPipeline() {
           on_registry: Task<boolean>;
           type: string;
           comments: GithubIssueComment[];
+          emailTask_id?: ObjectId;
         }
       >()
+  );
+}
+export function analyzePullsStatusPipeline() {
+  return (
+    baseCRPullStatusPipeline()
+      // fetch author email from Authors collection
+      .set({ ownername: "$base.user.login" })
+      .lookup({
+        from: "Authors",
+        // let: { <var_1>: <expression>, …, <var_n>: <expression> },
+        localField: "ownername",
+        foreignField: "githubId",
+        as: "author",
+        pipeline: [{ $project: { email: 1 } }],
+      })
+      .unwind({ path: "$author", preserveNullAndEmptyArrays: true })
+      .with<{ author: Author }>()
+      .lookup({
+        from: "EmailTasks",
+        // let: { <var_1>: <expression>, …, <var_n>: <expression> },
+        localField: "emailTask_id",
+        foreignField: "_id",
+        as: "emailTask",
+      })
+      .unwind({ path: "$emailTask", preserveNullAndEmptyArrays: true })
+      .set({ emailState: "$emailTask.state" }) //could be undefined or "waiting" | "sending" | "sent" | "error";
+      .with<{ emailState: EmailTask["state"] }>()
+
+      // .set({ author: { $first: ["$author"] } })
+      .project({ authors: 0 })
+
+      .set({ lastwords: { $arrayElemAt: ["$comments", -1] } })
+      .set({ lastwords: { $ifNull: [{ $concat: ["$lastwords.user.login", ": ", "$lastwords.body"] }, ""] } })
+
+      // calculate update_at max( )
+      // date format convert
+      .set({ created_at: { $toDate: "$created_at" } })
+      .set({ updated_at: { $toDate: "$updated_at" } })
+      .set({ latest_comment_at: { $toDate: "$latest_comment_at" } })
+      .set({ updated_at: { $max: ["$latest_comment_at", "$updated_at"] } })
+
+      // // projecting
       .project({
-        created_at: { $toDate: "$created_at" },
-        updated_at: { $toDate: "$updated_at" },
+        created_at: 1,
+        updated_at: 1,
         repository: 1,
         on_registry: "$on_registry.data",
         on_registry_at: "$on_registry.mtime",
         state: { $toUpper: `$prState` },
         url: "$html_url",
-        author_email: "$base.user.email",
         ownername: "$base.user.login",
         nickName: "$base.user.name",
         head: { $concat: ["$user.login", ":", "$type"] },
@@ -93,16 +153,16 @@ export function analyzePullsStatusPipeline() {
             chars: " ",
           },
         },
-        lastwords: { $arrayElemAt: ["$comments", -1] },
-        latest_comment_at: { $toDate: "$latest_comment_at" },
+        lastwords: 1,
         actived_at: 1,
+
+        instagramId: "$author.instagramId",
+        discordId: "$author.discordId",
+        twitterId: "$author.twitterId",
+        email: { $ifNull: ["$author.email", ""] },
+        emailState: 1,
       })
-      // .project({ latest_comment_at: {$toDate: '$latest_comment_at'} })
-      .set({ updated_at: { $max: ["$latest_comment_at", "$updated_at"] } })
-      .project({ latest_comment_at: 0 })
-      .set({ lastwords: { $concat: ["$lastwords.user.login", ": ", "$lastwords.body"] } })
-      .set({ lastwords: { $ifNull: ["$lastwords", ""] } })
-      // .set({ state: { $nin: ["CLOSED"] } })
+      .unset("latest_comment_at")
       .set({
         CLOSED: { $eq: ["$state", "CLOSED"] },
         MERGED: { $eq: ["$state", "MERGED"] },
@@ -113,7 +173,7 @@ export function analyzePullsStatusPipeline() {
       .unset(["CLOSED", "MERGED", "OPEN"])
       .as<{
         actived_at: Date;
-        author_email: string;
+        email: string | "";
         comments: number;
         created_at: Date;
         head: string;
@@ -126,9 +186,8 @@ export function analyzePullsStatusPipeline() {
         repository: string;
         state: "OPEN" | "MERGED" | "CLOSED";
         updated_at: Date;
+        emailState: EmailTask["state"];
         url: string;
       }>()
-    // .stage({ ...(!!skip && { $skip: skip }) })
-    // .stage({ ...(!!limit && { $limit: limit }) })
   );
 }

package/src/analyzeTotals.ts

@@ -15,7 +15,7 @@ if (import.meta.main) {
   });
 }
 /**
- * @warning this function is heavy
+ * @warning this function is heavy, TODO: split into small chunk
  */
 export async function analyzeTotals() {
   "use server";
@@ -111,9 +111,7 @@ export async function analyzeTotals() {
 
     // // Follow Rules
     // "Follow Up Rules": (async function () {
-    //   await pMap($pipeline(FollowRuleSets).aggregate(), async (ruleset) => {
-
-    //   });
+    //   return TaskDataOrNull(await showFollowRuleSet({ name: "default" }));
     // })(),
   });
   return totals;

package/src/createGithubForkForRepo.ts

@@ -32,6 +32,9 @@ export async function createGithubForkForRepo(upstreamRepoUrl: string) {
   const forkDst = `${FORK_OWNER}/${forkRepoName}`;
   const forkUrl = `https://github.com/${forkDst}`;
   const forked = await createGithubFork(upstreamRepoUrl, forkUrl);
-  if (forked.html_url !== forkUrl) DIE("forked url not expected");
+  if (forked.html_url !== forkUrl)
+    DIE(
+      "forked url not expected, it's likely you already forked this repo in your account before, and now trying to fork it again with different salt. To recovery you could delete that repo forked before by manual. (the repo forked before is listed in FORK OK: .....)",
+    );
   return forked;
 }

package/src/followRuleSchema.ts

@@ -30,12 +30,22 @@ export const zAddCommentAction = z
     body: z.string(),
   })
   .strict();
+export const zSendEmailAction = z
+  .object({
+    provider: z.enum(["google"]),
+    name: z.string(),
+    from: z.string(),
+    to: z.string(),
+    subject: z.string(),
+    body: z.string(),
+  })
+  .strict();
 const zFollowUpRule = z.object({
   name: z.string(),
   $match: z
     .object({
       actived_at: mDate,
-      author_email: mString,
+      email: mString,
       comments: mNumber,
       created_at: mDate,
       comments_author: mString,
@@ -53,15 +63,7 @@ const zFollowUpRule = z.object({
   action: z
     .object({
       "add-comment": zAddCommentAction,
-      "send-email": z
-        .object({
-          provider: z.string(),
-          from: z.string(),
-          to: z.string(),
-          subject: z.string(),
-          body: z.string(),
-        })
-        .strict(), // WARN: not implementd
+      "send-email": zSendEmailAction, // WARN: not implementd
       "update-issue": z
         .object({
           tags: mAny,
@@ -69,8 +71,8 @@ const zFollowUpRule = z.object({
         .strict(),
       // close: z.any(),
     })
-    .partial()
-    .strict(),
+    .strict()
+    .partial(),
 });
 // zPullsStatus
 export const zFollowUpRules = zFollowUpRule.array();

package/src/index.ts

@@ -1,7 +1,8 @@
-#!/usr/bin/env bun
 import "dotenv/config";
 import { checkComfyActivated } from "./checkComfyActivated";
+import { updateEmailTasks } from "./EmailTasks";
 import { initializeFollowRules } from "./initializeFollowRules";
+import { updateAuthors } from "./updateAuthors";
 import { updateCNRepos } from "./updateCNRepos";
 import { runFollowRuleSet } from "./updateFollowRuleSet";
 import { updateSlackMessages } from "./updateSlackMessages";
@@ -13,6 +14,8 @@ if (import.meta.main) {
     updateSlackMessages(),
     checkComfyActivated(), // needed if make pr
     updateCNRepos(),
+    updateAuthors(),
+    updateEmailTasks(),
   ]);
   await initializeFollowRules();
   await tLog("runFollowRuleSet", runFollowRuleSet);

package/src/sendEmailAction.ts

@@ -0,0 +1,56 @@
+"use server";
+import { $elemMatch } from "@/packages/mongodb-pipeline-ts/$elemMatch";
+import { TaskDataOrNull, type Task } from "@/packages/mongodb-pipeline-ts/Task";
+import DIE from "@snomiao/die";
+import pMap from "p-map";
+import { snoflow } from "snoflow";
+import type { z } from "zod";
+import type { PullStatusShown } from "./analyzePullsStatus";
+import { CNRepos } from "./CNRepos";
+import { $filaten } from "./db";
+import { enqueueEmailTask } from "./EmailTasks";
+import { zSendEmailAction } from "./followRuleSchema";
+import { yaml } from "./utils/yaml";
+
+export async function sendEmailAction({
+  matched,
+  action,
+  runAction,
+  rule,
+}: {
+  matched: Task<PullStatusShown[]>;
+  action: z.infer<typeof zSendEmailAction>;
+  runAction: boolean;
+  rule: { name: string };
+}) {
+  return await pMap(
+    TaskDataOrNull(matched) ?? DIE("NO-PAYLOAD-AVAILABLE"),
+    async (payload) => {
+      if (action.provider !== "google") DIE("Currently we only support gmail sender");
+      const loadedAction = await snoflow([action])
+        .map((e) => yaml.stringify(e))
+        .map((e) =>
+          // replace {{var}} s
+          e.replace(
+            /{{\$([_A-Za-z0-9]+)}}/,
+            (_, key: string) =>
+              (payload as any)[key] || DIE("Missing key: " + key + " in payload: " + JSON.stringify(payload)),
+          ),
+        )
+        .map((e) => yaml.parse(e))
+        .map((y) => zSendEmailAction.parse(y))
+        .map((a) => ({ ...a, action: "send-email" }))
+        .toLast();
+
+      if (runAction) {
+        const task = await enqueueEmailTask(loadedAction);
+        console.log(rule.name + " email enqueued :" + yaml.stringify(loadedAction));
+        await CNRepos.updateOne($filaten({ crPulls: { data: $elemMatch({ pull: { html_url: payload.url } }) } }), {
+          $set: { "crPulls.data.$.emailTask_id": task._id },
+        });
+      }
+      return loadedAction;
+    },
+    { concurrency: 1 },
+  );
+}

package/src/sendGmail.ts

@@ -0,0 +1,104 @@
+import DIE from "@snomiao/die";
+import type { Credentials, OAuth2Client } from "google-auth-library";
+import { GoogleApis } from "googleapis";
+import { createMimeMessage } from "mail-mime-builder";
+import markdownIt from "markdown-it";
+import {
+  GCloudOAuth2Credentials,
+  getGCloudOAuth2Client,
+  handleGCloudOAuth2Callback,
+} from "./gcloud/GCloudOAuth2Credentials";
+
+if (import.meta.main) {
+  // send test email
+  const name = "snomiao";
+  const from = "snomiao@gmail.com";
+  const to = "snomiao@gmail.com";
+  const subject = "Hello! snomiao";
+  const markdown = `
+# hello from sno
+  
+You've <b>just</b> received an *email* from snomiao.
+
+Thank you for receiving this email!!!!
+  `;
+
+  const html = markdownIt().render(markdown);
+  console.log(html);
+
+  const sent = await sendGmail({
+    name,
+    from,
+    to,
+    subject,
+    html,
+    auth: await getGCloudOAuth2Client({
+      email: from,
+      scope: ["https://www.googleapis.com/auth/gmail.compose"],
+      authorize: async (url) => {
+        // check saved credential in db
+        const getCred = async () =>
+          (
+            await GCloudOAuth2Credentials.findOne({
+              scopes: "https://www.googleapis.com/auth/gmail.compose",
+              email: from,
+              credentials: { $exists: true },
+            })
+          )?.credentials;
+        const cred = await getCred();
+        if (cred) return cred;
+
+        // otherwise wait for user approve
+        (await (await import("open")).default(url)).unref();
+        // setup one time server to receive ?code={{....}}
+        return await new Promise<Credentials>((r) => {
+          const server = Bun.serve({
+            fetch: async (req) => {
+              // wait for code, and then save to db
+              const res = await handleGCloudOAuth2Callback(req);
+              const cred = await getCred();
+              if (cred) r(cred);
+              server.stop();
+              return res;
+            },
+          });
+        });
+      },
+    }),
+  });
+  console.log(sent);
+}
+
+export async function sendGmail({
+  name,
+  from,
+  to,
+  subject,
+  text,
+  html,
+  auth,
+}: {
+  name: string;
+  from: string;
+  to: string;
+  subject: string;
+  text?: string;
+  html?: string;
+  auth: OAuth2Client;
+}) {
+  const msg = createMimeMessage();
+  msg.setSender({ name: name, addr: from });
+  msg.setRecipient(to);
+  msg.setSubject(subject);
+  text && msg.addMessage({ contentType: "text/plain", data: text });
+  html && msg.addMessage({ contentType: "text/html", data: html });
+  text || html || DIE("Missing msg body");
+
+  const result = await new GoogleApis().gmail("v1").users.messages.send({
+    auth,
+    userId: from,
+    requestBody: { raw: btoa(msg.asRaw()) },
+  });
+  const sent = result.data;
+  return sent;
+}

package/src/showFollowRuleSet.ts

@@ -0,0 +1,12 @@
+"use server";
+import DIE from "@snomiao/die";
+import { FollowRuleSets } from "./FollowRules";
+import { updateFollowRuleSet } from "./updateFollowRuleSet";
+
+export async function showFollowRuleSet({ name = "default" } = {}) {
+  const ruleset = (await FollowRuleSets.findOne({ name })) ?? DIE("default ruleset not found");
+  return await updateFollowRuleSet({
+    name: ruleset.name,
+    yaml: ruleset.yamlWhenEnabled ?? DIE("Rule not enabled"),
+  });
+}

package/src/updateAuthors.ts

@@ -0,0 +1,7 @@
+import { updateAuthorsForGithub } from "./updateAuthorsForGithub";
+import { updateAuthorsFromCNRepo } from "./updateAuthorsFromCNRepo";
+
+export async function updateAuthors() {
+  await updateAuthorsFromCNRepo();
+  await updateAuthorsForGithub();
+}

package/src/updateAuthorsForGithub.ts

@@ -0,0 +1,54 @@
+import { $OK, TaskDataOrNull, TaskError, TaskOK } from "@/packages/mongodb-pipeline-ts/Task";
+import DIE from "@snomiao/die";
+import console from "console";
+import { peekYaml } from "peek-log";
+import { snoflow } from "snoflow";
+import { Authors, GithubUsers } from "./Authors";
+import { $stale } from "./db";
+import { gh } from "./gh";
+
+if (import.meta.main) {
+  await updateAuthorsForGithub();
+}
+export async function updateAuthorsForGithub() {
+  await snoflow(Authors.find({ githubMtime: $stale("7d") }))
+    .map((e) => e.githubId)
+    .filter()
+    .pMap(2, async (username) => {
+      const cached = await GithubUsers.findOne({ username, mtime: $stale("1d"), ...$OK });
+      if (cached) return cached;
+      console.log("Fetching github user " + username);
+      const result = await gh.users
+        .getByUsername({ username })
+        .then((e) => e.data)
+        .then(TaskOK)
+        .catch(TaskError);
+      return (
+        (await GithubUsers.findOneAndUpdate(
+          { username },
+          { $set: result },
+          { returnDocument: "after", upsert: true },
+        )) ?? DIE(`fail to udpate gh users`)
+      );
+    })
+    .map((e) => TaskDataOrNull(e))
+    .filter()
+    .map(({ email, avatar_url, blog, updated_at, location, company, hireable, bio, login }) =>
+      Authors.findOneAndUpdate(
+        { githubId: login },
+        {
+          $set: { githubMtime: new Date(), ...(email && { email }), ...(null != hireable && { hireable }) },
+          $addToSet: {
+            ...(bio && { bios: bio }),
+            avatars: avatar_url,
+            ...(location && { locations: location }),
+            ...(blog && { blogs: blog }),
+            ...(company && { companies: company }),
+          },
+        },
+        { upsert: true, returnDocument: "after" },
+      ),
+    )
+    .peek(peekYaml)
+    .done();
+}

package/src/updateAuthorsFromCNRepo.ts

@@ -0,0 +1,30 @@
+import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
+import { snoflow } from "snoflow";
+import { Authors } from "./Authors";
+import { CNRepos } from "./CNRepos";
+import { $filaten } from "./db";
+
+/** Update authors for gh users, collecting emails/username/hireable */
+export async function updateAuthorsFromCNRepo() {
+  return await snoflow(
+    $pipeline(CNRepos)
+      .match($filaten({ info: { data: { owner: { login: { $exists: true } } } } }))
+      .group({
+        _id: "$info.data.owner.login",
+        // author: "$info.data.owner.login",
+        cm: { $sum: { $cond: [{ $eq: [{ $type: "$cm" }, "missing"] }, 0, 1] } },
+        cr: { $sum: { $cond: [{ $eq: [{ $type: "$cr" }, "missing"] }, 0, 1] } },
+
+        // TODO: get totals open/closed/merged
+        // pulls:{
+        //   OPEN: {"$crPulls.data.pull.prState", "open"}
+        // }
+        All: { $sum: 1 },
+      })
+      .set({ githubId: "$_id" })
+      .project({ _id: 0 })
+      .aggregate(),
+  )
+    .map(({ githubId, ...$set }) => Authors.updateOne({ githubId }, { $set }, { upsert: true }))
+    .done();
+}

package/src/updateComfyTotals.ts

@@ -11,7 +11,7 @@ if (import.meta.main) {
 }
 
 export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
-  await Totals.createIndex({today: 1, "totals.mtime": 1, "totals.state": 1})
+  await Totals.createIndex({ today: 1, "totals.mtime": 1, "totals.state": 1 });
   const today = new Date().toISOString().split("T")[0];
   const cached = await Totals.findOne($filaten({ today, totals: { mtime: $fresh(fresh), ...$OK } }));
   if (cached?.totals?.state === "ok")
@@ -24,14 +24,14 @@ export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
 
   // notify if today is not already notify
   if (notify) {
-    if(!await Totals.findOne($filaten({ today, totals: { mtime: $fresh('0.9d'), ...$OK } }))) 
-    // ignore today
-    await match(totals)
-      .with($OK, async (totals) => {
-        const msg = `Totals: \n${"```"}\n${YAML.stringify(totals)}\n${"```"}`;
-        await notifySlack(msg, { unique: true });
-      })
-      .otherwise(() => null);
+    if (!(await Totals.findOne($filaten({ today, totals: { mtime: $fresh("1d"), ...$OK } }))))
+      // ignore today
+      await match(totals)
+        .with($OK, async (totals) => {
+          const msg = `Totals: \n${"```"}\n${YAML.stringify(totals)}\n${"```"}`;
+          await notifySlack(msg, { unique: true });
+        })
+        .otherwise(() => null);
   }
 
   const insertResult = await Totals.insertOne({ totals });