comfy-pr 0.2.26 → 1.0.1

package/src/updateCRRepos.ts

@@ -7,7 +7,7 @@ if (import.meta.main) {
 }
 export async function updateCRRepos() {
   await CRNodes.createIndex({ repo_id: 1 });
-  return await pMap(CRNodes.find({ repo_id: { $exists: false } }), async (cr, i) => {
+  return await pMap(CRNodes.find({ repo_id: { $exists: false } }), async (cr, _i) => {
     const { repository, _id } = cr;
     await CNRepos.updateOne(
       { repository },

package/src/updateComfyTotals.ts

@@ -3,8 +3,8 @@ import YAML from "yaml";
 import { $OK, TaskError, TaskOK, tsmatch } from "../packages/mongodb-pipeline-ts/Task";
 import { Totals } from "./Totals";
 import { analyzeTotals } from "./analyzeTotals";
-import { $filaten, $fresh } from "./db";
-import { notifySlack } from "./slack/notifySlack";
+import { $flatten, $fresh } from "./db";
+import { notifySlack } from "@/lib/slack/notifySlack";
 
 if (import.meta.main) {
   await updateComfyTotals();
@@ -13,7 +13,9 @@ if (import.meta.main) {
 export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
   await Totals.createIndex({ today: 1, "totals.mtime": 1, "totals.state": 1 });
   const today = new Date().toISOString().split("T")[0];
-  const cached = await Totals.findOne($filaten({ today, totals: { mtime: $fresh(fresh), ...$OK } }));
+  const cached = await Totals.findOne(
+    $flatten({ today, totals: { mtime: $fresh(fresh), ...$OK } }),
+  );
   if (cached?.totals?.state === "ok")
     return [
       tsmatch(cached.totals)
@@ -24,7 +26,7 @@ export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
 
   // notify if today is not already notify
   if (notify) {
-    if (!(await Totals.findOne($filaten({ today, totals: { mtime: $fresh("1d"), ...$OK } }))))
+    if (!(await Totals.findOne($flatten({ today, totals: { mtime: $fresh("1d"), ...$OK } }))))
       // ignore today
       await match(totals)
         .with($OK, async (totals) => {
@@ -34,7 +36,7 @@ export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
         .otherwise(() => null);
   }
 
-  const insertResult = await Totals.insertOne({ totals });
+  const _insertResult = await Totals.insertOne({ totals });
   return [
     tsmatch(totals)
       .with($OK, ({ data }) => data)

package/src/updateFollowRuleSet.ts

@@ -8,9 +8,9 @@ import { CNRepos } from "./CNRepos";
 import { FollowRuleSets } from "./FollowRules";
 import { addCommentAction } from "./addCommentAction";
 import { analyzePullsStatus, analyzePullsStatusPipeline } from "./analyzePullsStatus";
-import { $filaten } from "./db";
+import { $flatten } from "./db";
 import { zAddCommentAction, zFollowUpRules, zSendEmailAction } from "./followRuleSchema";
-import { fetchIssueComments } from "./gh/fetchIssueComments";
+import { fetchIssueComments } from "@/lib/github/fetchIssueComments";
 import { stringifyGithubRepoUrl } from "./parseOwnerRepo";
 import { parsePullUrl } from "./parsePullUrl";
 import { sendEmailAction } from "./sendEmailAction";
@@ -56,19 +56,23 @@ export async function updateFollowRuleSet({
   return await (async function () {
     if (enable === false) {
       await FollowRuleSets.updateOne({ name }, { $set: { enabled: false, yamlWhenEnabled: "" } });
-      DIE`ruleset is disabled, no need to run`;
+      throw new Error("ruleset is disabled, no need to run");
     }
 
     // must parse while run, because the date in code is dynamic generated
     const rules = zFollowUpRules.parse(yaml.parse(code || DIE`yaml is empty`));
     // save if parse succ
     // await FollowRuleSets.updateOne({ name }, { $set: { yaml: code, rules } });
+
+    // todo: enhance performance
     const parseResult = await pMap(
       rules,
       async (rule) => {
         if (runAction) {
           // pre-fetch comments before run a rule -> match -> action, to prevent comment on a outdated pull state
-          const preMatched = await analyzePullsStatus({ pipeline: analyzePullsStatusPipeline().match(rule.$match) })
+          const preMatched = await analyzePullsStatus({
+            pipeline: analyzePullsStatusPipeline().match(rule.$match),
+          })
             .then(TaskOK)
             .catch(TaskError);
 
@@ -82,15 +86,19 @@ export async function updateFollowRuleSet({
             const comments = await fetchIssueComments(repository, { number: pull_number })
               .then(TaskOK)
               .catch(TaskError);
-            (
-              await CNRepos.updateOne($filaten({ repository, crPulls: { data: { pull: { html_url } } } }), {
+            const updateResult = await CNRepos.updateOne(
+              $flatten({ repository, crPulls: { data: { pull: { html_url } } } }),
+              {
                 $set: { "crPulls.data.$.comments": comments },
-              })
-            ).matchedCount ?? DIE("pre-matched comments is not found");
+              },
+            );
+            if (!updateResult.matchedCount) DIE("pre-matched comments is not found");
           });
         }
 
-        const matched = await analyzePullsStatus({ pipeline: analyzePullsStatusPipeline().match(rule.$match) })
+        const matched = await analyzePullsStatus({
+          pipeline: analyzePullsStatusPipeline().match(rule.$match),
+        })
           .then(TaskOK)
           .catch(TaskError);
 

package/src/updateOutdatedPullsTemplates.ts

@@ -1,54 +1,55 @@
 import { $elemMatch } from "@/packages/mongodb-pipeline-ts/$elemMatch";
 import DIE from "@snomiao/die";
+import gitDiff from "git-diff";
+import isCI from "is-ci";
+import stableStringify from "json-stable-stringify";
 import pMap from "p-map";
+import sflow from "sflow";
 import { match } from "ts-pattern";
+import { glob } from "zx";
 import { $OK, TaskError, TaskOK, tsmatch } from "../packages/mongodb-pipeline-ts/Task";
 import { CNRepos, type CRPull } from "./CNRepos";
-import { $filaten, $fresh, $stale } from "./db";
-import { gh } from "./gh";
-import { parsePull } from "./gh/parsePull";
+import { $flatten, $stale } from "./db";
+import { gh } from "@/lib/github";
+import { parsePull } from "@/lib/github/parsePull";
 import { ghUser } from "./ghUser";
-import { parseUrlRepoOwner } from "./parseOwnerRepo";
+import { parseGithubRepoUrl } from "./parseOwnerRepo";
 import { readTemplate } from "./readTemplateTitle";
-import { notifySlackLinks } from "./slack/notifySlackLinks";
+import { notifySlackLinks } from "@/lib/slack/notifySlackLinks";
 import { tLog } from "./utils/tLog";
 if (import.meta.main) {
   await tLog("updateOutdatedPullsTemplates", updateOutdatedPullsTemplates);
+  if (isCI) process.exit(0);
 }
 
 export async function updateOutdatedPullsTemplates() {
-  const pyproject = await readTemplate("add-toml.md");
-  const publishcr = await readTemplate("add-action.md");
-  const toml = await readTemplate("update-toml-license.md");
-  const outdated_toml = await readTemplate("outdated/update-toml-license.md");
-  const outdated_pyproject = await readTemplate("outdated/add-toml.md");
-  const outdated_publishcr = await readTemplate("outdated/add-action.md");
-  const outdated_pyproject_v2 = await readTemplate("outdated/add-toml-v2.md");
-  const outdated_publishcr_v2 = await readTemplate("outdated/add-action-v2.md");
-  const outdated_pyproject_v3 = await readTemplate("outdated/add-toml-v3.md");
-  const outdated_publishcr_v3 = await readTemplate("outdated/add-action-v3.md");
-  const outdated_pyproject_v4 = await readTemplate("outdated/add-toml-v4.md");
+  const pyproject = await readTemplate("./templates/add-toml.md");
+  const publishcr = await readTemplate("./templates/add-action.md");
+  // update toml license
+  const toml = await readTemplate("./templates/update-toml-license.md");
+  const outdated_toml = await readTemplate("./templates/outdated/update-toml-license.md");
+  // publishcr
+  const outdated_publishcr_templates = await sflow(
+    await glob("./templates/outdated/add-action*.md"),
+  )
+    .map((file) => readTemplate(file))
+    .toArray();
+  // pyproject
+  const outdated_pyproject_templates = await sflow(await glob("./templates/outdated/add-toml*.md"))
+    .map((file) => readTemplate(file))
+    .toArray();
+
   const outdateTitles = [
     outdated_toml.title,
-    outdated_pyproject.title,
-    outdated_publishcr.title,
-    outdated_pyproject_v2.title,
-    outdated_publishcr_v2.title,
-    outdated_pyproject_v3.title,
-    outdated_publishcr_v3.title,
-    outdated_pyproject_v4.title,
+    ...outdated_publishcr_templates.map((e) => e.title),
+    ...outdated_pyproject_templates.map((e) => e.title),
   ];
   const outdateBodies = [
     outdated_toml.body,
-    outdated_pyproject.body,
-    outdated_publishcr.body,
-    outdated_pyproject_v2.body,
-    outdated_publishcr_v2.body,
-    outdated_pyproject_v3.body,
-    outdated_publishcr_v3.body,
-    outdated_pyproject_v4.body,
+    ...outdated_publishcr_templates.map((e) => e.body),
+    ...outdated_pyproject_templates.map((e) => e.body),
   ];
-
+  console.log("Outdated PR templates: ", outdateTitles.length);
   // const templateOutdate = new Date("2024-06-13T09:02:56.630Z");
 
   await CNRepos.createIndex({
@@ -62,9 +63,9 @@ export async function updateOutdatedPullsTemplates() {
   });
   return await pMap(
     CNRepos.find(
-      $filaten({
+      $flatten({
         crPulls: {
-          mtime: $fresh("1h"), // retry if update fails
+          // mtime: $fresh("1h"), // retry if update fails
           data: $elemMatch({
             edited: {
               mtime: $stale("30m"), // retry if update fails
@@ -72,7 +73,7 @@ export async function updateOutdatedPullsTemplates() {
               error: { $nin: ["up to date", "body mismatch"] },
             },
             pull: {
-              user: { login: ghUser.login },
+              user: { login: (await ghUser()).login },
               title: { $in: outdateTitles },
               body: { $in: outdateBodies },
             },
@@ -96,9 +97,9 @@ export async function updateOutdatedPullsTemplates() {
       //   },
       // }),
     ),
-    async (repo) => {
+    async (repo, i) => {
       const { repository } = repo;
-      console.log("Updating PR templates for: " + repository);
+      console.log(i + " Updating PR templates for: " + repository);
       const crPulls = match(repo.crPulls)
         .with($OK, (e) => e.data)
         .otherwise(() => null)!;
@@ -106,28 +107,37 @@ export async function updateOutdatedPullsTemplates() {
       // edit CRPulls templates to latest
       const crPullsEdited = await pMap(
         crPulls,
-        async (data, i): Promise<CRPull> => {
+        async (data, pullIndex): Promise<CRPull> => {
           const { pull, type } = data;
           const { number } = pull;
-          if (pull.user.login !== ghUser.login) DIE("not editable");
+          if (pull.user.login !== (await ghUser()).login) DIE("not editable");
           const replacement = match(pull)
             .with(pyproject, () => DIE("Is already latest, should never happen here"))
             .with(publishcr, () => DIE("Is already latest, should never happen here"))
             .with(outdated_toml, () => toml)
-            .with(outdated_pyproject, () => pyproject)
-            .with(outdated_pyproject_v2, () => pyproject)
-            .with(outdated_pyproject_v3, () => pyproject)
-            .with(outdated_pyproject_v4, () => pyproject)
-            .with(outdated_publishcr, () => publishcr)
-            .with(outdated_publishcr_v2, () => publishcr)
-            .with(outdated_publishcr_v3, () => publishcr)
+            .when(
+              (pull) =>
+                outdated_pyproject_templates
+                  .map((e) => stableStringify(e))
+                  .includes(stableStringify({ title: pull.title, body: pull.body })),
+              () => pyproject,
+            )
+            .when(
+              (pull) =>
+                outdated_publishcr_templates
+                  .map((e) => stableStringify(e))
+                  .includes(stableStringify({ title: pull.title, body: pull.body })),
+              () => publishcr,
+            )
             // in case author clicked some task as completed, body will be different, may cause template mismatch
             .otherwise(() => DIE("Template not found: " + pull.title));
           if (!replacement) return { ...data, edited: TaskError("Template mismatch") };
+          const diff = gitDiff(pull.body!, replacement.body);
+          console.log(diff);
 
           const edited = await gh.issues
             .update({
-              ...parseUrlRepoOwner(repository),
+              ...parseGithubRepoUrl(repository),
               issue_number: number,
               ...(pull.title === replacement.title ? {} : { title: replacement.title }),
               ...(pull.body === replacement.body ? {} : { body: replacement.body }),
@@ -137,11 +147,14 @@ export async function updateOutdatedPullsTemplates() {
             .catch(TaskError);
           const updatedPull = (
             await gh.pulls.get({
-              ...parseUrlRepoOwner(repository),
+              ...parseGithubRepoUrl(repository),
               pull_number: number,
             })
           ).data;
-          await CNRepos.updateOne({ repository }, { $set: { [`crPulls.data.${i}.edited`]: edited } });
+          await CNRepos.updateOne(
+            { repository },
+            { $set: { [`crPulls.data.${pullIndex}.edited`]: edited } },
+          );
           return { pull: parsePull(updatedPull), type, edited };
         },
         { concurrency: 2 },

package/src/updateSlackMessages.ts

@@ -1,14 +1,17 @@
 import { $fresh, $stale } from "@/packages/mongodb-pipeline-ts";
-import { $filaten } from "@/packages/mongodb-pipeline-ts/$filaten";
+import { $flatten } from "@/packages/mongodb-pipeline-ts/$flatten";
 import pMap from "p-map";
 import { postSlackMessage } from "./postSlackMessage";
-import { SlackMsgs } from "./slack/SlackMsgs";
+import { SlackMsgs } from "@/lib/slack/SlackMsgs";
 
+/**
+ * @deprecated use upsertSlackMessage instead
+ */
 export async function updateSlackMessages() {
   // send
   return await pMap(
     SlackMsgs.find(
-      $filaten({
+      $flatten({
         $or: [
           { status: { $exists: false } },
           {
@@ -32,7 +35,8 @@ export async function updateSlackMessages() {
         }
       }
       console.log("SLACK POST MSG: " + JSON.stringify(text));
-      if (silent) return await SlackMsgs.updateOne({ _id }, { $set: { status: "sent", mtime: new Date() } });
+      if (silent)
+        return await SlackMsgs.updateOne({ _id }, { $set: { status: "sent", mtime: new Date() } });
       const sent = await postSlackMessage(text)
         .then((e) => ({ ...e, error: undefined, status: "sent" as const }))
         .catch((e) => ({ error: e.message ?? String(e), status: "error" as const }));

package/tailwind.config.ts

@@ -2,7 +2,12 @@ import type { Config } from "tailwindcss";
 
 const config = {
   darkMode: ["class"],
-  content: ["./pages/**/*.{ts,tsx}", "./components/**/*.{ts,tsx}", "./app/**/*.{ts,tsx}", "./src/**/*.{ts,tsx}"],
+  content: [
+    "./pages/**/*.{ts,tsx}",
+    "./components/**/*.{ts,tsx}",
+    "./app/**/*.{ts,tsx}",
+    "./src/**/*.{ts,tsx}",
+  ],
   prefix: "",
   theme: {
     container: {
@@ -70,6 +75,7 @@ const config = {
     },
   },
   plugins: [require("daisyui"), require("tailwindcss-animate")],
+  daisyui: { logs: false },
 } satisfies Config;
 
 export default config;

package/next-env.d.ts

@@ -1,5 +0,0 @@
-/// <reference types="next" />
-/// <reference types="next/image-types/global" />
-
-// NOTE: This file should not be edited
-// see https://nextjs.org/docs/basic-features/typescript for more information.

package/src/GIT_USEREMAIL.ts

@@ -1,5 +0,0 @@
-import DIE from "@snomiao/die";
-import { ghUser } from "./ghUser";
-
-export const GIT_USEREMAIL =
-  process.env.GIT_USEREMAIL || (ghUser.email && ghUser.email) || DIE("Missing env.GIT_USEREMAIL");

package/src/GIT_USERNAME.ts

@@ -1,9 +0,0 @@
-import DIE from "@snomiao/die";
-import { GIT_USEREMAIL } from "./GIT_USEREMAIL";
-import { ghUser } from "./ghUser";
-
-export const GIT_USERNAME =
-  process.env.GIT_USERNAME || (ghUser.email && ghUser.name) || DIE("Missing env.GIT_USERNAME");
-
-// read env/parameters
-console.log(`GIT COMMIT USER: ${GIT_USERNAME} <${GIT_USEREMAIL}>`);

package/src/clone_modify_push_Branches.ts

@@ -1,21 +0,0 @@
-import { makePublishcrBranch } from "./makePublishBranch";
-import { makePyprojectBranch } from "./makeTomlBranch";
-
-export async function clone_modify_push_Branches(upstreamUrl: string, forkUrl: string) {
-  return (
-    await Promise.all([
-      makePyprojectBranch(upstreamUrl, forkUrl),
-      makePublishcrBranch(upstreamUrl, forkUrl),
-      // makeLicenseUpdateBranch(upstreamUrl, forkUrl),
-    ])
-  )
-    .flatMap((e) => (e ? [e] : []))
-    .map(({ body, branch, title, type }) => ({
-      body,
-      branch,
-      title,
-      type,
-      srcUrl: forkUrl,
-      dstUrl: upstreamUrl,
-    }));
-}

package/src/tomlFillDescription.ts

@@ -1,17 +0,0 @@
-import { writeFile, readFile } from "fs/promises";
-import DIE from "@snomiao/die";
-import toml from "toml";
-import { fetchRepoDescriptionMap } from "./fetchRepoDescriptionMap";
-
-export async function tomlFillDescription(referenceUrl: string, pyprojectToml: string) {
-  const repoDescriptionMap = await fetchRepoDescriptionMap();
-  const matchedDescription = repoDescriptionMap[referenceUrl]?.toString() ||
-    DIE("Warn: missing description for " + referenceUrl);
-  const replaced = (await readFile(pyprojectToml, "utf8")).replace(
-    `description = ""`,
-    `description = ${JSON.stringify(matchedDescription)}`
-  );
-  // check validity
-  toml.parse(replaced);
-  await writeFile(pyprojectToml, replaced);
-}