comfy-pr 0.2.20 → 0.2.24

package/src/updateAuthorsForGithub.ts

@@ -0,0 +1,50 @@
+import { $OK, TaskDataOrNull, TaskError, TaskOK } from "@/packages/mongodb-pipeline-ts/Task";
+import DIE from "@snomiao/die";
+import console from "console";
+import { peekYaml } from "peek-log";
+import { snoflow } from "snoflow";
+import { Authors, GithubUsers } from "./Authors";
+import { $stale } from "./db";
+import { gh } from "./gh";
+
+export async function updateAuthorsForGithub() {
+  await snoflow(Authors.find({ githubMtime: $stale("7d") }))
+    .map((e) => e.githubId)
+    .filter()
+    .pMap(2, async (username) => {
+      const cached = await GithubUsers.findOne({ username, mtime: $stale("1d"), ...$OK });
+      if (cached) return cached;
+      console.log("Fetching github user " + username);
+      const result = await gh.users
+        .getByUsername({ username })
+        .then((e) => e.data)
+        .then(TaskOK)
+        .catch(TaskError);
+      return (
+        (await GithubUsers.findOneAndUpdate(
+          { username },
+          { $set: result },
+          { returnDocument: "after", upsert: true }
+        )) ?? DIE(`fail to udpate gh users`)
+      );
+    })
+    .map((e) => TaskDataOrNull(e))
+    .filter()
+    .map(({ email, avatar_url, blog, updated_at, location, company, hireable, bio, login }) => Authors.findOneAndUpdate(
+      { githubId: login },
+      {
+        $set: { githubMtime: new Date(), ...(email && { email }), ...(null != hireable && { hireable }) },
+        $addToSet: {
+          ...(bio && { bios: bio }),
+          avatars: avatar_url,
+          ...(location && { locations: location }),
+          ...(blog && { blogs: blog }),
+          ...(company && { companies: company }),
+        },
+      },
+      { upsert: true, returnDocument: "after" }
+    )
+    )
+    .peek(peekYaml)
+    .done();
+}

package/src/updateAuthorsFromCNRepo.ts

@@ -0,0 +1,30 @@
+import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
+import { snoflow } from "snoflow";
+import { Authors } from "./Authors";
+import { CNRepos } from "./CNRepos";
+import { $filaten } from "./db";
+
+/** Update authors for gh users, collecting emails/username/hireable */
+export async function updateAuthorsFromCNRepo() {
+  return await snoflow(
+    $pipeline(CNRepos)
+      .match($filaten({ info: { data: { owner: { login: { $exists: true } } } } }))
+      .group({
+        _id: "$info.data.owner.login",
+        // author: "$info.data.owner.login",
+        cm: { $sum: { $cond: [{ $eq: [{ $type: "$cm" }, "missing"] }, 0, 1] } },
+        cr: { $sum: { $cond: [{ $eq: [{ $type: "$cr" }, "missing"] }, 0, 1] } },
+
+        // TODO: get totals open/closed/merged
+        // pulls:{
+        //   OPEN: {"$crPulls.data.pull.prState", "open"}
+        // }
+        All: { $sum: 1 },
+      })
+      .set({ login: "$_id" })
+      .project({ _id: 0 })
+      .aggregate()
+  )
+    .peek(({ login, ...$set }) => Authors.updateOne({ login }, { $set }, { upsert: true }))
+    .done();
+}

package/src/updateCMNodesDuplicationWarnings.ts

@@ -1,56 +1,56 @@
-import { unary } from "lodash-es";
-import pMap from "p-map";
-import { dissoc, filter, groupBy, map, prop, toPairs } from "rambda";
-import YAML from "yaml";
-import { CMNodes, type CMNode } from "./CMNodes";
-import { notifySlack } from "./slack/notifySlack";
-
-export async function updateCMNodesDuplicationWarnings(nodes: CMNode[]) {
-  console.log("CMNodes checking duplicates");
-  // prettier-ignore
-  const dups = {
-    ID: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.id, nodes)),
-    TITLE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.title, nodes)),
-    REFERENCE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.reference, nodes)),
-  };
-  const dupsSummary = JSON.stringify(map((x) => map((x) => x.length, x), dups));
-  await notifySlack(
-    `[WARN] CMNodes duplicates: ${dupsSummary}\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json`,
-  );
-
-  await pMap(
-    toPairs(dups),
-    async ([topic, nodes]) =>
-      await pMap(
-        toPairs(nodes),
-        async ([key, nodesRaw]) => {
-          const nodes = nodesRaw.map(unary(dissoc("hash")));
-          const hashes = nodesRaw.map(prop("hash"));
-          // check sent
-          const someDuplicateSent = await CMNodes.findOne({
-            hash: { $in: hashes },
-            [`duplicated.${topic}`]: { $exists: true },
-          });
-          if (someDuplicateSent) return;
-          // send slack notification
-          const slackNotification = await notifySlack(
-            `[ACTION NEEDED WARNING]: please resolve duplicated node in ${topic}: ${key}\n` +
-              "```\n" +
-              YAML.stringify(nodes) +
-              "```" +
-              "\n\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json",
-            { unique: true },
-          );
-          // mark duplicates
-          await CMNodes.updateMany(
-            { hash: { $in: hashes } },
-            {
-              $set: { [`duplicated.${topic}`]: { hashes, slackNotification } },
-            },
-          );
-        },
-        { concurrency: 2 },
-      ),
-    { concurrency: 2 },
-  );
-}
+import { unary } from "lodash-es";
+import pMap from "p-map";
+import { dissoc, filter, groupBy, map, prop, toPairs } from "rambda";
+import YAML from "yaml";
+import { CMNodes, type CMNode } from "./CMNodes";
+import { notifySlack } from "./slack/notifySlack";
+
+export async function updateCMNodesDuplicationWarnings(nodes: CMNode[]) {
+  console.log("CMNodes checking duplicates");
+  // prettier-ignore
+  const dups = {
+    ID: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.id, nodes)),
+    TITLE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.title, nodes)),
+    REFERENCE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.reference, nodes)),
+  };
+  const dupsSummary = JSON.stringify(map((x) => map((x) => x.length, x), dups));
+  await notifySlack(
+    `[WARN] CMNodes duplicates: ${dupsSummary}\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json`,
+  );
+
+  await pMap(
+    toPairs(dups),
+    async ([topic, nodes]) =>
+      await pMap(
+        toPairs(nodes),
+        async ([key, nodesRaw]) => {
+          const nodes = nodesRaw.map(unary(dissoc("hash")));
+          const hashes = nodesRaw.map(prop("hash"));
+          // check sent
+          const someDuplicateSent = await CMNodes.findOne({
+            hash: { $in: hashes },
+            [`duplicated.${topic}`]: { $exists: true },
+          });
+          if (someDuplicateSent) return;
+          // send slack notification
+          const slackNotification = await notifySlack(
+            `[ACTION NEEDED WARNING]: please resolve duplicated node in ${topic}: ${key}\n` +
+              "```\n" +
+              YAML.stringify(nodes) +
+              "```" +
+              "\n\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json",
+            { unique: true },
+          );
+          // mark duplicates
+          await CMNodes.updateMany(
+            { hash: { $in: hashes } },
+            {
+              $set: { [`duplicated.${topic}`]: { hashes, slackNotification } },
+            },
+          );
+        },
+        { concurrency: 2 },
+      ),
+    { concurrency: 2 },
+  );
+}

package/src/updateCMRepos.ts

@@ -1,27 +1,27 @@
-import pMap from "p-map";
-import { CMNodes } from "./CMNodes";
-import { CNRepos } from "./CNRepos";
-import { tLog } from "./utils/tLog";
-if (import.meta.main) {
-  await tLog("Update Repos from ComfyUI Manager", updateCMRepos);
-  console.log(await CMNodes.estimatedDocumentCount());
-}
-export async function updateCMRepos() {
-  await CMNodes.createIndex({ repo_id: 1 });
-  return await pMap(CMNodes.find({ repo_id: { $exists: false } }), async (cm, i) => {
-    const { reference: repository, _id } = cm;
-    return await CNRepos.updateOne(
-      { repository },
-      {
-        $set: {
-          cm: (await CMNodes.findOneAndUpdate(
-            { _id },
-            { $set: { repo_id: cm._id } },
-            { upsert: true, returnDocument: "after" },
-          ))!,
-        },
-      },
-      { upsert: true },
-    );
-  });
-}
+import pMap from "p-map";
+import { CMNodes } from "./CMNodes";
+import { CNRepos } from "./CNRepos";
+import { tLog } from "./utils/tLog";
+if (import.meta.main) {
+  await tLog("Update Repos from ComfyUI Manager", updateCMRepos);
+  console.log(await CMNodes.estimatedDocumentCount());
+}
+export async function updateCMRepos() {
+  await CMNodes.createIndex({ repo_id: 1 });
+  return await pMap(CMNodes.find({ repo_id: { $exists: false } }), async (cm, i) => {
+    const { reference: repository, _id } = cm;
+    return await CNRepos.updateOne(
+      { repository },
+      {
+        $set: {
+          cm: (await CMNodes.findOneAndUpdate(
+            { _id },
+            { $set: { repo_id: cm._id } },
+            { upsert: true, returnDocument: "after" },
+          ))!,
+        },
+      },
+      { upsert: true },
+    );
+  });
+}

package/src/updateCNRepos.ts

@@ -1,58 +1,58 @@
-import { updateCMNodes } from "./CMNodes";
-import { getWorkerInstance } from "./WorkerInstances";
-import { createComfyRegistryPRsFromCandidates } from "./createComfyRegistryPRsFromCandidates";
-import { notifySlack } from "./slack/notifySlack";
-import { updateCMRepos } from "./updateCMRepos";
-import { updateCNReposCRPullsComments } from "./updateCNReposCRPullsComments";
-import { updateCNReposInfo } from "./updateCNReposInfo";
-import { updateCNReposPRCandidate } from "./updateCNReposPRCandidate";
-import { updateCNReposPulls } from "./updateCNReposPulls";
-import { updateCNRepoPullsDashboard } from "./updateCNReposPullsDashboard";
-import { updateCNReposRelatedPulls } from "./updateCNReposRelatedPulls";
-import { updateCRNodes } from "./updateCRNodes";
-import { updateCRRepos } from "./updateCRRepos";
-import { updateComfyTotals } from "./updateComfyTotals";
-import { updateOutdatedPullsTemplates } from "./updateOutdatedPullsTemplates";
-import { tLog } from "./utils/tLog";
-
-if (import.meta.main) {
-  await getWorkerInstance("Updating CNRepos");
-  // await cacheHealthReport();
-  await updateCNRepos();
-  // updateCNReposPRTasks
-  // await scanCNRepoThenPRs();
-  // await pMap(candidates, (e) => updateCNRepoPRStatus(e.repository), { concurrency: 4 });
-  // candidates
-}
-
-export async function updateCNRepos() {
-  await Promise.all([
-    tLog("Report Worker Status", async () => {
-      const worker = await getWorkerInstance("ComfyPR Bot Running");
-      const workerInfo = `${worker.geo.countryCode}/${worker.geo.region}/${worker.geo.city}`;
-      const msg = `COMFY-PR BOT RUNNING ${new Date().toISOString()}\nWorker: ${workerInfo}`;
-      return [await notifySlack(msg, { unique: true, silent: true })];
-    }),
-    // stage 1: get repos
-    await tLog("Update Nodes from ComfyUI Manager", updateCMNodes),
-    await tLog("Update Repos from ComfyUI Manager", updateCMRepos),
-    await tLog("Update Nodes from ComfyRegistry", updateCRNodes),
-    await tLog("Update Repos from ComfyRegistry", updateCRRepos),
-    // stage 2: update repo info & pulls
-    await tLog("Update CNRepos for Repo Infos", updateCNReposInfo),
-    await tLog("Update CNRepos for Github Pulls", updateCNReposPulls),
-    await tLog("Update Pulls Dashboard", updateCNRepoPullsDashboard),
-    // stage 3: update related pulls and comments
-    await tLog("Update CNRepos for Related Pulls", updateCNReposRelatedPulls),
-    await tLog("Update Outdated Pulls Templates", updateOutdatedPullsTemplates),
-    // stage 4: update related comments
-    await tLog("Update CNRepos for Related Comments", updateCNReposCRPullsComments),
-    // stage 5: mark and create PRs
-    await tLog("Update CNRepos PR Candidates", updateCNReposPRCandidate),
-    await tLog("Create ComfyRegistry PRs", createComfyRegistryPRsFromCandidates),
-    // final
-    await tLog("Update Comfy Totals", updateComfyTotals),
-  ]);
-
-  console.log("All repo updated");
-}
+import { updateCMNodes } from "./CMNodes";
+import { getWorkerInstance } from "./WorkerInstances";
+import { createComfyRegistryPRsFromCandidates } from "./createComfyRegistryPRsFromCandidates";
+import { notifySlack } from "./slack/notifySlack";
+import { updateCMRepos } from "./updateCMRepos";
+import { updateCNReposCRPullsComments } from "./updateCNReposCRPullsComments";
+import { updateCNReposInfo } from "./updateCNReposInfo";
+import { updateCNReposPRCandidate } from "./updateCNReposPRCandidate";
+import { updateCNReposPulls } from "./updateCNReposPulls";
+import { updateCNRepoPullsDashboard } from "./updateCNReposPullsDashboard";
+import { updateCNReposRelatedPulls } from "./updateCNReposRelatedPulls";
+import { updateCRNodes } from "./updateCRNodes";
+import { updateCRRepos } from "./updateCRRepos";
+import { updateComfyTotals } from "./updateComfyTotals";
+import { updateOutdatedPullsTemplates } from "./updateOutdatedPullsTemplates";
+import { tLog } from "./utils/tLog";
+
+if (import.meta.main) {
+  await getWorkerInstance("Updating CNRepos");
+  // await cacheHealthReport();
+  await updateCNRepos();
+  // updateCNReposPRTasks
+  // await scanCNRepoThenPRs();
+  // await pMap(candidates, (e) => updateCNRepoPRStatus(e.repository), { concurrency: 4 });
+  // candidates
+}
+
+export async function updateCNRepos() {
+  await Promise.all([
+    tLog("Report Worker Status", async () => {
+      const worker = await getWorkerInstance("ComfyPR Bot Running");
+      const workerInfo = `${worker.geo.countryCode}/${worker.geo.region}/${worker.geo.city}`;
+      const msg = `COMFY-PR BOT RUNNING ${new Date().toISOString()}\nWorker: ${workerInfo}`;
+      return [await notifySlack(msg, { unique: true, silent: true })];
+    }),
+    // stage 1: get repos
+    await tLog("Update Nodes from ComfyUI Manager", updateCMNodes),
+    await tLog("Update Repos from ComfyUI Manager", updateCMRepos),
+    await tLog("Update Nodes from ComfyRegistry", updateCRNodes),
+    await tLog("Update Repos from ComfyRegistry", updateCRRepos),
+    // stage 2: update repo info & pulls
+    await tLog("Update CNRepos for Repo Infos", updateCNReposInfo),
+    await tLog("Update CNRepos for Github Pulls", updateCNReposPulls),
+    await tLog("Update Pulls Dashboard", updateCNRepoPullsDashboard),
+    // stage 3: update related pulls and comments
+    await tLog("Update CNRepos for Related Pulls", updateCNReposRelatedPulls),
+    await tLog("Update Outdated Pulls Templates", updateOutdatedPullsTemplates),
+    // stage 4: update related comments
+    await tLog("Update CNRepos for Related Comments", updateCNReposCRPullsComments),
+    // stage 5: mark and create PRs
+    await tLog("Update CNRepos PR Candidates", updateCNReposPRCandidate),
+    await tLog("Create ComfyRegistry PRs", createComfyRegistryPRsFromCandidates),
+    // final
+    await tLog("Update Comfy Totals", updateComfyTotals),
+  ]);
+
+  console.log("All repo updated");
+}

package/src/updateCNReposCRPullsComments.ts

@@ -1,40 +1,40 @@
-import pMap from "p-map";
-import { CNRepos, type CRPull } from "./CNRepos";
-import { $fresh, $stale } from "./db";
-// import { $filaten } from "./db";
-// import { $pipeline } from "./db/$pipeline";
-import { $filaten } from "@/packages/mongodb-pipeline-ts/$filaten";
-import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
-import { TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
-import { getWorkerInstance } from "./WorkerInstances";
-import { fetchIssueComments } from "./gh/fetchIssueComments";
-
-if (import.meta.main) {
-  await getWorkerInstance("updateCNReposCRPullsComments");
-  await updateCNReposCRPullsComments();
-  console.log("All done");
-}
-
-export async function updateCNReposCRPullsComments() {
-  return await pMap(
-    $pipeline(CNRepos)
-      .unwind("$crPulls.data")
-      .match($filaten({ crPulls: { mtime: $fresh("7d"), data: { comments: { mtime: $stale("1d") } } } }))
-      .set($filaten({ "crPulls.data": { repository: "$repository" } }))
-      .replaceRoot({ newRoot: "$crPulls.data" })
-      .aggregate(),
-    async (data) => {
-      const { repository, pull } = data as unknown as { repository: string } & CRPull;
-      const html_url = pull.html_url;
-      const comments = await fetchIssueComments(repository, pull).then(TaskOK).catch(TaskError);
-      return [
-        await CNRepos.findOneAndUpdate(
-          $filaten({ repository, crPulls: { data: { pull: { html_url } } } }),
-          { $set: { "crPulls.data.$.comments": comments } },
-          { upsert: true, returnDocument: "after" },
-        ),
-      ];
-    },
-    { concurrency: 2 },
-  );
-}
+import pMap from "p-map";
+import { CNRepos, type CRPull } from "./CNRepos";
+import { $fresh, $stale } from "./db";
+// import { $filaten } from "./db";
+// import { $pipeline } from "./db/$pipeline";
+import { $filaten } from "@/packages/mongodb-pipeline-ts/$filaten";
+import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
+import { TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
+import { getWorkerInstance } from "./WorkerInstances";
+import { fetchIssueComments } from "./gh/fetchIssueComments";
+
+if (import.meta.main) {
+  await getWorkerInstance("updateCNReposCRPullsComments");
+  await updateCNReposCRPullsComments();
+  console.log("All done");
+}
+
+export async function updateCNReposCRPullsComments() {
+  return await pMap(
+    $pipeline(CNRepos)
+      .unwind("$crPulls.data")
+      .match($filaten({ crPulls: { mtime: $fresh("7d"), data: { comments: { mtime: $stale("1d") } } } }))
+      .set($filaten({ "crPulls.data": { repository: "$repository" } }))
+      .replaceRoot({ newRoot: "$crPulls.data" })
+      .aggregate(),
+    async (data) => {
+      const { repository, pull } = data as unknown as { repository: string } & CRPull;
+      const html_url = pull.html_url;
+      const comments = await fetchIssueComments(repository, pull).then(TaskOK).catch(TaskError);
+      return [
+        await CNRepos.findOneAndUpdate(
+          $filaten({ repository, crPulls: { data: { pull: { html_url } } } }),
+          { $set: { "crPulls.data.$.comments": comments } },
+          { upsert: true, returnDocument: "after" },
+        ),
+      ];
+    },
+    { concurrency: 2 },
+  );
+}

package/src/updateCNReposInfo.ts

@@ -1,57 +1,57 @@
-import pMap from "p-map";
-import { dissoc } from "rambda";
-import { match } from "ts-pattern";
-import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
-import { CNRepos } from "./CNRepos";
-import { getWorkerInstance } from "./WorkerInstances";
-import { $filaten, $stale } from "./db";
-import { gh } from "./gh";
-import { parseUrlRepoOwner } from "./parseOwnerRepo";
-import { tLog } from "./utils/tLog";
-
-if (import.meta.main) {
-  await getWorkerInstance("updateCNReposInfo");
-  await tLog("updateCNReposInfo", updateCNReposInfo);
-}
-
-export async function updateCNReposInfo() {
-  await CNRepos.createIndex($filaten({ info: { mtime: 1 } }));
-  return await pMap(
-    CNRepos.find($filaten({ info: { mtime: $stale("1d") } })),
-    async (repo) => {
-      const { repository } = repo;
-      console.log("[INFO] Fetching meta info from " + repository);
-      const _info = await gh.repos
-        .get({ ...parseUrlRepoOwner(repository) })
-        .then(({ data }) => data)
-        .then(TaskOK)
-        .catch(TaskError);
-      // Handle renamed repos
-      const { info, url } = await match(_info)
-        .with($OK, async (info) => {
-          const url = info.data.html_url;
-          if (url === repository) return { info, url: repository };
-
-          // console.log(info.data.use_squash_pr_title_as_default)
-          console.log("[INFO] Migrating renamed repo: \nfrom: ", repository + "\n  to: " + url);
-          // migrate data into new CNRepo
-          await CNRepos.updateOne(
-            { repository: url },
-            {
-              $set: {
-                ...dissoc("_id", { ...(await CNRepos.findOneAndDelete({ repository })) }),
-                repository: url,
-                oldUrls: { $addToSet: repository },
-              },
-            },
-            { upsert: true },
-          );
-          return { info, url };
-        })
-        .otherwise((info) => ({ info, url: repository }));
-
-      return await CNRepos.updateOne({ repository: url }, { $set: { info } }, { upsert: true });
-    },
-    { concurrency: 2 },
-  );
-}
+import pMap from "p-map";
+import { dissoc } from "rambda";
+import { match } from "ts-pattern";
+import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
+import { CNRepos } from "./CNRepos";
+import { getWorkerInstance } from "./WorkerInstances";
+import { $filaten, $stale } from "./db";
+import { gh } from "./gh";
+import { parseUrlRepoOwner } from "./parseOwnerRepo";
+import { tLog } from "./utils/tLog";
+
+if (import.meta.main) {
+  await getWorkerInstance("updateCNReposInfo");
+  await tLog("updateCNReposInfo", updateCNReposInfo);
+}
+
+export async function updateCNReposInfo() {
+  await CNRepos.createIndex($filaten({ info: { mtime: 1 } }));
+  return await pMap(
+    CNRepos.find($filaten({ info: { mtime: $stale("1d") } })),
+    async (repo) => {
+      const { repository } = repo;
+      console.log("[INFO] Fetching meta info from " + repository);
+      const _info = await gh.repos
+        .get({ ...parseUrlRepoOwner(repository) })
+        .then(({ data }) => data)
+        .then(TaskOK)
+        .catch(TaskError);
+      // Handle renamed repos
+      const { info, url } = await match(_info)
+        .with($OK, async (info) => {
+          const url = info.data.html_url;
+          if (url === repository) return { info, url: repository };
+
+          // console.log(info.data.use_squash_pr_title_as_default)
+          console.log("[INFO] Migrating renamed repo: \nfrom: ", repository + "\n  to: " + url);
+          // migrate data into new CNRepo
+          await CNRepos.updateOne(
+            { repository: url },
+            {
+              $set: {
+                ...dissoc("_id", { ...(await CNRepos.findOneAndDelete({ repository })) }),
+                repository: url,
+                oldUrls: { $addToSet: repository },
+              },
+            },
+            { upsert: true },
+          );
+          return { info, url };
+        })
+        .otherwise((info) => ({ info, url: repository }));
+
+      return await CNRepos.updateOne({ repository: url }, { $set: { info } }, { upsert: true });
+    },
+    { concurrency: 2 },
+  );
+}