comfy-pr 0.2.7

package/src/fetchCMNodes.ts

@@ -0,0 +1,22 @@
+import { fetchJson } from "./utils/fetchJson";
+
+if (import.meta.main) {
+  console.log(await fetchCMNodes());
+}
+export async function fetchCMNodes() {
+  const customNodeListSource =
+    process.env.CUSTOM_LIST_SOURCE ||
+    "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/custom-node-list.json";
+  const nodeList = (await fetchJson(customNodeListSource)) as {
+    custom_nodes: {
+      author: "Dr.Lt.Data" | string;
+      title: "ComfyUI-Manager" | string;
+      id: "manager" | string;
+      reference: "https://github.com/ltdrdata/ComfyUI-Manager" | string;
+      files: ["https://github.com/ltdrdata/ComfyUI-Manager"] | string[];
+      install_type: "git-clone" | string;
+      description: "ComfyUI-Manager itself is also a custom node." | string;
+    }[];
+  };
+  return nodeList.custom_nodes;
+}

package/src/fetchComfyRegistryNodes.ts

@@ -0,0 +1,11 @@
+import type { mockPublishedNodes } from "../mocks/mockPublishedNodes";
+import DIE from "@snomiao/die";
+import { fetchJson } from "./utils/fetchJson";
+
+export async function fetchCRNodes() {
+  const r = (await fetchJson<typeof mockPublishedNodes>(
+    "https://api.comfy.org/nodes?page=1&limit=99999999"
+  )) as typeof mockPublishedNodes;
+  r.totalPages === 1 || DIE("FAIL TO FETCH ALL NODES");
+  return r.nodes;
+}

package/src/fetchCurrentGeoInfo.ts

@@ -0,0 +1,6 @@
+import { fetchJson } from "./utils/fetchJson";
+export async function fetchCurrentGeoInfo() {
+  const { query, city, iat, lon, countryCode, region, regionName } = (await fetchJson("http://ip-api.com/json")) as any;
+  const geo = { ip: query, city, iat, lon, countryCode, region, regionName };
+  return geo;
+}

package/src/fetchRelatedPullWithComments.ts

@@ -0,0 +1,15 @@
+import pMap from "p-map";
+import { fetchIssueComments } from "./gh/fetchIssueComments";
+import { matchRelatedPulls } from "./matchRelatedPulls";
+import type { GithubPullParsed } from "./parsePullsState";
+import { summaryLastPullComment } from "./summaryLastPullComment";
+/** @deprecated */
+export async function fetchRelatedPullWithComments(repository: string, pulls: GithubPullParsed[]) {
+  const relatedPulls = await matchRelatedPulls(pulls);
+  const relatedPullsWithComment = await pMap(relatedPulls, async (data) => {
+    const comments = await fetchIssueComments(repository, data.pull);
+    const lastText = summaryLastPullComment(comments);
+    return { ...data, comments, lastText };
+  });
+  return relatedPullsWithComment;
+}

package/src/fetchRepoDescriptionMap.ts

@@ -0,0 +1,15 @@
+import { fromPairs } from "rambda";
+import { fetchCMNodes } from "./fetchCMNodes";
+
+
+export async function fetchRepoDescriptionMap() {
+  const nodeList = await fetchCMNodes();
+  const repoDescriptionMap = fromPairs(
+    nodeList.map((e) => [e.reference, e.description])
+  );
+  repoDescriptionMap["https://github.com/snomiao/ComfyNode-Registry-test"] =
+    "ComfyNode-Registry-test-description";
+
+  console.log("Fetched " + nodeList.length + " CustomNode descriptions");
+  return repoDescriptionMap;
+}

package/src/followRuleSchema.ts

@@ -0,0 +1,69 @@
+import { $before, $fresh, $stale } from "@/packages/mongodb-pipeline-ts/$fresh";
+import { tryCatch } from "rambda";
+import { z } from "zod";
+
+const znot_include = z.object({ "not-include": z.string() }).transform((x) => ({ $nin: x["not-include"] }));
+const mString = z.string().transform((x) =>
+  tryCatch<string, string | RegExp>(
+    (x) => new RegExp(x) as RegExp,
+    (x) => x as string | RegExp,
+  )(x),
+);
+const zbefore = z
+  .object({ $before: z.coerce.date().or(z.string()).or(z.number()) })
+  .transform((x) => $before(x.$before));
+const zfresh = z.object({ $fresh: z.coerce.date().or(z.string()).or(z.number()) }).transform((x) => $fresh(x.$fresh));
+const mstale = z.object({ $stale: z.coerce.date().or(z.string()).or(z.number()) }).transform((x) => $stale(x.$stale));
+const mDate = z.date().or(zbefore).or(zfresh).or(mstale);
+const mAny = z.number().or(znot_include).or(mString).or(zbefore).or(zfresh);
+const mNumber = z
+  .number()
+  .or(z.object({ $gt: z.number() }))
+  .or(z.object({ $lt: z.number() }))
+  .or(z.object({ $eq: z.number() }))
+  .or(z.object({ $ne: z.number() }))
+  .or(z.object({ $gte: z.number() }))
+  .or(z.object({ $lte: z.number() }));
+export const zAddCommentAction = z
+  .object({
+    by: z.string(),
+    body: z.string(),
+  })
+  .strict();
+const zFollowUpRule = z.object({
+  name: z.string(),
+  $match: z
+    .object({
+      state: z.enum(["OPEN", "CLOSED", "MERGED"]),
+      on_registry: z.boolean(),
+      updated_at: mDate,
+      lastwords: mString,
+      head: mString,
+      comments: mNumber,
+    })
+    .strict()
+    .partial(),
+  action: z
+    .object({
+      "add-comment": zAddCommentAction,
+      "send-email": z
+        .object({
+          provider: z.string(),
+          from: z.string(),
+          to: z.string(),
+          subject: z.string(),
+          body: z.string(),
+        })
+        .strict(), // WARN: not implementd
+      "update-issue": z
+        .object({
+          tags: mAny,
+        })
+        .strict(),
+      // close: z.any(),
+    })
+    .partial()
+    .strict(),
+});
+// zPullsStatus
+export const zFollowUpRules = zFollowUpRule.array();

package/src/getActivatedShell.ts

@@ -0,0 +1,18 @@
+import { $ as zx } from "zx";
+import { getActivateCMD } from "./cli/getActivateCMD";
+
+if (import.meta.main) {
+  // await checkComfyActivated();
+  zx.verbose = true;
+  const $ = getActivatedShell();
+  const p = await $`comfy-cli --version`;
+  console.log(p.stdout);
+  // zx({ prefix:  })`comfy-cli --help`;
+}
+
+export function getActivatedShell() {
+  const activate = getActivateCMD();
+  return zx({
+    prefix: `echo Comfy CLI version: $(comfy-cli --version) || (apt-get install -y python3-venv && python -m venv .venv && ${activate} && pip install comfy-cli); `,
+  });
+}

package/src/getBranchWorkingDir.ts

@@ -0,0 +1,16 @@
+import { rm } from "fs/promises";
+import { getRepoWorkingDir } from "./getRepoWorkingDir";
+import { parseUrlRepoOwner } from "./parseOwnerRepo";
+
+export async function getBranchWorkingDir(
+  upstreamUrl: string,
+  forkUrl: string,
+  branch: string
+) {
+  const src = parseUrlRepoOwner(upstreamUrl);
+  const dir = getRepoWorkingDir(forkUrl);
+  const packageName = src.repo;
+  const cwd = `${dir}/${branch}/${packageName}`;
+  await rm(cwd, { recursive: true }).catch(() => null);
+  return cwd;
+}

package/src/getRepoWorkingDir.ts

@@ -0,0 +1,6 @@
+import { parseUrlRepoOwner } from "./parseOwnerRepo";
+
+export function getRepoWorkingDir(forkUrl: string) {
+  return `prs/${parseUrlRepoOwner(forkUrl).repo}`;
+}
+  

package/src/ghUser.ts

@@ -0,0 +1,6 @@
+import { gh } from "./gh";
+
+export const ghUser = (await gh.users.getAuthenticated()).data;
+
+console.log("Fetch Current Github User...");
+console.log(`Current Github User: ${ghUser.login} <${ghUser.email}>`);

package/src/index.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env bun
+import "dotenv/config";
+import { $ as zx } from "zx";
+import { checkComfyActivated } from "./checkComfyActivated";
+import { initializeFollowRules } from "./initializeFollowRules";
+import { updateCNRepos } from "./updateCNRepos";
+import { runFollowRuleSet } from "./updateFollowRuleSet";
+import { updateSlackMessages } from "./updateSlackMessages";
+
+if (import.meta.main) {
+  zx.verbose = true;
+  await Promise.all([
+    // try send msgs that didn't send in last run
+    updateSlackMessages(),
+    checkComfyActivated(), // needed if make pr
+    initializeFollowRules(),
+    runFollowRuleSet(),
+    updateCNRepos(),
+  ]);
+  console.log("All done");
+  process.exit(0);
+}

package/src/initializeFollowRules.ts

@@ -0,0 +1,26 @@
+import DIE from "@snomiao/die";
+import { readFile } from "fs/promises";
+import { FollowRuleSets } from "./FollowRules";
+
+if (import.meta.main) {
+  const followRules = await initializeFollowRules();
+  const ruleset = (await FollowRuleSets.findOne({ name: "default" })) ?? DIE("default ruleset not found");
+  if (!ruleset?.enabled) DIE("default ruleset not enabled");
+
+  console.log("initialized follow rules");
+}
+/**
+ *
+ * @author: snomiao <snomiao@gmail.com>
+ */
+export async function initializeFollowRules() {
+  return await updateFollowRules("default", await readFile("./templates/follow-rules.yaml", "utf8"));
+}
+
+export async function updateFollowRules(name: string, rawRuleYaml: string) {
+  return (await FollowRuleSets.findOneAndUpdate(
+    { name },
+    { $setOnInsert: { yaml: rawRuleYaml, enabled: false } },
+    { upsert: true, returnDocument: "after" },
+  ))!;
+}

package/src/makePublishBranch.ts

@@ -0,0 +1,58 @@
+import { readFile } from "fs/promises";
+import { dirname } from "path";
+import { GIT_USEREMAIL } from "./GIT_USEREMAIL";
+import { GIT_USERNAME } from "./GIT_USERNAME";
+import { $ } from "./cli/echoBunShell";
+import { getBranchWorkingDir } from "./getBranchWorkingDir";
+import { gh } from "./gh";
+import { parseUrlRepoOwner, stringifyGithubOrigin, stringifyGithubRepoUrl } from "./parseOwnerRepo";
+import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
+
+/**
+ * Clone from upstream
+ * push to fork url
+ * @param dir
+ * @param upstreamUrl
+ * @param origin
+ * @returns
+ */
+export async function makePublishcrBranch(upstreamUrl: string, forkUrl: Readonly<string>) {
+  const type = "publishcr" as const;
+
+  const origin = await stringifyGithubOrigin(parseUrlRepoOwner(forkUrl));
+  const branch = "publish";
+  const tmpl = await readFile("./templates/add-action.md", "utf8");
+  const { title, body } = parseTitleBodyOfMarkdown(tmpl);
+  const repo = parseUrlRepoOwner(origin);
+
+  if (await gh.repos.getBranch({ ...repo, branch }).catch(() => null)) {
+    console.log("Skip changes as branch existed: " + branch);
+    return { type, title, body, branch };
+  }
+
+  const cwd = await getBranchWorkingDir(upstreamUrl, forkUrl, branch);
+
+  const file = `${cwd}/.github/workflows/publish.yml`;
+  const publishYmlPath = "./templates/publish.yaml";
+
+  // commit & push changes
+  await $`
+git clone ${upstreamUrl} ${cwd}
+
+mkdir -p ${dirname(file)}
+cat ${publishYmlPath} > ${file}
+
+cd ${cwd}
+
+git config user.name ${GIT_USERNAME} && \
+git config user.email ${GIT_USEREMAIL} && \
+git checkout -b ${branch} && \
+git add . && \ 
+git commit -am "chore(${branch}): ${title}" && \
+git push "${origin}" ${branch}:${branch}
+    `;
+
+  const branchUrl = `${stringifyGithubRepoUrl(repo)}/tree/${branch}`;
+  console.log(`Branch Push OK: ${branchUrl}`);
+  return { type, title, body, branch };
+}

package/src/makeTomlBranch.ts

@@ -0,0 +1,53 @@
+import { readFile } from "fs/promises";
+import { GIT_USEREMAIL } from "./GIT_USEREMAIL";
+import { GIT_USERNAME } from "./GIT_USERNAME";
+import { $ } from "./cli/echoBunShell";
+import { getBranchWorkingDir } from "./getBranchWorkingDir";
+import { gh } from "./gh";
+import { parseUrlRepoOwner, stringifyGithubOrigin } from "./parseOwnerRepo";
+import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
+import { tomlFillDescription } from "./tomlFillDescription";
+
+export async function makePyprojectBranch(upstreamUrl: string, forkUrl: string) {
+  const type = "pyproject" as const;
+  const origin = await stringifyGithubOrigin(parseUrlRepoOwner(forkUrl));
+  const branch = "pyproject";
+  const tmpl = await readFile("./templates/add-toml.md", "utf8");
+  const { title, body } = parseTitleBodyOfMarkdown(tmpl);
+  const repo = parseUrlRepoOwner(forkUrl);
+
+  if (await gh.repos.getBranch({ ...repo, branch }).catch(() => null)) {
+    console.log("Skip changes as branch existed: " + branch);
+    return { type, title, body, branch };
+  }
+  const src = parseUrlRepoOwner(upstreamUrl);
+  const cwd = await getBranchWorkingDir(upstreamUrl, forkUrl, branch);
+
+  // commit changes
+  await $`
+git clone ${upstreamUrl} ${cwd}
+
+cd ${cwd}
+echo N | comfy node init
+`;
+
+  // Try fill description from ComfyUI-manager
+  const referenceUrl = `https://github.com/${src.owner}/${src.repo}`;
+  const pyprojectToml = cwd + "/pyproject.toml";
+  await tomlFillDescription(referenceUrl, pyprojectToml).catch((e) => {
+    console.error(e);
+  });
+
+  await $`
+cd ${cwd}
+git config user.name ${GIT_USERNAME} && \
+git config user.email ${GIT_USEREMAIL} && \
+git checkout -b ${branch} && \
+git add . && \
+git commit -am ${`chore(${branch}): ${title}`} && \
+git push "${origin}" ${branch}:${branch}
+`;
+  const branchUrl = `https://github.com/${repo.owner}/${repo.repo}/tree/${branch}`;
+  console.log(`Branch Push OK: ${branchUrl}`);
+  return { type, title, body, branch };
+}

package/src/matchRelatedPulls.ts

@@ -0,0 +1,25 @@
+import pMap from "p-map";
+import { match } from "ts-pattern";
+import { fetchRelatedPullWithComments } from "./fetchRelatedPullWithComments";
+import type { GithubPullParsed } from "./parsePullsState";
+import { readTemplateTitle } from "./readTemplateTitle";
+
+export type RelatedPullsWithComments = Awaited<ReturnType<typeof fetchRelatedPullWithComments>>;
+export type RelatedPull = Awaited<ReturnType<typeof matchRelatedPulls>>[number];
+export async function matchRelatedPulls(pulls: GithubPullParsed[]) {
+  const pyproject = await readTemplateTitle("add-toml.md");
+  const publishcr = await readTemplateTitle("add-action.md");
+  const relatedPulls = await pMap(pulls, async (pull) =>
+    match(pull)
+      .with({ title: pyproject }, (pull) => ({
+        type: "pyproject" as const,
+        pull,
+      }))
+      .with({ title: publishcr }, (pull) => ({
+        type: "publishcr" as const,
+        pull,
+      }))
+      .otherwise(() => null),
+  );
+  return relatedPulls.flatMap((e) => (e ? [e] : []));
+}

package/src/muteInstances.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env bun
+import { readFile, writeFile } from "fs/promises";
+import { globby } from "globby";
+import pMap from "p-map";
+if (import.meta.main) {
+  const files = await globby("node_modules/**/*instance*.js");
+  await pMap(files, async (file) => {
+    const x = await readFile(file, "utf8");
+    const y = x.replace(/console.warn/, ";");
+    if (x !== y) {
+      await writeFile(file, y);
+      console.log("muted: " + file);
+    }
+  });
+  console.log("done");
+}

package/src/parseIssueUrl.ts

@@ -0,0 +1,6 @@
+export function parseIssueUrl(issueUrl: string) {
+  const [owner, repo, strNumber] = issueUrl
+    .match(/^https:\/\/github\.com\/([\w-]+)\/([\w-]+)\/(?:pull|issues)\/(\d+)$/)!
+    .slice(1);
+  return { owner, repo, issue_number: Number(strNumber) };
+}

package/src/parseOwnerRepo.ts

@@ -0,0 +1,33 @@
+import { basename, dirname } from "path";
+
+/**
+ * Parse owner and repo obj
+ * @param gitUrl git@github.ocm:owner/repo or https://github.ocm/owner/repo
+ */
+export function parseUrlRepoOwner(gitUrl: string) {
+  return {
+    owner: basename(dirname(gitUrl.replace(/:/, "/"))),
+    repo: basename(gitUrl.replace(/:/, "/")).replace(/\.git$/, ""),
+  };
+}
+export function stringifyOwnerRepo({ owner, repo }: ReturnType<typeof parseUrlRepoOwner>) {
+  return owner + "/" + repo;
+}
+export function stringifyGithubRepoUrl({ owner, repo }: ReturnType<typeof parseUrlRepoOwner>) {
+  return "https://github.com/" + owner + "/" + repo;
+}
+export async function stringifyGithubOrigin({ owner, repo }: ReturnType<typeof parseUrlRepoOwner>) {
+  const PR_TOKEN = process.env.GH_TOKEN_COMFY_PR;
+  if (PR_TOKEN) {
+    // fails: maybe permission issue
+    // const USERNAME = (
+    //   await new Octokit({
+    //     auth: PR_TOKEN,
+    //   }).rest.users.getAuthenticated()
+    // ).data.login;
+    // return `https://${USERNAME}:${PR_TOKEN}@github.com/${owner}/${repo}`;
+
+    return `git@github.com:${owner}/${repo}`;
+  }
+  return `git@github.com:${owner}/${repo}`;
+}

package/src/parsePullUrl.ts

@@ -0,0 +1,6 @@
+export function parsePullUrl(issueUrl: string) {
+  const [owner, repo, strNumber] = issueUrl
+    .match(/^https:\/\/github\.com\/([\w-]+)\/([\w-]+)\/(?:pull)\/(\d+)$/)!
+    .slice(1);
+  return { owner, repo, pull_number: Number(strNumber) };
+}

package/src/parsePullsState.ts

@@ -0,0 +1,6 @@
+import type { GithubPull } from "./gh/GithubPull";
+import { parsePull } from "./gh/parsePull";
+export type GithubPullParsed = ReturnType<typeof parsePulls>[number];
+export function parsePulls(data: GithubPull[]) {
+  return data.map((e) => parsePull(e));
+}

package/src/parseTitleBodyOfMarkdown.ts

@@ -0,0 +1,8 @@
+import DIE from "@snomiao/die";
+
+export function parseTitleBodyOfMarkdown(tmpl: string) {
+  tmpl.startsWith("# ") || DIE("Unrecognized template format:" + tmpl);
+  const title = tmpl.split("\n")[0].slice(1).trim();
+  const body = tmpl.split("\n").slice(1).join("\n").trim();
+  return { title, body };
+}

package/src/postSlackMessage.ts

@@ -0,0 +1,21 @@
+import DIE from "@snomiao/die";
+import { slack } from "./slack";
+
+export async function postSlackMessage(text: string) {
+  const channel = process.env.SLACK_BOT_CHANNEL || DIE(new Error("missing env.SLACK_BOT_CHANNEL"));
+  // this api will auto retry if failed
+  const response = await slack.chat.postMessage({
+    channel,
+    text,
+    blocks: [
+      {
+        type: "section",
+        text: {
+          type: "mrkdwn",
+          text: text,
+        },
+      },
+    ],
+  });
+  return { channel, ts: response.ts, text };
+}

package/src/preload.ts

@@ -0,0 +1,30 @@
+await Bun.plugin({
+  name: "YAML",
+  async setup(build) {
+    const { load } = await import("js-yaml");
+
+    // when a .yaml file is imported...
+    build.onLoad({ filter: /\.(yaml|yml)$/ }, async (args) => {
+      // read and parse the file
+      const text = await Bun.file(args.path).text();
+      const exports = load(text) as Record<string, any>;
+
+      // and returns it as a module
+      return {
+        exports,
+        loader: "object", // special loader for JS objects
+      };
+    });
+  },
+});
+
+Bun.plugin({
+  name: "preload-plugin",
+  setup(builder) {
+    builder.onLoad({ filter: /\.ts$/ }, async (args) => {
+      const text = await Bun.file(args.path).text();
+      // console.log("text", text);
+      return { contents: text, loader: args.loader };
+    });
+  },
+});

package/src/pullStatusFollowSchema.ts

@@ -0,0 +1,12 @@
+import { z } from "zod";
+
+export const zPullStatusFollow = z.array(
+  z.object({
+    updated: z.string(),
+    state: z.string(),
+    url: z.string(),
+    head: z.string(),
+    comments: z.number(),
+    lastwords: z.string(),
+  }),
+);

package/src/readTemplateTitle.ts

@@ -0,0 +1,11 @@
+import { readFile } from "fs/promises";
+import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
+
+export async function readTemplateTitle(filename: string) {
+  return await readTemplate(filename).then((e) => e.title);
+}
+export async function readTemplate(filename: string) {
+  return readFile("./templates/" + filename, "utf8").then(
+    parseTitleBodyOfMarkdown,
+  );
+}

package/src/summaryLastPullComment.ts

@@ -0,0 +1,8 @@
+import { type GithubIssueComment } from "./GithubIssueComments";
+
+export function summaryLastPullComment(comments: GithubIssueComment[]) {
+  // assume ascending order
+  const last = comments.toReversed()[0];
+  const lastText = (last ?? "") && "@" + last.user!.name + ":" + last.body!.replace(/\s+/gim, " ");
+  return lastText;
+}

package/src/tomlFillDescription.ts

@@ -0,0 +1,17 @@
+import { writeFile, readFile } from "fs/promises";
+import DIE from "@snomiao/die";
+import toml from "toml";
+import { fetchRepoDescriptionMap } from "./fetchRepoDescriptionMap";
+
+export async function tomlFillDescription(referenceUrl: string, pyprojectToml: string) {
+  const repoDescriptionMap = await fetchRepoDescriptionMap();
+  const matchedDescription = repoDescriptionMap[referenceUrl]?.toString() ||
+    DIE("Warn: missing description for " + referenceUrl);
+  const replaced = (await readFile(pyprojectToml, "utf8")).replace(
+    `description = ""`,
+    `description = ${JSON.stringify(matchedDescription)}`
+  );
+  // check validity
+  toml.parse(replaced);
+  await writeFile(pyprojectToml, replaced);
+}

package/src/updateCMNodesDuplicationWarnings.ts

@@ -0,0 +1,56 @@
+import { unary } from "lodash-es";
+import pMap from "p-map";
+import { dissoc, filter, groupBy, map, prop, toPairs } from "rambda";
+import YAML from "yaml";
+import { CMNodes, type CMNode } from "./CMNodes";
+import { notifySlack } from "./slack/notifySlack";
+
+export async function updateCMNodesDuplicationWarnings(nodes: CMNode[]) {
+  console.log("CMNodes checking duplicates");
+  // prettier-ignore
+  const dups = {
+    ID: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.id, nodes)),
+    TITLE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.title, nodes)),
+    REFERENCE: filter((e: typeof nodes) => e.length > 1, groupBy((e) => e.reference, nodes)),
+  };
+  const dupsSummary = JSON.stringify(map((x) => map((x) => x.length, x), dups));
+  await notifySlack(
+    `[WARN] CMNodes duplicates: ${dupsSummary}\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json`,
+  );
+
+  await pMap(
+    toPairs(dups),
+    async ([topic, nodes]) =>
+      await pMap(
+        toPairs(nodes),
+        async ([key, nodesRaw]) => {
+          const nodes = nodesRaw.map(unary(dissoc("hash")));
+          const hashes = nodesRaw.map(prop("hash"));
+          // check sent
+          const someDuplicateSent = await CMNodes.findOne({
+            hash: { $in: hashes },
+            [`duplicated.${topic}`]: { $exists: true },
+          });
+          if (someDuplicateSent) return;
+          // send slack notification
+          const slackNotification = await notifySlack(
+            `[ACTION NEEDED WARNING]: please resolve duplicated node in ${topic}: ${key}\n` +
+              "```\n" +
+              YAML.stringify(nodes) +
+              "```" +
+              "\n\nSolve them in https://github.com/ltdrdata/ComfyUI-Manager/blob/main/custom-node-list.json",
+            { unique: true },
+          );
+          // mark duplicates
+          await CMNodes.updateMany(
+            { hash: { $in: hashes } },
+            {
+              $set: { [`duplicated.${topic}`]: { hashes, slackNotification } },
+            },
+          );
+        },
+        { concurrency: 2 },
+      ),
+    { concurrency: 2 },
+  );
+}

package/src/updateCMRepos.ts

@@ -0,0 +1,27 @@
+import pMap from "p-map";
+import { CMNodes } from "./CMNodes";
+import { CNRepos } from "./CNRepos";
+import { tLog } from "./utils/tLog";
+if (import.meta.main) {
+  await tLog("Update Repos from ComfyUI Manager", updateCMRepos);
+  console.log(await CMNodes.estimatedDocumentCount());
+}
+export async function updateCMRepos() {
+  await CMNodes.createIndex({ repo_id: 1 });
+  return await pMap(CMNodes.find({ repo_id: { $exists: false } }), async (cm, i) => {
+    const { reference: repository, _id } = cm;
+    return await CNRepos.updateOne(
+      { repository },
+      {
+        $set: {
+          cm: (await CMNodes.findOneAndUpdate(
+            { _id },
+            { $set: { repo_id: cm._id } },
+            { upsert: true, returnDocument: "after" },
+          ))!,
+        },
+      },
+      { upsert: true },
+    );
+  });
+}