comfy-pr 0.2.24 → 0.2.26

package/src/bypassRepos.spec.ts

@@ -0,0 +1,10 @@
+import { isRepoBypassed } from "./bypassRepos";
+
+it("bypass", async () => {
+  const bypassRepo = "https://github.com/loopyd/ComfyUI-FD-Tagger";
+  expect(isRepoBypassed(bypassRepo)).toBeTruthy();
+});
+it("allow", async () => {
+  const allowRepo = "https://github.com/snomiao/ComfyUI-FD-Tagger";
+  expect(isRepoBypassed(allowRepo)).toBeFalsy();
+});

package/src/bypassRepos.ts

@@ -0,0 +1,8 @@
+import { z } from "zod";
+import { yaml } from "./utils/yaml";
+
+export const bypassRepos = z
+  .string()
+  .array()
+  .parse(yaml.parse(await Bun.file("./bypass.yaml").text()).bypass_repos);
+export const isRepoBypassed = (repo: string) => bypassRepos.some((reg) => repo.match(reg));

package/src/checkComfyActivated.ts

@@ -1,39 +1,39 @@
-import DIE from "@snomiao/die";
-import { $ as bunSh } from "bun";
-import { os } from "zx";
-import { getActivateCMD } from "./cli/getActivateCMD";
-
-export async function checkComfyActivated() {
-  console.log("Checking ComfyUI Activated...");
-
-  if (!(await bunSh`comfy --help`.quiet().catch(() => null))) {
-    const activate = getActivateCMD();
-    // apt-get install -y python3 python3-venv
-    const installPython =
-      os.platform() === "win32"
-        ? "python3 --version || winget install python3 || choco install -y python3"
-        : "apt-get install -y python3 python3-venv";
-
-    await bunSh`
-${installPython}
-python -m venv .venv
-${activate}
-pip install comfy-cli
-comfy-cli --help
-`.catch(console.error);
-
-    DIE(
-      `
-Cound not found comfy-cli.
-Please install comfy-cli before run "bunx comfy-pr" here.
-
-$ >>>>>>>>>>>>>>>>>>>>>>>>>>
-${installPython}
-python -m venv .venv
-${activate}
-pip install comfy-cli
-comfy-cli --help
-`.trim(),
-    );
-  }
-}
+import DIE from "@snomiao/die";
+import { $ as bunSh } from "bun";
+import { os } from "zx";
+import { getActivateCMD } from "./cli/getActivateCMD";
+
+export async function checkComfyActivated() {
+  console.log("Checking ComfyUI Activated...");
+
+  if (!(await bunSh`comfy --help`.quiet().catch(() => null))) {
+    const activate = getActivateCMD();
+    // apt-get install -y python3 python3-venv
+    const installPython =
+      os.platform() === "win32"
+        ? "python3 --version || winget install python3 || choco install -y python3"
+        : "apt-get install -y python3 python3-venv";
+
+    await bunSh`
+${installPython}
+python -m venv .venv
+${activate}
+pip install comfy-cli
+comfy-cli --help
+`.catch(console.error);
+
+    DIE(
+      `
+Cound not found comfy-cli.
+Please install comfy-cli before run "bunx comfy-pr" here.
+
+$ >>>>>>>>>>>>>>>>>>>>>>>>>>
+${installPython}
+python -m venv .venv
+${activate}
+pip install comfy-cli
+comfy-cli --help
+`.trim(),
+    );
+  }
+}

package/src/checkPRsFailures.ts

@@ -0,0 +1,18 @@
+import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
+import { csvFormatBody } from "d3";
+import sflow from "sflow";
+import { CNRepos } from "./CNRepos";
+
+if (import.meta.main) {
+  const pipeline = $pipeline(CNRepos)
+    .match({ "createdPulls.state": "error" })
+    .set({ error: "$createdPulls.error" })
+    .project({ _id: 0, repository: 1, error: 1 })
+    .as<{ repository: string; error: string }>()
+    .aggregate();
+  const csv = await sflow(pipeline)
+    .map((e, i) => ((!i && "repository,error\n") || "") + csvFormatBody([e]) + "\n")
+    .log()
+    .text();
+  await Bun.write(".cache/pr-errors.csv", csv);
+}

package/src/cli.ts

@@ -1,40 +1,40 @@
-#!bun
-import DIE from "@snomiao/die";
-import { readFile } from "fs/promises";
-import { argv, $ as zx } from "zx";
-import { checkComfyActivated } from "./checkComfyActivated";
-import { createComfyRegistryPullRequests } from "./createComfyRegistryPullRequests";
-zx.verbose = true;
-
-if (argv.help) {
-  console.log(
-    `
-  bunx comfy-pr --repolist repos.txt       one repo per-line
-  bunx comfy-pr [...GITHUB_REPO_URLS]      github repos
-  bunx cross-env REPO=https://github.com/OWNER/REPO bunx comfy-pr
-    `.trim(),
-  );
-}
-
-{
-  await checkComfyActivated();
-
-  const envRepos =
-    process.env.REPO?.split("\n")
-      .map((e) => e.trim())
-      .filter(Boolean) || [];
-  const argvRepos = argv._.filter((a) => !a.endsWith(import.meta.filename));
-  const listRepos =
-    (argv.repolist &&
-      (await readFile(argv.repolist, "utf8").catch(() => ""))
-        .split("\n")
-        .map((e) => e.trim())
-        .filter(Boolean)) ||
-    [];
-  const repos = (listRepos.length && listRepos) ||
-    (argvRepos.length && argvRepos) ||
-    (envRepos.length && envRepos) || [DIE("Missing PR target, please set env.REPO")];
-  for await (const upstreamUrl of repos) {
-    await createComfyRegistryPullRequests(upstreamUrl);
-  }
-}
+#!bun
+import DIE from "@snomiao/die";
+import { readFile } from "fs/promises";
+import { argv, $ as zx } from "zx";
+import { checkComfyActivated } from "./checkComfyActivated";
+import { createComfyRegistryPullRequests } from "./createComfyRegistryPullRequests";
+zx.verbose = true;
+
+if (argv.help) {
+  console.log(
+    `
+  bunx comfy-pr --repolist repos.txt       one repo per-line
+  bunx comfy-pr [...GITHUB_REPO_URLS]      github repos
+  bunx cross-env REPO=https://github.com/OWNER/REPO bunx comfy-pr
+    `.trim(),
+  );
+}
+
+{
+  await checkComfyActivated();
+
+  const envRepos =
+    process.env.REPO?.split("\n")
+      .map((e) => e.trim())
+      .filter(Boolean) || [];
+  const argvRepos = argv._.filter((a) => !a.endsWith(import.meta.filename));
+  const listRepos =
+    (argv.repolist &&
+      (await readFile(argv.repolist, "utf8").catch(() => ""))
+        .split("\n")
+        .map((e) => e.trim())
+        .filter(Boolean)) ||
+    [];
+  const repos = (listRepos.length && listRepos) ||
+    (argvRepos.length && argvRepos) ||
+    (envRepos.length && envRepos) || [DIE("Missing PR target, please set env.REPO")];
+  for await (const upstreamUrl of repos) {
+    await createComfyRegistryPullRequests(upstreamUrl);
+  }
+}

package/src/clone_modify_push_Branches.ts

@@ -1,21 +1,21 @@
-import { makePublishcrBranch } from "./makePublishBranch";
-import { makePyprojectBranch } from "./makeTomlBranch";
-
-export async function clone_modify_push_Branches(
-  upstreamUrl: string,
-  forkUrl: string,
-) {
-  return (
-    await Promise.all([
-      makePyprojectBranch(upstreamUrl, forkUrl),
-      makePublishcrBranch(upstreamUrl, forkUrl),
-    ])
-  ).map(({ body, branch, title, type }) => ({
-    body,
-    branch,
-    title,
-    type,
-    srcUrl: forkUrl,
-    dstUrl: upstreamUrl,
-  }));
-}
+import { makePublishcrBranch } from "./makePublishBranch";
+import { makePyprojectBranch } from "./makeTomlBranch";
+
+export async function clone_modify_push_Branches(upstreamUrl: string, forkUrl: string) {
+  return (
+    await Promise.all([
+      makePyprojectBranch(upstreamUrl, forkUrl),
+      makePublishcrBranch(upstreamUrl, forkUrl),
+      // makeLicenseUpdateBranch(upstreamUrl, forkUrl),
+    ])
+  )
+    .flatMap((e) => (e ? [e] : []))
+    .map(({ body, branch, title, type }) => ({
+      body,
+      branch,
+      title,
+      type,
+      srcUrl: forkUrl,
+      dstUrl: upstreamUrl,
+    }));
+}

package/src/createComfyRegistryPRsFromCandidates.ts

@@ -1,55 +1,55 @@
-import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
-import pMap from "p-map";
-import { match } from "ts-pattern";
-import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
-import { CNRepos } from "./CNRepos";
-import { createComfyRegistryPullRequests } from "./createComfyRegistryPullRequests";
-import { $filaten, $stale } from "./db";
-import { parseUrlRepoOwner, stringifyOwnerRepo } from "./parseOwnerRepo";
-import { notifySlackLinks } from "./slack/notifySlackLinks";
-import { tLog } from "./utils/tLog";
-if (import.meta.main) {
-  await tLog("createComfyRegistryPRsFromCandidates", createComfyRegistryPRsFromCandidates);
-  console.log("all done");
-}
-export async function createComfyRegistryPRsFromCandidates() {
-  await CNRepos.createIndex($filaten({ candidate: { data: 1 } }));
-  await CNRepos.createIndex(
-    $filaten({
-      candidate: { data: 1 },
-      createdPulls: { state: 1, mtime: 1 },
-    }),
-  );
-  return await pMap(
-    $pipeline(CNRepos)
-      .match(
-        $filaten({
-          candidate: { data: { $eq: true } },
-          createdPulls: { state: { $ne: "ok" }, mtime: $stale("5m") },
-        }),
-      )
-      .aggregate(),
-    async (repo) => {
-      const { repository } = repo;
-      console.log("Making PRs for " + repository);
-      const createdPulls = await createComfyRegistryPullRequests(repository).then(TaskOK).catch(TaskError);
-      match(createdPulls).with($OK, async ({ data }) => {
-        const links = data.map((e) => ({
-          href: e.html_url,
-          name: stringifyOwnerRepo(parseUrlRepoOwner(e.html_url.replace(/\/pull\/.*$/, ""))) + " #" + e.title,
-        }));
-        await notifySlackLinks("PR just Created, @HaoHao check plz", links);
-        await pMap(data, async (pull) => {
-          const { html_url } = pull;
-          // also update to crPulls
-          await CNRepos.updateOne($filaten({ repository, crPulls: { data: { pull: { html_url } } } }), {
-            $set: { "crPulls.data.$.pull": pull },
-          });
-        });
-      });
-
-      return await CNRepos.updateOne({ repository }, { $set: { createdPulls } });
-    },
-    { concurrency: 2, stopOnError: false },
-  );
-}
+import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
+import pMap from "p-map";
+import { match } from "ts-pattern";
+import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
+import { CNRepos } from "./CNRepos";
+import { createComfyRegistryPullRequests } from "./createComfyRegistryPullRequests";
+import { $filaten, $stale } from "./db";
+import { parseUrlRepoOwner, stringifyOwnerRepo } from "./parseOwnerRepo";
+import { notifySlackLinks } from "./slack/notifySlackLinks";
+import { tLog } from "./utils/tLog";
+if (import.meta.main) {
+  await tLog("createComfyRegistryPRsFromCandidates", createComfyRegistryPRsFromCandidates);
+  console.log("all done");
+}
+export async function createComfyRegistryPRsFromCandidates() {
+  await CNRepos.createIndex($filaten({ candidate: { data: 1 } }));
+  await CNRepos.createIndex(
+    $filaten({
+      candidate: { data: 1 },
+      createdPulls: { state: 1, mtime: 1 },
+    }),
+  );
+  return await pMap(
+    $pipeline(CNRepos)
+      .match(
+        $filaten({
+          candidate: { data: { $eq: true } },
+          createdPulls: { state: { $ne: "ok" }, mtime: $stale("5m") },
+        }),
+      )
+      .aggregate(),
+    async (repo) => {
+      const { repository } = repo;
+      console.log("Making PRs for " + repository);
+      const createdPulls = await createComfyRegistryPullRequests(repository).then(TaskOK).catch(TaskError);
+      match(createdPulls).with($OK, async ({ data }) => {
+        const links = data.map((e) => ({
+          href: e.html_url,
+          name: stringifyOwnerRepo(parseUrlRepoOwner(e.html_url.replace(/\/pull\/.*$/, ""))) + " #" + e.title,
+        }));
+        await notifySlackLinks("PR just Created, @HaoHao check plz", links);
+        await pMap(data, async (pull) => {
+          const { html_url } = pull;
+          // also update to crPulls
+          await CNRepos.updateOne($filaten({ repository, crPulls: { data: { pull: { html_url } } } }), {
+            $set: { "crPulls.data.$.pull": pull },
+          });
+        });
+      });
+
+      return await CNRepos.updateOne({ repository }, { $set: { createdPulls } });
+    },
+    { concurrency: 2, stopOnError: false },
+  );
+}

package/src/createComfyRegistryPullRequests.ts

@@ -1,22 +1,32 @@
-import pMap from "p-map";
-import yaml from "yaml";
-// import { chalk } from "zx";
-import { clone_modify_push_Branches } from "./clone_modify_push_Branches";
-import { createGithubForkForRepo } from "./createGithubForkForRepo";
-import { createGithubPullRequest } from "./createGithubPullRequest";
-import type { GithubPull } from "./gh/GithubPull";
-import { parsePulls } from "./parsePullsState";
-
-export async function createComfyRegistryPullRequests(upstreamRepoUrl: string) {
-  const forkedRepo = await createGithubForkForRepo(upstreamRepoUrl);
-  const PR_REQUESTS = await clone_modify_push_Branches(upstreamRepoUrl, forkedRepo.html_url);
-  // branch is ready in fork now
-  // create prs for each branch
-  console.log("Going to create PRs for the following branches:");
-  // console.log(chalk.green(yaml.stringify({ PR_REQUESTS })));
-  console.log(yaml.stringify({ PR_REQUESTS }));
-  // prs
-  const prs = await pMap(PR_REQUESTS, async ({ type, ...prInfo }) => await createGithubPullRequest({ ...prInfo }));
-  console.log("ALL PRs DONE");
-  return (prs as GithubPull[]).map((e) => parsePulls([e])[0]);
-}
+import pMap from "p-map";
+// import { chalk } from "zx";
+import { clone_modify_push_Branches } from "./clone_modify_push_Branches";
+import { createGithubForkForRepo } from "./createGithubForkForRepo";
+import { createGithubPullRequest } from "./createGithubPullRequest";
+import type { GithubPull } from "./gh/GithubPull";
+import { makeUpdateTomlLicenseBranch } from "./makeUpdateTomlLicenseBranch";
+import { parsePulls } from "./parsePullsState";
+
+export async function createComfyRegistryPullRequests(upstreamRepoUrl: string) {
+  const forkedRepo = await createGithubForkForRepo(upstreamRepoUrl);
+
+  const PR_REQUESTS = await clone_modify_push_Branches(upstreamRepoUrl, forkedRepo.html_url);
+  const prs = await pMap(PR_REQUESTS, async ({ type, ...prInfo }) => await createGithubPullRequest({ ...prInfo }));
+
+  console.log("Registry PRs DONE");
+
+  return ([...prs] as GithubPull[]).map((e) => parsePulls([e])[0]);
+}
+
+export async function clone_modify_push_Branches_for_updateTomlLicense(upstreamUrl: string, forkUrl: string) {
+  return (await Promise.all([makeUpdateTomlLicenseBranch(upstreamUrl, forkUrl)]))
+    .flatMap((e) => (e ? [e] : []))
+    .map(({ body, branch, title, type }) => ({
+      body,
+      branch,
+      title,
+      type,
+      srcUrl: forkUrl,
+      dstUrl: upstreamUrl,
+    }));
+}

package/src/createGithubForkForRepo.ts

@@ -1,37 +1,54 @@
-import DIE from "@snomiao/die";
-import md5 from "md5";
-import minimist from "minimist";
-import { FORK_OWNER } from "./FORK_OWNER";
-import { FORK_PREFIX } from "./FORK_PREFIX";
-import { createGithubFork } from "./gh/createGithubFork";
-import { ghUser } from "./ghUser";
-import { parseUrlRepoOwner } from "./parseOwnerRepo";
-
-/**
- * this function creates a fork of the upstream repo,
- * fork to the FORK_OWNER, and add a prefix to the repo name
- * - the prefix is optional, if not provided, the repo name will be the same as the upstream repo
- * - the prefix is useful to distinguish the forked repo from the other repo in the same owner
- * SALT is used to generate a unique repo name, so that the forked repo will not conflict with other forks
- *
- * @author snomiao <snomiao@gmail.com>
- * @param upstreamRepoUrl
- * @returns forked repo info
- */
-export async function createGithubForkForRepo(upstreamRepoUrl: string) {
-  // debug
-  // console.log(`* Change env.SALT=${salt} will fork into a different repo`);
-  // console.log("PR_SRC: ", forkSSHUrl);
-  // console.log("PR_DST: ", upstreamUrl);
-  // console.log(forkSSHUrl);
-  const upstream = parseUrlRepoOwner(upstreamRepoUrl);
-  const argv = minimist(process.argv.slice(2));
-  const salt = argv.salt || process.env.SALT || "m3KMgZ2AeZGWYh7W";
-  const repo_hash = md5(`${salt}-${ghUser.name}-${upstream.owner}/${upstream.repo}`).slice(0, 8);
-  const forkRepoName = (FORK_PREFIX && `${FORK_PREFIX}${upstream.repo}-${repo_hash}`) || upstream.repo;
-  const forkDst = `${FORK_OWNER}/${forkRepoName}`;
-  const forkUrl = `https://github.com/${forkDst}`;
-  const forked = await createGithubFork(upstreamRepoUrl, forkUrl);
-  if (forked.html_url !== forkUrl) DIE("forked url not expected");
-  return forked;
-}
+import DIE from "@snomiao/die";
+import md5 from "md5";
+import minimist from "minimist";
+import { FORK_OWNER } from "./FORK_OWNER";
+import { FORK_PREFIX } from "./FORK_PREFIX";
+import { createGithubFork } from "./gh/createGithubFork";
+import { ghUser } from "./ghUser";
+import { parseUrlRepoOwner } from "./parseOwnerRepo";
+
+if (import.meta.main) {
+  console.log(await createGithubForkForRepo("https://github.com/comfyanonymous/ComfyUI_TensorRT"));
+}
+
+/**
+ * this function creates a fork of the upstream repo,
+ * fork to the FORK_OWNER, and add a prefix to the repo name
+ * - the prefix is optional, if not provided, the repo name will be the same as the upstream repo
+ * - the prefix is useful to distinguish the forked repo from the other repo in the same owner
+ * SALT is used to generate a unique repo name, so that the forked repo will not conflict with other forks
+ *
+ * @author snomiao <snomiao@gmail.com>
+ * @param upstreamRepoUrl
+ * @returns forked repo info
+ */
+export async function createGithubForkForRepo(
+  upstreamRepoUrl: string,
+  { forkUrl = createGithubForkUrlForRepo(upstreamRepoUrl) } = {},
+) {
+  console.debug(
+    `
+Forking ${upstreamRepoUrl}
+   into ${forkUrl}
+`.trim(),
+  );
+  const forked = await createGithubFork(upstreamRepoUrl, forkUrl);
+  if (forked.html_url !== forkUrl)
+    DIE(
+      new Error(
+        "forked url not expected, it's likely you already forked this repo in your account before, and now trying to fork it again with different salt. To recovery you could delete that repo forked before by manual. (the repo forked before is listed in FORK OK: .....)",
+      ),
+    );
+  return forked;
+}
+export function createGithubForkUrlForRepo(upstreamRepoUrl: string) {
+  // console.log(`* Change env.SALT=${salt} will fork into a different repo`);
+  const upstream = parseUrlRepoOwner(upstreamRepoUrl);
+  const argv = minimist(process.argv.slice(2));
+  const salt = argv.salt || process.env.SALT || "m3KMgZ2AeZGWYh7W";
+  const repo_hash = md5(`${salt}-${ghUser.name}-${upstream.owner}/${upstream.repo}`).slice(0, 8);
+  const forkRepoName = (FORK_PREFIX && `${FORK_PREFIX}${upstream.repo}-${repo_hash}`) || upstream.repo;
+  const forkDst = `${FORK_OWNER}/${forkRepoName}`;
+  const forkUrl = `https://github.com/${forkDst}`;
+  return forkUrl;
+}