comfy-pr 0.2.26 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +258 -157
- package/bot/IdleWaiter.ts +31 -0
- package/bot/RestartManager.spec.ts +163 -0
- package/bot/RestartManager.ts +190 -0
- package/bot/WorkingTasksManager.spec.ts +154 -0
- package/bot/cli.ts +1171 -0
- package/bot/codesearch-ai-wip.ts +351 -0
- package/bot/error-collector.ts +133 -0
- package/bot/index.ts +14 -0
- package/bot/restart-example.ts +99 -0
- package/bot/slack-bolt.ts +688 -0
- package/bot/slack-bot.ts +1575 -0
- package/bot/state.ts +19 -0
- package/bot/templateLoader.test.ts +84 -0
- package/bot/templateLoader.ts +92 -0
- package/next.config.ts +26 -0
- package/package.json +161 -41
- package/post-summary.ts +44 -0
- package/src/Authors.ts +2 -2
- package/src/CMNodes.ts +3 -3
- package/src/CNRepos.ts +30 -7
- package/src/CRNodes.ts +11 -9
- package/src/CRPulls.ts +3 -0
- package/src/EmailTasks.ts +27 -9
- package/src/FORK_OWNER.ts +3 -1
- package/src/FollowRules.ts +3 -3
- package/src/GithubIssueComments.ts +1 -1
- package/src/Totals.ts +9 -8
- package/src/WorkerInstances.ts +31 -8
- package/src/addCommentAction.ts +19 -11
- package/src/analyzePullsStatus.ts +39 -17
- package/src/analyzeTotals.ts +19 -11
- package/src/bypassRepos.ts +6 -5
- package/src/checkPRsFailures.ts +13 -8
- package/src/cli.test.ts +2 -0
- package/src/cli.ts +1 -1
- package/src/constants.ts +2 -0
- package/src/createComfyRegistryPRsFromCandidates.ts +20 -12
- package/src/createComfyRegistryPullRequests.ts +35 -10
- package/src/createGithubForkForRepo.ts +36 -11
- package/src/createGithubPullRequest.ts +83 -33
- package/src/createIssueComment.ts +4 -4
- package/src/fetchComfyRegistryNodes.ts +34 -4
- package/src/fetchCurrentGeoInfo.ts +3 -1
- package/src/fetchRelatedPullWithComments.ts +1 -1
- package/src/fetchRepoDescriptionMap.ts +1 -4
- package/src/followRuleSchema.ts +10 -4
- package/src/getBranchWorkingDir.ts +3 -7
- package/src/getRepoWorkingDir.ts +2 -3
- package/src/gh.spec.ts +571 -0
- package/src/ghData.ts +13 -0
- package/src/ghPageFlow.ts +33 -0
- package/src/ghSubscriber.ts +76 -0
- package/src/ghUser.ts +28 -4
- package/src/index.ts +13 -4
- package/src/initializeFollowRules.ts +11 -4
- package/src/logger.spec.ts +95 -0
- package/src/logger.ts +46 -0
- package/src/makeGpl3LicenseBranch.ts +66 -0
- package/src/makePublishBranch.ts +21 -15
- package/src/makeTomlBranch.ts +48 -14
- package/src/makeUpdateTomlLicenseBranch.ts +40 -44
- package/src/matchRelatedPulls.ts +9 -4
- package/src/normalizeGithubUrl.spec.ts +131 -0
- package/src/normalizeGithubUrl.ts +64 -0
- package/src/parseIssueUrl.spec.ts +95 -0
- package/src/parseIssueUrl.ts +17 -3
- package/src/parseOwnerRepo.spec.ts +147 -0
- package/src/parseOwnerRepo.ts +8 -5
- package/src/parsePullsState.ts +2 -2
- package/src/parseTitleBodyOfMarkdown.ts +1 -1
- package/src/pickRepoInfo.ts +37 -0
- package/src/postSlackMessage.ts +5 -1
- package/src/preload.ts +30 -27
- package/src/readTemplateTitle.ts +1 -3
- package/src/sendEmailAction.ts +14 -10
- package/src/sendGmail.ts +3 -3
- package/src/updateAuthorsForGithub.ts +53 -35
- package/src/updateAuthorsFromCNRepo.ts +12 -5
- package/src/updateCMNodesDuplicationWarnings.ts +14 -9
- package/src/updateCMRepos.ts +1 -1
- package/src/updateCNRepos.ts +14 -14
- package/src/updateCNReposCRPullsComments.ts +7 -5
- package/src/updateCNReposInfo.ts +49 -40
- package/src/updateCNReposPRCandidate.ts +3 -3
- package/src/updateCNReposPulls.ts +9 -5
- package/src/updateCNReposPullsDashboard.ts +11 -9
- package/src/updateCNReposRelatedPulls.ts +3 -3
- package/src/updateCRNodes.ts +24 -5
- package/src/updateCRRepos.ts +1 -1
- package/src/updateComfyTotals.ts +7 -5
- package/src/updateFollowRuleSet.ts +17 -9
- package/src/updateOutdatedPullsTemplates.ts +61 -48
- package/src/updateSlackMessages.ts +8 -4
- package/tailwind.config.ts +7 -1
- package/next-env.d.ts +0 -5
- package/src/GIT_USEREMAIL.ts +0 -5
- package/src/GIT_USERNAME.ts +0 -9
- package/src/clone_modify_push_Branches.ts +0 -21
- package/src/tomlFillDescription.ts +0 -17
package/src/ghUser.ts
CHANGED
|
@@ -1,9 +1,33 @@
|
|
|
1
1
|
import type { Task } from "@/packages/mongodb-pipeline-ts/Task";
|
|
2
|
-
import
|
|
2
|
+
import DIE from "@snomiao/die";
|
|
3
|
+
import { gh } from "@/lib/github";
|
|
3
4
|
import type { AwaitedReturnType } from "./types/AwaitedReturnType";
|
|
4
5
|
|
|
5
|
-
|
|
6
|
+
const g = globalThis as typeof globalThis & {
|
|
7
|
+
ghUserCache: AwaitedReturnType<typeof gh.users.getAuthenticated>["data"] | null;
|
|
8
|
+
};
|
|
9
|
+
export const ghUser = async () =>
|
|
10
|
+
(g.ghUserCache ??= (
|
|
11
|
+
await gh.users.getAuthenticated().catch((error) => {
|
|
12
|
+
throw new Error(
|
|
13
|
+
`FAIL TO GET AUTHENTICATED USER INFO, CHECK ${process.env.GH_TOKEN ? "[?]" : "[ ]"}GH_TOKEN and ${process.env.GH_TOKEN_COMFY_PR ? "[?]" : "[ ]"}GH_TOKEN_COMFY_PR`,
|
|
14
|
+
{ cause: error },
|
|
15
|
+
);
|
|
16
|
+
})
|
|
17
|
+
).data);
|
|
18
|
+
|
|
19
|
+
// console.log(`GH_TOKEN User: ${(await ghUser()).login} <${(await ghUser()).email}>`);
|
|
6
20
|
|
|
7
|
-
console.log("Fetch Current Github User...");
|
|
8
|
-
console.log(`Current Github User: ${ghUser.login} <${ghUser.email}>`);
|
|
9
21
|
export type GHUser = Task<AwaitedReturnType<typeof gh.users.getByUsername>["data"]>;
|
|
22
|
+
|
|
23
|
+
export const GIT_USEREMAIL = async () =>
|
|
24
|
+
process.env.GIT_USEREMAIL ||
|
|
25
|
+
((await ghUser()).email && (await ghUser()).email) ||
|
|
26
|
+
DIE("Missing env.GIT_USEREMAIL");
|
|
27
|
+
export const GIT_USERNAME = async () =>
|
|
28
|
+
process.env.GIT_USERNAME ||
|
|
29
|
+
((await ghUser()).email && (await ghUser()).name) ||
|
|
30
|
+
DIE("Missing env.GIT_USERNAME");
|
|
31
|
+
|
|
32
|
+
// read env/parameters
|
|
33
|
+
console.log(`GIT COMMIT USER: ${await GIT_USERNAME()} <${await GIT_USEREMAIL()}>`);
|
package/src/index.ts
CHANGED
|
@@ -2,25 +2,34 @@ import "dotenv/config";
|
|
|
2
2
|
import { checkComfyActivated } from "./checkComfyActivated";
|
|
3
3
|
import { updateEmailTasks } from "./EmailTasks";
|
|
4
4
|
import { initializeFollowRules } from "./initializeFollowRules";
|
|
5
|
-
import {
|
|
5
|
+
import { createLogger } from "./logger";
|
|
6
6
|
import { updateAuthors } from "./updateAuthors";
|
|
7
7
|
import { updateCNRepos } from "./updateCNRepos";
|
|
8
8
|
import { runFollowRuleSet } from "./updateFollowRuleSet";
|
|
9
9
|
import { updateSlackMessages } from "./updateSlackMessages";
|
|
10
|
+
import { getWorkerInstance } from "./WorkerInstances";
|
|
10
11
|
import { tLog } from "./utils/tLog";
|
|
11
12
|
|
|
13
|
+
const logger = createLogger("main");
|
|
14
|
+
|
|
12
15
|
if (import.meta.main) {
|
|
16
|
+
// Register task name immediately so watchWorkerInstancesLoop captures me.task correctly.
|
|
17
|
+
// Without this, me.task is undefined and any other process calling getWorkerInstance()
|
|
18
|
+
// without a task name would incorrectly evict this process.
|
|
19
|
+
await getWorkerInstance("ComfyPR Bot Running");
|
|
13
20
|
await Promise.all([
|
|
14
21
|
// try send msgs that didn't send in last run
|
|
15
22
|
updateSlackMessages(),
|
|
16
|
-
checkComfyActivated(), // needed if
|
|
23
|
+
checkComfyActivated(), // needed if making pr
|
|
17
24
|
updateCNRepos(),
|
|
18
25
|
updateAuthors(),
|
|
19
26
|
updateEmailTasks(),
|
|
20
|
-
|
|
27
|
+
|
|
28
|
+
// 2025-03-14 temporary disable due to the standard has updated:
|
|
29
|
+
// updateTomlLicenseTasks(),
|
|
21
30
|
]);
|
|
22
31
|
await initializeFollowRules();
|
|
23
32
|
await tLog("runFollowRuleSet", runFollowRuleSet);
|
|
24
|
-
|
|
33
|
+
logger.info("All done");
|
|
25
34
|
process.exit(0);
|
|
26
35
|
}
|
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
import DIE from "@snomiao/die";
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
3
|
import { FollowRuleSets } from "./FollowRules";
|
|
4
|
+
import { createLogger } from "./logger";
|
|
5
|
+
|
|
6
|
+
const logger = createLogger("initializeFollowRules");
|
|
4
7
|
|
|
5
8
|
if (import.meta.main) {
|
|
6
|
-
const
|
|
7
|
-
const ruleset =
|
|
9
|
+
const _followRules = await initializeFollowRules();
|
|
10
|
+
const ruleset =
|
|
11
|
+
(await FollowRuleSets.findOne({ name: "default" })) ?? DIE("default ruleset not found");
|
|
8
12
|
if (!ruleset?.enabled) DIE("default ruleset not enabled");
|
|
9
13
|
|
|
10
|
-
|
|
14
|
+
logger.info("initialized follow rules");
|
|
11
15
|
}
|
|
12
16
|
/**
|
|
13
17
|
*
|
|
14
18
|
* @author: snomiao <snomiao@gmail.com>
|
|
15
19
|
*/
|
|
16
20
|
export async function initializeFollowRules() {
|
|
17
|
-
return await updateFollowRules(
|
|
21
|
+
return await updateFollowRules(
|
|
22
|
+
"default",
|
|
23
|
+
await readFile("./templates/follow-rules.yaml", "utf8"),
|
|
24
|
+
);
|
|
18
25
|
}
|
|
19
26
|
|
|
20
27
|
export async function updateFollowRules(name: string, rawRuleYaml: string) {
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import winston from "winston";
|
|
3
|
+
import { createLogger, logger } from "./logger";
|
|
4
|
+
|
|
5
|
+
describe("Logger", () => {
|
|
6
|
+
describe("logger instance", () => {
|
|
7
|
+
it("should be a winston logger instance", () => {
|
|
8
|
+
expect(logger).toBeInstanceOf(winston.Logger);
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("should have default log level from environment or 'info'", () => {
|
|
12
|
+
const expectedLevel = process.env.LOG_LEVEL || "info";
|
|
13
|
+
expect(logger.level).toBe(expectedLevel);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it("should have console transport", () => {
|
|
17
|
+
expect(logger.transports.length).toBeGreaterThan(0);
|
|
18
|
+
expect(logger.transports[0]).toBeInstanceOf(winston.transports.Console);
|
|
19
|
+
});
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
describe("createLogger", () => {
|
|
23
|
+
it("should create a child logger with module context", () => {
|
|
24
|
+
const moduleLogger = createLogger("test-module");
|
|
25
|
+
expect(moduleLogger).toBeInstanceOf(winston.Logger);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it("should include module name in metadata", () => {
|
|
29
|
+
const moduleLogger = createLogger("test-module");
|
|
30
|
+
expect(moduleLogger).toBeInstanceOf(winston.Logger);
|
|
31
|
+
// Child loggers in Winston add metadata
|
|
32
|
+
expect(moduleLogger.defaultMeta).toBeDefined();
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
describe("log levels", () => {
|
|
37
|
+
it("should have info method", () => {
|
|
38
|
+
expect(logger.info).toBeDefined();
|
|
39
|
+
expect(typeof logger.info).toBe("function");
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("should have error method", () => {
|
|
43
|
+
expect(logger.error).toBeDefined();
|
|
44
|
+
expect(typeof logger.error).toBe("function");
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it("should have warning method", () => {
|
|
48
|
+
expect(logger.warn).toBeDefined();
|
|
49
|
+
expect(typeof logger.warn).toBe("function");
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("should have debug method", () => {
|
|
53
|
+
expect(logger.debug).toBeDefined();
|
|
54
|
+
expect(typeof logger.debug).toBe("function");
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("should respect log level settings", () => {
|
|
58
|
+
const originalLevel = logger.level;
|
|
59
|
+
logger.level = "error";
|
|
60
|
+
expect(logger.level).toBe("error");
|
|
61
|
+
logger.level = originalLevel;
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe("metadata handling", () => {
|
|
66
|
+
it("should accept metadata objects", () => {
|
|
67
|
+
const metadata = { userId: 123, action: "test" };
|
|
68
|
+
// This should not throw
|
|
69
|
+
expect(() => logger.info("test with metadata", metadata)).not.toThrow();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("should handle complex objects as metadata", () => {
|
|
73
|
+
const complexData = {
|
|
74
|
+
nested: {
|
|
75
|
+
value: "test",
|
|
76
|
+
array: [1, 2, 3],
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
// This should not throw
|
|
80
|
+
expect(() => logger.info("complex metadata", complexData)).not.toThrow();
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
describe("logger configuration", () => {
|
|
85
|
+
it("should have timestamp format configured", () => {
|
|
86
|
+
const formats = logger.format;
|
|
87
|
+
expect(formats).toBeDefined();
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("should have error stack handling", () => {
|
|
91
|
+
const formats = logger.format;
|
|
92
|
+
expect(formats).toBeDefined();
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
});
|
package/src/logger.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import winston from "winston";
|
|
2
|
+
|
|
3
|
+
const { combine, timestamp, printf, colorize, errors } = winston.format;
|
|
4
|
+
|
|
5
|
+
// Custom format for console output
|
|
6
|
+
const consoleFormat = printf(({ level, message, timestamp, ...metadata }) => {
|
|
7
|
+
let msg = `${timestamp} [${level}]: ${message}`;
|
|
8
|
+
|
|
9
|
+
// Add metadata if present
|
|
10
|
+
if (Object.keys(metadata).length > 0) {
|
|
11
|
+
// Remove error stack from metadata display if it exists
|
|
12
|
+
const { stack, ...cleanMetadata } = metadata;
|
|
13
|
+
if (Object.keys(cleanMetadata).length > 0) {
|
|
14
|
+
msg += ` ${JSON.stringify(cleanMetadata)}`;
|
|
15
|
+
}
|
|
16
|
+
// Add stack trace on new line if present
|
|
17
|
+
if (stack) {
|
|
18
|
+
msg += `\n${stack}`;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
return msg;
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// Create the logger instance
|
|
26
|
+
export const logger = winston.createLogger({
|
|
27
|
+
level: process.env.LOG_LEVEL || "info",
|
|
28
|
+
format: combine(errors({ stack: true }), timestamp({ format: "YYYY-MM-DD HH:mm:ss.SSS" })),
|
|
29
|
+
transports: [
|
|
30
|
+
// Console transport for development
|
|
31
|
+
new winston.transports.Console({
|
|
32
|
+
format: combine(colorize({ all: true }), consoleFormat),
|
|
33
|
+
}),
|
|
34
|
+
],
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
// Create child loggers for different modules
|
|
38
|
+
export const createLogger = (module: string) => {
|
|
39
|
+
return logger.child({ module });
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
// Convenience exports for common log levels
|
|
43
|
+
export const logInfo = logger.info.bind(logger);
|
|
44
|
+
export const logError = logger.error.bind(logger);
|
|
45
|
+
export const logWarn = logger.warn.bind(logger);
|
|
46
|
+
export const logDebug = logger.debug.bind(logger);
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { readFile, writeFile } from "fs/promises";
|
|
2
|
+
import { $ } from "./cli/echoBunShell";
|
|
3
|
+
import { getBranchWorkingDir } from "./getBranchWorkingDir";
|
|
4
|
+
import { gh } from "@/lib/github";
|
|
5
|
+
import { GIT_USEREMAIL, GIT_USERNAME } from "./ghUser";
|
|
6
|
+
import {
|
|
7
|
+
parseGithubRepoUrl,
|
|
8
|
+
stringifyGithubOrigin,
|
|
9
|
+
stringifyGithubRepoUrl,
|
|
10
|
+
} from "./parseOwnerRepo";
|
|
11
|
+
import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
|
|
12
|
+
|
|
13
|
+
export async function makeGpl3LicenseBranch(upstreamUrl: string, forkUrl: Readonly<string>) {
|
|
14
|
+
const type = "gpl3-license" as const;
|
|
15
|
+
|
|
16
|
+
const upstreamRepo = parseGithubRepoUrl(upstreamUrl);
|
|
17
|
+
|
|
18
|
+
// Check if the repo already has a license — skip if it does
|
|
19
|
+
const existingLicense = await gh.licenses
|
|
20
|
+
.getForRepo({ owner: upstreamRepo.owner, repo: upstreamRepo.repo })
|
|
21
|
+
.catch(() => null);
|
|
22
|
+
if (existingLicense) {
|
|
23
|
+
console.log("Skip: license already exists for " + upstreamUrl);
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const origin = await stringifyGithubOrigin(parseGithubRepoUrl(forkUrl));
|
|
28
|
+
const branch = "gpl3-license";
|
|
29
|
+
const tmpl = await readFile("./templates/add-gpl3-license.md", "utf8");
|
|
30
|
+
const { title, body } = parseTitleBodyOfMarkdown(tmpl);
|
|
31
|
+
const repo = parseGithubRepoUrl(origin);
|
|
32
|
+
|
|
33
|
+
if (await gh.repos.getBranch({ ...repo, branch }).catch(() => null)) {
|
|
34
|
+
// prevent unrelated history
|
|
35
|
+
console.log("Skip changes as branch existed: " + branch);
|
|
36
|
+
return { type, title, body, branch };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Fetch canonical GPL-3.0 text from GitHub API
|
|
40
|
+
const licenseData = await gh.licenses.get({ license: "gpl-3.0" });
|
|
41
|
+
const licenseText = licenseData.data.body;
|
|
42
|
+
|
|
43
|
+
const cwd = await getBranchWorkingDir(upstreamUrl, forkUrl, branch);
|
|
44
|
+
|
|
45
|
+
// Clone the repo
|
|
46
|
+
await $`git clone --single-branch ${upstreamUrl} ${cwd}`;
|
|
47
|
+
|
|
48
|
+
// Write LICENSE file via Node.js to avoid shell escaping issues with license text
|
|
49
|
+
await writeFile(`${cwd}/LICENSE`, licenseText, "utf8");
|
|
50
|
+
|
|
51
|
+
// commit & push changes
|
|
52
|
+
await $`
|
|
53
|
+
cd ${cwd}
|
|
54
|
+
|
|
55
|
+
git config user.name ${GIT_USERNAME} && \
|
|
56
|
+
git config user.email ${GIT_USEREMAIL} && \
|
|
57
|
+
git checkout -b ${branch} && \
|
|
58
|
+
git add LICENSE && \
|
|
59
|
+
git commit -m "chore(${branch}): ${title}" && \
|
|
60
|
+
git push -f "${origin}" ${branch}:${branch}
|
|
61
|
+
`;
|
|
62
|
+
|
|
63
|
+
const branchUrl = `${stringifyGithubRepoUrl(repo)}/tree/${branch}`;
|
|
64
|
+
console.log(`Branch Push OK: ${branchUrl}`);
|
|
65
|
+
return { type, title, body, branch };
|
|
66
|
+
}
|
package/src/makePublishBranch.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { readFile } from "fs/promises";
|
|
2
2
|
import { dirname } from "path";
|
|
3
|
-
import { GIT_USEREMAIL } from "./GIT_USEREMAIL";
|
|
4
|
-
import { GIT_USERNAME } from "./GIT_USERNAME";
|
|
5
3
|
import { $ } from "./cli/echoBunShell";
|
|
6
4
|
import { getBranchWorkingDir } from "./getBranchWorkingDir";
|
|
7
|
-
import { gh } from "
|
|
8
|
-
import {
|
|
5
|
+
import { gh } from "@/lib/github";
|
|
6
|
+
import { GIT_USEREMAIL, GIT_USERNAME } from "./ghUser";
|
|
7
|
+
import {
|
|
8
|
+
parseGithubRepoUrl,
|
|
9
|
+
stringifyGithubOrigin,
|
|
10
|
+
stringifyGithubRepoUrl,
|
|
11
|
+
} from "./parseOwnerRepo";
|
|
9
12
|
import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
|
|
10
13
|
|
|
11
14
|
/**
|
|
@@ -19,38 +22,41 @@ import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
|
|
|
19
22
|
export async function makePublishcrBranch(upstreamUrl: string, forkUrl: Readonly<string>) {
|
|
20
23
|
const type = "publishcr" as const;
|
|
21
24
|
|
|
22
|
-
const origin = await stringifyGithubOrigin(
|
|
25
|
+
const origin = await stringifyGithubOrigin(parseGithubRepoUrl(forkUrl));
|
|
23
26
|
const branch = "publish";
|
|
24
27
|
const tmpl = await readFile("./templates/add-action.md", "utf8");
|
|
25
28
|
const { title, body } = parseTitleBodyOfMarkdown(tmpl);
|
|
26
|
-
const repo =
|
|
29
|
+
const repo = parseGithubRepoUrl(origin);
|
|
27
30
|
|
|
28
31
|
if (await gh.repos.getBranch({ ...repo, branch }).catch(() => null)) {
|
|
32
|
+
// prevent unrelated history
|
|
29
33
|
console.log("Skip changes as branch existed: " + branch);
|
|
30
34
|
return { type, title, body, branch };
|
|
31
35
|
}
|
|
32
36
|
|
|
33
37
|
const cwd = await getBranchWorkingDir(upstreamUrl, forkUrl, branch);
|
|
34
|
-
|
|
35
|
-
const file =
|
|
38
|
+
const upsreamOwner = parseGithubRepoUrl(upstreamUrl).owner;
|
|
39
|
+
const file = `.github/workflows/publish.yml`;
|
|
36
40
|
const publishYmlPath = "./templates/publish.yaml";
|
|
37
|
-
|
|
41
|
+
const publishYmlTemplate = await readFile(publishYmlPath, "utf8");
|
|
42
|
+
const repalcedContent = publishYmlTemplate.replace("NODE_AUTHOR_OWNER", upsreamOwner);
|
|
43
|
+
if (publishYmlTemplate === repalcedContent) throw new Error("fail to replace NODE_AUTHOR_OWNER");
|
|
38
44
|
// commit & push changes
|
|
39
45
|
await $`
|
|
40
|
-
git clone ${upstreamUrl} ${cwd}
|
|
41
|
-
|
|
42
|
-
mkdir -p ${dirname(file)}
|
|
43
|
-
cat ${publishYmlPath} > ${file}
|
|
46
|
+
git clone --single-branch ${upstreamUrl} ${cwd}
|
|
44
47
|
|
|
45
48
|
cd ${cwd}
|
|
46
49
|
|
|
50
|
+
mkdir -p ${dirname(file)}
|
|
51
|
+
echo "${repalcedContent}" > ${file}
|
|
52
|
+
|
|
47
53
|
git config user.name ${GIT_USERNAME} && \
|
|
48
54
|
git config user.email ${GIT_USEREMAIL} && \
|
|
49
55
|
git checkout -b ${branch} && \
|
|
50
56
|
git add . && \
|
|
51
57
|
git commit -am "chore(${branch}): ${title}" && \
|
|
52
|
-
git push "${origin}" ${branch}:${branch}
|
|
53
|
-
|
|
58
|
+
git push -f "${origin}" ${branch}:${branch}
|
|
59
|
+
`;
|
|
54
60
|
|
|
55
61
|
const branchUrl = `${stringifyGithubRepoUrl(repo)}/tree/${branch}`;
|
|
56
62
|
console.log(`Branch Push OK: ${branchUrl}`);
|
package/src/makeTomlBranch.ts
CHANGED
|
@@ -1,31 +1,51 @@
|
|
|
1
|
-
import
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
1
|
+
import DIE from "@snomiao/die";
|
|
2
|
+
import { readFile, writeFile } from "fs/promises";
|
|
3
|
+
import toml from "toml";
|
|
4
4
|
import { $ } from "./cli/echoBunShell";
|
|
5
|
+
import { fetchRepoDescriptionMap } from "./fetchRepoDescriptionMap";
|
|
5
6
|
import { getBranchWorkingDir } from "./getBranchWorkingDir";
|
|
6
|
-
import { gh } from "
|
|
7
|
-
import {
|
|
7
|
+
import { gh } from "@/lib/github";
|
|
8
|
+
import { GIT_USEREMAIL, GIT_USERNAME } from "./ghUser";
|
|
9
|
+
import { parseGithubRepoUrl, stringifyGithubOrigin } from "./parseOwnerRepo";
|
|
8
10
|
import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
|
|
9
|
-
import { tomlFillDescription } from "./tomlFillDescription";
|
|
10
11
|
|
|
11
12
|
export async function makePyprojectBranch(upstreamUrl: string, forkUrl: string) {
|
|
12
13
|
const type = "pyproject" as const;
|
|
13
|
-
const origin = await stringifyGithubOrigin(
|
|
14
|
+
const origin = await stringifyGithubOrigin(parseGithubRepoUrl(forkUrl));
|
|
14
15
|
const branch = "pyproject";
|
|
15
16
|
const tmpl = await readFile("./templates/add-toml.md", "utf8");
|
|
16
17
|
const { title, body } = parseTitleBodyOfMarkdown(tmpl);
|
|
17
|
-
const repo =
|
|
18
|
+
const repo = parseGithubRepoUrl(forkUrl);
|
|
18
19
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
const existingBranch = await gh.repos.getBranch({ ...repo, branch }).catch(() => null);
|
|
21
|
+
if (existingBranch) {
|
|
22
|
+
console.log("Branch exists, checking if outdated: " + branch);
|
|
23
|
+
|
|
24
|
+
// Get upstream HEAD commit
|
|
25
|
+
const upstreamRepo = parseGithubRepoUrl(upstreamUrl);
|
|
26
|
+
const upstreamRepoInfo = await gh.repos.get(upstreamRepo);
|
|
27
|
+
const defaultBranch = upstreamRepoInfo.data.default_branch;
|
|
28
|
+
const upstreamHead = await gh.repos.getBranch({ ...upstreamRepo, branch: defaultBranch });
|
|
29
|
+
|
|
30
|
+
// Get the base commit of the existing branch
|
|
31
|
+
const branchBaseCommit = existingBranch.data.commit.sha;
|
|
32
|
+
const upstreamHeadCommit = upstreamHead.data.commit.sha;
|
|
33
|
+
|
|
34
|
+
// Check if branch is outdated (base commit differs from upstream HEAD)
|
|
35
|
+
if (branchBaseCommit === upstreamHeadCommit) {
|
|
36
|
+
console.log("Branch is up to date, skipping: " + branch);
|
|
37
|
+
return { type, title, body, branch };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
console.log("Branch is outdated, recreating and force-pushing: " + branch);
|
|
41
|
+
// Continue with the branch creation logic below (will force-push)
|
|
22
42
|
}
|
|
23
|
-
const src =
|
|
43
|
+
const src = parseGithubRepoUrl(upstreamUrl);
|
|
24
44
|
const cwd = await getBranchWorkingDir(upstreamUrl, forkUrl, branch);
|
|
25
45
|
|
|
26
46
|
// commit changes
|
|
27
47
|
await $`
|
|
28
|
-
git clone ${upstreamUrl} ${cwd}
|
|
48
|
+
git clone --single-branch ${upstreamUrl} ${cwd}
|
|
29
49
|
|
|
30
50
|
cd ${cwd}
|
|
31
51
|
echo N | comfy node init
|
|
@@ -45,9 +65,23 @@ git config user.email ${GIT_USEREMAIL} && \
|
|
|
45
65
|
git checkout -b ${branch} && \
|
|
46
66
|
git add . && \
|
|
47
67
|
git commit -am ${`chore(${branch}): ${title}`} && \
|
|
48
|
-
git push "${origin}" ${branch}:${branch}
|
|
68
|
+
git push ${existingBranch ? "--force" : ""} "${origin}" ${branch}:${branch}
|
|
49
69
|
`;
|
|
50
70
|
const branchUrl = `https://github.com/${repo.owner}/${repo.repo}/tree/${branch}`;
|
|
51
71
|
console.log(`Branch Push OK: ${branchUrl}`);
|
|
52
72
|
return { type, title, body, branch };
|
|
53
73
|
}
|
|
74
|
+
|
|
75
|
+
async function tomlFillDescription(referenceUrl: string, pyprojectToml: string) {
|
|
76
|
+
const repoDescriptionMap = await fetchRepoDescriptionMap();
|
|
77
|
+
const matchedDescription =
|
|
78
|
+
repoDescriptionMap[referenceUrl]?.toString() ||
|
|
79
|
+
DIE("Warn: missing description for " + referenceUrl);
|
|
80
|
+
const replaced = (await readFile(pyprojectToml, "utf8")).replace(
|
|
81
|
+
`description = ""`,
|
|
82
|
+
`description = ${JSON.stringify(matchedDescription)}`,
|
|
83
|
+
);
|
|
84
|
+
// check validity
|
|
85
|
+
toml.parse(replaced);
|
|
86
|
+
await writeFile(pyprojectToml, replaced);
|
|
87
|
+
}
|
|
@@ -6,25 +6,23 @@ import { readFile } from "fs/promises";
|
|
|
6
6
|
import type { WithId } from "mongodb";
|
|
7
7
|
import { basename, dirname } from "path";
|
|
8
8
|
import sflow, { nil } from "sflow";
|
|
9
|
-
import { CNRepos } from "./CNRepos";
|
|
10
|
-
import { GIT_USEREMAIL } from "./GIT_USEREMAIL";
|
|
11
|
-
import { GIT_USERNAME } from "./GIT_USERNAME";
|
|
12
9
|
import { isRepoBypassed } from "./bypassRepos";
|
|
13
10
|
import { $ } from "./cli/echoBunShell";
|
|
14
|
-
import {
|
|
11
|
+
import { CNRepos } from "./CNRepos";
|
|
12
|
+
import { createGithubForkForRepoEx } from "./createGithubForkForRepo";
|
|
15
13
|
import { createGithubPullRequest } from "./createGithubPullRequest";
|
|
16
|
-
import { $
|
|
14
|
+
import { $flatten, $stale, db } from "./db";
|
|
17
15
|
import { getBranchWorkingDir } from "./getBranchWorkingDir";
|
|
18
|
-
import { gh } from "
|
|
19
|
-
import type { GithubPull } from "
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
16
|
+
import { gh } from "@/lib/github";
|
|
17
|
+
import type { GithubPull } from "@/lib/github/GithubPull";
|
|
18
|
+
import { GIT_USEREMAIL, GIT_USERNAME } from "./ghUser";
|
|
19
|
+
import { parseGithubRepoUrl, stringifyGithubOrigin } from "./parseOwnerRepo";
|
|
22
20
|
import { parseTitleBodyOfMarkdown } from "./parseTitleBodyOfMarkdown";
|
|
23
21
|
|
|
24
22
|
type LicenseUpdateTask = {
|
|
25
23
|
repository: string;
|
|
26
24
|
tomlUpdated?: boolean;
|
|
27
|
-
prTask: Task<
|
|
25
|
+
prTask: Task<unknown>;
|
|
28
26
|
updatedAt?: Date;
|
|
29
27
|
};
|
|
30
28
|
|
|
@@ -32,18 +30,12 @@ const LicenseTasks = db.collection<LicenseUpdateTask>("LicenseTasks");
|
|
|
32
30
|
await LicenseTasks.createIndex({ repository: 1 }, { unique: true });
|
|
33
31
|
|
|
34
32
|
if (import.meta.main) {
|
|
35
|
-
|
|
36
|
-
// await LicenseTasks.deleteMany({});
|
|
37
|
-
// await _pullTemplate();
|
|
38
|
-
// const testUpstreamRepo = "https://github.com/haohaocreates/ComfyUI-HH-Image-Selector";
|
|
39
|
-
// await _testMakeUpdateTomlLicenseBranch();
|
|
40
|
-
// await _listReposWithNoLicense();
|
|
41
|
-
const testMapping = {
|
|
33
|
+
const _testMapping = {
|
|
42
34
|
[`license = "MIT"`]: `license = { text = "MIT" }`,
|
|
43
35
|
[`license = "LICENSE.txt"`]: `license = { file = "LICENSE.txt" }`,
|
|
44
36
|
};
|
|
45
37
|
|
|
46
|
-
const
|
|
38
|
+
const _repoExamples = {
|
|
47
39
|
"https://github.com/MuziekMagie/ComfyUI-Matchering": "license already updated",
|
|
48
40
|
// - [ComfyUI_FizzNodes/LICENCE.txt at main · FizzleDorf/ComfyUI_FizzNodes]( https://github.com/FizzleDorf/ComfyUI_FizzNodes/blob/main/LICENCE.txt )
|
|
49
41
|
"https://github.com/FizzleDorf/ComfyUI_FizzNodes": "licenCe",
|
|
@@ -68,17 +60,26 @@ export async function updateTomlLicenseTasks() {
|
|
|
68
60
|
await $pipeline(CNRepos)
|
|
69
61
|
.match({ cr: { $exists: true } })
|
|
70
62
|
.project({ _id: 0, repository: 1 })
|
|
63
|
+
.match({
|
|
64
|
+
repository: { $exists: true, $ne: null as unknown as string, $type: "string" },
|
|
65
|
+
})
|
|
71
66
|
.merge({ into: LicenseTasks.collectionName, on: "repository" })
|
|
72
67
|
.aggregate()
|
|
73
68
|
.next();
|
|
74
69
|
|
|
75
70
|
// reset retry-able errors with a cd interval
|
|
76
|
-
await LicenseTasks.updateMany(
|
|
77
|
-
$
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
71
|
+
await LicenseTasks.updateMany(
|
|
72
|
+
$flatten({ prTask: { error: /was submitted too quickly/, mtime: $stale("5m") } }),
|
|
73
|
+
{
|
|
74
|
+
$unset: { prTask: 1 },
|
|
75
|
+
},
|
|
76
|
+
);
|
|
77
|
+
await LicenseTasks.updateMany(
|
|
78
|
+
$flatten({ prTask: { error: /Missing env.GH_TOKEN_COMFY_PR/, mtime: $stale("1h") } }),
|
|
79
|
+
{
|
|
80
|
+
$unset: { prTask: 1 },
|
|
81
|
+
},
|
|
82
|
+
);
|
|
82
83
|
console.log((await LicenseTasks.estimatedDocumentCount()) + " license tasks");
|
|
83
84
|
|
|
84
85
|
// update tasks
|
|
@@ -109,13 +110,14 @@ async function _testMakeUpdateTomlLicenseBranch() {
|
|
|
109
110
|
const testUpstreamRepo = "https://github.com/snomiao/comfy-malicious-node-test";
|
|
110
111
|
|
|
111
112
|
const prTask = await createTomlLicensePR(testUpstreamRepo).then(TaskOK).catch(TaskError);
|
|
112
|
-
if (TaskErrorOrNull(prTask)?.match("Not matched outdated case"))
|
|
113
|
+
if (TaskErrorOrNull(prTask)?.match("Not matched outdated case"))
|
|
114
|
+
console.log("not matched outdated case");
|
|
113
115
|
console.log(prTask);
|
|
114
116
|
console.log("prs_updateTomlLicense PRs DONE");
|
|
115
117
|
}
|
|
116
118
|
|
|
117
119
|
async function createTomlLicensePR(upstreamUrl: string): Promise<GithubPull> {
|
|
118
|
-
const { html_url: forkUrl } = await
|
|
120
|
+
const { html_url: forkUrl } = await createGithubForkForRepoEx(upstreamUrl);
|
|
119
121
|
const branchInfo = await makeUpdateTomlLicenseBranch(upstreamUrl, forkUrl);
|
|
120
122
|
// console.log(forkUrl); // note: this forkUrl may not be final forked url
|
|
121
123
|
console.log({ branchInfo });
|
|
@@ -126,7 +128,7 @@ async function createTomlLicensePR(upstreamUrl: string): Promise<GithubPull> {
|
|
|
126
128
|
srcUrl: forkUrl,
|
|
127
129
|
dstUrl: upstreamUrl,
|
|
128
130
|
}))
|
|
129
|
-
.map(async ({ type, ...prInfo }) => await createGithubPullRequest({ ...prInfo }))
|
|
131
|
+
.map(async ({ type: _type, ...prInfo }) => await createGithubPullRequest({ ...prInfo }))
|
|
130
132
|
.forEach((e) => e || DIE("missing pr result"))
|
|
131
133
|
.toOne()) ?? DIE("never")
|
|
132
134
|
);
|
|
@@ -139,15 +141,15 @@ export async function makeUpdateTomlLicenseBranch(upstreamUrl: string, forkUrl:
|
|
|
139
141
|
const { title, body } = parseTitleBodyOfMarkdown(tmpl);
|
|
140
142
|
|
|
141
143
|
// check forked repo if target branch existed
|
|
142
|
-
const origin = await stringifyGithubOrigin(
|
|
143
|
-
const repo =
|
|
144
|
+
const origin = await stringifyGithubOrigin(parseGithubRepoUrl(forkUrl));
|
|
145
|
+
const repo = parseGithubRepoUrl(forkUrl);
|
|
144
146
|
const existedBranch = await gh.repos.getBranch({ ...repo, branch }).catch(() => null);
|
|
145
147
|
|
|
146
148
|
const cwd = await getBranchWorkingDir(upstreamUrl, forkUrl, branch);
|
|
147
149
|
// commit changes
|
|
148
150
|
await $`
|
|
149
151
|
rm -rf ${cwd}
|
|
150
|
-
git clone ${upstreamUrl} ${cwd}
|
|
152
|
+
git clone --single-branch ${upstreamUrl} ${cwd}
|
|
151
153
|
`;
|
|
152
154
|
|
|
153
155
|
// // also pull from existed forked branch
|
|
@@ -188,7 +190,8 @@ git push "${origin}" ${branch}:${branch}
|
|
|
188
190
|
|
|
189
191
|
export async function pyprojectTomlUpdateLicenses(tomlFile: string, upstreamRepoUrl: string) {
|
|
190
192
|
const raw =
|
|
191
|
-
(await Bun.file(tomlFile).text().catch(nil)) ||
|
|
193
|
+
(await Bun.file(tomlFile).text().catch(nil)) ||
|
|
194
|
+
DIE(new Error("pyproject.toml file not existed or got empty file"));
|
|
192
195
|
const m = raw.match(/^license\s*=(.*)/im);
|
|
193
196
|
|
|
194
197
|
const licenseLine = m?.[0];
|
|
@@ -196,7 +199,9 @@ export async function pyprojectTomlUpdateLicenses(tomlFile: string, upstreamRepo
|
|
|
196
199
|
const outdatedDesiredLicense = license?.match(/^"([^"\n\r]+)"$/i)?.[1];
|
|
197
200
|
const isOutdated = !!outdatedDesiredLicense;
|
|
198
201
|
// const isOutdated = !!raw.match(outdated);
|
|
199
|
-
|
|
202
|
+
if (license) {
|
|
203
|
+
await LicenseTasks.updateOne({ repository: upstreamRepoUrl }, { $set: { license: license } });
|
|
204
|
+
}
|
|
200
205
|
if (!licenseLine) throw new Error("no license line was found, please check toml file");
|
|
201
206
|
if (!isOutdated) return { updated: false }; // not outdated
|
|
202
207
|
|
|
@@ -216,7 +221,7 @@ export async function pyprojectTomlUpdateLicenses(tomlFile: string, upstreamRepo
|
|
|
216
221
|
|
|
217
222
|
// - [Writing your pyproject.toml - Python Packaging User Guide]( https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license )
|
|
218
223
|
updated ||= await (async function () {
|
|
219
|
-
const resp = await gh.repos.get({ ...
|
|
224
|
+
const resp = await gh.repos.get({ ...parseGithubRepoUrl(upstreamRepoUrl) });
|
|
220
225
|
const license = resp.data.license;
|
|
221
226
|
if (!license) return null;
|
|
222
227
|
return `license = { text = "${license?.name}" }`;
|
|
@@ -228,18 +233,9 @@ export async function pyprojectTomlUpdateLicenses(tomlFile: string, upstreamRepo
|
|
|
228
233
|
);
|
|
229
234
|
|
|
230
235
|
const replaced = raw.replace(licenseLine, () => updated);
|
|
231
|
-
if (replaced === raw)
|
|
236
|
+
if (replaced === raw)
|
|
237
|
+
DIE(new Error("licenseLine not matched", { cause: { raw, licenseLine, updated } }));
|
|
232
238
|
await LicenseTasks.updateOne({ repository: upstreamRepoUrl }, { $set: { updateLine: updated } });
|
|
233
239
|
await Bun.write(tomlFile, replaced);
|
|
234
240
|
return { updated: true, license: replaced.match(/^license\s*=(.*)/i)?.[1]?.trim() };
|
|
235
241
|
}
|
|
236
|
-
|
|
237
|
-
/** use when template updated */
|
|
238
|
-
async function _pullTemplate() {
|
|
239
|
-
const referenceTemplate = "https://github.com/haohaocreates/ComfyUI-HH-Image-Selector/pull/3";
|
|
240
|
-
const template = await sflow(gh.pulls.get(parsePullUrl(referenceTemplate)))
|
|
241
|
-
.map((e) => e.data.body!)
|
|
242
|
-
.text();
|
|
243
|
-
await Bun.write("./templates/update-toml-license.md", template);
|
|
244
|
-
console.log(template);
|
|
245
|
-
}
|