comfy-pr 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +225 -0
- package/next-env.d.ts +5 -0
- package/package.json +115 -0
- package/src/CMNodes.ts +60 -0
- package/src/CNRepos.ts +72 -0
- package/src/CRNodes.ts +26 -0
- package/src/CRPulls.ts +4 -0
- package/src/FORK_OWNER.ts +5 -0
- package/src/FORK_PREFIX.ts +5 -0
- package/src/FollowRules.ts +25 -0
- package/src/GIT_USEREMAIL.ts +5 -0
- package/src/GIT_USERNAME.ts +9 -0
- package/src/GithubIssueComments.ts +3 -0
- package/src/PushedBranch.ts +3 -0
- package/src/Totals.ts +9 -0
- package/src/TrackingPRs.ts +12 -0
- package/src/WorkerInstances.ts +89 -0
- package/src/analyzePullsStatus.ts +115 -0
- package/src/analyzeTotals.test.ts +5 -0
- package/src/analyzeTotals.ts +114 -0
- package/src/checkComfyActivated.ts +39 -0
- package/src/cli.test.ts +0 -0
- package/src/cli.ts +40 -0
- package/src/clone_modify_push_Branches.ts +21 -0
- package/src/createComfyRegistryPRsFromCandidates.ts +55 -0
- package/src/createComfyRegistryPullRequests.ts +22 -0
- package/src/createGithubForkForRepo.ts +37 -0
- package/src/createGithubPullRequest.ts +94 -0
- package/src/createIssueComment.ts +29 -0
- package/src/fetchCMNodes.ts +22 -0
- package/src/fetchComfyRegistryNodes.ts +11 -0
- package/src/fetchCurrentGeoInfo.ts +6 -0
- package/src/fetchRelatedPullWithComments.ts +15 -0
- package/src/fetchRepoDescriptionMap.ts +15 -0
- package/src/followRuleSchema.ts +69 -0
- package/src/getActivatedShell.ts +18 -0
- package/src/getBranchWorkingDir.ts +16 -0
- package/src/getRepoWorkingDir.ts +6 -0
- package/src/ghUser.ts +6 -0
- package/src/index.ts +22 -0
- package/src/initializeFollowRules.ts +26 -0
- package/src/makePublishBranch.ts +58 -0
- package/src/makeTomlBranch.ts +53 -0
- package/src/matchRelatedPulls.ts +25 -0
- package/src/muteInstances.ts +16 -0
- package/src/parseIssueUrl.ts +6 -0
- package/src/parseOwnerRepo.ts +33 -0
- package/src/parsePullUrl.ts +6 -0
- package/src/parsePullsState.ts +6 -0
- package/src/parseTitleBodyOfMarkdown.ts +8 -0
- package/src/postSlackMessage.ts +21 -0
- package/src/preload.ts +30 -0
- package/src/pullStatusFollowSchema.ts +12 -0
- package/src/readTemplateTitle.ts +11 -0
- package/src/summaryLastPullComment.ts +8 -0
- package/src/tomlFillDescription.ts +17 -0
- package/src/updateCMNodesDuplicationWarnings.ts +56 -0
- package/src/updateCMRepos.ts +27 -0
- package/src/updateCNRepos.ts +58 -0
- package/src/updateCNReposCRPullsComments.ts +41 -0
- package/src/updateCNReposInfo.ts +57 -0
- package/src/updateCNReposPRCandidate.ts +85 -0
- package/src/updateCNReposPulls.ts +28 -0
- package/src/updateCNReposPullsDashboard.ts +49 -0
- package/src/updateCNReposRelatedPulls.ts +25 -0
- package/src/updateCRNodes.ts +50 -0
- package/src/updateCRRepos.ts +26 -0
- package/src/updateComfyTotals.ts +40 -0
- package/src/updateFollowRuleSet.ts +153 -0
- package/src/updateOutdatedPullsTemplates.ts +137 -0
- package/src/updateSlackMessages.ts +43 -0
- package/tailwind.config.ts +20 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { updateCMNodes } from "./CMNodes";
|
|
2
|
+
import { getWorkerInstance } from "./WorkerInstances";
|
|
3
|
+
import { createComfyRegistryPRsFromCandidates } from "./createComfyRegistryPRsFromCandidates";
|
|
4
|
+
import { notifySlack } from "./slack/notifySlack";
|
|
5
|
+
import { updateCMRepos } from "./updateCMRepos";
|
|
6
|
+
import { updateCNReposCRPullsComments } from "./updateCNReposCRPullsComments";
|
|
7
|
+
import { updateCNReposInfo } from "./updateCNReposInfo";
|
|
8
|
+
import { updateCNReposPRCandidate } from "./updateCNReposPRCandidate";
|
|
9
|
+
import { updateCNReposPulls } from "./updateCNReposPulls";
|
|
10
|
+
import { updateCNRepoPullsDashboard } from "./updateCNReposPullsDashboard";
|
|
11
|
+
import { updateCNReposRelatedPulls } from "./updateCNReposRelatedPulls";
|
|
12
|
+
import { updateCRNodes } from "./updateCRNodes";
|
|
13
|
+
import { updateCRRepos } from "./updateCRRepos";
|
|
14
|
+
import { updateComfyTotals } from "./updateComfyTotals";
|
|
15
|
+
import { updateOutdatedPullsTemplates } from "./updateOutdatedPullsTemplates";
|
|
16
|
+
import { tLog } from "./utils/tLog";
|
|
17
|
+
|
|
18
|
+
if (import.meta.main) {
|
|
19
|
+
await getWorkerInstance("Updating CNRepos");
|
|
20
|
+
// await cacheHealthReport();
|
|
21
|
+
await updateCNRepos();
|
|
22
|
+
// updateCNReposPRTasks
|
|
23
|
+
// await scanCNRepoThenPRs();
|
|
24
|
+
// await pMap(candidates, (e) => updateCNRepoPRStatus(e.repository), { concurrency: 4 });
|
|
25
|
+
// candidates
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export async function updateCNRepos() {
|
|
29
|
+
await Promise.all([
|
|
30
|
+
tLog("Report Worker Status", async () => {
|
|
31
|
+
const worker = await getWorkerInstance("ComfyPR Bot Running");
|
|
32
|
+
const workerInfo = `${worker.geo.countryCode}/${worker.geo.region}/${worker.geo.city}`;
|
|
33
|
+
const msg = `COMFY-PR BOT RUNNING ${new Date().toISOString()}\nWorker: ${workerInfo}`;
|
|
34
|
+
return [await notifySlack(msg, { unique: true, silent: true })];
|
|
35
|
+
}),
|
|
36
|
+
// stage 1: get repos
|
|
37
|
+
tLog("Update Nodes from ComfyUI Manager", updateCMNodes),
|
|
38
|
+
tLog("Update Repos from ComfyUI Manager", updateCMRepos),
|
|
39
|
+
tLog("Update Nodes from ComfyRegistry", updateCRNodes),
|
|
40
|
+
tLog("Update Repos from ComfyRegistry", updateCRRepos),
|
|
41
|
+
// stage 2: update repo info & pulls
|
|
42
|
+
tLog("Update CNRepos for Repo Infos", updateCNReposInfo),
|
|
43
|
+
tLog("Update CNRepos for Github Pulls", updateCNReposPulls),
|
|
44
|
+
tLog("Update Pulls Dashboard", updateCNRepoPullsDashboard),
|
|
45
|
+
// stage 3: update related pulls and comments
|
|
46
|
+
tLog("Update CNRepos for Related Pulls", updateCNReposRelatedPulls),
|
|
47
|
+
tLog("Update Outdated Pulls Templates", updateOutdatedPullsTemplates),
|
|
48
|
+
// stage 4: update related comments
|
|
49
|
+
tLog("Update CNRepos for Related Comments", updateCNReposCRPullsComments),
|
|
50
|
+
// stage 5: mark and create PRs
|
|
51
|
+
tLog("Update CNRepos PR Candidates", updateCNReposPRCandidate),
|
|
52
|
+
tLog("Create ComfyRegistry PRs", createComfyRegistryPRsFromCandidates),
|
|
53
|
+
// final
|
|
54
|
+
tLog("Update Comfy Totals", updateComfyTotals),
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
console.log("All repo updated");
|
|
58
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import pMap from "p-map";
|
|
2
|
+
import { peekYaml } from "peek-log";
|
|
3
|
+
import { CNRepos, type CRPull } from "./CNRepos";
|
|
4
|
+
import { $fresh, $stale } from "./db";
|
|
5
|
+
// import { $filaten } from "./db";
|
|
6
|
+
// import { $pipeline } from "./db/$pipeline";
|
|
7
|
+
import { $filaten } from "@/packages/mongodb-pipeline-ts/$filaten";
|
|
8
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
9
|
+
import { getWorkerInstance } from "./WorkerInstances";
|
|
10
|
+
import { fetchIssueComments } from "./gh/fetchIssueComments";
|
|
11
|
+
import { TaskError, TaskOK } from "./utils/Task";
|
|
12
|
+
|
|
13
|
+
if (import.meta.main) {
|
|
14
|
+
await getWorkerInstance("updateCNReposCRPullsComments");
|
|
15
|
+
await updateCNReposCRPullsComments();
|
|
16
|
+
console.log("All done");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function updateCNReposCRPullsComments() {
|
|
20
|
+
return await pMap(
|
|
21
|
+
$pipeline(CNRepos)
|
|
22
|
+
.unwind("$crPulls.data")
|
|
23
|
+
.match($filaten({ crPulls: { mtime: $fresh("7d"), data: { comments: { mtime: $stale("1d") } } } }))
|
|
24
|
+
.set($filaten({ "crPulls.data": { repository: "$repository" } }))
|
|
25
|
+
.replaceRoot({ newRoot: "$crPulls.data" })
|
|
26
|
+
.aggregate(),
|
|
27
|
+
async (data) => {
|
|
28
|
+
const { repository, pull } = data as unknown as { repository: string } & CRPull;
|
|
29
|
+
const html_url = pull.html_url;
|
|
30
|
+
const comments = await fetchIssueComments(repository, pull).then(TaskOK).catch(TaskError);
|
|
31
|
+
return peekYaml([
|
|
32
|
+
await CNRepos.findOneAndUpdate(
|
|
33
|
+
$filaten({ repository, crPulls: { data: { pull: { html_url } } } }),
|
|
34
|
+
{ $set: { "crPulls.data.$.comments": comments } },
|
|
35
|
+
{ upsert: true, returnDocument: "after" },
|
|
36
|
+
),
|
|
37
|
+
]);
|
|
38
|
+
},
|
|
39
|
+
{ concurrency: 2 },
|
|
40
|
+
);
|
|
41
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import pMap from "p-map";
|
|
2
|
+
import { dissoc } from "rambda";
|
|
3
|
+
import { match } from "ts-pattern";
|
|
4
|
+
import { CNRepos } from "./CNRepos";
|
|
5
|
+
import { getWorkerInstance } from "./WorkerInstances";
|
|
6
|
+
import { $filaten, $stale } from "./db";
|
|
7
|
+
import { gh } from "./gh";
|
|
8
|
+
import { parseUrlRepoOwner } from "./parseOwnerRepo";
|
|
9
|
+
import { $OK, TaskError, TaskOK } from "./utils/Task";
|
|
10
|
+
import { tLog } from "./utils/tLog";
|
|
11
|
+
|
|
12
|
+
if (import.meta.main) {
|
|
13
|
+
await getWorkerInstance("updateCNReposInfo");
|
|
14
|
+
await tLog("updateCNReposInfo", updateCNReposInfo);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function updateCNReposInfo() {
|
|
18
|
+
await CNRepos.createIndex($filaten({ info: { mtime: 1 } }));
|
|
19
|
+
return await pMap(
|
|
20
|
+
CNRepos.find($filaten({ info: { mtime: $stale("1d") } })),
|
|
21
|
+
async (repo) => {
|
|
22
|
+
const { repository } = repo;
|
|
23
|
+
console.log("[INFO] Fetching meta info from " + repository);
|
|
24
|
+
const _info = await gh.repos
|
|
25
|
+
.get({ ...parseUrlRepoOwner(repository) })
|
|
26
|
+
.then(({ data }) => data)
|
|
27
|
+
.then(TaskOK)
|
|
28
|
+
.catch(TaskError);
|
|
29
|
+
|
|
30
|
+
// Handle renamed repos
|
|
31
|
+
const { info, url } = await match(_info)
|
|
32
|
+
.with($OK, async (info) => {
|
|
33
|
+
const url = info.data.html_url;
|
|
34
|
+
if (url === repository) return { info, url: repository };
|
|
35
|
+
|
|
36
|
+
console.log("[INFO] Migrating renamed repo: \nfrom: ", repository + "\n to: " + url);
|
|
37
|
+
// migrate data into new CNRepo
|
|
38
|
+
await CNRepos.updateOne(
|
|
39
|
+
{ repository: url },
|
|
40
|
+
{
|
|
41
|
+
$set: {
|
|
42
|
+
...dissoc("_id", { ...(await CNRepos.findOneAndDelete({ repository })) }),
|
|
43
|
+
repository: url,
|
|
44
|
+
oldUrls: { $addToSet: repository },
|
|
45
|
+
},
|
|
46
|
+
},
|
|
47
|
+
{ upsert: true },
|
|
48
|
+
);
|
|
49
|
+
return { info, url };
|
|
50
|
+
})
|
|
51
|
+
.otherwise((info) => ({ info, url: repository }));
|
|
52
|
+
|
|
53
|
+
return await CNRepos.updateOne({ repository: url }, { $set: { info } }, { upsert: true });
|
|
54
|
+
},
|
|
55
|
+
{ concurrency: 2 },
|
|
56
|
+
);
|
|
57
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
import { peekYaml } from "peek-log";
|
|
4
|
+
import { match } from "ts-pattern";
|
|
5
|
+
import { CNRepos } from "./CNRepos";
|
|
6
|
+
import { $filaten, $fresh, $stale } from "./db";
|
|
7
|
+
import { updateCNReposPulls } from "./updateCNReposPulls";
|
|
8
|
+
import { updateCNReposRelatedPulls } from "./updateCNReposRelatedPulls";
|
|
9
|
+
import { $OK, TaskOK } from "./utils/Task";
|
|
10
|
+
import { tLog } from "./utils/tLog";
|
|
11
|
+
if (import.meta.main) {
|
|
12
|
+
console.log(await updateCNReposPRCandidate());
|
|
13
|
+
// show candidates
|
|
14
|
+
// console.log(
|
|
15
|
+
// YAML.stringify(
|
|
16
|
+
// await CNRepos.find($filaten({ candidate: { data: { $eq: true } } }))
|
|
17
|
+
// .map((e) => ({
|
|
18
|
+
// // candidate: match(e.candidate)
|
|
19
|
+
// // .with($OK, (e) => e)
|
|
20
|
+
// // .otherwise(() => DIE("")).data,
|
|
21
|
+
// repo: e.repository + "/pulls?q=",
|
|
22
|
+
// }))
|
|
23
|
+
// .toArray(),
|
|
24
|
+
// ),
|
|
25
|
+
// );
|
|
26
|
+
tLog("updateCNReposPulls", updateCNReposPulls);
|
|
27
|
+
tLog("updateCNReposRelatedPulls", updateCNReposRelatedPulls);
|
|
28
|
+
|
|
29
|
+
await CNRepos.createIndex("crPulls.mtime");
|
|
30
|
+
// show candidates
|
|
31
|
+
peekYaml(
|
|
32
|
+
await $pipeline(CNRepos)
|
|
33
|
+
.match(
|
|
34
|
+
$filaten({
|
|
35
|
+
crPulls: { mtime: $stale("7d") },
|
|
36
|
+
// info: { mtime: $fresh("7d"), data: { private: false, archived: false } },
|
|
37
|
+
// createdPulls: { mtime: $stale("7d"), data: { $exists: false } },
|
|
38
|
+
}),
|
|
39
|
+
)
|
|
40
|
+
.aggregate()
|
|
41
|
+
.map((e) => ({
|
|
42
|
+
repo: e.repository + "/pulls?q=",
|
|
43
|
+
pulls: match(e.pulls)
|
|
44
|
+
.with($OK, ({ data }) => data)
|
|
45
|
+
.otherwise(() => null)?.length,
|
|
46
|
+
crPulls: match(e.crPulls)
|
|
47
|
+
.with($OK, ({ data }) => data)
|
|
48
|
+
.otherwise(() => null)?.length,
|
|
49
|
+
}))
|
|
50
|
+
.toArray(),
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function updateCNReposPRCandidate() {
|
|
55
|
+
return await pMap(
|
|
56
|
+
$pipeline(CNRepos)
|
|
57
|
+
.match(
|
|
58
|
+
$filaten({
|
|
59
|
+
crPulls: { mtime: $fresh("1d"), ...$OK },
|
|
60
|
+
// info: { mtime: $fresh("7d"), ...$OK, data: { private: false, archived: false } },
|
|
61
|
+
info: { mtime: $fresh("7d"), ...$OK },
|
|
62
|
+
// createdPulls: { state: { $ne: "ok" }, mtime: $stale("5m") },
|
|
63
|
+
candidate: { mtime: $stale("1d") },
|
|
64
|
+
}),
|
|
65
|
+
)
|
|
66
|
+
.aggregate(),
|
|
67
|
+
async (repo) => {
|
|
68
|
+
const crPulls = match(repo.crPulls)
|
|
69
|
+
.with($OK, (e) => e.data)
|
|
70
|
+
.otherwise(() => null)!;
|
|
71
|
+
const info = match(repo.info)
|
|
72
|
+
.with($OK, (e) => e.data)
|
|
73
|
+
.otherwise(() => null)!;
|
|
74
|
+
|
|
75
|
+
const isCandidate = !info.private && !info.archived && crPulls.length === 0;
|
|
76
|
+
const candidate = TaskOK(isCandidate);
|
|
77
|
+
|
|
78
|
+
// silent
|
|
79
|
+
// if (isCandidate)
|
|
80
|
+
// await notifySlackLinks("Found new PR candidate", [repo.repository]);
|
|
81
|
+
await CNRepos.updateOne({ repository: repo.repository }, { $set: { candidate } });
|
|
82
|
+
return repo;
|
|
83
|
+
},
|
|
84
|
+
);
|
|
85
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
import { match } from "ts-pattern";
|
|
4
|
+
import { CNRepos } from "./CNRepos";
|
|
5
|
+
import { $filaten, $stale } from "./db";
|
|
6
|
+
import { fetchGithubPulls } from "./gh/fetchGithubPulls";
|
|
7
|
+
import { $OK, TaskError, TaskOK } from "./utils/Task";
|
|
8
|
+
import { tLog } from "./utils/tLog";
|
|
9
|
+
if (import.meta.main) {
|
|
10
|
+
console.log(await tLog("Update CNRepos for Github Pulls", updateCNReposPulls));
|
|
11
|
+
}
|
|
12
|
+
export async function updateCNReposPulls() {
|
|
13
|
+
await CNRepos.createIndex("pulls.mtime");
|
|
14
|
+
return await pMap(
|
|
15
|
+
$pipeline(CNRepos)
|
|
16
|
+
.match($filaten({ pulls: { mtime: $stale("1d") } }))
|
|
17
|
+
.project({ repository: 1 })
|
|
18
|
+
.aggregate(),
|
|
19
|
+
async ({ repository }) => {
|
|
20
|
+
const pulls = await fetchGithubPulls(repository).then(TaskOK).catch(TaskError);
|
|
21
|
+
match(pulls)
|
|
22
|
+
.with($OK, ({ data }) => console.debug(`[DEBUG] updated ${data.length} Pulls from ${repository}`))
|
|
23
|
+
.otherwise(() => {});
|
|
24
|
+
return await CNRepos.updateOne({ repository }, { $set: { pulls } });
|
|
25
|
+
},
|
|
26
|
+
{ concurrency: 2 },
|
|
27
|
+
);
|
|
28
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import DIE from "@snomiao/die";
|
|
2
|
+
import { match } from "ts-pattern";
|
|
3
|
+
import { CNRepos } from "./CNRepos";
|
|
4
|
+
import { $filaten, $fresh } from "./db";
|
|
5
|
+
import { gh } from "./gh";
|
|
6
|
+
import { ghUser } from "./ghUser";
|
|
7
|
+
import { parseUrlRepoOwner, stringifyOwnerRepo } from "./parseOwnerRepo";
|
|
8
|
+
import { $OK } from "./utils/Task";
|
|
9
|
+
|
|
10
|
+
export async function updateCNRepoPullsDashboard() {
|
|
11
|
+
if (ghUser.login !== "snomiao") return [];
|
|
12
|
+
const dashBoardIssue = process.env.DASHBOARD_ISSUE_URL || DIE("DASHBOARD_ISSUE_URL not found");
|
|
13
|
+
const dashBoardRepo = dashBoardIssue.replace(/\/issues\/\d+$/, "");
|
|
14
|
+
const dashBoardIssueNumber = Number(dashBoardIssue.match(/\/issues\/(\d+)$/)?.[1] || DIE("Issue number not found"));
|
|
15
|
+
// update dashboard issue if run by @snomiao
|
|
16
|
+
const repos = await CNRepos.find($filaten({ crPulls: { mtime: $fresh("1d") } })).toArray();
|
|
17
|
+
const result = repos
|
|
18
|
+
.map((repo) => {
|
|
19
|
+
const crPulls = match(repo.crPulls)
|
|
20
|
+
.with($OK, ({ data }) => data)
|
|
21
|
+
.otherwise(() => DIE("CR Pulls not found"));
|
|
22
|
+
const repoName = stringifyOwnerRepo(parseUrlRepoOwner(repo.repository));
|
|
23
|
+
const body = crPulls
|
|
24
|
+
.filter((e) => e.pull.prState !== "closed")
|
|
25
|
+
.map((e) => {
|
|
26
|
+
const date = new Date(e.pull.createdAt).toISOString().slice(0, 10);
|
|
27
|
+
const state = e.pull.prState.toUpperCase();
|
|
28
|
+
return {
|
|
29
|
+
href: e.pull.html_url,
|
|
30
|
+
name: `PR ${date} ${state}: ${repoName} #${e.type}`,
|
|
31
|
+
};
|
|
32
|
+
})
|
|
33
|
+
.map(({ href, name }) => `- [${name}](${href})`)
|
|
34
|
+
.toSorted()
|
|
35
|
+
.join("\n");
|
|
36
|
+
return body;
|
|
37
|
+
})
|
|
38
|
+
.filter(Boolean)
|
|
39
|
+
.join("\n");
|
|
40
|
+
const body = result;
|
|
41
|
+
|
|
42
|
+
return [
|
|
43
|
+
await gh.issues.update({
|
|
44
|
+
...parseUrlRepoOwner(dashBoardRepo),
|
|
45
|
+
issue_number: dashBoardIssueNumber,
|
|
46
|
+
body,
|
|
47
|
+
}),
|
|
48
|
+
];
|
|
49
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import DIE from "@snomiao/die";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
import { match } from "ts-pattern";
|
|
4
|
+
import { CNRepos } from "./CNRepos";
|
|
5
|
+
import { $filaten, $stale } from "./db";
|
|
6
|
+
import { matchRelatedPulls } from "./matchRelatedPulls";
|
|
7
|
+
import { $OK, TaskError, TaskOK } from "./utils/Task";
|
|
8
|
+
import { tLog } from "./utils/tLog";
|
|
9
|
+
if (import.meta.main) {
|
|
10
|
+
await tLog("updateCNReposRelatedPulls", updateCNReposRelatedPulls);
|
|
11
|
+
}
|
|
12
|
+
export async function updateCNReposRelatedPulls() {
|
|
13
|
+
await CNRepos.createIndex({ "pulls.state": 1, "crPulls.mtime": 1 });
|
|
14
|
+
return await pMap(
|
|
15
|
+
CNRepos.find($filaten({ pulls: { state: "ok" }, crPulls: { mtime: $stale("1d") } })),
|
|
16
|
+
async (repo, i) => {
|
|
17
|
+
const { repository } = repo;
|
|
18
|
+
const pulls = match(repo.pulls)
|
|
19
|
+
.with($OK, (e) => e.data)
|
|
20
|
+
.otherwise(() => DIE("Pulls not found"));
|
|
21
|
+
const crPulls = await matchRelatedPulls(pulls).then(TaskOK).catch(TaskError);
|
|
22
|
+
return await CNRepos.updateOne({ repository }, { $set: { crPulls } });
|
|
23
|
+
},
|
|
24
|
+
);
|
|
25
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
+
import { peekYaml } from "peek-log";
|
|
3
|
+
import { filter, groupBy, values } from "rambda";
|
|
4
|
+
import YAML from "yaml";
|
|
5
|
+
import { CNRepos } from "./CNRepos";
|
|
6
|
+
import { CRNodes } from "./CRNodes";
|
|
7
|
+
import { fetchCRNodes } from "./fetchComfyRegistryNodes";
|
|
8
|
+
import { notifySlack } from "./slack/notifySlack";
|
|
9
|
+
import { TaskOK } from "./utils/Task";
|
|
10
|
+
import { tLog } from "./utils/tLog";
|
|
11
|
+
|
|
12
|
+
if (import.meta.main) {
|
|
13
|
+
peekYaml(await tLog(updateCRNodes));
|
|
14
|
+
console.log("CRNodes updated");
|
|
15
|
+
|
|
16
|
+
peekYaml(
|
|
17
|
+
await $pipeline(CNRepos).project({ repository: 1, on_registry: 1 }).sample({ size: 8 }).aggregate().toArray(),
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
export async function updateCRNodes() {
|
|
21
|
+
const nodes = await fetchCRNodes();
|
|
22
|
+
|
|
23
|
+
// check src duplicated
|
|
24
|
+
const group = groupBy((e) => e.repository, nodes);
|
|
25
|
+
const duplicates = filter((e) => e.length > 1, group);
|
|
26
|
+
if (values(duplicates).length) {
|
|
27
|
+
const msg =
|
|
28
|
+
"[WARN] Same repo but different ids found in comfyregistry:\n" + "```\n" + YAML.stringify(duplicates) + "\n```";
|
|
29
|
+
await notifySlack(msg, { unique: true });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return [
|
|
33
|
+
await $pipeline(CNRepos)
|
|
34
|
+
.project({ repository: 1 })
|
|
35
|
+
.set({ on_registry: TaskOK({ $in: ["$repository", nodes.map(({ repository }) => repository).filter(Boolean)] }) })
|
|
36
|
+
.merge({ into: "CNRepos", on: "repository", whenMatched: "merge", whenNotMatched: "insert" })
|
|
37
|
+
.aggregate()
|
|
38
|
+
.next(),
|
|
39
|
+
await CRNodes.bulkWrite(
|
|
40
|
+
nodes.map((node) => ({
|
|
41
|
+
updateOne: {
|
|
42
|
+
filter: { id: node.id },
|
|
43
|
+
update: { $set: node },
|
|
44
|
+
upsert: true,
|
|
45
|
+
},
|
|
46
|
+
})),
|
|
47
|
+
{ ordered: false },
|
|
48
|
+
),
|
|
49
|
+
];
|
|
50
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import pMap from "p-map";
|
|
2
|
+
import { CNRepos } from "./CNRepos";
|
|
3
|
+
import { CRNodes } from "./CRNodes";
|
|
4
|
+
import { tLog } from "./utils/tLog";
|
|
5
|
+
if (import.meta.main) {
|
|
6
|
+
await tLog("updateCRRepos", updateCRRepos);
|
|
7
|
+
}
|
|
8
|
+
export async function updateCRRepos() {
|
|
9
|
+
await CRNodes.createIndex({ repo_id: 1 });
|
|
10
|
+
return await pMap(CRNodes.find({ repo_id: { $exists: false } }), async (cr, i) => {
|
|
11
|
+
const { repository, _id } = cr;
|
|
12
|
+
await CNRepos.updateOne(
|
|
13
|
+
{ repository },
|
|
14
|
+
{
|
|
15
|
+
$set: {
|
|
16
|
+
cr: (await CRNodes.findOneAndUpdate(
|
|
17
|
+
{ _id },
|
|
18
|
+
{ $set: { repo_id: cr._id } },
|
|
19
|
+
{ upsert: true, returnDocument: "after" },
|
|
20
|
+
))!,
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
{ upsert: true },
|
|
24
|
+
);
|
|
25
|
+
});
|
|
26
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { match } from "ts-pattern";
|
|
2
|
+
import YAML from "yaml";
|
|
3
|
+
import { Totals } from "./Totals";
|
|
4
|
+
import { analyzeTotals } from "./analyzeTotals";
|
|
5
|
+
import { $filaten, $fresh } from "./db";
|
|
6
|
+
import { notifySlack } from "./slack/notifySlack";
|
|
7
|
+
import { $OK, TaskError, TaskOK } from "./utils/Task";
|
|
8
|
+
import { tsmatch } from "./utils/tsmatch";
|
|
9
|
+
|
|
10
|
+
if (import.meta.main) {
|
|
11
|
+
await updateComfyTotals();
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
|
|
15
|
+
const today = new Date().toISOString().split("T")[0];
|
|
16
|
+
const cached = await Totals.findOne($filaten({ today, totals: { mtime: $fresh(fresh), ...$OK } }));
|
|
17
|
+
if (cached?.totals?.state === "ok")
|
|
18
|
+
return [
|
|
19
|
+
tsmatch(cached.totals)
|
|
20
|
+
.with($OK, ({ data }) => data)
|
|
21
|
+
.otherwise(() => null),
|
|
22
|
+
].flatMap((e) => (e ? [e] : []));
|
|
23
|
+
const totals = await analyzeTotals().then(TaskOK).catch(TaskError);
|
|
24
|
+
|
|
25
|
+
// notify
|
|
26
|
+
notify &&
|
|
27
|
+
(await match(totals)
|
|
28
|
+
.with($OK, async (totals) => {
|
|
29
|
+
const msg = `Totals: \n${"```"}\n${YAML.stringify(totals)}\n${"```"}`;
|
|
30
|
+
await notifySlack(msg, { unique: true });
|
|
31
|
+
})
|
|
32
|
+
.otherwise(() => null));
|
|
33
|
+
|
|
34
|
+
const insertResult = await Totals.insertOne({ totals });
|
|
35
|
+
return [
|
|
36
|
+
tsmatch(totals)
|
|
37
|
+
.with($OK, ({ data }) => data)
|
|
38
|
+
.otherwise(() => null),
|
|
39
|
+
].flatMap((e) => (e ? [e] : []));
|
|
40
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use server";
|
|
2
|
+
import { TaskDataOrNull } from "@/app/(dashboard)/rules/TaskDataOrNull";
|
|
3
|
+
import { $elemMatch } from "@/packages/mongodb-pipeline-ts/$elemMatch";
|
|
4
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
5
|
+
import DIE from "@snomiao/die";
|
|
6
|
+
import pMap from "p-map";
|
|
7
|
+
import { CNRepos, type CRPull } from "./CNRepos";
|
|
8
|
+
import { FollowRuleSets } from "./FollowRules";
|
|
9
|
+
import type { GithubIssueComment } from "./GithubIssueComments";
|
|
10
|
+
import { analyzePullsStatusPipeline, type PullStatus } from "./analyzePullsStatus";
|
|
11
|
+
import { createIssueComment } from "./createIssueComment";
|
|
12
|
+
import { $filaten } from "./db";
|
|
13
|
+
import { zAddCommentAction, zFollowUpRules } from "./followRuleSchema";
|
|
14
|
+
import { notifySlackLinks } from "./slack/notifySlackLinks";
|
|
15
|
+
import { TaskError, TaskOK } from "./utils/Task";
|
|
16
|
+
import { prettyMs } from "./utils/tLog";
|
|
17
|
+
import { yaml } from "./utils/yaml";
|
|
18
|
+
|
|
19
|
+
if (import.meta.main) {
|
|
20
|
+
// updateFollowRuleSet({yaml: 'default'})
|
|
21
|
+
await runFollowRuleSet();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export async function runFollowRuleSet({ name = "default" } = {}) {
|
|
25
|
+
const ruleset = (await FollowRuleSets.findOne({ name })) ?? DIE("default ruleset not found");
|
|
26
|
+
return await updateFollowRuleSet({
|
|
27
|
+
name: ruleset.name,
|
|
28
|
+
enable: ruleset.enabled ?? DIE("Ruleset is not enabled"),
|
|
29
|
+
yaml: ruleset.yamlWhenEnabled ?? DIE("Enabled yaml is not found"),
|
|
30
|
+
runAction: true,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
export type updateFollowRuleSet = typeof updateFollowRuleSet;
|
|
34
|
+
export async function updateFollowRuleSet({
|
|
35
|
+
name,
|
|
36
|
+
yaml: code,
|
|
37
|
+
enable,
|
|
38
|
+
runAction = false,
|
|
39
|
+
}: {
|
|
40
|
+
name: string;
|
|
41
|
+
/** update yaml if provided */
|
|
42
|
+
yaml?: string;
|
|
43
|
+
/** update enable if provided */
|
|
44
|
+
enable?: boolean | undefined;
|
|
45
|
+
/** run action if set to true */
|
|
46
|
+
runAction?: boolean;
|
|
47
|
+
}) {
|
|
48
|
+
"use server";
|
|
49
|
+
return await (async function () {
|
|
50
|
+
if (enable === false) {
|
|
51
|
+
await FollowRuleSets.updateOne({ name }, { $set: { enabled: false, yamlWhenEnabled: "" } });
|
|
52
|
+
DIE("ruleset is disabled, no need to run");
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// must parse while run, because the date in code is dynamic generated
|
|
56
|
+
const rules = zFollowUpRules.parse(yaml.parse(code || DIE("yaml is empty")));
|
|
57
|
+
// save if parse succ
|
|
58
|
+
// await FollowRuleSets.updateOne({ name }, { $set: { yaml: code, rules } });
|
|
59
|
+
const parseResult = await pMap(
|
|
60
|
+
rules,
|
|
61
|
+
async (rule) => {
|
|
62
|
+
const matched = await analyzePullsStatusPipeline()
|
|
63
|
+
.match(rule.$match)
|
|
64
|
+
.aggregate()
|
|
65
|
+
.map(({ updated_at, created_at, on_registry_at, ...pull }) => {
|
|
66
|
+
const updated = prettyMs(+new Date() - +new Date(updated_at), { compact: true }) + " ago";
|
|
67
|
+
return {
|
|
68
|
+
updated, //: updated === created ? "never" : updated,
|
|
69
|
+
...pull,
|
|
70
|
+
lastwords: pull.lastwords?.replace(/\s+/g, " ").replace(/\*\*\*.*/g, "..."),
|
|
71
|
+
} as PullStatus;
|
|
72
|
+
})
|
|
73
|
+
.toArray()
|
|
74
|
+
.then(TaskOK)
|
|
75
|
+
.catch(TaskError);
|
|
76
|
+
const actions = await (async function () {
|
|
77
|
+
return await pMap(
|
|
78
|
+
Object.entries(rule.action),
|
|
79
|
+
async ([name, _action]) => {
|
|
80
|
+
if (name === "add-comment") {
|
|
81
|
+
const action = zAddCommentAction.parse(_action);
|
|
82
|
+
const matchedData = TaskDataOrNull(matched) ?? DIE("NO-PAYLOAD-AVAILABLE");
|
|
83
|
+
return await pMap(
|
|
84
|
+
matchedData,
|
|
85
|
+
async (payload) => {
|
|
86
|
+
const loadedAction = {
|
|
87
|
+
action: "add-comment",
|
|
88
|
+
url: payload.url,
|
|
89
|
+
by: action.by,
|
|
90
|
+
body: action.body.replace(
|
|
91
|
+
/{{\$(\w+)}}/,
|
|
92
|
+
(_, key: string) =>
|
|
93
|
+
(payload as any)[key] ||
|
|
94
|
+
DIE("Missing key: " + key + " in payload: " + JSON.stringify(payload)),
|
|
95
|
+
),
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
if (runAction) {
|
|
99
|
+
const existedComments =
|
|
100
|
+
(await $pipeline(CNRepos)
|
|
101
|
+
.unwind("$crPulls.data")
|
|
102
|
+
.match({ "crPulls.data.pull.html_url": loadedAction.url })
|
|
103
|
+
.with<{ "crPulls.data": CRPull }>()
|
|
104
|
+
.replaceRoot({ newRoot: "$crPulls.data.comments.data" })
|
|
105
|
+
.as<GithubIssueComment[]>()
|
|
106
|
+
.aggregate()
|
|
107
|
+
.next()) ??
|
|
108
|
+
DIE("comments is not fetched before, plz check " + loadedAction.url + " in CNRepos");
|
|
109
|
+
|
|
110
|
+
const existedComment = existedComments.find((e) => e.body === loadedAction.body);
|
|
111
|
+
if (!existedComment) {
|
|
112
|
+
const { comments, comment } = await createIssueComment(
|
|
113
|
+
loadedAction.url,
|
|
114
|
+
loadedAction.body,
|
|
115
|
+
loadedAction.by,
|
|
116
|
+
);
|
|
117
|
+
const updateResult = await CNRepos.updateOne(
|
|
118
|
+
$filaten({ crPulls: { data: $elemMatch({ pull: { html_url: loadedAction.url } }) } }),
|
|
119
|
+
{ $set: { "crPulls.data.$.comments": comments } },
|
|
120
|
+
);
|
|
121
|
+
if (!updateResult.matchedCount) DIE("created issue not matched");
|
|
122
|
+
await notifySlackLinks("A New issue comment are created from rule " + rule.name, [
|
|
123
|
+
comment.html_url,
|
|
124
|
+
]);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
return loadedAction;
|
|
129
|
+
},
|
|
130
|
+
{ stopOnError: false, concurrency: 1 },
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
{ concurrency: 1 },
|
|
135
|
+
);
|
|
136
|
+
})()
|
|
137
|
+
.then(TaskOK)
|
|
138
|
+
.catch(TaskError);
|
|
139
|
+
|
|
140
|
+
return { name: rule.name, matched, actions };
|
|
141
|
+
},
|
|
142
|
+
{ concurrency: 1 },
|
|
143
|
+
);
|
|
144
|
+
if (enable) {
|
|
145
|
+
await FollowRuleSets.updateOne({ name }, { $set: { enabled: true, yamlWhenEnabled: code, yaml: code } });
|
|
146
|
+
} else {
|
|
147
|
+
await FollowRuleSets.updateOne({ name }, { $set: { yaml: code } });
|
|
148
|
+
}
|
|
149
|
+
return parseResult;
|
|
150
|
+
})()
|
|
151
|
+
.then(TaskOK)
|
|
152
|
+
.catch(TaskError);
|
|
153
|
+
}
|