comfy-pr 0.2.19 → 0.2.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +225 -225
- package/next-env.d.ts +5 -5
- package/package.json +10 -2
- package/src/Authors.ts +48 -0
- package/src/CMNodes.ts +60 -60
- package/src/CNRepos.ts +74 -72
- package/src/CRNodes.ts +27 -27
- package/src/CRPulls.ts +4 -4
- package/src/FORK_OWNER.ts +5 -5
- package/src/FORK_PREFIX.ts +5 -5
- package/src/FollowRules.ts +25 -25
- package/src/GIT_USEREMAIL.ts +5 -5
- package/src/GIT_USERNAME.ts +9 -9
- package/src/GithubIssueComments.ts +3 -3
- package/src/PushedBranch.ts +3 -3
- package/src/Totals.ts +9 -9
- package/src/TrackingPRs.ts +12 -12
- package/src/WorkerInstances.ts +89 -89
- package/src/addCommentAction.ts +65 -0
- package/src/analyzePullsStatus.ts +170 -125
- package/src/analyzeTotals.test.ts +5 -5
- package/src/analyzeTotals.ts +118 -120
- package/src/checkComfyActivated.ts +39 -39
- package/src/cli.ts +40 -40
- package/src/clone_modify_push_Branches.ts +21 -21
- package/src/createComfyRegistryPRsFromCandidates.ts +55 -55
- package/src/createComfyRegistryPullRequests.ts +22 -22
- package/src/createGithubForkForRepo.ts +37 -37
- package/src/createGithubPullRequest.ts +94 -94
- package/src/createIssueComment.ts +29 -29
- package/src/fetchCMNodes.ts +22 -22
- package/src/fetchComfyRegistryNodes.ts +11 -11
- package/src/fetchCurrentGeoInfo.ts +6 -6
- package/src/fetchRelatedPullWithComments.ts +15 -15
- package/src/fetchRepoDescriptionMap.ts +15 -15
- package/src/followRuleSchema.ts +76 -75
- package/src/getActivatedShell.ts +18 -18
- package/src/getBranchWorkingDir.ts +16 -16
- package/src/getRepoWorkingDir.ts +5 -5
- package/src/ghUser.ts +6 -6
- package/src/index.ts +22 -21
- package/src/initializeFollowRules.ts +26 -26
- package/src/makePublishBranch.ts +58 -58
- package/src/makeTomlBranch.ts +53 -53
- package/src/matchRelatedPulls.ts +60 -25
- package/src/muteInstances.ts +16 -16
- package/src/parseIssueUrl.ts +6 -6
- package/src/parseOwnerRepo.ts +33 -33
- package/src/parsePullUrl.ts +6 -6
- package/src/parsePullsState.ts +6 -6
- package/src/parseTitleBodyOfMarkdown.ts +8 -8
- package/src/postSlackMessage.ts +21 -21
- package/src/preload.ts +30 -30
- package/src/pullStatusFollowSchema.ts +12 -12
- package/src/readTemplateTitle.ts +11 -11
- package/src/showFollowRuleSet.ts +12 -0
- package/src/summaryLastPullComment.ts +8 -8
- package/src/tomlFillDescription.ts +17 -17
- package/src/updateAuthors.ts +7 -0
- package/src/updateAuthorsForGithub.ts +50 -0
- package/src/updateAuthorsFromCNRepo.ts +30 -0
- package/src/updateCMNodesDuplicationWarnings.ts +56 -56
- package/src/updateCMRepos.ts +27 -27
- package/src/updateCNRepos.ts +58 -58
- package/src/updateCNReposCRPullsComments.ts +40 -40
- package/src/updateCNReposInfo.ts +57 -57
- package/src/updateCNReposPRCandidate.ts +85 -85
- package/src/updateCNReposPulls.ts +28 -28
- package/src/updateCNReposPullsDashboard.ts +49 -49
- package/src/updateCNReposRelatedPulls.ts +25 -25
- package/src/updateCRNodes.ts +50 -50
- package/src/updateCRRepos.ts +26 -26
- package/src/updateComfyTotals.ts +43 -43
- package/src/updateFollowRuleSet.ts +127 -182
- package/src/updateOutdatedPullsTemplates.ts +160 -156
- package/src/updateSlackMessages.ts +43 -43
- package/tailwind.config.ts +20 -20
- package/CHANGELOG.md +0 -109
|
@@ -1,85 +1,85 @@
|
|
|
1
|
-
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
-
import pMap from "p-map";
|
|
3
|
-
import { peekYaml } from "peek-log";
|
|
4
|
-
import { match } from "ts-pattern";
|
|
5
|
-
import { $OK, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
6
|
-
import { CNRepos } from "./CNRepos";
|
|
7
|
-
import { $filaten, $fresh, $stale } from "./db";
|
|
8
|
-
import { updateCNReposPulls } from "./updateCNReposPulls";
|
|
9
|
-
import { updateCNReposRelatedPulls } from "./updateCNReposRelatedPulls";
|
|
10
|
-
import { tLog } from "./utils/tLog";
|
|
11
|
-
if (import.meta.main) {
|
|
12
|
-
console.log(await updateCNReposPRCandidate());
|
|
13
|
-
// show candidates
|
|
14
|
-
// console.log(
|
|
15
|
-
// YAML.stringify(
|
|
16
|
-
// await CNRepos.find($filaten({ candidate: { data: { $eq: true } } }))
|
|
17
|
-
// .map((e) => ({
|
|
18
|
-
// // candidate: match(e.candidate)
|
|
19
|
-
// // .with($OK, (e) => e)
|
|
20
|
-
// // .otherwise(() => DIE("")).data,
|
|
21
|
-
// repo: e.repository + "/pulls?q=",
|
|
22
|
-
// }))
|
|
23
|
-
// .toArray(),
|
|
24
|
-
// ),
|
|
25
|
-
// );
|
|
26
|
-
tLog("updateCNReposPulls", updateCNReposPulls);
|
|
27
|
-
tLog("updateCNReposRelatedPulls", updateCNReposRelatedPulls);
|
|
28
|
-
|
|
29
|
-
await CNRepos.createIndex("crPulls.mtime");
|
|
30
|
-
// show candidates
|
|
31
|
-
peekYaml(
|
|
32
|
-
await $pipeline(CNRepos)
|
|
33
|
-
.match(
|
|
34
|
-
$filaten({
|
|
35
|
-
crPulls: { mtime: $stale("7d") },
|
|
36
|
-
// info: { mtime: $fresh("7d"), data: { private: false, archived: false } },
|
|
37
|
-
// createdPulls: { mtime: $stale("7d"), data: { $exists: false } },
|
|
38
|
-
}),
|
|
39
|
-
)
|
|
40
|
-
.aggregate()
|
|
41
|
-
.map((e) => ({
|
|
42
|
-
repo: e.repository + "/pulls?q=",
|
|
43
|
-
pulls: match(e.pulls)
|
|
44
|
-
.with($OK, ({ data }) => data)
|
|
45
|
-
.otherwise(() => null)?.length,
|
|
46
|
-
crPulls: match(e.crPulls)
|
|
47
|
-
.with($OK, ({ data }) => data)
|
|
48
|
-
.otherwise(() => null)?.length,
|
|
49
|
-
}))
|
|
50
|
-
.toArray(),
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export async function updateCNReposPRCandidate() {
|
|
55
|
-
return await pMap(
|
|
56
|
-
$pipeline(CNRepos)
|
|
57
|
-
.match(
|
|
58
|
-
$filaten({
|
|
59
|
-
crPulls: { mtime: $fresh("1d"), ...$OK },
|
|
60
|
-
// info: { mtime: $fresh("7d"), ...$OK, data: { private: false, archived: false } },
|
|
61
|
-
info: { mtime: $fresh("7d"), ...$OK },
|
|
62
|
-
// createdPulls: { state: { $ne: "ok" }, mtime: $stale("5m") },
|
|
63
|
-
candidate: { mtime: $stale("1d") },
|
|
64
|
-
}),
|
|
65
|
-
)
|
|
66
|
-
.aggregate(),
|
|
67
|
-
async (repo) => {
|
|
68
|
-
const crPulls = match(repo.crPulls)
|
|
69
|
-
.with($OK, (e) => e.data)
|
|
70
|
-
.otherwise(() => null)!;
|
|
71
|
-
const info = match(repo.info)
|
|
72
|
-
.with($OK, (e) => e.data)
|
|
73
|
-
.otherwise(() => null)!;
|
|
74
|
-
|
|
75
|
-
const isCandidate = !info.private && !info.archived && crPulls.length === 0;
|
|
76
|
-
const candidate = TaskOK(isCandidate);
|
|
77
|
-
|
|
78
|
-
// silent
|
|
79
|
-
// if (isCandidate)
|
|
80
|
-
// await notifySlackLinks("Found new PR candidate", [repo.repository]);
|
|
81
|
-
await CNRepos.updateOne({ repository: repo.repository }, { $set: { candidate } });
|
|
82
|
-
return repo;
|
|
83
|
-
},
|
|
84
|
-
);
|
|
85
|
-
}
|
|
1
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
import { peekYaml } from "peek-log";
|
|
4
|
+
import { match } from "ts-pattern";
|
|
5
|
+
import { $OK, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
6
|
+
import { CNRepos } from "./CNRepos";
|
|
7
|
+
import { $filaten, $fresh, $stale } from "./db";
|
|
8
|
+
import { updateCNReposPulls } from "./updateCNReposPulls";
|
|
9
|
+
import { updateCNReposRelatedPulls } from "./updateCNReposRelatedPulls";
|
|
10
|
+
import { tLog } from "./utils/tLog";
|
|
11
|
+
if (import.meta.main) {
|
|
12
|
+
console.log(await updateCNReposPRCandidate());
|
|
13
|
+
// show candidates
|
|
14
|
+
// console.log(
|
|
15
|
+
// YAML.stringify(
|
|
16
|
+
// await CNRepos.find($filaten({ candidate: { data: { $eq: true } } }))
|
|
17
|
+
// .map((e) => ({
|
|
18
|
+
// // candidate: match(e.candidate)
|
|
19
|
+
// // .with($OK, (e) => e)
|
|
20
|
+
// // .otherwise(() => DIE("")).data,
|
|
21
|
+
// repo: e.repository + "/pulls?q=",
|
|
22
|
+
// }))
|
|
23
|
+
// .toArray(),
|
|
24
|
+
// ),
|
|
25
|
+
// );
|
|
26
|
+
tLog("updateCNReposPulls", updateCNReposPulls);
|
|
27
|
+
tLog("updateCNReposRelatedPulls", updateCNReposRelatedPulls);
|
|
28
|
+
|
|
29
|
+
await CNRepos.createIndex("crPulls.mtime");
|
|
30
|
+
// show candidates
|
|
31
|
+
peekYaml(
|
|
32
|
+
await $pipeline(CNRepos)
|
|
33
|
+
.match(
|
|
34
|
+
$filaten({
|
|
35
|
+
crPulls: { mtime: $stale("7d") },
|
|
36
|
+
// info: { mtime: $fresh("7d"), data: { private: false, archived: false } },
|
|
37
|
+
// createdPulls: { mtime: $stale("7d"), data: { $exists: false } },
|
|
38
|
+
}),
|
|
39
|
+
)
|
|
40
|
+
.aggregate()
|
|
41
|
+
.map((e) => ({
|
|
42
|
+
repo: e.repository + "/pulls?q=",
|
|
43
|
+
pulls: match(e.pulls)
|
|
44
|
+
.with($OK, ({ data }) => data)
|
|
45
|
+
.otherwise(() => null)?.length,
|
|
46
|
+
crPulls: match(e.crPulls)
|
|
47
|
+
.with($OK, ({ data }) => data)
|
|
48
|
+
.otherwise(() => null)?.length,
|
|
49
|
+
}))
|
|
50
|
+
.toArray(),
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function updateCNReposPRCandidate() {
|
|
55
|
+
return await pMap(
|
|
56
|
+
$pipeline(CNRepos)
|
|
57
|
+
.match(
|
|
58
|
+
$filaten({
|
|
59
|
+
crPulls: { mtime: $fresh("1d"), ...$OK },
|
|
60
|
+
// info: { mtime: $fresh("7d"), ...$OK, data: { private: false, archived: false } },
|
|
61
|
+
info: { mtime: $fresh("7d"), ...$OK },
|
|
62
|
+
// createdPulls: { state: { $ne: "ok" }, mtime: $stale("5m") },
|
|
63
|
+
candidate: { mtime: $stale("1d") },
|
|
64
|
+
}),
|
|
65
|
+
)
|
|
66
|
+
.aggregate(),
|
|
67
|
+
async (repo) => {
|
|
68
|
+
const crPulls = match(repo.crPulls)
|
|
69
|
+
.with($OK, (e) => e.data)
|
|
70
|
+
.otherwise(() => null)!;
|
|
71
|
+
const info = match(repo.info)
|
|
72
|
+
.with($OK, (e) => e.data)
|
|
73
|
+
.otherwise(() => null)!;
|
|
74
|
+
|
|
75
|
+
const isCandidate = !info.private && !info.archived && crPulls.length === 0;
|
|
76
|
+
const candidate = TaskOK(isCandidate);
|
|
77
|
+
|
|
78
|
+
// silent
|
|
79
|
+
// if (isCandidate)
|
|
80
|
+
// await notifySlackLinks("Found new PR candidate", [repo.repository]);
|
|
81
|
+
await CNRepos.updateOne({ repository: repo.repository }, { $set: { candidate } });
|
|
82
|
+
return repo;
|
|
83
|
+
},
|
|
84
|
+
);
|
|
85
|
+
}
|
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
-
import pMap from "p-map";
|
|
3
|
-
import { match } from "ts-pattern";
|
|
4
|
-
import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
5
|
-
import { CNRepos } from "./CNRepos";
|
|
6
|
-
import { $filaten, $stale } from "./db";
|
|
7
|
-
import { fetchGithubPulls } from "./gh/fetchGithubPulls";
|
|
8
|
-
import { tLog } from "./utils/tLog";
|
|
9
|
-
if (import.meta.main) {
|
|
10
|
-
console.log(await tLog("Update CNRepos for Github Pulls", updateCNReposPulls));
|
|
11
|
-
}
|
|
12
|
-
export async function updateCNReposPulls() {
|
|
13
|
-
await CNRepos.createIndex("pulls.mtime");
|
|
14
|
-
return await pMap(
|
|
15
|
-
$pipeline(CNRepos)
|
|
16
|
-
.match($filaten({ pulls: { mtime: $stale("1d") } }))
|
|
17
|
-
.project({ repository: 1 })
|
|
18
|
-
.aggregate(),
|
|
19
|
-
async ({ repository }) => {
|
|
20
|
-
const pulls = await fetchGithubPulls(repository).then(TaskOK).catch(TaskError);
|
|
21
|
-
match(pulls)
|
|
22
|
-
.with($OK, ({ data }) => console.debug(`[DEBUG] updated ${data.length} Pulls from ${repository}`))
|
|
23
|
-
.otherwise(() => {});
|
|
24
|
-
return await CNRepos.updateOne({ repository }, { $set: { pulls } });
|
|
25
|
-
},
|
|
26
|
-
{ concurrency: 2 },
|
|
27
|
-
);
|
|
28
|
-
}
|
|
1
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
import { match } from "ts-pattern";
|
|
4
|
+
import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
5
|
+
import { CNRepos } from "./CNRepos";
|
|
6
|
+
import { $filaten, $stale } from "./db";
|
|
7
|
+
import { fetchGithubPulls } from "./gh/fetchGithubPulls";
|
|
8
|
+
import { tLog } from "./utils/tLog";
|
|
9
|
+
if (import.meta.main) {
|
|
10
|
+
console.log(await tLog("Update CNRepos for Github Pulls", updateCNReposPulls));
|
|
11
|
+
}
|
|
12
|
+
export async function updateCNReposPulls() {
|
|
13
|
+
await CNRepos.createIndex("pulls.mtime");
|
|
14
|
+
return await pMap(
|
|
15
|
+
$pipeline(CNRepos)
|
|
16
|
+
.match($filaten({ pulls: { mtime: $stale("1d") } }))
|
|
17
|
+
.project({ repository: 1 })
|
|
18
|
+
.aggregate(),
|
|
19
|
+
async ({ repository }) => {
|
|
20
|
+
const pulls = await fetchGithubPulls(repository).then(TaskOK).catch(TaskError);
|
|
21
|
+
match(pulls)
|
|
22
|
+
.with($OK, ({ data }) => console.debug(`[DEBUG] updated ${data.length} Pulls from ${repository}`))
|
|
23
|
+
.otherwise(() => {});
|
|
24
|
+
return await CNRepos.updateOne({ repository }, { $set: { pulls } });
|
|
25
|
+
},
|
|
26
|
+
{ concurrency: 2 },
|
|
27
|
+
);
|
|
28
|
+
}
|
|
@@ -1,49 +1,49 @@
|
|
|
1
|
-
import DIE from "@snomiao/die";
|
|
2
|
-
import { match } from "ts-pattern";
|
|
3
|
-
import { $OK } from "../packages/mongodb-pipeline-ts/Task";
|
|
4
|
-
import { CNRepos } from "./CNRepos";
|
|
5
|
-
import { $filaten, $fresh } from "./db";
|
|
6
|
-
import { gh } from "./gh";
|
|
7
|
-
import { ghUser } from "./ghUser";
|
|
8
|
-
import { parseUrlRepoOwner, stringifyOwnerRepo } from "./parseOwnerRepo";
|
|
9
|
-
|
|
10
|
-
export async function updateCNRepoPullsDashboard() {
|
|
11
|
-
if (ghUser.login !== "snomiao") return [];
|
|
12
|
-
const dashBoardIssue = process.env.DASHBOARD_ISSUE_URL || DIE("DASHBOARD_ISSUE_URL not found");
|
|
13
|
-
const dashBoardRepo = dashBoardIssue.replace(/\/issues\/\d+$/, "");
|
|
14
|
-
const dashBoardIssueNumber = Number(dashBoardIssue.match(/\/issues\/(\d+)$/)?.[1] || DIE("Issue number not found"));
|
|
15
|
-
// update dashboard issue if run by @snomiao
|
|
16
|
-
const repos = await CNRepos.find($filaten({ crPulls: { mtime: $fresh("1d") } })).toArray();
|
|
17
|
-
const result = repos
|
|
18
|
-
.map((repo) => {
|
|
19
|
-
const crPulls = match(repo.crPulls)
|
|
20
|
-
.with($OK, ({ data }) => data)
|
|
21
|
-
.otherwise(() => DIE("CR Pulls not found"));
|
|
22
|
-
const repoName = stringifyOwnerRepo(parseUrlRepoOwner(repo.repository));
|
|
23
|
-
const body = crPulls
|
|
24
|
-
.filter((e) => e.pull.prState !== "closed")
|
|
25
|
-
.map((e) => {
|
|
26
|
-
const date = new Date(e.pull.createdAt).toISOString().slice(0, 10);
|
|
27
|
-
const state = e.pull.prState.toUpperCase();
|
|
28
|
-
return {
|
|
29
|
-
href: e.pull.html_url,
|
|
30
|
-
name: `PR ${date} ${state}: ${repoName} #${e.type}`,
|
|
31
|
-
};
|
|
32
|
-
})
|
|
33
|
-
.map(({ href, name }) => `- [${name}](${href})`)
|
|
34
|
-
.toSorted()
|
|
35
|
-
.join("\n");
|
|
36
|
-
return body;
|
|
37
|
-
})
|
|
38
|
-
.filter(Boolean)
|
|
39
|
-
.join("\n");
|
|
40
|
-
const body = result;
|
|
41
|
-
|
|
42
|
-
return [
|
|
43
|
-
await gh.issues.update({
|
|
44
|
-
...parseUrlRepoOwner(dashBoardRepo),
|
|
45
|
-
issue_number: dashBoardIssueNumber,
|
|
46
|
-
body,
|
|
47
|
-
}),
|
|
48
|
-
];
|
|
49
|
-
}
|
|
1
|
+
import DIE from "@snomiao/die";
|
|
2
|
+
import { match } from "ts-pattern";
|
|
3
|
+
import { $OK } from "../packages/mongodb-pipeline-ts/Task";
|
|
4
|
+
import { CNRepos } from "./CNRepos";
|
|
5
|
+
import { $filaten, $fresh } from "./db";
|
|
6
|
+
import { gh } from "./gh";
|
|
7
|
+
import { ghUser } from "./ghUser";
|
|
8
|
+
import { parseUrlRepoOwner, stringifyOwnerRepo } from "./parseOwnerRepo";
|
|
9
|
+
|
|
10
|
+
export async function updateCNRepoPullsDashboard() {
|
|
11
|
+
if (ghUser.login !== "snomiao") return [];
|
|
12
|
+
const dashBoardIssue = process.env.DASHBOARD_ISSUE_URL || DIE("DASHBOARD_ISSUE_URL not found");
|
|
13
|
+
const dashBoardRepo = dashBoardIssue.replace(/\/issues\/\d+$/, "");
|
|
14
|
+
const dashBoardIssueNumber = Number(dashBoardIssue.match(/\/issues\/(\d+)$/)?.[1] || DIE("Issue number not found"));
|
|
15
|
+
// update dashboard issue if run by @snomiao
|
|
16
|
+
const repos = await CNRepos.find($filaten({ crPulls: { mtime: $fresh("1d") } })).toArray();
|
|
17
|
+
const result = repos
|
|
18
|
+
.map((repo) => {
|
|
19
|
+
const crPulls = match(repo.crPulls)
|
|
20
|
+
.with($OK, ({ data }) => data)
|
|
21
|
+
.otherwise(() => DIE("CR Pulls not found"));
|
|
22
|
+
const repoName = stringifyOwnerRepo(parseUrlRepoOwner(repo.repository));
|
|
23
|
+
const body = crPulls
|
|
24
|
+
.filter((e) => e.pull.prState !== "closed")
|
|
25
|
+
.map((e) => {
|
|
26
|
+
const date = new Date(e.pull.createdAt).toISOString().slice(0, 10);
|
|
27
|
+
const state = e.pull.prState.toUpperCase();
|
|
28
|
+
return {
|
|
29
|
+
href: e.pull.html_url,
|
|
30
|
+
name: `PR ${date} ${state}: ${repoName} #${e.type}`,
|
|
31
|
+
};
|
|
32
|
+
})
|
|
33
|
+
.map(({ href, name }) => `- [${name}](${href})`)
|
|
34
|
+
.toSorted()
|
|
35
|
+
.join("\n");
|
|
36
|
+
return body;
|
|
37
|
+
})
|
|
38
|
+
.filter(Boolean)
|
|
39
|
+
.join("\n");
|
|
40
|
+
const body = result;
|
|
41
|
+
|
|
42
|
+
return [
|
|
43
|
+
await gh.issues.update({
|
|
44
|
+
...parseUrlRepoOwner(dashBoardRepo),
|
|
45
|
+
issue_number: dashBoardIssueNumber,
|
|
46
|
+
body,
|
|
47
|
+
}),
|
|
48
|
+
];
|
|
49
|
+
}
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import DIE from "@snomiao/die";
|
|
2
|
-
import pMap from "p-map";
|
|
3
|
-
import { match } from "ts-pattern";
|
|
4
|
-
import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
5
|
-
import { CNRepos } from "./CNRepos";
|
|
6
|
-
import { $filaten, $stale } from "./db";
|
|
7
|
-
import { matchRelatedPulls } from "./matchRelatedPulls";
|
|
8
|
-
import { tLog } from "./utils/tLog";
|
|
9
|
-
if (import.meta.main) {
|
|
10
|
-
await tLog("updateCNReposRelatedPulls", updateCNReposRelatedPulls);
|
|
11
|
-
}
|
|
12
|
-
export async function updateCNReposRelatedPulls() {
|
|
13
|
-
await CNRepos.createIndex({ "pulls.state": 1, "crPulls.mtime": 1 });
|
|
14
|
-
return await pMap(
|
|
15
|
-
CNRepos.find($filaten({ pulls: { state: "ok" }, crPulls: { mtime: $stale("1d") } })),
|
|
16
|
-
async (repo, i) => {
|
|
17
|
-
const { repository } = repo;
|
|
18
|
-
const pulls = match(repo.pulls)
|
|
19
|
-
.with($OK, (e) => e.data)
|
|
20
|
-
.otherwise(() => DIE("Pulls not found"));
|
|
21
|
-
const crPulls = await matchRelatedPulls(pulls).then(TaskOK).catch(TaskError);
|
|
22
|
-
return await CNRepos.updateOne({ repository }, { $set: { crPulls } });
|
|
23
|
-
},
|
|
24
|
-
);
|
|
25
|
-
}
|
|
1
|
+
import DIE from "@snomiao/die";
|
|
2
|
+
import pMap from "p-map";
|
|
3
|
+
import { match } from "ts-pattern";
|
|
4
|
+
import { $OK, TaskError, TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
5
|
+
import { CNRepos } from "./CNRepos";
|
|
6
|
+
import { $filaten, $stale } from "./db";
|
|
7
|
+
import { matchRelatedPulls } from "./matchRelatedPulls";
|
|
8
|
+
import { tLog } from "./utils/tLog";
|
|
9
|
+
if (import.meta.main) {
|
|
10
|
+
await tLog("updateCNReposRelatedPulls", updateCNReposRelatedPulls);
|
|
11
|
+
}
|
|
12
|
+
export async function updateCNReposRelatedPulls() {
|
|
13
|
+
await CNRepos.createIndex({ "pulls.state": 1, "crPulls.mtime": 1 });
|
|
14
|
+
return await pMap(
|
|
15
|
+
CNRepos.find($filaten({ pulls: { state: "ok" }, crPulls: { mtime: $stale("1d") } })),
|
|
16
|
+
async (repo, i) => {
|
|
17
|
+
const { repository } = repo;
|
|
18
|
+
const pulls = match(repo.pulls)
|
|
19
|
+
.with($OK, (e) => e.data)
|
|
20
|
+
.otherwise(() => DIE("Pulls not found"));
|
|
21
|
+
const crPulls = await matchRelatedPulls(pulls).then(TaskOK).catch(TaskError);
|
|
22
|
+
return await CNRepos.updateOne({ repository }, { $set: { crPulls } });
|
|
23
|
+
},
|
|
24
|
+
);
|
|
25
|
+
}
|
package/src/updateCRNodes.ts
CHANGED
|
@@ -1,50 +1,50 @@
|
|
|
1
|
-
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
-
import { peekYaml } from "peek-log";
|
|
3
|
-
import { filter, groupBy, values } from "rambda";
|
|
4
|
-
import YAML from "yaml";
|
|
5
|
-
import { TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
6
|
-
import { CNRepos } from "./CNRepos";
|
|
7
|
-
import { CRNodes } from "./CRNodes";
|
|
8
|
-
import { fetchCRNodes } from "./fetchComfyRegistryNodes";
|
|
9
|
-
import { notifySlack } from "./slack/notifySlack";
|
|
10
|
-
import { tLog } from "./utils/tLog";
|
|
11
|
-
|
|
12
|
-
if (import.meta.main) {
|
|
13
|
-
peekYaml(await tLog(updateCRNodes));
|
|
14
|
-
console.log("CRNodes updated");
|
|
15
|
-
|
|
16
|
-
peekYaml(
|
|
17
|
-
await $pipeline(CNRepos).project({ repository: 1, on_registry: 1 }).sample({ size: 8 }).aggregate().toArray(),
|
|
18
|
-
);
|
|
19
|
-
}
|
|
20
|
-
export async function updateCRNodes() {
|
|
21
|
-
const nodes = await fetchCRNodes();
|
|
22
|
-
|
|
23
|
-
// check src duplicated
|
|
24
|
-
const group = groupBy((e) => e.repository, nodes);
|
|
25
|
-
const duplicates = filter((e) => e.length > 1, group);
|
|
26
|
-
if (values(duplicates).length) {
|
|
27
|
-
const msg =
|
|
28
|
-
"[WARN] Same repo but different ids found in comfyregistry:\n" + "```\n" + YAML.stringify(duplicates) + "\n```";
|
|
29
|
-
await notifySlack(msg, { unique: true });
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
return [
|
|
33
|
-
await $pipeline(CNRepos)
|
|
34
|
-
.project({ repository: 1 })
|
|
35
|
-
.set({ on_registry: TaskOK({ $in: ["$repository", nodes.map(({ repository }) => repository).filter(Boolean)] }) })
|
|
36
|
-
.merge({ into: "CNRepos", on: "repository", whenMatched: "merge", whenNotMatched: "insert" })
|
|
37
|
-
.aggregate()
|
|
38
|
-
.next(),
|
|
39
|
-
await CRNodes.bulkWrite(
|
|
40
|
-
nodes.map((node) => ({
|
|
41
|
-
updateOne: {
|
|
42
|
-
filter: { id: node.id },
|
|
43
|
-
update: { $set: node },
|
|
44
|
-
upsert: true,
|
|
45
|
-
},
|
|
46
|
-
})),
|
|
47
|
-
{ ordered: false },
|
|
48
|
-
),
|
|
49
|
-
];
|
|
50
|
-
}
|
|
1
|
+
import { $pipeline } from "@/packages/mongodb-pipeline-ts/$pipeline";
|
|
2
|
+
import { peekYaml } from "peek-log";
|
|
3
|
+
import { filter, groupBy, values } from "rambda";
|
|
4
|
+
import YAML from "yaml";
|
|
5
|
+
import { TaskOK } from "../packages/mongodb-pipeline-ts/Task";
|
|
6
|
+
import { CNRepos } from "./CNRepos";
|
|
7
|
+
import { CRNodes } from "./CRNodes";
|
|
8
|
+
import { fetchCRNodes } from "./fetchComfyRegistryNodes";
|
|
9
|
+
import { notifySlack } from "./slack/notifySlack";
|
|
10
|
+
import { tLog } from "./utils/tLog";
|
|
11
|
+
|
|
12
|
+
if (import.meta.main) {
|
|
13
|
+
peekYaml(await tLog(updateCRNodes));
|
|
14
|
+
console.log("CRNodes updated");
|
|
15
|
+
|
|
16
|
+
peekYaml(
|
|
17
|
+
await $pipeline(CNRepos).project({ repository: 1, on_registry: 1 }).sample({ size: 8 }).aggregate().toArray(),
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
export async function updateCRNodes() {
|
|
21
|
+
const nodes = await fetchCRNodes();
|
|
22
|
+
|
|
23
|
+
// check src duplicated
|
|
24
|
+
const group = groupBy((e) => e.repository, nodes);
|
|
25
|
+
const duplicates = filter((e) => e.length > 1, group);
|
|
26
|
+
if (values(duplicates).length) {
|
|
27
|
+
const msg =
|
|
28
|
+
"[WARN] Same repo but different ids found in comfyregistry:\n" + "```\n" + YAML.stringify(duplicates) + "\n```";
|
|
29
|
+
await notifySlack(msg, { unique: true });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return [
|
|
33
|
+
await $pipeline(CNRepos)
|
|
34
|
+
.project({ repository: 1 })
|
|
35
|
+
.set({ on_registry: TaskOK({ $in: ["$repository", nodes.map(({ repository }) => repository).filter(Boolean)] }) })
|
|
36
|
+
.merge({ into: "CNRepos", on: "repository", whenMatched: "merge", whenNotMatched: "insert" })
|
|
37
|
+
.aggregate()
|
|
38
|
+
.next(),
|
|
39
|
+
await CRNodes.bulkWrite(
|
|
40
|
+
nodes.map((node) => ({
|
|
41
|
+
updateOne: {
|
|
42
|
+
filter: { id: node.id },
|
|
43
|
+
update: { $set: node },
|
|
44
|
+
upsert: true,
|
|
45
|
+
},
|
|
46
|
+
})),
|
|
47
|
+
{ ordered: false },
|
|
48
|
+
),
|
|
49
|
+
];
|
|
50
|
+
}
|
package/src/updateCRRepos.ts
CHANGED
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
import pMap from "p-map";
|
|
2
|
-
import { CNRepos } from "./CNRepos";
|
|
3
|
-
import { CRNodes } from "./CRNodes";
|
|
4
|
-
import { tLog } from "./utils/tLog";
|
|
5
|
-
if (import.meta.main) {
|
|
6
|
-
await tLog("updateCRRepos", updateCRRepos);
|
|
7
|
-
}
|
|
8
|
-
export async function updateCRRepos() {
|
|
9
|
-
await CRNodes.createIndex({ repo_id: 1 });
|
|
10
|
-
return await pMap(CRNodes.find({ repo_id: { $exists: false } }), async (cr, i) => {
|
|
11
|
-
const { repository, _id } = cr;
|
|
12
|
-
await CNRepos.updateOne(
|
|
13
|
-
{ repository },
|
|
14
|
-
{
|
|
15
|
-
$set: {
|
|
16
|
-
cr: (await CRNodes.findOneAndUpdate(
|
|
17
|
-
{ _id },
|
|
18
|
-
{ $set: { repo_id: cr._id } },
|
|
19
|
-
{ upsert: true, returnDocument: "after" },
|
|
20
|
-
))!,
|
|
21
|
-
},
|
|
22
|
-
},
|
|
23
|
-
{ upsert: true },
|
|
24
|
-
);
|
|
25
|
-
});
|
|
26
|
-
}
|
|
1
|
+
import pMap from "p-map";
|
|
2
|
+
import { CNRepos } from "./CNRepos";
|
|
3
|
+
import { CRNodes } from "./CRNodes";
|
|
4
|
+
import { tLog } from "./utils/tLog";
|
|
5
|
+
if (import.meta.main) {
|
|
6
|
+
await tLog("updateCRRepos", updateCRRepos);
|
|
7
|
+
}
|
|
8
|
+
export async function updateCRRepos() {
|
|
9
|
+
await CRNodes.createIndex({ repo_id: 1 });
|
|
10
|
+
return await pMap(CRNodes.find({ repo_id: { $exists: false } }), async (cr, i) => {
|
|
11
|
+
const { repository, _id } = cr;
|
|
12
|
+
await CNRepos.updateOne(
|
|
13
|
+
{ repository },
|
|
14
|
+
{
|
|
15
|
+
$set: {
|
|
16
|
+
cr: (await CRNodes.findOneAndUpdate(
|
|
17
|
+
{ _id },
|
|
18
|
+
{ $set: { repo_id: cr._id } },
|
|
19
|
+
{ upsert: true, returnDocument: "after" },
|
|
20
|
+
))!,
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
{ upsert: true },
|
|
24
|
+
);
|
|
25
|
+
});
|
|
26
|
+
}
|
package/src/updateComfyTotals.ts
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
|
-
import { match } from "ts-pattern";
|
|
2
|
-
import YAML from "yaml";
|
|
3
|
-
import { $OK, TaskError, TaskOK, tsmatch } from "../packages/mongodb-pipeline-ts/Task";
|
|
4
|
-
import { Totals } from "./Totals";
|
|
5
|
-
import { analyzeTotals } from "./analyzeTotals";
|
|
6
|
-
import { $filaten, $fresh } from "./db";
|
|
7
|
-
import { notifySlack } from "./slack/notifySlack";
|
|
8
|
-
|
|
9
|
-
if (import.meta.main) {
|
|
10
|
-
await updateComfyTotals();
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
|
|
14
|
-
await Totals.createIndex({today: 1, "totals.mtime": 1, "totals.state": 1})
|
|
15
|
-
const today = new Date().toISOString().split("T")[0];
|
|
16
|
-
const cached = await Totals.findOne($filaten({ today, totals: { mtime: $fresh(fresh), ...$OK } }));
|
|
17
|
-
if (cached?.totals?.state === "ok")
|
|
18
|
-
return [
|
|
19
|
-
tsmatch(cached.totals)
|
|
20
|
-
.with($OK, ({ data }) => data)
|
|
21
|
-
.otherwise(() => null),
|
|
22
|
-
].flatMap((e) => (e ? [e] : []));
|
|
23
|
-
const totals = await analyzeTotals().then(TaskOK).catch(TaskError);
|
|
24
|
-
|
|
25
|
-
// notify if today is not already notify
|
|
26
|
-
if (notify) {
|
|
27
|
-
if(!await Totals.findOne($filaten({ today, totals: { mtime: $fresh(
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
const insertResult = await Totals.insertOne({ totals });
|
|
38
|
-
return [
|
|
39
|
-
tsmatch(totals)
|
|
40
|
-
.with($OK, ({ data }) => data)
|
|
41
|
-
.otherwise(() => null),
|
|
42
|
-
].flatMap((e) => (e ? [e] : []));
|
|
43
|
-
}
|
|
1
|
+
import { match } from "ts-pattern";
|
|
2
|
+
import YAML from "yaml";
|
|
3
|
+
import { $OK, TaskError, TaskOK, tsmatch } from "../packages/mongodb-pipeline-ts/Task";
|
|
4
|
+
import { Totals } from "./Totals";
|
|
5
|
+
import { analyzeTotals } from "./analyzeTotals";
|
|
6
|
+
import { $filaten, $fresh } from "./db";
|
|
7
|
+
import { notifySlack } from "./slack/notifySlack";
|
|
8
|
+
|
|
9
|
+
if (import.meta.main) {
|
|
10
|
+
await updateComfyTotals();
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export async function updateComfyTotals({ notify = true, fresh = "30m" } = {}) {
|
|
14
|
+
await Totals.createIndex({ today: 1, "totals.mtime": 1, "totals.state": 1 });
|
|
15
|
+
const today = new Date().toISOString().split("T")[0];
|
|
16
|
+
const cached = await Totals.findOne($filaten({ today, totals: { mtime: $fresh(fresh), ...$OK } }));
|
|
17
|
+
if (cached?.totals?.state === "ok")
|
|
18
|
+
return [
|
|
19
|
+
tsmatch(cached.totals)
|
|
20
|
+
.with($OK, ({ data }) => data)
|
|
21
|
+
.otherwise(() => null),
|
|
22
|
+
].flatMap((e) => (e ? [e] : []));
|
|
23
|
+
const totals = await analyzeTotals().then(TaskOK).catch(TaskError);
|
|
24
|
+
|
|
25
|
+
// notify if today is not already notify
|
|
26
|
+
if (notify) {
|
|
27
|
+
if (!(await Totals.findOne($filaten({ today, totals: { mtime: $fresh("1d"), ...$OK } }))))
|
|
28
|
+
// ignore today
|
|
29
|
+
await match(totals)
|
|
30
|
+
.with($OK, async (totals) => {
|
|
31
|
+
const msg = `Totals: \n${"```"}\n${YAML.stringify(totals)}\n${"```"}`;
|
|
32
|
+
await notifySlack(msg, { unique: true });
|
|
33
|
+
})
|
|
34
|
+
.otherwise(() => null);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const insertResult = await Totals.insertOne({ totals });
|
|
38
|
+
return [
|
|
39
|
+
tsmatch(totals)
|
|
40
|
+
.with($OK, ({ data }) => data)
|
|
41
|
+
.otherwise(() => null),
|
|
42
|
+
].flatMap((e) => (e ? [e] : []));
|
|
43
|
+
}
|