issue-timeline 0.0.0-stage → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,313 @@
1
+ #!/usr/bin/env node
2
+ // Read-only issue and pull/merge request timeline for GitHub and GitLab repositories.
3
+ import { createServer } from "node:http";
4
+ import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
5
+ import { extname, join, normalize } from "node:path";
6
+ import { createHash } from "node:crypto";
7
+ import { fileURLToPath } from "node:url";
8
+ import { Store } from "./store.js";
9
+ import { createClient, resolveToken } from "./github.js";
10
+ import { gitlabCommitFiles, gitlabCommitPage, gitlabMergeMethod, gitlabPipelines, gitlabProjectExists, gitlabWorkflowChanges, parseGitLabRepo, syncGitLab } from "./gitlab.js";
11
+ import { sync } from "./sync.js";
12
+ import { HELP, defaultStoreDir, authSummary, openBrowser, parseArgs, shouldOpen, startBackground, stopBackground } from "./cli.js";
13
+ import { buildPayload } from "./derive.js";
14
+ import { buildCommitLinks } from "./links.js";
15
+ import { CiStore, summarize, windowFloor, readFrom, githubRuns, githubWorkflowChanges } from "./ci.js";
16
+ import { CommitStore, FULL_SHA, diffDirFor, finishFiles, githubCommitFiles, githubHistoryPager, githubMergeMethod, readCachedDiff, syncCommits, writeCachedDiff } from "./commits.js";
17
+
18
+ const args = parseArgs(process.argv.slice(2));
19
+ if (args.help) { console.log(HELP); process.exit(0); }
20
+ if (args.stop) {
21
+ const pid = stopBackground(defaultStoreDir());
22
+ console.log(pid ? `issue-timeline stopped (pid ${pid}).` : "issue-timeline is not running in the background.");
23
+ process.exit(0);
24
+ }
25
+ if (args.background && !process.env.ISSUE_TIMELINE_CHILD) {
26
+ const dir = defaultStoreDir();
27
+ const pid = startBackground({ dir, argv: process.argv.slice(2), script: fileURLToPath(import.meta.url) });
28
+ const shown = `http://${(process.env.HOST ?? "127.0.0.1") === "0.0.0.0" ? "localhost" : process.env.HOST ?? "127.0.0.1"}:${process.env.PORT ?? 4310}`;
29
+ console.log(`issue-timeline is running in the background (pid ${pid}).\n Open: ${shown}\n Logs: ${dir}/issue-timeline.log\n Stop: issue-timeline stop`);
30
+ if (shouldOpen(args)) openBrowser(shown);
31
+ process.exit(0);
32
+ }
33
+
34
+ const root = fileURLToPath(new URL("../..", import.meta.url));
35
+ const port = Number(process.env.PORT ?? 4310);
36
+ const host = process.env.HOST ?? "127.0.0.1";
37
+ const intervalMs = Number(process.env.POLL_MINUTES ?? 5) * 60 * 1000;
38
+ // Data goes to STORE_DIR, a user data folder, or the ./data of a previous run (see defaultStoreDir).
39
+ const storeDir = defaultStoreDir();
40
+ const repoConfigFile = join(storeDir, "repos.json");
41
+ const webDir = join(root, "web", "dist");
42
+ const entries = new Map();
43
+ const mergeMethods = new Map(); // "repo#number" -> how the PR was merged (immutable once known)
44
+ const savedRepos = await readFile(repoConfigFile, "utf8").then((s) => JSON.parse(s)).catch(() => []);
45
+ const validRepoKey = (r) => {
46
+ if (typeof r !== "string") return false;
47
+ try { if (r.startsWith("gitlab:")) { parseGitLabRepo(r); return true; } return /^[\w.-]+\/[\w.-]+$/.test(r); }
48
+ catch { return false; }
49
+ };
50
+ let repos = [...new Set((Array.isArray(savedRepos) ? savedRepos : []).filter(validRepoKey))];
51
+ const storeFileFor = (repo) => repo.startsWith("gitlab:")
52
+ ? join(storeDir, `gitlab-${createHash("sha256").update(repo).digest("hex").slice(0, 16)}.json`)
53
+ : join(storeDir, `${repo.replace("/", "__")}.json`);
54
+ let graphql;
55
+ let token;
56
+ function getToken() {
57
+ if (!token) token = resolveToken();
58
+ return token;
59
+ }
60
+ function getGraphql() {
61
+ if (!graphql) graphql = createClient({ token: getToken() });
62
+ return graphql;
63
+ }
64
+ // The commits of a repo live next to its store file: <store>.commits.json.
65
+ const commitsFileOf = (storeFile) => storeFile.replace(/\.json$/, ".commits.json");
66
+ const ciFileOf = (storeFile) => storeFile.replace(/\.json$/, ".ci.json");
67
+ const ownerAndName = (repo) => repo.split("/");
68
+
69
+ // A repository is only saved once the host confirms it exists and the login can read it.
70
+ async function checkRepoExists(repo) {
71
+ if (repo.startsWith("gitlab:")) {
72
+ if (!(await gitlabProjectExists(repo))) throw new Error(`GitLab project not found, or the glab login cannot see it: ${repo.slice("gitlab:".length)}`);
73
+ return;
74
+ }
75
+ const [owner, name] = ownerAndName(repo);
76
+ try {
77
+ const data = await getGraphql()("query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { id } }", { owner, name });
78
+ if (!data.repository) throw new Error("not found");
79
+ } catch (e) {
80
+ if (/Could not resolve|not found/i.test(String(e.message))) throw new Error(`GitHub repository not found, or no access to it: ${repo}`);
81
+ throw new Error(`Could not check ${repo} on GitHub: ${e.message}`);
82
+ }
83
+ }
84
+
85
+ async function addConfiguredRepo(repo) {
86
+ const store = await new Store(process.env.STORE_FILE && repos.length === 0 ? process.env.STORE_FILE : storeFileFor(repo)).load();
87
+ const commits = await new CommitStore(commitsFileOf(store.file)).load();
88
+ const ci = await new CiStore(ciFileOf(store.file)).load();
89
+ const state = { syncing: false, lastError: null, nextSyncAt: null, progress: null, commitsError: null, ciError: null };
90
+ entries.set(repo, { store, state, commits, ci });
91
+ repos = [...repos, repo];
92
+ await mkdir(storeDir, { recursive: true });
93
+ const tmp = `${repoConfigFile}.tmp`;
94
+ await writeFile(tmp, JSON.stringify(repos));
95
+ await rename(tmp, repoConfigFile);
96
+ runSync(repo);
97
+ }
98
+
99
+ for (const repo of repos) {
100
+ const store = await new Store(process.env.STORE_FILE && repos.length === 1 ? process.env.STORE_FILE : storeFileFor(repo)).load();
101
+ const commits = await new CommitStore(commitsFileOf(store.file)).load();
102
+ const ci = await new CiStore(ciFileOf(store.file)).load();
103
+ const state = { syncing: false, lastError: null, nextSyncAt: null, progress: null, commitsError: null, ciError: null };
104
+ entries.set(repo, { store, state, commits, ci });
105
+ }
106
+
107
+ // The default branch's commits, read after the issues and PRs. Incremental: usually one page, the rest is stored.
108
+ // The CI runs of the window: GitHub Actions runs, or GitLab pipelines. Only summaries reach the page.
109
+ async function syncRepoCi(repo) {
110
+ const { ci } = entries.get(repo);
111
+ const now = Date.now();
112
+ const since = readFrom(now, ci.data.meta);
113
+ const fresh = repo.startsWith("gitlab:")
114
+ ? await gitlabPipelines({ repoKey: repo, since })
115
+ : await githubRuns({ token: getToken(), owner: ownerAndName(repo)[0], name: ownerAndName(repo)[1], since });
116
+ ci.merge(repo, fresh, windowFloor(now), new Date(now).toISOString());
117
+ // The commits that changed the CI definition in the window: the PR panel draws them as a line.
118
+ ci.data.changes = repo.startsWith("gitlab:")
119
+ ? await gitlabWorkflowChanges({ repoKey: repo, since: windowFloor(now) })
120
+ : await githubWorkflowChanges({ token: getToken(), owner: ownerAndName(repo)[0], name: ownerAndName(repo)[1], since: windowFloor(now) });
121
+ await ci.save();
122
+ return { runs: ci.data.runs.length };
123
+ }
124
+
125
+ async function syncRepoCommits(repo) {
126
+ const { commits } = entries.get(repo);
127
+ const fetchPage = repo.startsWith("gitlab:")
128
+ ? (cursor) => gitlabCommitPage({ repoKey: repo, page: cursor })
129
+ : githubHistoryPager({ graphql: getGraphql(), owner: ownerAndName(repo)[0], name: ownerAndName(repo)[1] });
130
+ return syncCommits({ store: commits, repo, fetchPage });
131
+ }
132
+
133
+ async function runSync(repo, full = false) {
134
+ const { store, state } = entries.get(repo);
135
+ if (state.syncing) return { skipped: true };
136
+ state.syncing = true;
137
+ state.progress = { phase: "Preparing", done: 0, total: null };
138
+ try {
139
+ const log = (m) => console.log(new Date().toISOString(), repo, m);
140
+ const onProgress = (progress) => { state.progress = progress; };
141
+ const r = repo.startsWith("gitlab:")
142
+ ? await syncGitLab({ repoKey: repo, store, full, log, onProgress })
143
+ : await sync({ graphql: getGraphql(), store, repo, full, log, onProgress });
144
+ state.lastError = null;
145
+ try {
146
+ state.progress = { phase: "Reading commits", done: 0, total: null };
147
+ await syncRepoCommits(repo);
148
+ state.commitsError = null;
149
+ } catch (e) {
150
+ state.commitsError = String(e.message ?? e);
151
+ console.error(new Date().toISOString(), repo, "commits sync failed:", state.commitsError);
152
+ }
153
+ try {
154
+ state.progress = { phase: "Reading CI runs", done: 0, total: null };
155
+ await syncRepoCi(repo);
156
+ state.ciError = null;
157
+ } catch (e) {
158
+ state.ciError = String(e.message ?? e);
159
+ console.error(new Date().toISOString(), repo, "CI sync failed:", state.ciError);
160
+ }
161
+ return r;
162
+ } catch (e) {
163
+ state.lastError = String(e.message ?? e);
164
+ console.error(new Date().toISOString(), repo, "sync failed:", state.lastError);
165
+ return { error: state.lastError };
166
+ } finally {
167
+ state.syncing = false;
168
+ state.nextSyncAt = new Date(Date.now() + intervalMs).toISOString();
169
+ state.progress = null;
170
+ }
171
+ }
172
+
173
+ const json = (res, code, body) => { res.writeHead(code, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); res.end(JSON.stringify(body)); };
174
+ const MIME = { ".html": "text/html; charset=utf-8", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".json": "application/json" };
175
+ async function serveStatic(req, res, pathname) {
176
+ const rel = normalize(pathname === "/" ? "/index.html" : pathname).replace(/^(\.\.[/\\])+/, "");
177
+ try {
178
+ const file = join(webDir, rel);
179
+ if (!file.startsWith(webDir)) throw new Error("outside");
180
+ const body = await readFile(file);
181
+ res.writeHead(200, { "content-type": MIME[extname(file)] ?? "application/octet-stream" }); res.end(body);
182
+ } catch {
183
+ try {
184
+ const body = await readFile(join(webDir, "index.html"));
185
+ res.writeHead(200, { "content-type": MIME[".html"] }); res.end(body);
186
+ } catch {
187
+ res.writeHead(404, { "content-type": "text/plain" });
188
+ res.end("The frontend is not built: run `npm run build` (or `npm run dev:web` for the dev server on :5173).");
189
+ }
190
+ }
191
+ }
192
+
193
+ const server = createServer(async (req, res) => {
194
+ const url = new URL(req.url, "http://localhost");
195
+ try {
196
+ if (url.pathname === "/api/repos" && req.method === "GET") return json(res, 200, { repos });
197
+ if (url.pathname === "/api/repos" && req.method === "POST") {
198
+ let body = "";
199
+ for await (const chunk of req) body += chunk;
200
+ const input = JSON.parse(body);
201
+ let repo;
202
+ try {
203
+ if (input.provider === "gitlab") {
204
+ const source = input.repo?.trim() ?? "";
205
+ const parsed = parseGitLabRepo(source);
206
+ repo = `gitlab:${parsed.origin}/${parsed.project}`;
207
+ } else {
208
+ if (input.provider && input.provider !== "github") throw new Error("Unsupported provider");
209
+ repo = (input.repo?.trim() ?? "").replace(/^https?:\/\/[^/]+\//, "").replace(/\.git$/, "").replace(/\/+$/, "");
210
+ if (!/^[\w.-]+\/[\w.-]+$/.test(repo)) throw new Error("Use the owner/repo format");
211
+ }
212
+ } catch (e) { return json(res, 400, { error: e.message }); }
213
+ if (entries.has(repo)) return json(res, 200, { repos, repo, added: false });
214
+ try { await checkRepoExists(repo); } catch (e) { return json(res, 400, { error: e.message }); }
215
+ await addConfiguredRepo(repo);
216
+ return json(res, 201, { repos, repo, added: true });
217
+ }
218
+ const repo = url.searchParams.get("repo") ?? repos[0];
219
+ const entry = entries.get(repo);
220
+ if (url.pathname.startsWith("/api/") && !entry) return json(res, 400, { error: "repo not configured" });
221
+ if (url.pathname === "/api/data" && req.method === "GET") return json(res, 200, { ...buildPayload(entry.store.data), ci: { ...summarize({ runs: entry.ci.data.runs, pulls: entry.store.data.pulls }), changes: entry.ci.data.changes ?? [] }, status: { ...entry.state } });
222
+ const one = url.pathname.match(/^\/api\/(issues|pulls)\/(\d+)$/);
223
+ if (one && req.method === "GET") {
224
+ const record = entry.store.data[one[1]][one[2]];
225
+ return record ? json(res, 200, record) : json(res, 404, { error: "not found" });
226
+ }
227
+ // The list of commits: only read when the Commits tab asks for it, a page at a time, filtered on the server.
228
+ if (url.pathname === "/api/commits" && req.method === "GET") {
229
+ const q = (url.searchParams.get("q") ?? "").trim().toLowerCase();
230
+ const author = (url.searchParams.get("author") ?? "").trim().toLowerCase();
231
+ const limit = Math.min(500, Math.max(1, Number(url.searchParams.get("limit")) || 100));
232
+ const offset = Math.max(0, Number(url.searchParams.get("offset")) || 0);
233
+ const pr = Number(url.searchParams.get("pr")) || null;
234
+ const issue = Number(url.searchParams.get("issue")) || null;
235
+ const { items, meta } = entry.commits.data;
236
+ const { pulls, issues } = entry.store.data;
237
+ // A PR's own commits join the list, even those that never reached the default branch.
238
+ const known = new Set(items.map((c) => c.sha));
239
+ const own = pr && pulls[pr]
240
+ ? (pulls[pr].commits ?? []).filter((c) => !known.has(c.oid)).map((c) => ({ sha: c.oid, title: c.message, author: c.author, at: c.at, additions: null, deletions: null }))
241
+ : [];
242
+ const all = [...items, ...own];
243
+ const links = buildCommitLinks({ commits: all, pulls, issues });
244
+ const matches = all
245
+ .map((c) => ({ ...c, ...links.get(c.sha), inHistory: known.has(c.sha) }))
246
+ .filter((c) =>
247
+ (!q || c.title.toLowerCase().includes(q) || c.sha.startsWith(q))
248
+ && (!author || (c.author ?? "").toLowerCase().includes(author))
249
+ && (!pr || c.pr === pr)
250
+ && (!issue || c.issues.includes(issue)));
251
+ return json(res, 200, { total: matches.length, synced: items.length, lastSync: meta.lastSync, items: matches.slice(offset, offset + limit) });
252
+ }
253
+ // One commit's files and diffs: fetched from the host the first time, then kept in the cache. Any commit of the repo
254
+ // works, not only the default branch's: a PR's own commits (squashed or never merged there) are on the host too.
255
+ if (url.pathname === "/api/commit" && req.method === "GET") {
256
+ const sha = (url.searchParams.get("sha") ?? "").toLowerCase();
257
+ const known = entry.commits.get(sha);
258
+ if (!known && !FULL_SHA.test(sha)) return json(res, 404, { error: "not a full commit sha of this repo" });
259
+ const dir = diffDirFor(storeDir, repo);
260
+ let value = await readCachedDiff(dir, sha);
261
+ if (!value) {
262
+ const host = repo.startsWith("gitlab:")
263
+ ? await gitlabCommitFiles({ repoKey: repo, sha })
264
+ : await githubCommitFiles({ token: getToken(), owner: ownerAndName(repo)[0], name: ownerAndName(repo)[1], sha });
265
+ value = { info: host.info, ...finishFiles(host.files) };
266
+ await writeCachedDiff(dir, sha, value);
267
+ }
268
+ const info = known
269
+ ? { title: known.title, author: known.author, at: known.at, additions: known.additions, deletions: known.deletions }
270
+ : { title: value.info?.title ?? "", author: value.info?.author ?? null, at: value.info?.at ?? null, additions: null, deletions: null };
271
+ const links = buildCommitLinks({ commits: [{ sha, title: info.title }], pulls: entry.store.data.pulls, issues: entry.store.data.issues }).get(sha);
272
+ return json(res, 200, { sha, ...info, ...links, inHistory: !!known, files: value.files, truncated: value.truncated });
273
+ }
274
+ // How a merged PR landed (squash, merge commit, rebase...). Asked of the host once per PR, then kept in memory.
275
+ if (url.pathname === "/api/merge" && req.method === "GET") {
276
+ const number = Number(url.searchParams.get("number"));
277
+ const record = entry.store.data.pulls[number];
278
+ if (!record) return json(res, 404, { error: "not found" });
279
+ if (!record.mergedAt) return json(res, 200, { method: null });
280
+ const key = `${repo}#${number}`;
281
+ if (!mergeMethods.has(key)) {
282
+ const method = repo.startsWith("gitlab:")
283
+ ? await gitlabMergeMethod({ repoKey: repo, iid: number })
284
+ : await githubMergeMethod({ token: getToken(), owner: ownerAndName(repo)[0], name: ownerAndName(repo)[1], number, record });
285
+ if (method) mergeMethods.set(key, method);
286
+ return json(res, 200, { method });
287
+ }
288
+ return json(res, 200, { method: mergeMethods.get(key) });
289
+ }
290
+ if (url.pathname === "/api/sync" && req.method === "POST") {
291
+ runSync(repo, url.searchParams.get("full") === "1");
292
+ return json(res, 202, { started: !entry.state.syncing });
293
+ }
294
+ if (url.pathname.startsWith("/api/")) return json(res, 404, { error: "not found" });
295
+ return serveStatic(req, res, decodeURIComponent(url.pathname));
296
+ } catch (e) { console.error(e); return json(res, 500, { error: String(e.message ?? e) }); }
297
+ });
298
+
299
+ server.listen(port, host, () => {
300
+ const url = `http://${host === "0.0.0.0" || host === "::" ? "localhost" : host}:${port}`;
301
+ console.log(`\nissue-timeline is running\n Open: ${url}\n Repos: ${repos.join(", ") || "none yet, add one from the page"}\n Data: ${storeDir}\n Auth: uses your gh and glab sessions (read-only)\n${authSummary().map((l) => ` ${l}\n`).join("")} Stop: Ctrl+C\n`);
302
+ if (shouldOpen(args)) openBrowser(url);
303
+ });
304
+ server.on("error", (e) => {
305
+ console.error(e.code === "EADDRINUSE" ? `Port ${port} is already in use. Is issue-timeline already running? Set PORT to use another.` : e.message);
306
+ process.exit(1);
307
+ });
308
+ for (const [repo, { store, state }] of entries) {
309
+ const stale = !store.data.meta.lastSync || Date.now() - Date.parse(store.data.meta.lastSync) > intervalMs;
310
+ if (stale) runSync(repo);
311
+ else state.nextSyncAt = new Date(Date.parse(store.data.meta.lastSync) + intervalMs).toISOString();
312
+ }
313
+ setInterval(() => { for (const repo of repos) runSync(repo); }, intervalMs).unref?.();
@@ -0,0 +1,38 @@
1
+ // Links between commits and the pull requests and issues they belong to. Everything is read from the stores (no call
2
+ // to the host): a PR's own commits are exact; for the others the title tells, as GitHub writes it ("... (#584)" for a
3
+ // squash, "Merge pull request #584" for a merge commit), and "closes #N" / "fixes #N" / "#N" name the issues.
4
+
5
+ // GitHub: "... (#584)" for a squash, "Merge pull request #584". GitLab: "See merge request group/project!584" (also the
6
+ // title of a merge commit of an MR), where the number after "!" is the MR's iid.
7
+ const PR_IN_TITLE = [/\(#(\d+)\)\s*$/, /^Merge pull request #(\d+)\b/, /See merge request \S+!(\d+)/];
8
+ const CLOSING = /\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi;
9
+ const ANY_REF = /#(\d+)\b/g;
10
+
11
+ /** Maps every known commit (by sha) to { pr, issues }. `pulls` and `issues` are the stores' records keyed by number. */
12
+ export function buildCommitLinks({ commits, pulls, issues }) {
13
+ // A PR's own commits: exact, and the PR's "closes" are the issues it closes.
14
+ const prOfSha = new Map();
15
+ for (const p of Object.values(pulls)) {
16
+ for (const c of p.commits ?? []) if (!prOfSha.has(c.oid)) prOfSha.set(c.oid, p.number);
17
+ // The commit a merge or a squash left on the branch: GitLab's MR says which one it is, whatever the title says.
18
+ if (p.mergeCommitSha && !prOfSha.has(p.mergeCommitSha)) prOfSha.set(p.mergeCommitSha, p.number);
19
+ }
20
+ const isPr = (n) => !!pulls[n];
21
+ const isIssue = (n) => !!issues[n] && !isPr(n);
22
+ const links = new Map();
23
+ for (const c of commits) {
24
+ const title = c.title ?? c.message ?? "";
25
+ let pr = prOfSha.get(c.sha) ?? null;
26
+ if (pr === null) {
27
+ for (const re of PR_IN_TITLE) {
28
+ const n = Number(re.exec(title)?.[1]);
29
+ if (n && isPr(n)) { pr = n; break; }
30
+ }
31
+ }
32
+ const found = new Set(pr ? pulls[pr].closes ?? [] : []);
33
+ for (const m of title.matchAll(CLOSING)) if (isIssue(Number(m[1]))) found.add(Number(m[1]));
34
+ for (const m of title.matchAll(ANY_REF)) if (isIssue(Number(m[1]))) found.add(Number(m[1]));
35
+ links.set(c.sha, { pr, issues: [...found].filter(isIssue).sort((a, b) => a - b) });
36
+ }
37
+ return links;
38
+ }
@@ -0,0 +1,39 @@
1
+ import { test } from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { buildCommitLinks } from "./links.js";
4
+
5
+ const issues = { 10: { number: 10 }, 20: { number: 20 }, 30: { number: 30 } };
6
+ const pulls = {
7
+ 5: { number: 5, closes: [10], commits: [{ oid: "aaa" }, { oid: "bbb" }] },
8
+ 7: { number: 7, closes: [], commits: [] },
9
+ // a PR number is never taken for an issue, even when mentioned
10
+ 50: { number: 50, closes: [], commits: [] },
11
+ };
12
+
13
+ test("a PR's own commits map exactly, and the PR's closes come with them", () => {
14
+ const links = buildCommitLinks({ commits: [{ sha: "aaa", title: "wip" }], pulls, issues });
15
+ assert.deepEqual(links.get("aaa"), { pr: 5, issues: [10] });
16
+ });
17
+
18
+ test("a squash title names its PR", () => {
19
+ const links = buildCommitLinks({ commits: [{ sha: "x1", title: "feat: thing (#7)" }], pulls, issues });
20
+ assert.deepEqual(links.get("x1"), { pr: 7, issues: [] });
21
+ });
22
+
23
+ test("a merge commit names its PR", () => {
24
+ const links = buildCommitLinks({ commits: [{ sha: "m1", title: "Merge pull request #7 from a/b" }], pulls, issues });
25
+ assert.equal(links.get("m1").pr, 7);
26
+ });
27
+
28
+ test("closes / fixes keywords and plain #N name known issues, not PRs", () => {
29
+ const links = buildCommitLinks({
30
+ commits: [{ sha: "c1", title: "fix login, closes #20 and see #30 (#50)" }],
31
+ pulls, issues,
32
+ });
33
+ assert.deepEqual(links.get("c1"), { pr: 50, issues: [20, 30] });
34
+ });
35
+
36
+ test("unknown numbers are ignored", () => {
37
+ const links = buildCommitLinks({ commits: [{ sha: "u1", title: "fixes #999" }], pulls, issues });
38
+ assert.deepEqual(links.get("u1"), { pr: null, issues: [] });
39
+ });
@@ -0,0 +1,127 @@
1
+ // What is asked of GitHub (GraphQL selections) and how an answer becomes the store's records.
2
+ const login = "author{login}";
3
+
4
+ const COMMENT = `id ${login} createdAt body`;
5
+ const ISSUE_EVENTS = [
6
+ "LABELED_EVENT", "UNLABELED_EVENT", "CLOSED_EVENT", "REOPENED_EVENT", "ASSIGNED_EVENT",
7
+ "MILESTONED_EVENT", "RENAMED_TITLE_EVENT", "CROSS_REFERENCED_EVENT",
8
+ ];
9
+ const PULL_EVENTS = [
10
+ "LABELED_EVENT", "UNLABELED_EVENT", "CLOSED_EVENT", "REOPENED_EVENT", "MERGED_EVENT",
11
+ "READY_FOR_REVIEW_EVENT", "CONVERT_TO_DRAFT_EVENT", "REVIEW_REQUESTED_EVENT", "CROSS_REFERENCED_EVENT",
12
+ ];
13
+ // An issue's timeline union and a pull request's differ: a fragment on a type the union lacks is an error.
14
+ const ISSUE_EVENT_SEL = `__typename
15
+ ... on LabeledEvent{createdAt actor{login} label{name}}
16
+ ... on UnlabeledEvent{createdAt actor{login} label{name}}
17
+ ... on ClosedEvent{createdAt actor{login} stateReason}
18
+ ... on ReopenedEvent{createdAt actor{login}}
19
+ ... on AssignedEvent{createdAt actor{login} assignee{... on User{login}}}
20
+ ... on MilestonedEvent{createdAt milestoneTitle}
21
+ ... on RenamedTitleEvent{createdAt currentTitle previousTitle}
22
+ ... on CrossReferencedEvent{createdAt isCrossRepository source{__typename ... on Issue{number} ... on PullRequest{number}}}`;
23
+ const PULL_EVENT_SEL = `${ISSUE_EVENT_SEL}
24
+ ... on MergedEvent{createdAt actor{login}}
25
+ ... on ReadyForReviewEvent{createdAt actor{login}}
26
+ ... on ConvertToDraftEvent{createdAt actor{login}}
27
+ ... on ReviewRequestedEvent{createdAt requestedReviewer{... on User{login}}}`;
28
+
29
+ const events = (list, sel) => ({ args: `itemTypes:[${list.join(",")}]`, sel });
30
+
31
+ export const ISSUE_CONNECTIONS = {
32
+ comments: { args: "", sel: COMMENT },
33
+ timelineItems: events(ISSUE_EVENTS, ISSUE_EVENT_SEL),
34
+ };
35
+ export const PULL_CONNECTIONS = {
36
+ comments: { args: "", sel: COMMENT },
37
+ timelineItems: events(PULL_EVENTS, PULL_EVENT_SEL),
38
+ reviews: { args: "", sel: `${login} state submittedAt body` },
39
+ commits: { args: "", sel: "commit{oid messageHeadline committedDate authoredDate author{user{login}}}" },
40
+ };
41
+
42
+ export const connectionSelection = (name, c) =>
43
+ `${name}(first:100${c.args ? "," + c.args : ""}){ totalCount pageInfo{hasNextPage endCursor} nodes{ ${c.sel} } }`;
44
+
45
+ const COMMON = `title body url state createdAt closedAt ${login}
46
+ labels(first:50){nodes{name color}}
47
+ assignees(first:20){nodes{login}}
48
+ milestone{title dueOn}`;
49
+ export const ISSUE_FIELDS = `${COMMON} stateReason parent{number} subIssues(first:100){nodes{number}}
50
+ blockedBy(first:50){nodes{number}} blocking(first:50){nodes{number}}`;
51
+ export const PULL_FIELDS = `${COMMON} isDraft merged mergedAt additions deletions changedFiles baseRefName headRefName
52
+ mergedBy{login} reviewDecision closingIssuesReferences(first:50){nodes{number}}`;
53
+
54
+ const who = (u) => u?.login ?? null;
55
+ // A connection GitHub could not return (NOT_FOUND on a node the token cannot see) is null: it reads as empty.
56
+ const nodesOf = (c) => c?.nodes ?? [];
57
+
58
+ function normalizeEvent(e) {
59
+ const base = { type: e.__typename.replace(/Event$/, ""), at: e.createdAt, actor: who(e.actor) };
60
+ switch (e.__typename) {
61
+ case "LabeledEvent":
62
+ case "UnlabeledEvent": return { ...base, label: e.label?.name };
63
+ case "ClosedEvent": return { ...base, reason: e.stateReason ?? null };
64
+ case "AssignedEvent": return { ...base, assignee: who(e.assignee) };
65
+ case "MilestonedEvent": return { ...base, milestone: e.milestoneTitle };
66
+ case "RenamedTitleEvent": return { ...base, from: e.previousTitle, to: e.currentTitle };
67
+ case "ReviewRequestedEvent": return { ...base, reviewer: who(e.requestedReviewer) };
68
+ case "CrossReferencedEvent": return { ...base, source: e.source?.number ?? null, sourceKind: e.source?.__typename ?? null, external: !!e.isCrossRepository };
69
+ default: return base;
70
+ }
71
+ }
72
+
73
+ const common = (n) => ({
74
+ id: n.id,
75
+ number: n.number,
76
+ title: n.title,
77
+ body: n.body ?? "",
78
+ url: n.url,
79
+ author: who(n.author),
80
+ createdAt: n.createdAt,
81
+ updatedAt: n.updatedAt,
82
+ closedAt: n.closedAt ?? null,
83
+ labels: nodesOf(n.labels).map((l) => ({ name: l.name, color: l.color })),
84
+ assignees: nodesOf(n.assignees).map((a) => a.login),
85
+ milestone: n.milestone ? { title: n.milestone.title, dueOn: n.milestone.dueOn ?? null } : null,
86
+ commentsCount: n.comments?.totalCount ?? 0,
87
+ comments: nodesOf(n.comments).map((c) => ({ id: c.id, author: who(c.author), createdAt: c.createdAt, body: c.body })),
88
+ events: nodesOf(n.timelineItems).filter(Boolean).map(normalizeEvent),
89
+ });
90
+
91
+ export function normalizeIssue(n) {
92
+ return {
93
+ kind: "issue",
94
+ ...common(n),
95
+ state: n.state,
96
+ stateReason: n.stateReason ?? null,
97
+ // GitHub's own sub-issue relation (the checklists of a body are read apart, see derive.js)
98
+ parent: n.parent?.number ?? null,
99
+ subIssues: (n.subIssues?.nodes ?? []).map((s) => s.number),
100
+ // GitHub's own dependencies (the blocker words written in a body are read apart, see derive.js)
101
+ blockedBy: (n.blockedBy?.nodes ?? []).map((s) => s.number),
102
+ blocking: (n.blocking?.nodes ?? []).map((s) => s.number),
103
+ };
104
+ }
105
+
106
+ export function normalizePull(n) {
107
+ return {
108
+ kind: "pull",
109
+ ...common(n),
110
+ state: n.merged ? "MERGED" : n.state, // OPEN | CLOSED | MERGED
111
+ isDraft: n.isDraft,
112
+ mergedAt: n.mergedAt ?? null,
113
+ mergedBy: who(n.mergedBy),
114
+ reviewDecision: n.reviewDecision ?? null,
115
+ base: n.baseRefName,
116
+ head: n.headRefName,
117
+ additions: n.additions,
118
+ deletions: n.deletions,
119
+ changedFiles: n.changedFiles,
120
+ closes: nodesOf(n.closingIssuesReferences).map((i) => i.number),
121
+ commitsCount: n.commits?.totalCount ?? 0,
122
+ commits: nodesOf(n.commits).map((c) => ({
123
+ oid: c.commit.oid, message: c.commit.messageHeadline, at: c.commit.committedDate, author: c.commit.author?.user?.login ?? null,
124
+ })),
125
+ reviews: nodesOf(n.reviews).map((r) => ({ author: who(r.author), state: r.state, at: r.submittedAt, body: r.body })),
126
+ };
127
+ }
@@ -0,0 +1,47 @@
1
+ // A mini store in one JSON file: { meta, issues: {number: Issue}, pulls: {number: Pull} }.
2
+ // Writes are atomic (temp file + rename) and serialized, so a crash never leaves half a file.
3
+ import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
4
+ import { dirname } from "node:path";
5
+
6
+ const EMPTY = () => ({ meta: { repo: null, lastSync: null, fullSyncAt: null, syncs: [] }, issues: {}, pulls: {} });
7
+
8
+ export class Store {
9
+ /** @param {string} file */
10
+ constructor(file) {
11
+ this.file = file;
12
+ this.data = EMPTY();
13
+ this.writing = Promise.resolve();
14
+ }
15
+
16
+ async load() {
17
+ try {
18
+ const parsed = JSON.parse(await readFile(this.file, "utf8"));
19
+ this.data = { ...EMPTY(), ...parsed, meta: { ...EMPTY().meta, ...parsed.meta } };
20
+ } catch (e) {
21
+ if (e.code !== "ENOENT") throw e;
22
+ }
23
+ return this;
24
+ }
25
+
26
+ /** Inserts or replaces by number; returns how many were new. */
27
+ upsert(kind, items) {
28
+ const bucket = this.data[kind];
29
+ let added = 0;
30
+ for (const item of items) {
31
+ if (!bucket[item.number]) added++;
32
+ bucket[item.number] = item;
33
+ }
34
+ return added;
35
+ }
36
+
37
+ save() {
38
+ // Serialize writes: each one waits for the previous and writes the state at its own turn.
39
+ this.writing = this.writing.then(async () => {
40
+ await mkdir(dirname(this.file), { recursive: true });
41
+ const tmp = `${this.file}.tmp`;
42
+ await writeFile(tmp, JSON.stringify(this.data));
43
+ await rename(tmp, this.file);
44
+ });
45
+ return this.writing;
46
+ }
47
+ }
@@ -0,0 +1,98 @@
1
+ // Pulls every issue and pull request of a repo (history included) into the store, and keeps it current.
2
+ // A first run (or `full`) walks everything; later runs walk newest-updated first and stop at the first page
3
+ // older than the last sync, so the 5-minute poll is one or two small queries.
4
+ import { normalizeIssue, normalizePull, ISSUE_CONNECTIONS, PULL_CONNECTIONS, ISSUE_FIELDS, PULL_FIELDS, connectionSelection } from "./model.js";
5
+
6
+ const MARGIN_MS = 2 * 60 * 1000; // re-read a little before the last sync: clocks and eventual consistency
7
+
8
+ const listQuery = (kind, fields, conns, pageSize) => `
9
+ query($owner:String!,$name:String!,$after:String){
10
+ repository(owner:$owner,name:$name){
11
+ ${kind}(first:${pageSize},after:$after,orderBy:{field:UPDATED_AT,direction:DESC}){
12
+ totalCount
13
+ pageInfo{hasNextPage endCursor}
14
+ nodes{ id number updatedAt ${fields} ${Object.entries(conns).map(([n, c]) => connectionSelection(n, c)).join("\n")} }
15
+ }
16
+ }
17
+ }`;
18
+
19
+ const ISSUES_QUERY = listQuery("issues", ISSUE_FIELDS, ISSUE_CONNECTIONS, 30);
20
+ const PULLS_QUERY = listQuery("pullRequests", PULL_FIELDS, PULL_CONNECTIONS, 20);
21
+
22
+ /** Reads the rest of a connection a node had more of than one page of. */
23
+ async function drain(graphql, typename, id, name, conn, firstPage) {
24
+ const nodes = [...firstPage.nodes];
25
+ let { hasNextPage, endCursor } = firstPage.pageInfo;
26
+ while (hasNextPage) {
27
+ const q = `query($id:ID!,$after:String){ node(id:$id){ ... on ${typename}{
28
+ ${name}(first:100,after:$after${conn.args ? "," + conn.args : ""}){ pageInfo{hasNextPage endCursor} nodes{ ${conn.sel} } } } } }`;
29
+ const data = await graphql(q, { id, after: endCursor });
30
+ const page = data.node[name];
31
+ nodes.push(...page.nodes);
32
+ ({ hasNextPage, endCursor } = page.pageInfo);
33
+ }
34
+ return nodes;
35
+ }
36
+
37
+ async function completeNode(graphql, typename, conns, node) {
38
+ for (const [name, conn] of Object.entries(conns)) {
39
+ const page = node[name];
40
+ if (page?.pageInfo?.hasNextPage) node[name] = { ...page, nodes: await drain(graphql, typename, node.id, name, conn, page) };
41
+ }
42
+ return node;
43
+ }
44
+
45
+ async function walk({ graphql, owner, name, query, rootField, typename, conns, since, normalize, onProgress = () => {} }) {
46
+ const out = [];
47
+ let processed = 0;
48
+ let after = null;
49
+ for (;;) {
50
+ const data = await graphql(query, { owner, name, after });
51
+ const page = data.repository[rootField];
52
+ let reachedOld = false;
53
+ for (const node of page.nodes) {
54
+ if (since && Date.parse(node.updatedAt) < since - MARGIN_MS) {
55
+ reachedOld = true; // newest first: everything after this one is older too
56
+ break;
57
+ }
58
+ out.push(normalize(await completeNode(graphql, typename, conns, node)));
59
+ processed++;
60
+ onProgress({ phase: rootField === "issues" ? "Issues" : "Pull requests", done: processed, total: since ? null : page.totalCount });
61
+ }
62
+ if (reachedOld || !page.pageInfo.hasNextPage) return { items: out, total: page.totalCount };
63
+ after = page.pageInfo.endCursor;
64
+ }
65
+ }
66
+
67
+ /**
68
+ * One sync pass. Returns what it did. `full` ignores the last sync and rereads everything.
69
+ * @param {{graphql: Function, store: import("./store.js").Store, repo: string, full?: boolean, now?: () => number, log?: Function}} o
70
+ */
71
+ export async function sync({ graphql, store, repo, full = false, now = Date.now, log = () => {}, onProgress = () => {} }) {
72
+ const [owner, name] = repo.split("/");
73
+ const t0 = now();
74
+ const startedAt = new Date(t0).toISOString();
75
+ const doFull = full || !store.data.meta.fullSyncAt || store.data.meta.repo !== repo;
76
+ if (store.data.meta.repo && store.data.meta.repo !== repo) {
77
+ store.data.issues = {};
78
+ store.data.pulls = {};
79
+ }
80
+ const since = doFull ? null : Date.parse(store.data.meta.lastSync);
81
+
82
+ const common = { graphql, owner, name, since };
83
+ const issues = await walk({ ...common, query: ISSUES_QUERY, rootField: "issues", typename: "Issue", conns: ISSUE_CONNECTIONS, normalize: normalizeIssue, onProgress });
84
+ const pulls = await walk({ ...common, query: PULLS_QUERY, rootField: "pullRequests", typename: "PullRequest", conns: PULL_CONNECTIONS, normalize: normalizePull, onProgress });
85
+ store.upsert("issues", issues.items);
86
+ store.upsert("pulls", pulls.items);
87
+
88
+ const meta = store.data.meta;
89
+ meta.repo = repo;
90
+ meta.lastSync = startedAt;
91
+ if (doFull) meta.fullSyncAt = startedAt;
92
+ meta.totals = { issues: issues.total, pulls: pulls.total };
93
+ const entry = { at: startedAt, full: doFull, issues: issues.items.length, pulls: pulls.items.length, ms: now() - t0 };
94
+ meta.syncs = [...meta.syncs, entry].slice(-50);
95
+ await store.save();
96
+ log(`sync ${doFull ? "full" : "incremental"}: ${entry.issues} issues, ${entry.pulls} pulls in ${entry.ms} ms`);
97
+ return entry;
98
+ }