atris 3.78.0 → 3.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/commands/brief.js +8 -2
- package/commands/gm.js +3 -0
- package/commands/rating.js +2 -0
- package/lib/fleet.js +10 -0
- package/lib/held-autosweep.js +559 -0
- package/lib/held-pull-requests.js +42 -15
- package/lib/task-db.js +70 -27
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -475,6 +475,7 @@ atris rating <member> # one member's rating here, next to its engine's
|
|
|
475
475
|
- Engines are rated in the combine: Loop Bench tasks from the atris repos, each graded by the tests that shipped with it. No engine is a 99 yet.
|
|
476
476
|
- Each engine's card also rates its Python tasks, its JavaScript tasks, the tasks the Atris prompt was never tuned on, and how rarely a try broke another test, with its median time and pay-per-use price per finished task.
|
|
477
477
|
- Members are rated on the same scale from the jobs they landed in your workspace over the last 30 days. Landed means accepted or merged; drops and abandons count against them. `atris gm` shows the same numbers.
|
|
478
|
+
- When a person merges a builder's pull request, the builder is usually credited within the hour, by an `atris brief`, `atris rating`, `atris gm` or flight that reaches GitHub; `atris task sweep --held` does it right away. `ATRIS_HELD_SWEEP=off` stops the automatic check.
|
|
478
479
|
- The session boot shows the build engine's rating, for example `build opus 5.5 (86 ovr)`. The combine grades building only, so review and search engines show none.
|
|
479
480
|
|
|
480
481
|
## Benchmark Harness
|
package/commands/brief.js
CHANGED
|
@@ -479,7 +479,7 @@ function renderOwnerHtml(data) {
|
|
|
479
479
|
blocks: [
|
|
480
480
|
{ type: 'title', headline: 'atris brief', sub: `${needs} ${needs === 1 ? 'thing needs' : 'things need'} you, ${done} landed this week` },
|
|
481
481
|
...ownerBlocks(shown),
|
|
482
|
-
{ type: 'close', tagline: '
|
|
482
|
+
{ type: 'close', tagline: 'This screen only reads your jobs.' },
|
|
483
483
|
],
|
|
484
484
|
}, { title: 'atris brief' });
|
|
485
485
|
}
|
|
@@ -559,7 +559,11 @@ function showHelp() {
|
|
|
559
559
|
console.log(' atris brief outcome <brief_id> --result pass|fail|partial --note <text> [--json|--raw]');
|
|
560
560
|
console.log(' atris brief review [--lessons] [--json] [--limit N]');
|
|
561
561
|
console.log('One screen: what needs you, what got done this week, and how the team is doing.');
|
|
562
|
-
console.log('
|
|
562
|
+
console.log('It reads your jobs. Before the screen it may quietly check GitHub and credit or release held jobs');
|
|
563
|
+
console.log('whose pull request merged or closed. A merge is usually credited within the hour, by a command that');
|
|
564
|
+
console.log('reaches GitHub; atris task sweep --held does it right away.');
|
|
565
|
+
console.log('ATRIS_HELD_SWEEP=off turns that off.');
|
|
566
|
+
console.log('--days N sets the older landed list in --json.');
|
|
563
567
|
}
|
|
564
568
|
|
|
565
569
|
function isBriefLedgerSubcommand(value) {
|
|
@@ -795,6 +799,8 @@ async function briefCommand(args = [], root = process.cwd(), io = {}) {
|
|
|
795
799
|
showHelp();
|
|
796
800
|
return 0;
|
|
797
801
|
}
|
|
802
|
+
// A person's merge pays the builder with no command: the held-job check runs here.
|
|
803
|
+
require('../lib/held-autosweep').autoSweepHeld({ root });
|
|
798
804
|
|
|
799
805
|
const data = buildBriefData(root, { days: flags.days, owner: true });
|
|
800
806
|
if (flags.json) {
|
package/commands/gm.js
CHANGED
|
@@ -1907,6 +1907,9 @@ async function gmCommand(...args) {
|
|
|
1907
1907
|
showHelp();
|
|
1908
1908
|
return;
|
|
1909
1909
|
}
|
|
1910
|
+
// A person's merge pays the builder with no command: the held-job check runs here, for the workspace the
|
|
1911
|
+
// board shows.
|
|
1912
|
+
require('../lib/held-autosweep').autoSweepHeld({ root: boardWorkspaceRoot(args) });
|
|
1910
1913
|
|
|
1911
1914
|
if (positional(args)[0] === 'target') {
|
|
1912
1915
|
const at = args.indexOf('target');
|
package/commands/rating.js
CHANGED
|
@@ -152,6 +152,8 @@ function ratingCommand(args = [], deps = {}) {
|
|
|
152
152
|
write(`${USAGE}\n`);
|
|
153
153
|
return 0;
|
|
154
154
|
}
|
|
155
|
+
// A person's merge pays the builder with no command: the held-job check runs here.
|
|
156
|
+
require('../lib/held-autosweep').autoSweepHeld({ root: deps.root });
|
|
155
157
|
const json = hasFlag(args, '--json');
|
|
156
158
|
const name = args.filter((arg) => !arg.startsWith('-')).join(' ').trim();
|
|
157
159
|
const cards = rating.combineCards();
|
package/lib/fleet.js
CHANGED
|
@@ -3413,6 +3413,12 @@ async function runFleetFlight({
|
|
|
3413
3413
|
const lowDisk = lowDiskRefusal({ root, agent: 'fleet', task: 'flight', freeSpace, spaceGuard });
|
|
3414
3414
|
if (lowDisk) return stopForLowDisk(flight, lowDisk, staffed.map((s) => ({ task: s.task.display_id, engine: s.engine })), log);
|
|
3415
3415
|
|
|
3416
|
+
// A person's merge pays the builder with no command: the held-job check runs here, never before the disk look
|
|
3417
|
+
// above (a flight that refuses writes nothing to any job) and never on a dry run: a dry run, and a flight with
|
|
3418
|
+
// nothing staffed, end before this point. A pick that reads the results after this point (a retry choosing a
|
|
3419
|
+
// fallback builder, say) sees what the check credited or released, and so does any pick in a later flight.
|
|
3420
|
+
require('./held-autosweep').autoSweepHeld({ root });
|
|
3421
|
+
|
|
3416
3422
|
const liveLogPath = startLiveFlight(root, receiptPath, flight);
|
|
3417
3423
|
|
|
3418
3424
|
// Claim + cut a worktree per assignment, then dispatch all in parallel.
|
|
@@ -4318,6 +4324,10 @@ async function runDispatchFlight({
|
|
|
4318
4324
|
// the flight writes anything into its folder.
|
|
4319
4325
|
const workspace = leaseDb.workspaceRoot(root);
|
|
4320
4326
|
try { leaseDb.expireLeases(leaseConnection); } catch (err) { if (process.env.ATRIS_DEBUG) process.stderr.write(`lease sweep failed: ${String(err?.message ?? err).replace(/[\r\n]+/g, ' ')}\n`); }
|
|
4327
|
+
// A person's merge pays the builder with no command: the held-job check runs here, never before the disk look
|
|
4328
|
+
// above. A pick that reads the results after this point (the picker is made below, and a retry choosing a
|
|
4329
|
+
// fallback builder reads them) sees what the check credited or released, and so does any pick in a later flight.
|
|
4330
|
+
require('./held-autosweep').autoSweepHeld({ root });
|
|
4321
4331
|
// Cut worktrees from, and check self-landing against, the workspace's real
|
|
4322
4332
|
// protected branch, not a hard-coded origin/master (CLI-1298).
|
|
4323
4333
|
const localBase = reviewOnly ? '' : localDispatchBase(root);
|
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Held jobs, swept on their own. A held job is work that is done and handed in, waiting in Review under its
|
|
4
|
+
// builder's name for a person to land its pull request. When the person merges it, nothing in the task store
|
|
5
|
+
// hears about it: `atris task sweep --held` is the check that asks GitHub (lib/held-pull-requests.js), and
|
|
6
|
+
// nobody ran it, so a merged pull request sat for hours with its builder still rated 25. The commands that
|
|
7
|
+
// show the rating or start work now run that same check themselves: atris brief, atris rating and atris gm at
|
|
8
|
+
// their start, and both flights (the fleet flight and the dispatch flight) once they are past their disk look.
|
|
9
|
+
// So a merge is usually credited within the hour, by one of those that reaches GitHub, and `atris task sweep --held`
|
|
10
|
+
// does it right away. Only what always holds is said about the rest. Order: the check never runs before a flight's
|
|
11
|
+
// low-disk look (a flight that refuses writes nothing to any job) and never on a dry run, and any pick of a builder
|
|
12
|
+
// that reads the results after the check sees what the check credited or released, in that flight or in a later one.
|
|
13
|
+
//
|
|
14
|
+
// It is a favor to the command around it, never a cost:
|
|
15
|
+
// - Only this workspace, and only when it has a held job, which reads of the task store tell. The first is one
|
|
16
|
+
// plain question about every workspace, with no call to git: with no job held anywhere, which is the usual
|
|
17
|
+
// case, that is the end of it, and nothing is asked of GitHub or of git or written anywhere. Only when some
|
|
18
|
+
// job is held somewhere is git asked which checkouts are linked to this workspace (a held job may sit in
|
|
19
|
+
// one), and that call is held to what is left of the eight seconds and ended outright if it does not answer.
|
|
20
|
+
// - A run that GitHub gave a readable answer in, and that did not fail, is written down, and holds off the runs
|
|
21
|
+
// that would start within the hour after it. Its time is one small file in the CLI's state folder
|
|
22
|
+
// (state/held-sweep, one file for each workspace), never in the task store or the repository. Any other run
|
|
23
|
+
// writes nothing, so it holds nothing off.
|
|
24
|
+
// - One command at a time. A run first makes a small lock file beside that record, with the exclusive flag,
|
|
25
|
+
// so of two commands that start together one asks GitHub and the other goes on without a word. A lock
|
|
26
|
+
// more than a minute old belongs to a run that died, and is taken over under a second small file made the
|
|
27
|
+
// same way, so that only one run at a time removes it and a late second command cannot remove the new lock
|
|
28
|
+
// the first has made; a run that loses any step of that goes on without a word.
|
|
29
|
+
// - Eight seconds bound the waiting, not the whole run. They are counted from the start of the run and cover git and
|
|
30
|
+
// GitHub: git gets two seconds to tell the linked checkouts and two to say where the workspace came from; one
|
|
31
|
+
// question to GitHub gets four, and none is started with under a second and a half left. Once the eight are used
|
|
32
|
+
// up the run stops asking. The task store is opened with a wait of about a second for other writers. So a slow or
|
|
33
|
+
// missing gh, a hung git or a locked store holds the command for eight seconds at most. Reads of the local task
|
|
34
|
+
// list are not cut short, and on the real store they take well under a second. Known limit: the run lists the
|
|
35
|
+
// workspace's tasks once and never numbers them, but each credit numbers the list inside the task store to write
|
|
36
|
+
// the journal line, as the manual sweep does, which can take seconds on a very large list. Nothing else starts
|
|
37
|
+
// git. The sweep would, when it refuses a pull request that another job has and names that job in that job's own
|
|
38
|
+
// workspace (listing a workspace's tasks asks git for its linked checkouts), and that can be a workspace this run
|
|
39
|
+
// was never asked about. The run says nothing of a refusal, so it names that job by its id and looks at no other
|
|
40
|
+
// workspace.
|
|
41
|
+
// - gh missing, gh that cannot be run, gh stopped by a signal or timed out, not signed in, GitHub down, git that
|
|
42
|
+
// cannot answer (it hung, could not be run, or exited with an error), a store that stays locked: the run stops,
|
|
43
|
+
// says nothing and is not written down, so it holds nothing off. A job that cannot be settled does not stop the
|
|
44
|
+
// run, but the run is not written down either. All of that holds after GitHub has answered too. The jobs it
|
|
45
|
+
// settled before it stopped stay settled. A run is written down when GitHub gave it at least one readable
|
|
46
|
+
// answer and nothing in it failed, and using up the eight seconds is not a failure (the rule is stated where the
|
|
47
|
+
// record is written), so a run that asked nothing, or understood nothing, or failed, holds nothing off.
|
|
48
|
+
// - Silent unless it acts. It says one plain line on stderr for each job it credits or lets go, as soon as
|
|
49
|
+
// that job is settled, and nothing on stdout, so --json output is untouched. A job it cannot settle costs
|
|
50
|
+
// one short line, and the run goes on to the next job. A line that cannot be written (nobody reads stderr,
|
|
51
|
+
// or the pipe is full) is lost where it is written, or cut short when the pipe had room for only part of it,
|
|
52
|
+
// and the command ends as it would have without the check. On POSIX a pipe that is full is not waited for; a
|
|
53
|
+
// terminal that is paused holds this write as it holds every other line the command prints.
|
|
54
|
+
// - It never breaks the command around it: a failure of its own costs one short line on stderr.
|
|
55
|
+
// - ATRIS_HELD_SWEEP=off skips it entirely.
|
|
56
|
+
//
|
|
57
|
+
// The check is the manual sweep itself (sweepHeldJobs), called for one job at a time under the name the manual
|
|
58
|
+
// sweep records when it is not told one, so the rating reads a credit the same whichever way it was written.
|
|
59
|
+
// The workspace's task list is read once for the whole run and never numbered, since the run prints no task number, and
|
|
60
|
+
// each call is handed its job from that.
|
|
61
|
+
// The manual sweep takes no lock, so it can be running at the same moment. That is safe: a job is settled only
|
|
62
|
+
// while its row is still what the sweep read, and one pull request lands once, so the second run finds
|
|
63
|
+
// nothing left to do.
|
|
64
|
+
|
|
65
|
+
const fs = require('fs');
|
|
66
|
+
const os = require('os');
|
|
67
|
+
const path = require('path');
|
|
68
|
+
const crypto = require('crypto');
|
|
69
|
+
const { spawnSync } = require('child_process');
|
|
70
|
+
const { DatabaseSync } = require('node:sqlite');
|
|
71
|
+
const taskDb = require('./task-db');
|
|
72
|
+
const { atrisHome } = require('./state-home');
|
|
73
|
+
const { pullRequestOfLink } = require('./pull-request-link');
|
|
74
|
+
const { askGitHub, askDefaultBranch, heldJobsAmong, originRepoOf, sweepHeldJobs } = require('./held-pull-requests');
|
|
75
|
+
|
|
76
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
77
|
+
// TOTAL_MS bounds the waiting, not the whole run. It is counted from the start of the run, and the run stops asking
|
|
78
|
+
// once it is used up. Reads of the local task list are not cut short. One question to GitHub gets CALL_MS of it and is
|
|
79
|
+
// not started with less than LEAST_CALL_MS left, which could not answer it in time; each of git's questions (which
|
|
80
|
+
// checkouts are linked to the workspace, and where the workspace came from) gets GIT_MS and is not started with less
|
|
81
|
+
// than LEAST_GIT_MS left.
|
|
82
|
+
const TOTAL_MS = 8000;
|
|
83
|
+
const CALL_MS = 4000;
|
|
84
|
+
const LEAST_CALL_MS = 1500;
|
|
85
|
+
const GIT_MS = 2000;
|
|
86
|
+
const LEAST_GIT_MS = 500;
|
|
87
|
+
// A lock this far from now, either way, belongs to a run that died.
|
|
88
|
+
const STALE_LOCK_MS = 60 * 1000;
|
|
89
|
+
const OFF = /^(off|0|false|no)$/i;
|
|
90
|
+
// One plain question to the task store, about every workspace and with no call to git: is any job held? It asks
|
|
91
|
+
// for more than the check settles (a job in review with a claim and a pull request on its row, whether or not that
|
|
92
|
+
// pull request is still open) and never for less, so a no is a no for every workspace.
|
|
93
|
+
const ANY_HELD_JOB = "SELECT 1 FROM tasks WHERE status = 'review' AND claimed_by IS NOT NULL AND claimed_by <> '' AND metadata LIKE '%\"pull_request\"%' LIMIT 1";
|
|
94
|
+
|
|
95
|
+
// What gh says when it is not signed in, and when GitHub cannot answer right now. The sweep already stops on
|
|
96
|
+
// a gh that is missing or that times out; these are the other ways gh fails for the whole run rather than for
|
|
97
|
+
// one pull request. Anything else gh says (a pull request it cannot find, a repository it may not read) is an
|
|
98
|
+
// answer about that one job, and the run goes on.
|
|
99
|
+
const SIGNED_OUT = /gh auth login|bad credentials|HTTP 401|not logged in|authentication (?:is )?required|requires authentication|GH_TOKEN|GITHUB_TOKEN/i;
|
|
100
|
+
const GITHUB_DOWN = /HTTP 5\d\d|HTTP 429|bad gateway|service unavailable|gateway time-?out|internal server error|rate limit|error connecting to|check your internet|githubstatus|could not resolve host|no such host|dial tcp|network is unreachable|connection (?:refused|reset|timed out)|\bE(?:CONNREFUSED|CONNRESET|NOTFOUND|AI_AGAIN|TIMEDOUT)\b|i\/o timeout|TLS handshake|x509/i;
|
|
101
|
+
// The one way gh says, in so many words, that GitHub has no pull request with the number it was asked about, and the
|
|
102
|
+
// repository itself is there. gh 2.74.2 prints exactly this on its error output, nothing on its output, and exits 1:
|
|
103
|
+
// GraphQL: Could not resolve to a PullRequest with the number of 99999999. (repository.pullRequest)
|
|
104
|
+
// Nothing else is that, and a 404 least of all, since GitHub answers 404 for a repository the token cannot read too.
|
|
105
|
+
// A repository it cannot read ("Could not resolve to a Repository"), a message it has not shown before and a failure
|
|
106
|
+
// with no words at all say nothing about the pull request. So the match is the whole error output, less the one line
|
|
107
|
+
// break gh ends it with, and that is that message and nothing but it, for the very number asked about: the same
|
|
108
|
+
// sentence for another number, the sentence inside a longer message (words in front of it, another error behind it,
|
|
109
|
+
// other lines around it), the sentence with a space or a blank line in front or a second line break or spaces behind,
|
|
110
|
+
// a 404 or a longer status like "HTTP 4040" are all some other reply, which cannot be read. Another wording of the
|
|
111
|
+
// sentence, such as "GraphQL error:" in front of it and no path behind it, is not taken for it either: if a gh words
|
|
112
|
+
// it so, a deleted link is no answer, and holds nothing off, which is the safe side.
|
|
113
|
+
function noSuchPullRequestMessage(number) {
|
|
114
|
+
return `GraphQL: Could not resolve to a PullRequest with the number of ${number}. (repository.pullRequest)`;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// What is left of the eight seconds of waiting, in milliseconds.
|
|
118
|
+
function timeLeft(now, began) {
|
|
119
|
+
return TOTAL_MS - (now() - began);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// One line on stderr, written on the spot, and lost where it is written if it cannot be. A stream's write on a pipe is
|
|
123
|
+
// finished later, and a pipe whose reader has gone then fails with EPIPE as an error event that no try around the
|
|
124
|
+
// write can catch: it ends a command that goes on running (atris gm, a flight) with a different exit code, after a
|
|
125
|
+
// job was settled. So the line goes straight to the file (descriptor 2), and any error (EPIPE when nobody reads,
|
|
126
|
+
// EAGAIN when the pipe is full) costs the line and nothing else. No listener is added to process.stderr: that would
|
|
127
|
+
// answer for every other write the command makes as well.
|
|
128
|
+
//
|
|
129
|
+
// The line before the write makes Node create process.stderr first. On POSIX Node sets a pipe to not wait for room
|
|
130
|
+
// when it creates that stream, and until then the descriptor is whatever the parent handed down, usually one that
|
|
131
|
+
// waits. A write to a pipe that is full and that nobody reads would then never return, and would hold the command and
|
|
132
|
+
// the lock for as long as nobody read it, after a job was settled: the eight seconds bound the waiting on git and
|
|
133
|
+
// GitHub, not a write. With the stream made, the same write fails at once with EAGAIN, which the catch swallows. A
|
|
134
|
+
// write can also go through in part when the pipe has a little room. It is not tried again, so on a full pipe a line
|
|
135
|
+
// can be cut short or lost. A terminal stays as Node sets it (it waits for room), so a paused terminal holds this
|
|
136
|
+
// write exactly as it holds every other line the command prints.
|
|
137
|
+
function writeLine(line) {
|
|
138
|
+
try {
|
|
139
|
+
void process.stderr;
|
|
140
|
+
fs.writeSync(2, `${line}\n`);
|
|
141
|
+
} catch {
|
|
142
|
+
// The line is lost. The command goes on.
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function oneLine(text, max) {
|
|
147
|
+
const flat = String(text === undefined || text === null ? '' : text).replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
148
|
+
return flat.length > max ? `${flat.slice(0, max - 3)}...` : flat;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Who `atris task sweep --held` records when it is not told with --as (DEFAULT_OWNER in commands/task.js).
|
|
152
|
+
// The rating reads a name with "auto" in it as an autoland, so a credit written here must carry that same
|
|
153
|
+
// name, never one of its own.
|
|
154
|
+
function actorOf(env) {
|
|
155
|
+
const osName = () => { try { return os.userInfo().username; } catch { return ''; } };
|
|
156
|
+
return String(env.ATRIS_AGENT_ID || env.USER || osName() || 'unknown');
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Where the time of the last written-down run for a workspace is kept: one small file named by the workspace's path,
|
|
160
|
+
// so two workspaces never share a record and looking at one never writes into its repository.
|
|
161
|
+
function recordPath(workspace, env) {
|
|
162
|
+
const name = crypto.createHash('sha1').update(path.resolve(String(workspace))).digest('hex');
|
|
163
|
+
return path.join(atrisHome(env), 'state', 'held-sweep', `${name}.json`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function lastRunAt(file) {
|
|
167
|
+
try {
|
|
168
|
+
const at = Number(JSON.parse(fs.readFileSync(file, 'utf8')).last_run_at);
|
|
169
|
+
return Number.isFinite(at) && at > 0 ? at : null;
|
|
170
|
+
} catch {
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Whether the check ran within the last hour. A last run in the future (the clock was set back) does not hold
|
|
176
|
+
// the sweep off.
|
|
177
|
+
function ranRecently(file, now) {
|
|
178
|
+
const last = lastRunAt(file);
|
|
179
|
+
return last !== null && now() >= last && now() - last < HOUR_MS;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Written to a temp file then renamed, so a reader never sees half a file and two runs never clobber each other.
|
|
183
|
+
function writeRecord(file, workspace, at) {
|
|
184
|
+
const tmp = `${file}.${process.pid}.${crypto.randomBytes(4).toString('hex')}.tmp`;
|
|
185
|
+
try {
|
|
186
|
+
fs.writeFileSync(tmp, `${JSON.stringify({ workspace, last_run_at: at, last_run_iso: new Date(at).toISOString() })}\n`);
|
|
187
|
+
fs.renameSync(tmp, file);
|
|
188
|
+
} catch (err) {
|
|
189
|
+
try { fs.unlinkSync(tmp); } catch { /* nothing left to clean */ }
|
|
190
|
+
throw err;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Makes the lock file with the exclusive flag and puts the run's token in it. False when the file is already there.
|
|
195
|
+
function makeLock(file, token) {
|
|
196
|
+
let fd;
|
|
197
|
+
try {
|
|
198
|
+
fd = fs.openSync(file, 'wx');
|
|
199
|
+
} catch (err) {
|
|
200
|
+
if (err && err.code === 'EEXIST') return false;
|
|
201
|
+
throw err;
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
fs.writeSync(fd, `${token}\n`);
|
|
205
|
+
} catch (err) {
|
|
206
|
+
fs.closeSync(fd);
|
|
207
|
+
try { fs.unlinkSync(file); } catch { /* nothing left to clean */ }
|
|
208
|
+
throw err;
|
|
209
|
+
}
|
|
210
|
+
fs.closeSync(fd);
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// When a file was made (its own time), or null when there is no such file.
|
|
215
|
+
function madeAtOf(file) {
|
|
216
|
+
try {
|
|
217
|
+
return fs.statSync(file).mtimeMs;
|
|
218
|
+
} catch (err) {
|
|
219
|
+
if (err && err.code === 'ENOENT') return null;
|
|
220
|
+
throw err;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// A file this far from now, either way, belongs to a run that died.
|
|
225
|
+
const isDead = (madeAt, now) => Math.abs(now() - madeAt) >= STALE_LOCK_MS;
|
|
226
|
+
|
|
227
|
+
// The right to run the check, as a small file made with the exclusive flag: of two commands that start together
|
|
228
|
+
// only one can make it. Returns { release }, or null when another run holds it.
|
|
229
|
+
//
|
|
230
|
+
// A lock more than STALE_LOCK_MS old belongs to a run that died, and is taken over. Removing a lock by its name
|
|
231
|
+
// is not safe on its own: two runs can look at the same dead lock, and the slower one would then remove the new
|
|
232
|
+
// lock the faster one has made since. So the removal is a step one run at a time may take, guarded by a second
|
|
233
|
+
// small file made the same way beside the lock (its name and .takeover). The run that makes it looks at the lock
|
|
234
|
+
// again, removes it only if it is still that old, and makes the new lock. A run that finds the guard taken, finds
|
|
235
|
+
// the lock new after all, or loses the making of the new lock to another run, gives the check up. No run removes
|
|
236
|
+
// a lock it has not just seen to be dead, while holding the guard that keeps every other run from removing one.
|
|
237
|
+
// A guard more than STALE_LOCK_MS old is a run that died in the middle of a takeover: it is removed, once.
|
|
238
|
+
function takeLock(file, now) {
|
|
239
|
+
const token = crypto.randomBytes(8).toString('hex');
|
|
240
|
+
const mine = { release: () => letGo(file, token) };
|
|
241
|
+
if (makeLock(file, token)) return mine;
|
|
242
|
+
const madeAt = madeAtOf(file);
|
|
243
|
+
// The run that held it let it go in the moment between: it is free to make, once.
|
|
244
|
+
if (madeAt === null) return makeLock(file, token) ? mine : null;
|
|
245
|
+
if (!isDead(madeAt, now)) return null;
|
|
246
|
+
const guard = `${file}.takeover`;
|
|
247
|
+
if (!makeLock(guard, token)) {
|
|
248
|
+
const guardedAt = madeAtOf(guard);
|
|
249
|
+
if (guardedAt === null || !isDead(guardedAt, now)) return null;
|
|
250
|
+
try { fs.unlinkSync(guard); } catch (err) { if (!err || err.code !== 'ENOENT') throw err; }
|
|
251
|
+
if (!makeLock(guard, token)) return null;
|
|
252
|
+
}
|
|
253
|
+
try {
|
|
254
|
+
const nowMadeAt = madeAtOf(file);
|
|
255
|
+
if (nowMadeAt !== null) {
|
|
256
|
+
if (!isDead(nowMadeAt, now)) return null;
|
|
257
|
+
try { fs.unlinkSync(file); } catch (err) { if (!err || err.code !== 'ENOENT') throw err; }
|
|
258
|
+
}
|
|
259
|
+
return makeLock(file, token) ? mine : null;
|
|
260
|
+
} finally {
|
|
261
|
+
letGo(guard, token);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Lets the lock go, if it is still this run's: one taken over as dead is not removed by the run that lost it.
|
|
266
|
+
function letGo(file, token) {
|
|
267
|
+
try {
|
|
268
|
+
if (fs.readFileSync(file, 'utf8').trim() === token) fs.unlinkSync(file);
|
|
269
|
+
} catch { /* already gone */ }
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Reads the task store for the question `ask`, on a connection that opens nothing for writing and creates nothing,
|
|
273
|
+
// as the sweep of expired claims does. A store that cannot be read holds no job this check can settle (the
|
|
274
|
+
// command's own views report it): `unread` is the answer then.
|
|
275
|
+
function readStore({ env, say, ask, unread }) {
|
|
276
|
+
try {
|
|
277
|
+
const target = taskDb.getDbPath();
|
|
278
|
+
if (!fs.existsSync(target)) return unread;
|
|
279
|
+
const reader = new DatabaseSync(target, { readOnly: true });
|
|
280
|
+
try {
|
|
281
|
+
return ask(reader);
|
|
282
|
+
} finally {
|
|
283
|
+
reader.close();
|
|
284
|
+
}
|
|
285
|
+
} catch (err) {
|
|
286
|
+
if (env.ATRIS_DEBUG) say(`held jobs: could not read the task list (${oneLine(err && err.message, 120)})`);
|
|
287
|
+
return unread;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Whether any job is held, in any workspace: one plain question, with no call to git.
|
|
292
|
+
function anyJobHeld({ env, say }) {
|
|
293
|
+
return readStore({ env, say, unread: false, ask: (reader) => Boolean(reader.prepare(ANY_HELD_JOB).get()) });
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Whether this workspace has a job held for a pull request: held (it has), none (it has not) or blocked (git could
|
|
297
|
+
// not tell which checkouts are linked to the workspace, in the time there was). The workspace's jobs include those
|
|
298
|
+
// held in checkouts linked to it, and telling them is a call to git, held to what is left of the eight seconds and
|
|
299
|
+
// ended outright if it does not answer. Not asked at all when no job is held anywhere (see anyJobHeld).
|
|
300
|
+
function workspaceHolds(workspace, { env, say, left }) {
|
|
301
|
+
return readStore({
|
|
302
|
+
env,
|
|
303
|
+
say,
|
|
304
|
+
unread: 'none',
|
|
305
|
+
ask: (reader) => {
|
|
306
|
+
const time = Math.floor(Math.min(GIT_MS, left()));
|
|
307
|
+
if (time < LEAST_GIT_MS) return 'blocked';
|
|
308
|
+
if (taskDb.lookUpLinkedWorktrees(workspace, { timeoutMs: time }).failed) return 'blocked';
|
|
309
|
+
return taskDb.listTasks(reader, { workspaceRoot: workspace, status: 'review' }).some((row) => taskDb.heldPullRequestOf(row)) ? 'held' : 'none';
|
|
310
|
+
},
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// Whether what gh did says it cannot be used for any pull request right now, as an outage the sweep stops on:
|
|
315
|
+
// it could not be launched at all (a file that cannot be run, no memory, a runner that threw, a wrapper that
|
|
316
|
+
// exits 126), it was stopped by a signal before it said anything, it is not signed in, or GitHub is not
|
|
317
|
+
// answering.
|
|
318
|
+
function troubleWith(heard) {
|
|
319
|
+
if (!heard) return null;
|
|
320
|
+
if (heard.error) {
|
|
321
|
+
// Output past the buffer is a trouble with that one answer, not with gh. It comes first because the runner
|
|
322
|
+
// ends gh with a signal then, too.
|
|
323
|
+
return heard.error.code === 'ENOBUFS' ? null : { kind: 'gh_unusable', detail: 'gh could not be run, so nothing could be checked' };
|
|
324
|
+
}
|
|
325
|
+
// A gh with no exit status was ended by a signal, and with no error to say why it has told us nothing about the
|
|
326
|
+
// pull request, as one that timed out has not: the run stops, and nothing is written down.
|
|
327
|
+
if (heard.status === null) return { kind: 'gh_stopped', detail: 'gh was stopped before it answered, so nothing could be checked' };
|
|
328
|
+
if (typeof heard.status !== 'number' || heard.status === 0) return null;
|
|
329
|
+
if (heard.status === 126) return { kind: 'gh_unusable', detail: 'gh could not be run, so nothing could be checked' };
|
|
330
|
+
const said = String(heard.stderr || '').slice(0, 4000);
|
|
331
|
+
if (heard.status === 4 || SIGNED_OUT.test(said)) return { kind: 'not_signed_in', detail: 'gh is not signed in to GitHub, so nothing could be checked' };
|
|
332
|
+
if (GITHUB_DOWN.test(said)) return { kind: 'github_down', detail: 'GitHub is not answering, so nothing could be checked' };
|
|
333
|
+
return null;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Whether what gh did is GitHub saying that there is no pull request with this number, the one the question was about:
|
|
337
|
+
// gh exited 1 and its whole error output, less the one line break at the end ("\n", or "\r\n"), is the message above
|
|
338
|
+
// for this number. Only that one break is let go: a space or a blank line in front, a second line break behind, spaces
|
|
339
|
+
// anywhere around the line make it some other reply. (The pattern's "$" is the very end of the text here, since it has
|
|
340
|
+
// no "m" flag, so it never lets go of a break in the middle.) GitHub has answered then, though not with a pull request
|
|
341
|
+
// state. With no number to check against (the question was not about one pull request) nothing is.
|
|
342
|
+
function saysNoSuchPullRequest(heard, number) {
|
|
343
|
+
if (!heard || heard.status !== 1 || !Number.isSafeInteger(number) || number <= 0) return false;
|
|
344
|
+
return String(heard.stderr || '').replace(/\r?\n$/, '') === noSuchPullRequestMessage(number);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// What the sweep asks of GitHub and of git, each held to its own limit and to what is left of the eight seconds,
|
|
348
|
+
// and each asked once for the whole run (the sweep is called job by job, and would ask again every time).
|
|
349
|
+
// It counts the readable answers GitHub gave (answered), which is half of what a run is written down for; the other
|
|
350
|
+
// half is that nothing failed. When the time is spent GitHub is not asked, and the sweep stops on that outage: it is no
|
|
351
|
+
// answer, it is not a failure, and it does not undo the answers before it. Blocked is the other kind of stop, for a gh
|
|
352
|
+
// or a git that cannot be used at all, on any question: that is a failure, and an answer before it does not save the run.
|
|
353
|
+
function limited({ run, now, began }) {
|
|
354
|
+
let blocked = false;
|
|
355
|
+
let answered = 0;
|
|
356
|
+
let heard = null;
|
|
357
|
+
const left = () => timeLeft(now, began);
|
|
358
|
+
// Runs a command and keeps what it did. A runner that throws did not run the command.
|
|
359
|
+
const listen = (command, args, options) => {
|
|
360
|
+
try {
|
|
361
|
+
heard = run(command, args, options) || {};
|
|
362
|
+
} catch (error) {
|
|
363
|
+
heard = { error };
|
|
364
|
+
}
|
|
365
|
+
return heard;
|
|
366
|
+
};
|
|
367
|
+
const asked = new Map();
|
|
368
|
+
const askOnce = (key, ask) => {
|
|
369
|
+
if (!asked.has(key)) asked.set(key, ask());
|
|
370
|
+
return asked.get(key);
|
|
371
|
+
};
|
|
372
|
+
const github = (key, ask, number) => askOnce(key, () => {
|
|
373
|
+
const time = left();
|
|
374
|
+
if (time < LEAST_CALL_MS) return { outage: { kind: 'out_of_time', detail: 'the check ran out of time' } };
|
|
375
|
+
heard = null;
|
|
376
|
+
const answer = ask({ run: listen, timeoutMs: Math.floor(Math.min(CALL_MS, time)) });
|
|
377
|
+
const trouble = answer.outage || (answer.unknown ? troubleWith(heard) : null);
|
|
378
|
+
if (!trouble) {
|
|
379
|
+
// A reply the sweep parsed into a pull request state, or GitHub saying there is no pull request with the number
|
|
380
|
+
// the question was about. The default branch of a repository (askBranch) is neither, and is only asked after a
|
|
381
|
+
// merge has been read and counted.
|
|
382
|
+
if (answer.answer || (answer.unknown && saysNoSuchPullRequest(heard, number))) answered += 1;
|
|
383
|
+
return answer;
|
|
384
|
+
}
|
|
385
|
+
blocked = true;
|
|
386
|
+
return { outage: trouble };
|
|
387
|
+
});
|
|
388
|
+
return {
|
|
389
|
+
// The number is the one in the link, which is the one gh is asked about and the only one it can name in its reply.
|
|
390
|
+
ask: (url) => github(`pull request ${url}`, (options) => askGitHub(url, options), (pullRequestOfLink(url) || {}).number),
|
|
391
|
+
askBranch: (repoLink) => github(`repository ${repoLink.toLowerCase()}`, (options) => askDefaultBranch(repoLink, options)),
|
|
392
|
+
// Where the workspace came from. With too little time left git is not asked, and the job is left as it is.
|
|
393
|
+
// A git that did not exit with 0 (it hung, could not be run, exited with an error, or was stopped
|
|
394
|
+
// by a signal) has not answered, and says nothing about the origin, so the run stops.
|
|
395
|
+
askOrigin: (root) => askOnce(`origin ${root}`, () => {
|
|
396
|
+
const time = left();
|
|
397
|
+
if (time < LEAST_GIT_MS) return null;
|
|
398
|
+
heard = null;
|
|
399
|
+
const origin = originRepoOf(root, { run: listen, timeoutMs: Math.floor(Math.min(GIT_MS, time)) });
|
|
400
|
+
if (!heard || (!heard.error && heard.status === 0)) return origin;
|
|
401
|
+
blocked = true;
|
|
402
|
+
return null;
|
|
403
|
+
}),
|
|
404
|
+
blocked: () => blocked,
|
|
405
|
+
answered: () => answered,
|
|
406
|
+
left,
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// The pull request a job names, in words: "pull request 71", or "its pull request" when the row gives no number.
|
|
411
|
+
function pullRequestWords(pullRequest) {
|
|
412
|
+
const number = pullRequest.number || (pullRequestOfLink(pullRequest.url) || {}).number;
|
|
413
|
+
return number ? `pull request ${number}` : 'its pull request';
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// One plain line for each job the sweep settled: what was done for the builder, the pull request, what happened.
|
|
417
|
+
function linesFor(report) {
|
|
418
|
+
const lines = [];
|
|
419
|
+
for (const result of report.results) {
|
|
420
|
+
const named = pullRequestWords(result.pull_request);
|
|
421
|
+
const builder = oneLine(result.builder, 60);
|
|
422
|
+
if (result.action === 'landed') lines.push(`credited ${builder}: ${named} merged`);
|
|
423
|
+
else if (result.action === 'released') lines.push(`released ${builder}: ${named} closed without merging`);
|
|
424
|
+
}
|
|
425
|
+
return lines;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
// The workspace's held jobs, oldest first, one call of the sweep for each, and each job's line said as soon as
|
|
429
|
+
// that job is settled. A job the sweep throws on costs one short line and the run goes on to the next. A store
|
|
430
|
+
// that is busy ends the run (the error goes up to autoSweepHeld) with the lines already said standing.
|
|
431
|
+
// The task list is read once, before the first job, and nothing is numbered: the run's lines name the builder and the
|
|
432
|
+
// pull request, never a task number, and the sweep is called job by job. Each call is handed its job, read again by its
|
|
433
|
+
// id first, so that a job someone settled since is not asked about. Reading the list can take the time there was; then
|
|
434
|
+
// there is none to ask with, and nothing is asked, said or written down.
|
|
435
|
+
// Returns one word: ran (GitHub gave at least one readable answer, and nothing failed: running out of time after an
|
|
436
|
+
// answer is not a failure), blocked (gh, git or GitHub could not be used, on any question and whatever was answered
|
|
437
|
+
// before, or there was no time left to ask and nothing had been answered), failed (a job could not be settled) or idle
|
|
438
|
+
// (no readable answer came back and nothing stopped the run: the last held job was settled by someone else since the
|
|
439
|
+
// first look, no job had a link to ask about, or every reply was unreadable). Only a ran is written down (see sweep).
|
|
440
|
+
function checkJobs({ db, workspace, actor, limits, say }) {
|
|
441
|
+
let failed = false;
|
|
442
|
+
let outOfTime = false;
|
|
443
|
+
const held = heldJobsAmong(taskDb, taskDb.listTasks(db, { workspaceRoot: workspace }));
|
|
444
|
+
if (!held.length) return 'idle';
|
|
445
|
+
if (limits.left() < LEAST_CALL_MS) return 'blocked';
|
|
446
|
+
for (const job of held) {
|
|
447
|
+
if (limits.blocked()) break;
|
|
448
|
+
if (limits.left() < LEAST_CALL_MS) {
|
|
449
|
+
outOfTime = true;
|
|
450
|
+
break;
|
|
451
|
+
}
|
|
452
|
+
let report;
|
|
453
|
+
try {
|
|
454
|
+
const current = taskDb.getTask(db, job.id);
|
|
455
|
+
report = sweepHeldJobs(taskDb, db, {
|
|
456
|
+
workspaceRoot: workspace,
|
|
457
|
+
actor,
|
|
458
|
+
only: job.id,
|
|
459
|
+
listing: current ? [current] : [],
|
|
460
|
+
ask: limits.ask,
|
|
461
|
+
askBranch: limits.askBranch,
|
|
462
|
+
askOrigin: limits.askOrigin,
|
|
463
|
+
// The run says nothing of a refusal, and the sweep's own way of naming the other job lists that job's
|
|
464
|
+
// workspace, which can start git outside the eight seconds. The id is name enough.
|
|
465
|
+
nameOf: (id) => String(id),
|
|
466
|
+
});
|
|
467
|
+
} catch (err) {
|
|
468
|
+
if (taskDb.isBusyError(err)) throw err;
|
|
469
|
+
failed = true;
|
|
470
|
+
say(`held jobs: could not check ${oneLine(job.claimed_by, 60)}: ${pullRequestWords(taskDb.heldPullRequestOf(job))} (${oneLine(err && err.message, 100) || 'unknown error'})`);
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
for (const line of linesFor(report)) say(line);
|
|
474
|
+
// A question can find the time gone though the look above did not: the default branch is asked after the pull
|
|
475
|
+
// request has been read.
|
|
476
|
+
if (report.outage && report.outage.kind === 'out_of_time') {
|
|
477
|
+
outOfTime = true;
|
|
478
|
+
break;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
if (limits.blocked()) return 'blocked';
|
|
482
|
+
if (failed) return 'failed';
|
|
483
|
+
if (limits.answered() > 0) return 'ran';
|
|
484
|
+
return outOfTime ? 'blocked' : 'idle';
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// The check itself, on a connection to the task store of its own that waits about a second for other writers.
|
|
488
|
+
function check({ workspace, env, now, run, say, began }) {
|
|
489
|
+
const db = taskDb.open(undefined, { shortWait: true });
|
|
490
|
+
try {
|
|
491
|
+
return checkJobs({ db, workspace, actor: actorOf(env), limits: limited({ run, now, began }), say });
|
|
492
|
+
} finally {
|
|
493
|
+
try { db.close(); } catch { /* nothing left to close */ }
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
function sweep({ root, env, now, run, say }) {
|
|
498
|
+
if (OFF.test(String(env.ATRIS_HELD_SWEEP || '').trim())) return 'off';
|
|
499
|
+
const began = now();
|
|
500
|
+
const workspace = taskDb.workspaceRoot(root);
|
|
501
|
+
// One plain question first, about every workspace and with no call to git: with no job held anywhere, which is
|
|
502
|
+
// the usual case, that is all.
|
|
503
|
+
if (!anyJobHeld({ env, say })) return 'idle';
|
|
504
|
+
const file = recordPath(workspace, env);
|
|
505
|
+
if (ranRecently(file, now)) return 'recent';
|
|
506
|
+
const here = workspaceHolds(workspace, { env, say, left: () => timeLeft(now, began) });
|
|
507
|
+
if (here === 'blocked') return 'blocked';
|
|
508
|
+
if (here === 'none') return 'idle';
|
|
509
|
+
// A run that cannot be written down would start again with every command, so it does not start: the folder
|
|
510
|
+
// for the record, which holds the lock too, is made first.
|
|
511
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
512
|
+
const lock = takeLock(file.replace(/\.json$/, '.lock'), now);
|
|
513
|
+
if (!lock) return 'running';
|
|
514
|
+
try {
|
|
515
|
+
// A command that started with this one may have run the check in the time between the look above and the lock.
|
|
516
|
+
if (ranRecently(file, now)) return 'recent';
|
|
517
|
+
const outcome = check({ workspace, env, now, run, say, began });
|
|
518
|
+
// The rule for the record, and the only one: a run is written down when GitHub gave at least one readable answer in
|
|
519
|
+
// it and nothing in the run failed. Running out of time is not a failure.
|
|
520
|
+
// - A readable answer is a reply the sweep parsed into a pull request state (open, merged or closed), or GitHub
|
|
521
|
+
// saying in so many words that there is no pull request with the number asked about: gh's whole message
|
|
522
|
+
// "Could not resolve to a PullRequest with the number of N", for that very N, which it says only once the
|
|
523
|
+
// repository itself resolved. A 404, a repository gh cannot read, another number, the sentence inside a longer
|
|
524
|
+
// message and any other words are no answer. Answers are counted, not jobs or calls.
|
|
525
|
+
// - A failure is a gh or a git that could not be used (missing, could not be launched, killed, timed out, not
|
|
526
|
+
// signed in, GitHub not answering), on any question, the one about the default branch included; a task store that
|
|
527
|
+
// stayed busy; or a job that could not be settled. After a failure nothing is written down, even if GitHub had
|
|
528
|
+
// answered before it, because a prompt retry is what a real failure wants: nothing holds the retry off. What the
|
|
529
|
+
// run settled before it failed stays settled. A single reply that cannot be read (nothing, something that is not
|
|
530
|
+
// a pull request, too much, gh failing in some other way for that one job) is no failure of the run, only no
|
|
531
|
+
// answer, and the run goes on to the next job.
|
|
532
|
+
// - Using up the eight seconds is not a failure. Once an answer has come back, the time running out later still
|
|
533
|
+
// counts as a run, and the jobs not asked are left as they are. With no answer there is no run, and nothing is
|
|
534
|
+
// held off: no time was left to ask, every reply was unreadable, or the only jobs had no link.
|
|
535
|
+
// Written before the lock is let go, so a run that takes the lock after this one finds the record.
|
|
536
|
+
if (outcome === 'ran') writeRecord(file, workspace, now());
|
|
537
|
+
return outcome;
|
|
538
|
+
} finally {
|
|
539
|
+
lock.release();
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// Runs the held-job sweep for the workspace the command is in, if it is due. Never throws. Returns what
|
|
544
|
+
// happened as one word: off, idle (no held job, or no readable answer from GitHub), recent (ran within the hour),
|
|
545
|
+
// running (another command has the check), blocked (gh, git or GitHub could not be used, there was no time left to
|
|
546
|
+
// ask, or the store stayed locked), ran (at least one readable answer and nothing failed: written down), or failed (a
|
|
547
|
+
// job could not be settled). `env`, `now`, `run` and `say` are for tests.
|
|
548
|
+
function autoSweepHeld({ root = process.cwd(), env = process.env, now = Date.now, run = spawnSync, say = writeLine } = {}) {
|
|
549
|
+
try {
|
|
550
|
+
return sweep({ root, env, now, run, say });
|
|
551
|
+
} catch (err) {
|
|
552
|
+
// A store that stays busy past its short wait is not worth a line: nothing is written down.
|
|
553
|
+
if (taskDb.isBusyError(err)) return 'blocked';
|
|
554
|
+
try { say(`held jobs: automatic check failed (${oneLine(err && err.message, 120) || 'unknown error'})`); } catch { /* the command goes on */ }
|
|
555
|
+
return 'failed';
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
module.exports = { autoSweepHeld };
|
|
@@ -234,15 +234,16 @@ const GIT_REPOSITORY_VARIABLES = [
|
|
|
234
234
|
|
|
235
235
|
// Which repository a workspace came from: the owner and name its origin remote points at, asked of
|
|
236
236
|
// git in the workspace's own folder. Null when git cannot say (no folder, no origin) or the remote
|
|
237
|
-
// does not read as a GitHub repository.
|
|
238
|
-
|
|
237
|
+
// does not read as a GitHub repository. `run` and `timeoutMs` (10 seconds unless told) are for the
|
|
238
|
+
// caller that holds git to a limit of its own, and for tests.
|
|
239
|
+
function originRepoOf(root, { run = spawnSync, timeoutMs = 10000 } = {}) {
|
|
239
240
|
if (!root) return null;
|
|
240
241
|
const env = { ...process.env };
|
|
241
242
|
for (const key of GIT_REPOSITORY_VARIABLES) delete env[key];
|
|
242
243
|
const done = run('git', ['-C', String(root), 'remote', 'get-url', 'origin'], {
|
|
243
244
|
encoding: 'utf8',
|
|
244
245
|
env,
|
|
245
|
-
timeout:
|
|
246
|
+
timeout: timeoutMs,
|
|
246
247
|
killSignal: 'SIGKILL',
|
|
247
248
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
248
249
|
}) || {};
|
|
@@ -289,6 +290,14 @@ function leftAloneDetail(reason) {
|
|
|
289
290
|
return 'it changed while GitHub was being asked, so it was left alone';
|
|
290
291
|
}
|
|
291
292
|
|
|
293
|
+
// The jobs among `rows` that wait on a pull request, oldest first (by when they were made, then by row id).
|
|
294
|
+
// The order is the order the sweep goes through them in.
|
|
295
|
+
function heldJobsAmong(taskStore, rows) {
|
|
296
|
+
return rows
|
|
297
|
+
.filter((row) => row.status === 'review' && taskStore.heldPullRequestOf(row))
|
|
298
|
+
.sort((a, b) => (Number(a.created_at || 0) - Number(b.created_at || 0)) || String(a.id).localeCompare(String(b.id)));
|
|
299
|
+
}
|
|
300
|
+
|
|
292
301
|
// Check every held job against GitHub and, unless dryRun, write down what happened.
|
|
293
302
|
//
|
|
294
303
|
// taskStore lib/task-db
|
|
@@ -298,6 +307,21 @@ function leftAloneDetail(reason) {
|
|
|
298
307
|
// ask how to ask GitHub about a link (askGitHub); asked once per link per run
|
|
299
308
|
// askBranch how to ask GitHub for a repository's default branch (askDefaultBranch); asked
|
|
300
309
|
// once per repository per run, and only for a pull request that was merged
|
|
310
|
+
// askOrigin how to read the repository a workspace came from (originRepoOf); asked once per
|
|
311
|
+
// workspace per run, and only for a pull request that was merged or closed
|
|
312
|
+
// only a row id: sweep that one job and none of the others (the automatic run goes job
|
|
313
|
+
// by job, to say each result as it is made)
|
|
314
|
+
// listing task rows already read. Given, this call lists and numbers nothing itself, and `only`
|
|
315
|
+
// still picks from it. A job in it is named by its number if the row has one (display_id or
|
|
316
|
+
// legacy_ref) and by its id if it has not. The automatic run reads the workspace once for the
|
|
317
|
+
// whole run and hands each call its job, because it prints no task number and so has none to
|
|
318
|
+
// make. Left out, as the manual sweep leaves it, this call lists and numbers the workspace
|
|
319
|
+
// itself.
|
|
320
|
+
// nameOf how a refusal names the other job that has the pull request, given its row id. Left
|
|
321
|
+
// out, the name is the one that job has in its own workspace's numbering, which lists
|
|
322
|
+
// that workspace's tasks and so can start git when checkouts are linked to it. The
|
|
323
|
+
// automatic run, which says nothing of a refusal and keeps to a time limit, gives the
|
|
324
|
+
// row id itself.
|
|
301
325
|
//
|
|
302
326
|
// Returns { ok, action, dry_run, scope, actor, outage, summary, results }. ok is false only on
|
|
303
327
|
// an outage. Each result is { task_id, ref, title, builder, workspace_root, pull_request,
|
|
@@ -305,15 +329,18 @@ function leftAloneDetail(reason) {
|
|
|
305
329
|
// unknown or not_asked, and action is what was done: landed, released, would_land,
|
|
306
330
|
// would_release, none, left_alone, or refused (the pull request is not this job's build: nothing
|
|
307
331
|
// was written, and detail is the plain line saying why).
|
|
308
|
-
function sweepHeldJobs(taskStore, db, { workspaceRoot = null, actor, dryRun = false, ask = askGitHub, askBranch = askDefaultBranch } = {}) {
|
|
332
|
+
function sweepHeldJobs(taskStore, db, { workspaceRoot = null, actor, dryRun = false, ask = askGitHub, askBranch = askDefaultBranch, askOrigin = originRepoOf, only = null, nameOf = null, listing = null } = {}) {
|
|
309
333
|
if (!actor) throw new Error('actor required');
|
|
310
334
|
const scope = workspaceRoot ? 'workspace' : 'everywhere';
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
335
|
+
let named;
|
|
336
|
+
if (listing) {
|
|
337
|
+
named = heldJobsAmong(taskStore, listing).filter((row) => !only || row.id === only);
|
|
338
|
+
} else {
|
|
339
|
+
// No task list yet means nothing is held.
|
|
340
|
+
const inScope = db ? taskStore.listTasks(db, { workspaceRoot: workspaceRoot || null }) : [];
|
|
341
|
+
const held = heldJobsAmong(taskStore, inScope).filter((row) => !only || row.id === only);
|
|
342
|
+
named = taskStore.withTaskDisplayRefs(held, inScope);
|
|
343
|
+
}
|
|
317
344
|
|
|
318
345
|
// Each question goes to GitHub once per run, whatever the answer: a failure is not asked again.
|
|
319
346
|
const answers = new Map();
|
|
@@ -347,17 +374,17 @@ function sweepHeldJobs(taskStore, db, { workspaceRoot = null, actor, dryRun = fa
|
|
|
347
374
|
// A workspace's origin is read once per run, and only if a pull request needs it.
|
|
348
375
|
const origins = new Map();
|
|
349
376
|
const originOf = (root) => {
|
|
350
|
-
if (!origins.has(root)) origins.set(root,
|
|
377
|
+
if (!origins.has(root)) origins.set(root, askOrigin(root));
|
|
351
378
|
return origins.get(root);
|
|
352
379
|
};
|
|
353
380
|
|
|
354
|
-
// The name a person reads for a job, in its own workspace's numbering.
|
|
355
|
-
const refOfJob = (id) => {
|
|
381
|
+
// The name a person reads for a job, in its own workspace's numbering, unless the caller names jobs itself.
|
|
382
|
+
const refOfJob = nameOf || ((id) => {
|
|
356
383
|
const task = taskStore.getTask(db, id);
|
|
357
384
|
if (!task) return String(id);
|
|
358
385
|
const [shown] = taskStore.withTaskDisplayRefs([task], taskStore.listTasks(db, { workspaceRoot: task.workspace_root || null }));
|
|
359
386
|
return (shown && (shown.display_id || shown.legacy_ref || shown.id)) || String(id);
|
|
360
|
-
};
|
|
387
|
+
});
|
|
361
388
|
// Why this pull request is not this job's to settle, when another job has it.
|
|
362
389
|
const takenReason = (taken, answer) => (taken.how === 'landed'
|
|
363
390
|
? `pull request ${answer.number} in ${answer.repo} already landed with ${refOfJob(taken.id)}`
|
|
@@ -586,4 +613,4 @@ function describeHeldSweep(report) {
|
|
|
586
613
|
return lines;
|
|
587
614
|
}
|
|
588
615
|
|
|
589
|
-
module.exports = { ASK_TIMEOUT_MS, askGitHub, askDefaultBranch, branchNamesJob, originRepoOf, sweepHeldJobs, describeHeldSweep };
|
|
616
|
+
module.exports = { ASK_TIMEOUT_MS, askGitHub, askDefaultBranch, branchNamesJob, heldJobsAmong, originRepoOf, sweepHeldJobs, describeHeldSweep };
|
package/lib/task-db.js
CHANGED
|
@@ -197,24 +197,39 @@ function getDbPath() {
|
|
|
197
197
|
return process.env.ATRIS_TASKS_DB || path.join(atrisHome(), 'tasks.db');
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
-
|
|
200
|
+
// How many times each setup statement of a short-wait connection is tried while another process holds
|
|
201
|
+
// the write lock: with the connection's own busy wait left at nothing, about a second in all.
|
|
202
|
+
const SHORT_WAIT_TRIES = 7;
|
|
203
|
+
|
|
204
|
+
// shortWait is for a caller that must not hang behind other writers (the automatic held-job sweep). It
|
|
205
|
+
// gets a connection of its own, never the cached one, that the caller closes. The connection's own busy
|
|
206
|
+
// wait stays at nothing, so a store that stays locked is given up on after about a second, with the busy
|
|
207
|
+
// error (see isBusyError) for the caller to tell apart, and the handle is closed.
|
|
208
|
+
function open(dbPath, { shortWait = false } = {}) {
|
|
201
209
|
const target = dbPath || getDbPath();
|
|
202
|
-
if (_cachedDb && _cachedPath === target) return _cachedDb;
|
|
210
|
+
if (!shortWait && _cachedDb && _cachedPath === target) return _cachedDb;
|
|
203
211
|
const dir = path.dirname(target);
|
|
204
212
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
205
213
|
const db = new DatabaseSync(target);
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
214
|
+
const tries = shortWait ? SHORT_WAIT_TRIES : undefined;
|
|
215
|
+
try {
|
|
216
|
+
// Concurrency: WAL gives readers + one writer concurrency; busy_timeout
|
|
217
|
+
// makes contended writers wait instead of returning SQLITE_BUSY at the
|
|
218
|
+
// C library level. We additionally wrap the setup PRAGMAs + DDL in our
|
|
219
|
+
// own retry, under heavy spawn-storm contention, node:sqlite leaks
|
|
220
|
+
// SQLITE_BUSY past the busy_timeout for `db.exec()` calls.
|
|
221
|
+
withBusyRetry(() => db.exec('PRAGMA journal_mode = WAL'), tries);
|
|
222
|
+
if (!shortWait) withBusyRetry(() => db.exec('PRAGMA busy_timeout = 30000'));
|
|
223
|
+
withBusyRetry(() => db.exec('PRAGMA foreign_keys = ON'), tries);
|
|
224
|
+
withBusyRetry(() => db.exec(SCHEMA), tries);
|
|
225
|
+
// Schema version. Bump at every additive migration.
|
|
226
|
+
// Future migrations read this and apply diffs idempotently.
|
|
227
|
+
withBusyRetry(() => db.exec('PRAGMA user_version = 2'), tries);
|
|
228
|
+
} catch (err) {
|
|
229
|
+
try { db.close(); } catch (_) {}
|
|
230
|
+
throw err;
|
|
231
|
+
}
|
|
232
|
+
if (shortWait) return db;
|
|
218
233
|
_cachedDb = db;
|
|
219
234
|
_cachedPath = target;
|
|
220
235
|
return db;
|
|
@@ -476,21 +491,43 @@ function findUnfinishedTaskByTitle(db, { workspaceRoot: ws, title }) {
|
|
|
476
491
|
// A main checkout's view also covers its linked git worktrees, whose tasks keep
|
|
477
492
|
// the worktree as workspace_root so checks run where the work lives. Only a
|
|
478
493
|
// main checkout with linked worktrees spawns git; any error means exact match.
|
|
494
|
+
// A git that does not answer in time is ended outright (SIGKILL, because the
|
|
495
|
+
// polite stop can be ignored), so no git can hold a command for good.
|
|
496
|
+
const LINKED_WORKTREE_LOOKUP_MS = 5000;
|
|
479
497
|
const linkedWorktreeCache = new Map();
|
|
480
|
-
|
|
498
|
+
|
|
499
|
+
// { roots, failed }: the checkouts linked to ws, and whether git could not say (it
|
|
500
|
+
// did not answer within timeoutMs, could not be run, or exited with an error), in
|
|
501
|
+
// which case roots is empty. The task list ignores the failure and shows the
|
|
502
|
+
// workspace alone; a caller that must know asks here. A failure is remembered for the
|
|
503
|
+
// process, except one under a limit the caller chose that is shorter than the usual
|
|
504
|
+
// one: the task list that comes next still gets its own full try.
|
|
505
|
+
function lookUpLinkedWorktrees(ws, { timeoutMs = LINKED_WORKTREE_LOOKUP_MS } = {}) {
|
|
481
506
|
if (linkedWorktreeCache.has(ws)) return linkedWorktreeCache.get(ws);
|
|
482
|
-
let
|
|
507
|
+
let looked = { roots: [], failed: false };
|
|
483
508
|
try {
|
|
484
509
|
if (fs.existsSync(path.join(ws, '.git', 'worktrees'))) {
|
|
510
|
+
// A limit of 0 would mean no limit at all.
|
|
511
|
+
if (!(timeoutMs > 0)) throw new Error('no time left to ask git');
|
|
485
512
|
const out = require('child_process').execFileSync('git', ['worktree', 'list', '--porcelain'],
|
|
486
|
-
{ cwd: ws, encoding: 'utf8', timeout:
|
|
487
|
-
|
|
488
|
-
.
|
|
489
|
-
|
|
513
|
+
{ cwd: ws, encoding: 'utf8', timeout: timeoutMs, killSignal: 'SIGKILL', stdio: ['ignore', 'pipe', 'ignore'] });
|
|
514
|
+
looked = {
|
|
515
|
+
roots: out.split('\n').filter(l => l.startsWith('worktree '))
|
|
516
|
+
.map(l => { const p = l.slice(9); try { return fs.realpathSync(p); } catch { return p; } })
|
|
517
|
+
.filter(p => p !== ws),
|
|
518
|
+
failed: false,
|
|
519
|
+
};
|
|
490
520
|
}
|
|
491
|
-
} catch {
|
|
492
|
-
|
|
493
|
-
|
|
521
|
+
} catch {
|
|
522
|
+
looked = { roots: [], failed: true };
|
|
523
|
+
if (timeoutMs < LINKED_WORKTREE_LOOKUP_MS) return looked;
|
|
524
|
+
}
|
|
525
|
+
linkedWorktreeCache.set(ws, looked);
|
|
526
|
+
return looked;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
function linkedWorktreeRoots(ws) {
|
|
530
|
+
return lookUpLinkedWorktrees(ws).roots;
|
|
494
531
|
}
|
|
495
532
|
|
|
496
533
|
// The workspaces a workspace's task listing covers: the workspace itself and, for a main
|
|
@@ -3106,6 +3143,13 @@ function safeJSON(s) {
|
|
|
3106
3143
|
try { return JSON.parse(s); } catch { return null; }
|
|
3107
3144
|
}
|
|
3108
3145
|
|
|
3146
|
+
// Whether an error is the store being busy: another process holds the write lock.
|
|
3147
|
+
function isBusyError(e) {
|
|
3148
|
+
const msg = String(e && e.message || '');
|
|
3149
|
+
const code = e && (e.code || e.errcode);
|
|
3150
|
+
return /SQLITE_BUSY|database is locked/i.test(msg) || code === 'SQLITE_BUSY' || code === 5;
|
|
3151
|
+
}
|
|
3152
|
+
|
|
3109
3153
|
// Wrap a write op so SQLITE_BUSY (concurrent writers from other processes)
|
|
3110
3154
|
// retries with exponential backoff. busy_timeout pragma alone leaks busy
|
|
3111
3155
|
// errors under spawn-storm contention with node:sqlite (~3% raw lock rate
|
|
@@ -3118,10 +3162,7 @@ function withBusyRetry(fn, attempts = 8) {
|
|
|
3118
3162
|
try { return fn(); }
|
|
3119
3163
|
catch (e) {
|
|
3120
3164
|
lastErr = e;
|
|
3121
|
-
|
|
3122
|
-
const code = e && (e.code || e.errcode);
|
|
3123
|
-
const busy = /SQLITE_BUSY|database is locked/i.test(msg) || code === 'SQLITE_BUSY' || code === 5;
|
|
3124
|
-
if (!busy) throw e;
|
|
3165
|
+
if (!isBusyError(e)) throw e;
|
|
3125
3166
|
// Sleep synchronously, node:sqlite is sync; matches the rest of the API
|
|
3126
3167
|
sleepSync(delay + Math.floor(Math.random() * delay));
|
|
3127
3168
|
delay = Math.min(delay * 2, 500);
|
|
@@ -3133,6 +3174,7 @@ function withBusyRetry(fn, attempts = 8) {
|
|
|
3133
3174
|
module.exports = {
|
|
3134
3175
|
open,
|
|
3135
3176
|
withBusyRetry,
|
|
3177
|
+
isBusyError,
|
|
3136
3178
|
close,
|
|
3137
3179
|
getDbPath,
|
|
3138
3180
|
workspaceRoot,
|
|
@@ -3144,6 +3186,7 @@ module.exports = {
|
|
|
3144
3186
|
listTasks,
|
|
3145
3187
|
listTasksCached,
|
|
3146
3188
|
rowInWorkspace,
|
|
3189
|
+
lookUpLinkedWorktrees,
|
|
3147
3190
|
claimTask,
|
|
3148
3191
|
setLease,
|
|
3149
3192
|
renewLease,
|
package/package.json
CHANGED