@sdods/integrations 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/evidence.d.ts +150 -0
- package/dist/evidence.js +569 -0
- package/dist/gif.d.ts +36 -0
- package/dist/gif.js +85 -0
- package/dist/github.d.ts +28 -0
- package/dist/github.js +175 -13
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/registry.js +6 -2
- package/dist/secrets.d.ts +2 -0
- package/dist/secrets.js +3 -0
- package/dist/types.d.ts +34 -0
- package/package.json +2 -2
package/dist/evidence.js
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
/**
|
|
3
|
+
* Issue evidence committed to a branch (#82). Screenshots in an issue render inline in a private
|
|
4
|
+
* repository only when GitHub serves them to the reader's session; `blob/<branch>/<path>?raw=true`
|
|
5
|
+
* does, release assets and Actions artifacts do not.
|
|
6
|
+
*
|
|
7
|
+
* Everything goes through the git data API, so no clone is needed:
|
|
8
|
+
* - the branch is an orphan (a root commit with only a README) created on first use;
|
|
9
|
+
* - one commit per run adds `runs/<runId>/<scenario>/<file>` and updates `runs/index.json` on top of
|
|
10
|
+
* the current head, and the ref moves with `force: false`. When another job moved it first, GitHub
|
|
11
|
+
* answers 422 (not a fast-forward); the tree and commit are rebuilt on the new head and retried.
|
|
12
|
+
* - `prune` rewrites the branch as a new orphan commit holding only the runs to keep.
|
|
13
|
+
*
|
|
14
|
+
* Traces are never passed here: they carry credentials (#101).
|
|
15
|
+
*/
|
|
16
|
+
export const EVIDENCE_INDEX = 'runs/index.json';
|
|
17
|
+
/** The first line of the README the orphan branch is created with; prune requires it. */
|
|
18
|
+
const README_MARKER = '# SDODS evidence';
|
|
19
|
+
const README = `${README_MARKER}
|
|
20
|
+
|
|
21
|
+
Screenshots, videos and GIF previews linked from issues that SDODS opened for failing scenarios.
|
|
22
|
+
Files live under \`runs/<runId>/<scenario>/\`; \`runs/index.json\` records when each run was uploaded.
|
|
23
|
+
|
|
24
|
+
This branch is rewritten by \`sdods integrations evidence prune\`. Do not base work on it.
|
|
25
|
+
`;
|
|
26
|
+
const KIND_ORDER = { screenshot: 0, preview: 1, video: 2 };
|
|
27
|
+
/**
|
|
28
|
+
* Apply the size caps. Screenshots go first, then GIF previews, then videos, so a few large
|
|
29
|
+
* videos never crowd out the images that render inline.
|
|
30
|
+
*/
|
|
31
|
+
export function planEvidence(files, caps) {
|
|
32
|
+
const ordered = files
|
|
33
|
+
.map((f, i) => ({ f, i }))
|
|
34
|
+
.sort((a, b) => KIND_ORDER[a.f.kind] - KIND_ORDER[b.f.kind] || a.i - b.i)
|
|
35
|
+
.map(({ f }) => f);
|
|
36
|
+
const upload = [];
|
|
37
|
+
const skipped = [];
|
|
38
|
+
let total = 0;
|
|
39
|
+
for (const f of ordered) {
|
|
40
|
+
const base = { key: f.key, fingerprint: f.fingerprint, name: f.name, bytes: f.bytes };
|
|
41
|
+
if (f.bytes > caps.maxFileBytes) {
|
|
42
|
+
skipped.push({
|
|
43
|
+
...base,
|
|
44
|
+
reason: `larger than evidence.maxFileBytes (${formatBytes(caps.maxFileBytes)})`,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
else if (total + f.bytes > caps.maxRunBytes) {
|
|
48
|
+
skipped.push({
|
|
49
|
+
...base,
|
|
50
|
+
reason: `evidence.maxRunBytes (${formatBytes(caps.maxRunBytes)}) reached for this run`,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
total += f.bytes;
|
|
55
|
+
upload.push(f);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return { upload, skipped };
|
|
59
|
+
}
|
|
60
|
+
export class GitHubBranchEvidence {
|
|
61
|
+
owner;
|
|
62
|
+
repo;
|
|
63
|
+
branch;
|
|
64
|
+
octokit;
|
|
65
|
+
serverUrl;
|
|
66
|
+
maxAttempts;
|
|
67
|
+
sleep;
|
|
68
|
+
now;
|
|
69
|
+
/** replaced per run with the integration context's logger */
|
|
70
|
+
logger;
|
|
71
|
+
retainDays;
|
|
72
|
+
constructor(opts) {
|
|
73
|
+
this.octokit = opts.octokit;
|
|
74
|
+
this.owner = opts.owner;
|
|
75
|
+
this.repo = opts.repo;
|
|
76
|
+
this.branch = opts.branch;
|
|
77
|
+
this.serverUrl = (opts.serverUrl ?? 'https://github.com').replace(/\/+$/, '');
|
|
78
|
+
this.maxAttempts = Math.max(1, opts.maxAttempts ?? 5);
|
|
79
|
+
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
80
|
+
this.now = opts.now ?? (() => new Date());
|
|
81
|
+
this.logger = opts.logger;
|
|
82
|
+
this.retainDays = opts.retainDays;
|
|
83
|
+
}
|
|
84
|
+
get fullName() {
|
|
85
|
+
return `${this.owner}/${this.repo}`;
|
|
86
|
+
}
|
|
87
|
+
/** A link that renders inline for anyone who can read the repository. */
|
|
88
|
+
url(path) {
|
|
89
|
+
const enc = (p) => p.split('/').map(encodeURIComponent).join('/');
|
|
90
|
+
return `${this.serverUrl}/${this.owner}/${this.repo}/blob/${enc(this.branch)}/${enc(path)}?raw=true`;
|
|
91
|
+
}
|
|
92
|
+
/** Commit a run's files in one commit and return their links. Files over the caps are skipped. */
|
|
93
|
+
async publishRun(run, files, caps) {
|
|
94
|
+
const { upload, skipped } = planEvidence(files, caps);
|
|
95
|
+
const urls = new Map();
|
|
96
|
+
if (!upload.length)
|
|
97
|
+
return { urls, skipped };
|
|
98
|
+
// first, so a default branch or an empty repository fails with a clear message before any upload
|
|
99
|
+
await this.assertNotDefaultBranch();
|
|
100
|
+
await this.ensureBranch();
|
|
101
|
+
const runDir = `runs/${safeSegment(run.id)}`;
|
|
102
|
+
const entries = [];
|
|
103
|
+
let bytes = 0;
|
|
104
|
+
// blobs are content-addressed: created once, reused by every retry of the commit
|
|
105
|
+
for (const f of upload) {
|
|
106
|
+
const path = `${runDir}/${safeSegment(f.scenarioDir)}/${safeSegment(f.name)}`;
|
|
107
|
+
const { data } = await this.octokit.rest.git.createBlob({
|
|
108
|
+
owner: this.owner,
|
|
109
|
+
repo: this.repo,
|
|
110
|
+
content: readFileSync(f.localPath).toString('base64'),
|
|
111
|
+
encoding: 'base64',
|
|
112
|
+
});
|
|
113
|
+
entries.push({ path, mode: '100644', type: 'blob', sha: data.sha });
|
|
114
|
+
urls.set(f.key, this.url(path));
|
|
115
|
+
bytes += f.bytes;
|
|
116
|
+
}
|
|
117
|
+
const message = `evidence: run ${run.id} (${upload.length} file${upload.length === 1 ? '' : 's'})`;
|
|
118
|
+
const commitSha = await this.commitWithRetry(async (head) => {
|
|
119
|
+
const { data: commit } = await this.octokit.rest.git.getCommit({
|
|
120
|
+
owner: this.owner,
|
|
121
|
+
repo: this.repo,
|
|
122
|
+
commit_sha: head,
|
|
123
|
+
});
|
|
124
|
+
const layout = await this.readLayout(commit.tree.sha);
|
|
125
|
+
const index = layout.index;
|
|
126
|
+
index.runs[safeSegment(run.id)] = {
|
|
127
|
+
uploadedAt: this.now().toISOString(),
|
|
128
|
+
project: run.projectSlug,
|
|
129
|
+
files: upload.length,
|
|
130
|
+
bytes,
|
|
131
|
+
};
|
|
132
|
+
const stale = this.staleRuns(index);
|
|
133
|
+
if (stale)
|
|
134
|
+
this.logger?.info(`github evidence: ${stale} run(s) on ${this.fullName}@${this.branch} are older than retainDays; run \`sdods integrations evidence prune\``);
|
|
135
|
+
const indexSha = await this.blob(`${JSON.stringify(index, null, 2)}\n`);
|
|
136
|
+
const { data: tree } = await this.octokit.rest.git.createTree({
|
|
137
|
+
owner: this.owner,
|
|
138
|
+
repo: this.repo,
|
|
139
|
+
base_tree: commit.tree.sha,
|
|
140
|
+
tree: [...entries, { path: EVIDENCE_INDEX, mode: '100644', type: 'blob', sha: indexSha }],
|
|
141
|
+
});
|
|
142
|
+
const { data: created } = await this.octokit.rest.git.createCommit({
|
|
143
|
+
owner: this.owner,
|
|
144
|
+
repo: this.repo,
|
|
145
|
+
message,
|
|
146
|
+
tree: tree.sha,
|
|
147
|
+
parents: [head],
|
|
148
|
+
});
|
|
149
|
+
return created.sha;
|
|
150
|
+
});
|
|
151
|
+
return { urls, skipped, commitSha };
|
|
152
|
+
}
|
|
153
|
+
staleRuns(index) {
|
|
154
|
+
if (!this.retainDays)
|
|
155
|
+
return 0;
|
|
156
|
+
const cutoff = this.now().getTime() - this.retainDays * 86_400_000;
|
|
157
|
+
return Object.values(index.runs).filter((r) => Date.parse(r.uploadedAt) < cutoff).length;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Build a commit on the current head and move the ref with `force: false`. A 422 means another
|
|
161
|
+
* job moved the branch in between: re-read the head and rebuild, with backoff, a bounded number
|
|
162
|
+
* of times.
|
|
163
|
+
*/
|
|
164
|
+
async commitWithRetry(build) {
|
|
165
|
+
for (let attempt = 1;; attempt++) {
|
|
166
|
+
const head = await this.ensureBranch();
|
|
167
|
+
const sha = await build(head);
|
|
168
|
+
try {
|
|
169
|
+
await this.octokit.rest.git.updateRef({
|
|
170
|
+
owner: this.owner,
|
|
171
|
+
repo: this.repo,
|
|
172
|
+
ref: `heads/${this.branch}`,
|
|
173
|
+
sha,
|
|
174
|
+
force: false,
|
|
175
|
+
});
|
|
176
|
+
return sha;
|
|
177
|
+
}
|
|
178
|
+
catch (e) {
|
|
179
|
+
const status = statusOf(e);
|
|
180
|
+
if ((status !== 422 && status !== 409) || attempt >= this.maxAttempts) {
|
|
181
|
+
throw new Error(`github evidence: cannot update ${this.fullName}@${this.branch} after ${attempt} attempt(s): ${e.message}`);
|
|
182
|
+
}
|
|
183
|
+
const wait = 250 * 2 ** (attempt - 1) + Math.floor(Math.random() * 100);
|
|
184
|
+
this.logger?.debug(`github evidence: ${this.branch} moved (HTTP ${status}), retrying in ${wait}ms`);
|
|
185
|
+
await this.sleep(wait);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
/** The branch head, creating the orphan branch when it does not exist. */
|
|
190
|
+
async ensureBranch() {
|
|
191
|
+
const existing = await this.headSha();
|
|
192
|
+
if (existing)
|
|
193
|
+
return existing;
|
|
194
|
+
const { data: tree } = await this.octokit.rest.git
|
|
195
|
+
.createTree({
|
|
196
|
+
owner: this.owner,
|
|
197
|
+
repo: this.repo,
|
|
198
|
+
tree: [{ path: 'README.md', mode: '100644', type: 'blob', content: README }],
|
|
199
|
+
})
|
|
200
|
+
.catch((e) => {
|
|
201
|
+
// the git data API cannot write to a repository without any commit
|
|
202
|
+
if (statusOf(e) === 409)
|
|
203
|
+
throw new Error(this.emptyRepoMessage());
|
|
204
|
+
throw e;
|
|
205
|
+
});
|
|
206
|
+
const { data: commit } = await this.octokit.rest.git.createCommit({
|
|
207
|
+
owner: this.owner,
|
|
208
|
+
repo: this.repo,
|
|
209
|
+
message: 'evidence: create the SDODS evidence branch',
|
|
210
|
+
tree: tree.sha,
|
|
211
|
+
parents: [],
|
|
212
|
+
});
|
|
213
|
+
try {
|
|
214
|
+
await this.octokit.rest.git.createRef({
|
|
215
|
+
owner: this.owner,
|
|
216
|
+
repo: this.repo,
|
|
217
|
+
ref: `refs/heads/${this.branch}`,
|
|
218
|
+
sha: commit.sha,
|
|
219
|
+
});
|
|
220
|
+
this.logger?.info(`github evidence: created orphan branch ${this.fullName}@${this.branch}`);
|
|
221
|
+
return commit.sha;
|
|
222
|
+
}
|
|
223
|
+
catch (e) {
|
|
224
|
+
// another job created it first
|
|
225
|
+
if (statusOf(e) !== 422)
|
|
226
|
+
throw e;
|
|
227
|
+
const head = await this.headSha();
|
|
228
|
+
if (!head)
|
|
229
|
+
throw e;
|
|
230
|
+
return head;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
defaultBranch;
|
|
234
|
+
/** The repository's default branch, read once. */
|
|
235
|
+
async repoDefaultBranch() {
|
|
236
|
+
this.defaultBranch ??= this.octokit.rest.repos
|
|
237
|
+
.get({ owner: this.owner, repo: this.repo })
|
|
238
|
+
.then(({ data }) => data.default_branch)
|
|
239
|
+
.catch((e) => {
|
|
240
|
+
this.defaultBranch = undefined;
|
|
241
|
+
throw e;
|
|
242
|
+
});
|
|
243
|
+
return this.defaultBranch;
|
|
244
|
+
}
|
|
245
|
+
defaultBranchMessage() {
|
|
246
|
+
return `github evidence: ${this.fullName}@${this.branch} is the repository's default branch; evidence is committed to an orphan branch that \`evidence prune\` rewrites, so set integrations.github.evidence.branch to a branch of its own (default sdods-evidence)`;
|
|
247
|
+
}
|
|
248
|
+
/** Uploads and prune never touch the default branch: prune would erase its history. */
|
|
249
|
+
async assertNotDefaultBranch() {
|
|
250
|
+
if ((await this.repoDefaultBranch()) === this.branch)
|
|
251
|
+
throw new Error(this.defaultBranchMessage());
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Prune force-rewrites the branch, so it only runs on a branch SDODS created: one whose root
|
|
255
|
+
* README.md is the one written with the orphan commit.
|
|
256
|
+
*/
|
|
257
|
+
async assertCreatedBySdods(root) {
|
|
258
|
+
const readme = root.find((e) => e.path === 'README.md' && e.type === 'blob');
|
|
259
|
+
let text = '';
|
|
260
|
+
if (readme?.sha) {
|
|
261
|
+
const { data } = await this.octokit.rest.git.getBlob({
|
|
262
|
+
owner: this.owner,
|
|
263
|
+
repo: this.repo,
|
|
264
|
+
file_sha: readme.sha,
|
|
265
|
+
});
|
|
266
|
+
text = Buffer.from(data.content, 'base64').toString('utf8');
|
|
267
|
+
}
|
|
268
|
+
if (!text.startsWith(README_MARKER)) {
|
|
269
|
+
throw new Error(`github evidence: refusing to prune ${this.fullName}@${this.branch}: it was not created by SDODS (its README.md does not start with "${README_MARKER}"), and prune replaces the branch with a new orphan commit, erasing its history. Point integrations.github.evidence.branch at a branch SDODS creates.`);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
async isEmptyRepo() {
|
|
273
|
+
try {
|
|
274
|
+
await this.octokit.rest.repos.listCommits({
|
|
275
|
+
owner: this.owner,
|
|
276
|
+
repo: this.repo,
|
|
277
|
+
per_page: 1,
|
|
278
|
+
});
|
|
279
|
+
return false;
|
|
280
|
+
}
|
|
281
|
+
catch (e) {
|
|
282
|
+
if (statusOf(e) === 409)
|
|
283
|
+
return true;
|
|
284
|
+
throw e;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
emptyRepoMessage() {
|
|
288
|
+
return `evidence repo ${this.fullName} is empty: create it with a README (any first commit) before using it`;
|
|
289
|
+
}
|
|
290
|
+
async headSha() {
|
|
291
|
+
try {
|
|
292
|
+
const { data } = await this.octokit.rest.git.getRef({
|
|
293
|
+
owner: this.owner,
|
|
294
|
+
repo: this.repo,
|
|
295
|
+
ref: `heads/${this.branch}`,
|
|
296
|
+
});
|
|
297
|
+
return data.object.sha;
|
|
298
|
+
}
|
|
299
|
+
catch (e) {
|
|
300
|
+
if (statusOf(e) === 404)
|
|
301
|
+
return null;
|
|
302
|
+
// GitHub answers 409 ("repository is empty"), not 404, for a ref of a repo without a commit
|
|
303
|
+
if (statusOf(e) === 409)
|
|
304
|
+
throw new Error(this.emptyRepoMessage());
|
|
305
|
+
throw e;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
async blob(text) {
|
|
309
|
+
const { data } = await this.octokit.rest.git.createBlob({
|
|
310
|
+
owner: this.owner,
|
|
311
|
+
repo: this.repo,
|
|
312
|
+
content: Buffer.from(text).toString('base64'),
|
|
313
|
+
encoding: 'base64',
|
|
314
|
+
});
|
|
315
|
+
return data.sha;
|
|
316
|
+
}
|
|
317
|
+
/** Root entries, `runs/` entries and the parsed index of a tree. */
|
|
318
|
+
async readLayout(rootTreeSha) {
|
|
319
|
+
const root = await this.listTree(rootTreeSha);
|
|
320
|
+
const runsDir = root.find((e) => e.path === 'runs' && e.type === 'tree');
|
|
321
|
+
const runs = runsDir?.sha ? await this.listTree(runsDir.sha) : [];
|
|
322
|
+
const indexEntry = runs.find((e) => e.path === 'index.json' && e.type === 'blob');
|
|
323
|
+
let index = { runs: {} };
|
|
324
|
+
if (indexEntry?.sha) {
|
|
325
|
+
const { data } = await this.octokit.rest.git.getBlob({
|
|
326
|
+
owner: this.owner,
|
|
327
|
+
repo: this.repo,
|
|
328
|
+
file_sha: indexEntry.sha,
|
|
329
|
+
});
|
|
330
|
+
try {
|
|
331
|
+
const parsed = JSON.parse(Buffer.from(data.content, 'base64').toString('utf8'));
|
|
332
|
+
if (parsed && typeof parsed.runs === 'object')
|
|
333
|
+
index = parsed;
|
|
334
|
+
}
|
|
335
|
+
catch {
|
|
336
|
+
this.logger?.warn(`github evidence: ${EVIDENCE_INDEX} is not valid JSON; starting afresh`);
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
return { root, runs, index };
|
|
340
|
+
}
|
|
341
|
+
async listTree(sha) {
|
|
342
|
+
const { data } = await this.octokit.rest.git.getTree({
|
|
343
|
+
owner: this.owner,
|
|
344
|
+
repo: this.repo,
|
|
345
|
+
tree_sha: sha,
|
|
346
|
+
});
|
|
347
|
+
return data.tree.map((e) => ({
|
|
348
|
+
path: e.path ?? '',
|
|
349
|
+
mode: (e.mode ?? '100644'),
|
|
350
|
+
type: (e.type ?? 'blob'),
|
|
351
|
+
sha: e.sha ?? '',
|
|
352
|
+
}));
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Rewrite the branch as a new orphan commit without the runs uploaded before the cutoff.
|
|
356
|
+
* Kept run directories are reused by tree sha, so nothing is uploaded again. Just before the
|
|
357
|
+
* force update the head is compared with the one the new tree was built from, and the rewrite is
|
|
358
|
+
* rebuilt when it moved. That narrows the race with a concurrent upload but cannot close it: a
|
|
359
|
+
* force update has no compare-and-swap, so a run pushed between that check and the update is
|
|
360
|
+
* lost. Prune when CI is quiet. GitHub garbage-collects the dropped objects later, and issues
|
|
361
|
+
* that linked them stop rendering those images.
|
|
362
|
+
*/
|
|
363
|
+
async prune(opts) {
|
|
364
|
+
await this.assertNotDefaultBranch();
|
|
365
|
+
for (let attempt = 1;; attempt++) {
|
|
366
|
+
const head = await this.headSha();
|
|
367
|
+
const result = {
|
|
368
|
+
branch: `${this.fullName}@${this.branch}`,
|
|
369
|
+
removed: [],
|
|
370
|
+
kept: [],
|
|
371
|
+
unknown: [],
|
|
372
|
+
dryRun: Boolean(opts.dryRun),
|
|
373
|
+
};
|
|
374
|
+
if (!head)
|
|
375
|
+
return result;
|
|
376
|
+
const { data: commit } = await this.octokit.rest.git.getCommit({
|
|
377
|
+
owner: this.owner,
|
|
378
|
+
repo: this.repo,
|
|
379
|
+
commit_sha: head,
|
|
380
|
+
});
|
|
381
|
+
const { root, runs, index } = await this.readLayout(commit.tree.sha);
|
|
382
|
+
await this.assertCreatedBySdods(root);
|
|
383
|
+
const cutoff = this.now().getTime() - opts.olderThanMs;
|
|
384
|
+
const keptDirs = [];
|
|
385
|
+
for (const entry of runs) {
|
|
386
|
+
if (entry.type !== 'tree') {
|
|
387
|
+
// stray files next to the run directories stay; the index is rewritten below
|
|
388
|
+
if (entry.path !== 'index.json')
|
|
389
|
+
keptDirs.push(entry);
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
const meta = index.runs[entry.path];
|
|
393
|
+
if (!meta) {
|
|
394
|
+
result.unknown.push(entry.path);
|
|
395
|
+
keptDirs.push(entry);
|
|
396
|
+
}
|
|
397
|
+
else if (Date.parse(meta.uploadedAt) < cutoff) {
|
|
398
|
+
result.removed.push({ runId: entry.path, uploadedAt: meta.uploadedAt });
|
|
399
|
+
}
|
|
400
|
+
else {
|
|
401
|
+
result.kept.push(entry.path);
|
|
402
|
+
keptDirs.push(entry);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
if (!result.removed.length || opts.dryRun)
|
|
406
|
+
return result;
|
|
407
|
+
const removed = new Set(result.removed.map((r) => r.runId));
|
|
408
|
+
const nextIndex = {
|
|
409
|
+
runs: Object.fromEntries(Object.entries(index.runs).filter(([id]) => !removed.has(id))),
|
|
410
|
+
};
|
|
411
|
+
const indexSha = await this.blob(`${JSON.stringify(nextIndex, null, 2)}\n`);
|
|
412
|
+
const { data: runsTree } = await this.octokit.rest.git.createTree({
|
|
413
|
+
owner: this.owner,
|
|
414
|
+
repo: this.repo,
|
|
415
|
+
tree: [...keptDirs, { path: 'index.json', mode: '100644', type: 'blob', sha: indexSha }],
|
|
416
|
+
});
|
|
417
|
+
const { data: rootTree } = await this.octokit.rest.git.createTree({
|
|
418
|
+
owner: this.owner,
|
|
419
|
+
repo: this.repo,
|
|
420
|
+
tree: [
|
|
421
|
+
...root.filter((e) => e.path !== 'runs'),
|
|
422
|
+
{ path: 'runs', mode: '040000', type: 'tree', sha: runsTree.sha },
|
|
423
|
+
],
|
|
424
|
+
});
|
|
425
|
+
const { data: orphan } = await this.octokit.rest.git.createCommit({
|
|
426
|
+
owner: this.owner,
|
|
427
|
+
repo: this.repo,
|
|
428
|
+
message: `evidence: prune ${removed.size} run(s) uploaded before ${new Date(cutoff).toISOString()}`,
|
|
429
|
+
tree: rootTree.sha,
|
|
430
|
+
parents: [],
|
|
431
|
+
});
|
|
432
|
+
// a force update would silently drop a run committed since the head was read
|
|
433
|
+
if ((await this.headSha()) !== head) {
|
|
434
|
+
if (attempt >= this.maxAttempts)
|
|
435
|
+
throw new Error(`github evidence: ${this.fullName}@${this.branch} kept moving during prune; try again`);
|
|
436
|
+
await this.sleep(250 * 2 ** (attempt - 1));
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
439
|
+
await this.octokit.rest.git.updateRef({
|
|
440
|
+
owner: this.owner,
|
|
441
|
+
repo: this.repo,
|
|
442
|
+
ref: `heads/${this.branch}`,
|
|
443
|
+
sha: orphan.sha,
|
|
444
|
+
force: true,
|
|
445
|
+
});
|
|
446
|
+
return { ...result, commitSha: orphan.sha };
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* For `sdods integrations test`: the token can push to the evidence repository, and the branch
|
|
451
|
+
* exists or will be created on the first upload. Nothing is created here, except that a token
|
|
452
|
+
* whose repository response has no `permissions` is checked by writing one unreferenced blob.
|
|
453
|
+
*/
|
|
454
|
+
async check(issueRepoPrivate) {
|
|
455
|
+
let repoData;
|
|
456
|
+
try {
|
|
457
|
+
({ data: repoData } = await this.octokit.rest.repos.get({
|
|
458
|
+
owner: this.owner,
|
|
459
|
+
repo: this.repo,
|
|
460
|
+
}));
|
|
461
|
+
}
|
|
462
|
+
catch (e) {
|
|
463
|
+
return { ok: false, detail: `evidence repo ${this.fullName}: ${e.message}` };
|
|
464
|
+
}
|
|
465
|
+
if (repoData.default_branch === this.branch)
|
|
466
|
+
return { ok: false, detail: this.defaultBranchMessage().replace(/^github evidence: /, '') };
|
|
467
|
+
const push = repoData.permissions?.push;
|
|
468
|
+
let writable;
|
|
469
|
+
if (push === false) {
|
|
470
|
+
return {
|
|
471
|
+
ok: false,
|
|
472
|
+
detail: `evidence repo ${this.fullName}: the token cannot push (evidence.host: branch needs contents: write)`,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
else if (push === true) {
|
|
476
|
+
writable = 'writable';
|
|
477
|
+
}
|
|
478
|
+
else {
|
|
479
|
+
try {
|
|
480
|
+
await this.blob('sdods evidence write check\n');
|
|
481
|
+
writable = 'writable (checked by writing a blob)';
|
|
482
|
+
}
|
|
483
|
+
catch (e) {
|
|
484
|
+
return {
|
|
485
|
+
ok: false,
|
|
486
|
+
detail: `evidence repo ${this.fullName}: the token cannot write git objects (needs contents: write): ${e.message}`,
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
let branch;
|
|
491
|
+
try {
|
|
492
|
+
let head;
|
|
493
|
+
try {
|
|
494
|
+
head = await this.headSha();
|
|
495
|
+
}
|
|
496
|
+
catch (e) {
|
|
497
|
+
if (e.message === this.emptyRepoMessage())
|
|
498
|
+
return { ok: false, detail: this.emptyRepoMessage() };
|
|
499
|
+
throw e;
|
|
500
|
+
}
|
|
501
|
+
if (!head && (await this.isEmptyRepo()))
|
|
502
|
+
return { ok: false, detail: this.emptyRepoMessage() };
|
|
503
|
+
branch = head
|
|
504
|
+
? `branch ${this.branch} exists`
|
|
505
|
+
: `branch ${this.branch} will be created on the first upload`;
|
|
506
|
+
}
|
|
507
|
+
catch (e) {
|
|
508
|
+
return {
|
|
509
|
+
ok: false,
|
|
510
|
+
detail: `evidence repo ${this.fullName}: cannot read branch ${this.branch}: ${e.message}`,
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
const visibility = !repoData.private && issueRepoPrivate
|
|
514
|
+
? '; WARNING: the evidence repo is public, anyone can open the screenshots'
|
|
515
|
+
: repoData.private && issueRepoPrivate === false
|
|
516
|
+
? '; the evidence repo is private, so readers of this public repo will not see the images'
|
|
517
|
+
: '';
|
|
518
|
+
return {
|
|
519
|
+
ok: true,
|
|
520
|
+
detail: `evidence ${this.fullName}@${this.branch} ${writable}, ${branch}${visibility}`,
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* `90min`, `36h`, `14d`, `2w` or a bare number of days → milliseconds. A bare `m` is refused: it
|
|
526
|
+
* reads as months to some people and minutes to others, and `3m` meant as months prunes nearly
|
|
527
|
+
* everything.
|
|
528
|
+
*/
|
|
529
|
+
export function parseAge(text) {
|
|
530
|
+
const m = /^\s*(\d+(?:\.\d+)?)\s*(min|[a-z]*)\s*$/i.exec(text);
|
|
531
|
+
const unit = (m?.[2] || 'd').toLowerCase();
|
|
532
|
+
if (m && unit === 'm')
|
|
533
|
+
throw new Error(`Cannot parse age "${text}": "m" is ambiguous: use ${m[1]}min for minutes or ${Math.round(Number(m[1]) * 30)}d for about ${m[1]} month(s)`);
|
|
534
|
+
const ms = { min: 60_000, h: 3_600_000, d: 86_400_000, w: 604_800_000 }[unit];
|
|
535
|
+
if (!m || ms === undefined)
|
|
536
|
+
throw new Error(`Cannot parse age "${text}" (use e.g. 90min, 36h, 14d, 2w)`);
|
|
537
|
+
return Number(m[1]) * ms;
|
|
538
|
+
}
|
|
539
|
+
/** A path segment safe in a git tree and a URL. */
|
|
540
|
+
export function safeSegment(text) {
|
|
541
|
+
const s = text.replace(/[^\w.-]+/g, '-').replace(/^[.-]+|-+$/g, '');
|
|
542
|
+
return s || 'x';
|
|
543
|
+
}
|
|
544
|
+
/** `runs/<id>/<scenario-slug>/`: readable, and unique through the fingerprint. */
|
|
545
|
+
export function scenarioSlug(scenarioName, fingerprint) {
|
|
546
|
+
const name = scenarioName
|
|
547
|
+
.toLowerCase()
|
|
548
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
549
|
+
.replace(/^-+|-+$/g, '')
|
|
550
|
+
.slice(0, 60)
|
|
551
|
+
.replace(/-+$/, '');
|
|
552
|
+
const fp = fingerprint.replace(/[^\w]+/g, '').slice(0, 10);
|
|
553
|
+
return [name, fp].filter(Boolean).join('-') || 'scenario';
|
|
554
|
+
}
|
|
555
|
+
export function evidenceKey(fingerprint, relPath) {
|
|
556
|
+
return `${fingerprint}${relPath}`;
|
|
557
|
+
}
|
|
558
|
+
export function formatBytes(bytes) {
|
|
559
|
+
if (bytes >= 1024 * 1024)
|
|
560
|
+
return `${(bytes / (1024 * 1024)).toFixed(1).replace(/\.0$/, '')} MB`;
|
|
561
|
+
if (bytes >= 1024)
|
|
562
|
+
return `${Math.round(bytes / 1024)} KB`;
|
|
563
|
+
return `${bytes} B`;
|
|
564
|
+
}
|
|
565
|
+
function statusOf(e) {
|
|
566
|
+
const s = e?.status;
|
|
567
|
+
return typeof s === 'number' ? s : undefined;
|
|
568
|
+
}
|
|
569
|
+
//# sourceMappingURL=evidence.js.map
|
package/dist/gif.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { IntegrationLogger } from './types.js';
|
|
2
|
+
export interface CommandResult {
|
|
3
|
+
code: number;
|
|
4
|
+
stderr?: string;
|
|
5
|
+
}
|
|
6
|
+
/** Runs a command; rejects when the command cannot be started (for example ENOENT). */
|
|
7
|
+
export type CommandRunner = (command: string, args: string[], opts?: {
|
|
8
|
+
timeoutMs?: number;
|
|
9
|
+
}) => Promise<CommandResult>;
|
|
10
|
+
export declare const defaultCommandRunner: CommandRunner;
|
|
11
|
+
export interface GifPreviewSettings {
|
|
12
|
+
/** the last N seconds of the video, where the failure is */
|
|
13
|
+
seconds: number;
|
|
14
|
+
width: number;
|
|
15
|
+
fps: number;
|
|
16
|
+
/** a larger GIF is dropped rather than embedded */
|
|
17
|
+
maxBytes: number;
|
|
18
|
+
}
|
|
19
|
+
export declare const GIF_PREVIEW_DEFAULTS: GifPreviewSettings;
|
|
20
|
+
/**
|
|
21
|
+
* A short inline GIF of the end of a scenario's `video.webm`, made with ffmpeg when it is on PATH.
|
|
22
|
+
* A missing ffmpeg, a failing conversion or an oversized result skip the preview with a debug log.
|
|
23
|
+
*/
|
|
24
|
+
export declare class GifPreviewer {
|
|
25
|
+
private readonly runner;
|
|
26
|
+
private readonly ffmpeg;
|
|
27
|
+
private available?;
|
|
28
|
+
private readonly settings;
|
|
29
|
+
constructor(runner?: CommandRunner, settings?: Partial<GifPreviewSettings>, ffmpeg?: string);
|
|
30
|
+
get seconds(): number;
|
|
31
|
+
/** ffmpeg arguments for one preview (exported for the docs and tests). */
|
|
32
|
+
args(videoPath: string, outPath: string): string[];
|
|
33
|
+
make(videoPath: string, outPath: string, logger?: IntegrationLogger): Promise<string | null>;
|
|
34
|
+
private isAvailable;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=gif.d.ts.map
|
package/dist/gif.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { existsSync, statSync } from 'node:fs';
|
|
3
|
+
export const defaultCommandRunner = (command, args, opts = {}) => new Promise((resolve, reject) => {
|
|
4
|
+
execFile(command, args, { timeout: opts.timeoutMs ?? 60_000, maxBuffer: 4 * 1024 * 1024 }, (error, _stdout, stderr) => {
|
|
5
|
+
if (error && typeof error.code === 'string') {
|
|
6
|
+
reject(error);
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
const code = error ? (error.code ?? 1) : 0;
|
|
10
|
+
resolve({ code: typeof code === 'number' ? code : 1, stderr: String(stderr) });
|
|
11
|
+
});
|
|
12
|
+
});
|
|
13
|
+
export const GIF_PREVIEW_DEFAULTS = {
|
|
14
|
+
seconds: 5,
|
|
15
|
+
width: 480,
|
|
16
|
+
fps: 8,
|
|
17
|
+
maxBytes: 3 * 1024 * 1024,
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* A short inline GIF of the end of a scenario's `video.webm`, made with ffmpeg when it is on PATH.
|
|
21
|
+
* A missing ffmpeg, a failing conversion or an oversized result skip the preview with a debug log.
|
|
22
|
+
*/
|
|
23
|
+
export class GifPreviewer {
|
|
24
|
+
runner;
|
|
25
|
+
ffmpeg;
|
|
26
|
+
available;
|
|
27
|
+
settings;
|
|
28
|
+
constructor(runner = defaultCommandRunner, settings = {}, ffmpeg = 'ffmpeg') {
|
|
29
|
+
this.runner = runner;
|
|
30
|
+
this.ffmpeg = ffmpeg;
|
|
31
|
+
this.settings = { ...GIF_PREVIEW_DEFAULTS, ...settings };
|
|
32
|
+
}
|
|
33
|
+
get seconds() {
|
|
34
|
+
return this.settings.seconds;
|
|
35
|
+
}
|
|
36
|
+
/** ffmpeg arguments for one preview (exported for the docs and tests). */
|
|
37
|
+
args(videoPath, outPath) {
|
|
38
|
+
const { seconds, width, fps } = this.settings;
|
|
39
|
+
return [
|
|
40
|
+
'-hide_banner',
|
|
41
|
+
'-loglevel',
|
|
42
|
+
'error',
|
|
43
|
+
'-y',
|
|
44
|
+
'-sseof',
|
|
45
|
+
`-${seconds}`,
|
|
46
|
+
'-i',
|
|
47
|
+
videoPath,
|
|
48
|
+
'-vf',
|
|
49
|
+
`fps=${fps},scale=${width}:-2:flags=lanczos,split[a][b];[a]palettegen=max_colors=128[p];[b][p]paletteuse`,
|
|
50
|
+
'-loop',
|
|
51
|
+
'0',
|
|
52
|
+
outPath,
|
|
53
|
+
];
|
|
54
|
+
}
|
|
55
|
+
async make(videoPath, outPath, logger) {
|
|
56
|
+
if (!(await this.isAvailable())) {
|
|
57
|
+
logger?.debug('github evidence: ffmpeg not found on PATH, skipping the GIF preview');
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
try {
|
|
61
|
+
const res = await this.runner(this.ffmpeg, this.args(videoPath, outPath), {
|
|
62
|
+
timeoutMs: 60_000,
|
|
63
|
+
});
|
|
64
|
+
if (res.code !== 0 || !existsSync(outPath)) {
|
|
65
|
+
logger?.debug(`github evidence: ffmpeg could not make a GIF preview of ${videoPath} (exit ${res.code}): ${(res.stderr ?? '').trim().slice(0, 300)}`);
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
const size = statSync(outPath).size;
|
|
69
|
+
if (size > this.settings.maxBytes) {
|
|
70
|
+
logger?.debug(`github evidence: GIF preview of ${videoPath} is ${size} bytes, over ${this.settings.maxBytes}; skipped`);
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
return outPath;
|
|
74
|
+
}
|
|
75
|
+
catch (e) {
|
|
76
|
+
logger?.debug(`github evidence: GIF preview failed: ${e.message}`);
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
isAvailable() {
|
|
81
|
+
this.available ??= this.runner(this.ffmpeg, ['-version'], { timeoutMs: 10_000 }).then((r) => r.code === 0, () => false);
|
|
82
|
+
return this.available;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=gif.js.map
|