@fro.bot/systematic 3.25.2 → 3.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HARNESSES.md +49 -1
- package/README.md +1 -1
- package/dist/cli.js +1 -1
- package/dist/{index-bmsgc2em.js → index-1dyb2x0z.js} +1 -0
- package/dist/index.js +1 -1
- package/dist/lib/bundled-names.d.ts +1 -1
- package/dist/pi.js +1 -0
- package/package.json +2 -1
- package/skills/git-commit-push-pr/SKILL.md +79 -6
- package/skills/pr-attachments/SKILL.md +764 -0
- package/skills/pr-attachments/scripts/pr-attachments.mjs +3140 -0
|
@@ -0,0 +1,3140 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// Helper for the pr-attachments skill: the gates that must be exact.
|
|
4
|
+
//
|
|
5
|
+
// Every subcommand reads one JSON object on stdin and writes one JSON object
|
|
6
|
+
// on stdout. None prints file or body contents. Each reports only the paths
|
|
7
|
+
// the caller gave it, except `cleanup-set`, which reports the paths git
|
|
8
|
+
// found under docs/evidence. A PR body is decoded once, from the JSON string.
|
|
9
|
+
// A body the helper produces goes to the file named by `out_file`, never to
|
|
10
|
+
// stdout. The one other thing the helper changes is the index, through
|
|
11
|
+
// `git rm` in the `cleanup-set` removal step.
|
|
12
|
+
//
|
|
13
|
+
// Usage:
|
|
14
|
+
// node pr-attachments.mjs classify < input.json
|
|
15
|
+
// node pr-attachments.mjs splice < input.json
|
|
16
|
+
// node pr-attachments.mjs extract < input.json
|
|
17
|
+
// node pr-attachments.mjs urls < input.json
|
|
18
|
+
// node pr-attachments.mjs repin-rewrite < input.json
|
|
19
|
+
// node pr-attachments.mjs pins < input.json
|
|
20
|
+
// node pr-attachments.mjs cleanup-set < input.json
|
|
21
|
+
// node pr-attachments.mjs verify < input.json
|
|
22
|
+
//
|
|
23
|
+
// classify sorts explicit image paths into accepted entries with destination
|
|
24
|
+
// names, or rejects the whole batch.
|
|
25
|
+
// paths non-empty array of image file paths (relative to the cwd)
|
|
26
|
+
// branch branch name, used for the topic when `topic` is empty
|
|
27
|
+
// topic optional user-supplied topic; wins over `branch`
|
|
28
|
+
// existing optional map of destination path -> git blob SHA already
|
|
29
|
+
// present at that path on the head commit
|
|
30
|
+
// max_bytes optional size limit, default 10 MiB (injectable for tests).
|
|
31
|
+
// GitHub serves a PNG as image/png up to exactly 10 MiB and as
|
|
32
|
+
// application/octet-stream from one byte more, so a larger file
|
|
33
|
+
// is rejected as `too-large` before anything is published.
|
|
34
|
+
//
|
|
35
|
+
// splice puts a `## Demo` section into a PR body, replacing the one that is
|
|
36
|
+
// there or appending one, and leaves every byte outside it identical.
|
|
37
|
+
// body the PR body
|
|
38
|
+
// demo_file file holding the Demo section: one `## Demo` heading and its
|
|
39
|
+
// content, nothing else (`extract` output is valid input)
|
|
40
|
+
// out_file where the new body is written
|
|
41
|
+
//
|
|
42
|
+
// extract writes the existing `## Demo` section, heading through its last
|
|
43
|
+
// non-blank line, to `out_file`, or reports `absent`.
|
|
44
|
+
// body, out_file
|
|
45
|
+
//
|
|
46
|
+
// urls lists the managed embed URLs in a body. A URL is managed only when it
|
|
47
|
+
// is a raw or blob URL for `head_repo` whose path is
|
|
48
|
+
// docs/evidence/<slug>/<name> with a whitelisted slug and name.
|
|
49
|
+
// body, head_repo ("owner/repo")
|
|
50
|
+
//
|
|
51
|
+
// repin-rewrite replaces only the 40-hex ref of managed URLs, so captions and
|
|
52
|
+
// hand edits stay as they are.
|
|
53
|
+
// body, head_repo, out_file
|
|
54
|
+
// pins map of managed path -> new 40-hex lower-case SHA
|
|
55
|
+
//
|
|
56
|
+
// pins and cleanup-set run git themselves, from the top of the repository
|
|
57
|
+
// that holds the working directory (the skill fetches first). They never take
|
|
58
|
+
// git output as input. Every git call is an argv array with a timeout, no
|
|
59
|
+
// stdin, literal pathspecs and a sanitized environment (every GIT_* variable
|
|
60
|
+
// dropped but GIT_CONFIG_GLOBAL and GIT_CONFIG_NOSYSTEM). Both take an
|
|
61
|
+
// optional `git_timeout_ms` per call (default 10000, at most 120000).
|
|
62
|
+
//
|
|
63
|
+
// pins reports, for each managed path, whether the pins in the body still
|
|
64
|
+
// serve the image that is on the PR head.
|
|
65
|
+
// head_ref_oid the PR head commit (40 hex, lower case)
|
|
66
|
+
// urls non-empty array of { path, sha }; the `urls` array of the
|
|
67
|
+
// `urls` subcommand fits as it is, and other fields are ignored
|
|
68
|
+
// For each path it finds the newest commit reachable from `head_ref_oid`
|
|
69
|
+
// that added or modified it. A pin is valid when its commit exists, is
|
|
70
|
+
// reachable from the head, and holds the same blob at the path as that
|
|
71
|
+
// newest commit; reachability alone is not enough. `pins` in the report is
|
|
72
|
+
// the map `repin-rewrite` takes: the newest commit for a stale path, and the
|
|
73
|
+
// pin already in the body for a path whose pins are valid.
|
|
74
|
+
// result already-valid (0), repin-needed (0), partial (1: some paths never
|
|
75
|
+
// existed on the head), unresolved (1: none did), head-not-found (1)
|
|
76
|
+
//
|
|
77
|
+
// pins also takes `select` in place of `urls`, to choose the pin of an embed
|
|
78
|
+
// that does not exist yet. It picks the same newest commit for each path, and
|
|
79
|
+
// that pin must hold the blob the caller expects (the `blob_sha` classify
|
|
80
|
+
// reported), so a pin that would serve other bytes stops the run. It then
|
|
81
|
+
// writes the Demo text for the pins to `out_file`.
|
|
82
|
+
// select non-empty array of { path, blob_sha }, in the order of the
|
|
83
|
+
// embeds; a path may appear once
|
|
84
|
+
// head_repo "owner/repo", for the URLs
|
|
85
|
+
// form raw (https://raw.githubusercontent.com/...) or blob
|
|
86
|
+
// (https://github.com/.../blob/...?raw=true)
|
|
87
|
+
// out_file where the Demo text is written
|
|
88
|
+
// head_ref_oid optional; the checked-out HEAD when absent
|
|
89
|
+
// result selected (0: `pins`, `evidence_sha` -- the newest commit of any
|
|
90
|
+
// selected path -- and the file written), pin-mismatch (1:
|
|
91
|
+
// `mismatched` lists path, pin, blob_sha and expected_blob_sha),
|
|
92
|
+
// unresolved (1: a path has no commit on the head),
|
|
93
|
+
// head-not-found (1)
|
|
94
|
+
//
|
|
95
|
+
// cleanup-set lists the files the PR added under docs/evidence, and removes
|
|
96
|
+
// them in a second step.
|
|
97
|
+
// head_ref_oid the PR head commit
|
|
98
|
+
// base_ref_name + remote the base branch and the remote it was fetched
|
|
99
|
+
// from, read as refs/remotes/<remote>/<name>; or
|
|
100
|
+
// base_ref one fully qualified ref, such as refs/remotes/origin/main
|
|
101
|
+
// remove_digest optional; turns the call into the removal step
|
|
102
|
+
// The set is the files the PR added (rename detection off, so a rename in
|
|
103
|
+
// counts as an add), less every path present at the base tip, not the
|
|
104
|
+
// merge-base. It is sorted, with a digest of the set.
|
|
105
|
+
// Removal recomputes the set, refuses with digest-mismatch unless its digest
|
|
106
|
+
// equals `remove_digest`, refuses with head-not-checked-out unless HEAD is
|
|
107
|
+
// `head_ref_oid`, and only then runs `git rm` on the set. It stages the
|
|
108
|
+
// removals and commits nothing.
|
|
109
|
+
// result set (0), empty (0), removed (0), digest-mismatch (1),
|
|
110
|
+
// head-not-checked-out (1), head-not-found (1), base-not-found (1),
|
|
111
|
+
// no-merge-base (1)
|
|
112
|
+
//
|
|
113
|
+
// A git failure or timeout exits 1 with result git-failed or git-timeout and
|
|
114
|
+
// `reached`: the step it stopped at, and the progress made (paths done for
|
|
115
|
+
// pins, whether removal had started for cleanup-set).
|
|
116
|
+
//
|
|
117
|
+
// verify is the one subcommand that uses the network. It checks that each
|
|
118
|
+
// managed raw URL serves an image, retrying while GitHub's CDN catches up
|
|
119
|
+
// with a push. It sends a bare GET (no credentials), follows at most 5
|
|
120
|
+
// redirects and only to raw.githubusercontent.com or media.githubusercontent.com
|
|
121
|
+
// over https, reads only the status and headers, and drops the body unread.
|
|
122
|
+
// Nothing from a response is ever printed.
|
|
123
|
+
// urls non-empty array (at most 50) of managed raw URLs:
|
|
124
|
+
// https://raw.githubusercontent.com/<owner>/<repo>/
|
|
125
|
+
// <40-hex>/docs/evidence/<slug>/<name>, with no query or
|
|
126
|
+
// fragment. A blob URL is a usage error (`blob-url`):
|
|
127
|
+
// the skill checks those through `gh api`.
|
|
128
|
+
// head_repo optional "owner/repo"; every URL must belong to it
|
|
129
|
+
// max_wait_s optional, default 90, at most 300: the most this call
|
|
130
|
+
// spends waiting between attempts
|
|
131
|
+
// waited_s optional, default 0: the `waited_s` of the latest report
|
|
132
|
+
// for the same URLs, passed on unchanged. The schedule
|
|
133
|
+
// resumes there, and the 300 s total counts it.
|
|
134
|
+
// resume_at optional, epoch milliseconds (a positive integer): the
|
|
135
|
+
// `resume_at` of the latest report, passed on unchanged.
|
|
136
|
+
// No request goes out before it. The call sleeps until
|
|
137
|
+
// then first, within its own `max_wait_s`, and the sleep
|
|
138
|
+
// counts toward `waited_s`. If the deadline lies past the
|
|
139
|
+
// 300 s total, nothing is sent and the report is not
|
|
140
|
+
// retryable; if it lies past this call, the call sleeps
|
|
141
|
+
// its whole limit, sends nothing and returns the same
|
|
142
|
+
// deadline. If the sleep uses up the call limit or the
|
|
143
|
+
// total, nothing is sent either and no deadline is
|
|
144
|
+
// handed on. Such URLs report `rate-limited`, no HTTP
|
|
145
|
+
// status and 0 attempts.
|
|
146
|
+
// request_timeout_s optional, default 10, at most 300: per request
|
|
147
|
+
// Retried (404, 429, 5xx, network error, timeout): wait 2, 4, 8, 15, then 30 s
|
|
148
|
+
// again and again, each with +-20% jitter, and never less than a 429's
|
|
149
|
+
// Retry-After. A call stops when the next wait would pass `max_wait_s` or
|
|
150
|
+
// the 300 s total. Each request is cut short at the time the call has left,
|
|
151
|
+
// and no attempt starts once that time is gone, so a call ends at its
|
|
152
|
+
// `max_wait_s` and not one request later. (A call always makes its first
|
|
153
|
+
// request, even when earlier calls used up the whole total, unless the
|
|
154
|
+
// cooldown it served took the last of its time.) Final at once: a 200 that is not image/*, 401/403, any
|
|
155
|
+
// other status, a redirect off the allowed hosts, a missing Location, or a
|
|
156
|
+
// 6th redirect. Every URL is checked at the same time.
|
|
157
|
+
// result verified (0), failed (1)
|
|
158
|
+
// Each URL is { url, status, http_status, attempts, retryable }. `status` is
|
|
159
|
+
// ok, not-found, rate-limited, server-error, network-error, timeout,
|
|
160
|
+
// not-an-image, forbidden, unexpected-status, bad-redirect,
|
|
161
|
+
// redirect-off-host or too-many-redirects. `retryable` is true when waiting
|
|
162
|
+
// longer could help and the 300 s total has room for the next wait. The
|
|
163
|
+
// report has `failed` (the failing entries), top-level `retryable` (any
|
|
164
|
+
// failure is retryable) and `waited_s`: the longest wait of this call plus
|
|
165
|
+
// the `waited_s` it was given, rounded up. It is the running total, so the
|
|
166
|
+
// caller never adds reports up. When a 429 carries a Retry-After that the
|
|
167
|
+
// call cannot serve, the report also has `resume_at` (epoch milliseconds,
|
|
168
|
+
// the latest over the failures that can still be retried), and so does each
|
|
169
|
+
// such URL. The helper keeps no state: the caller hands `resume_at` back.
|
|
170
|
+
//
|
|
171
|
+
// Section rules (no markers; the heading is the only boundary):
|
|
172
|
+
// - The heading matches ^##[ \t]+Demo([ \t]+#+)?[ \t]*$: column 0, no
|
|
173
|
+
// leading spaces. An indented one is never the managed heading.
|
|
174
|
+
// - A section runs to the next H1 or H2 (ATX or setext) or to the end.
|
|
175
|
+
// - Fenced code hides headings. A fence opens on a run of three or more
|
|
176
|
+
// backticks or tildes after any leading whitespace, quote markers and
|
|
177
|
+
// list markers, so a fence inside a list item or a quote counts. A run
|
|
178
|
+
// indented four columns or more with no marker before it is indented
|
|
179
|
+
// code, never a fence. It closes on a run of the same character, at
|
|
180
|
+
// least as long, with no info string. An unclosed fence runs to the end.
|
|
181
|
+
// - Raw HTML blocks hide headings: `<pre>`, `<script>`, `<style>` and
|
|
182
|
+
// `<textarea>` (until the matching closing tag), and comments (until a
|
|
183
|
+
// literal `-->`, as GitHub Markdown reads them). Block quotes and
|
|
184
|
+
// indented code hide them too.
|
|
185
|
+
// - Fail closed: when a block that never closes began inside the Demo
|
|
186
|
+
// section, splice and extract stop with `unterminated-region` (kind
|
|
187
|
+
// `fence`, `comment` or `html`) and write nothing, because where the
|
|
188
|
+
// section ends is unknown. Splice also stops, rather than append, when a
|
|
189
|
+
// block is still open at the end of a body that has no Demo section.
|
|
190
|
+
// - A comment that holds `--!>` before its `-->` (or that holds `--!>` and
|
|
191
|
+
// no `-->`) is ambiguous: a browser ends a comment at `--!>`, GitHub
|
|
192
|
+
// Markdown does not. Splice and extract stop with `unterminated-region`
|
|
193
|
+
// (kind `comment`) and write nothing, wherever in the body it sits.
|
|
194
|
+
// - Shape guard: an existing Demo section is replaced (splice) or returned
|
|
195
|
+
// (extract) only when every non-blank line under its heading is one the
|
|
196
|
+
// skill writes (`demoText`): ``, where the URL is a
|
|
197
|
+
// raw or blob URL of a managed path (the forms `pins select` writes).
|
|
198
|
+
// Any other line, a heading, a fence, HTML, a table or prose, stops both
|
|
199
|
+
// with `unexpected-section-content` and writes nothing, because the
|
|
200
|
+
// section is not the managed one or its end was misjudged.
|
|
201
|
+
// - Trailing blank lines belong to the gap after a section, not to it.
|
|
202
|
+
// - Lines split on CRLF, LF or CR and keep their terminators. Lines the
|
|
203
|
+
// helper inserts use the body's dominant terminator; Demo text is
|
|
204
|
+
// inserted verbatim.
|
|
205
|
+
//
|
|
206
|
+
// Exit codes: 0 success, 1 domain failure (rejected batch, or a question
|
|
207
|
+
// the user must answer, or an output file that cannot be written, or an embed
|
|
208
|
+
// that does not serve an image), 2 usage error.
|
|
209
|
+
// A usage error carries `error` (a fixed code) and `field` (the input field
|
|
210
|
+
// that failed, by its schema name; `input` for the whole object and
|
|
211
|
+
// `operation` for the command line). Neither ever echoes caller text.
|
|
212
|
+
|
|
213
|
+
import { spawnSync } from 'node:child_process'
|
|
214
|
+
import { createHash } from 'node:crypto'
|
|
215
|
+
import {
|
|
216
|
+
closeSync,
|
|
217
|
+
constants as fsConstants,
|
|
218
|
+
fstatSync,
|
|
219
|
+
openSync,
|
|
220
|
+
readFileSync,
|
|
221
|
+
readSync,
|
|
222
|
+
realpathSync,
|
|
223
|
+
writeFileSync,
|
|
224
|
+
} from 'node:fs'
|
|
225
|
+
import path from 'node:path'
|
|
226
|
+
import { fileURLToPath } from 'node:url'
|
|
227
|
+
|
|
228
|
+
const SCHEMA_VERSION = 1
|
|
229
|
+
// The largest file raw.githubusercontent.com serves with an image type.
|
|
230
|
+
const DEFAULT_MAX_BYTES = 10 * 1024 * 1024
|
|
231
|
+
const MAX_SLUG_LENGTH = 60
|
|
232
|
+
const EVIDENCE_ROOT = 'docs/evidence'
|
|
233
|
+
const SVG_MESSAGE = 'SVG is not supported by this skill'
|
|
234
|
+
const USAGE =
|
|
235
|
+
'node pr-attachments.mjs <classify|splice|extract|urls|repin-rewrite|pins|cleanup-set|verify> < input.json'
|
|
236
|
+
|
|
237
|
+
const EXTENSIONS = { png: 'png', jpeg: 'jpg', gif: 'gif', webp: 'webp' }
|
|
238
|
+
|
|
239
|
+
const REJECTION_MESSAGES = {
|
|
240
|
+
missing: 'The path does not exist',
|
|
241
|
+
unreadable: 'The file cannot be read',
|
|
242
|
+
'not-a-file': 'The path is not a regular file',
|
|
243
|
+
'too-large': 'The file is over the size limit',
|
|
244
|
+
'svg-not-supported': SVG_MESSAGE,
|
|
245
|
+
'unsupported-type': 'The file is not a PNG, JPEG, GIF or WebP image',
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// ── Format detection ───────────────────────────────────────────────────────
|
|
249
|
+
|
|
250
|
+
const PNG_SIGNATURE = Buffer.from([
|
|
251
|
+
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
|
252
|
+
])
|
|
253
|
+
|
|
254
|
+
// Only this many leading bytes are decoded and scanned for SVG markup.
|
|
255
|
+
const SVG_SNIFF_BYTES = 4096
|
|
256
|
+
|
|
257
|
+
function hasPrefixIgnoreCase(text, index, prefix) {
|
|
258
|
+
return text.slice(index, index + prefix.length).toLowerCase() === prefix
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function skipWhitespace(text, index) {
|
|
262
|
+
let cursor = index
|
|
263
|
+
while (cursor < text.length && /\s/.test(text.charAt(cursor))) {
|
|
264
|
+
cursor += 1
|
|
265
|
+
}
|
|
266
|
+
return cursor
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* The index just past a doctype that starts its body at `from`, or -1 when it
|
|
271
|
+
* is unterminated. An optional `[ ... ]` internal subset may precede the `>`.
|
|
272
|
+
* @param {string} text
|
|
273
|
+
* @param {number} from
|
|
274
|
+
*/
|
|
275
|
+
function endOfDoctype(text, from) {
|
|
276
|
+
let cursor = from
|
|
277
|
+
while (
|
|
278
|
+
cursor < text.length &&
|
|
279
|
+
text.charAt(cursor) !== '[' &&
|
|
280
|
+
text.charAt(cursor) !== '>'
|
|
281
|
+
) {
|
|
282
|
+
cursor += 1
|
|
283
|
+
}
|
|
284
|
+
if (cursor >= text.length) {
|
|
285
|
+
return -1
|
|
286
|
+
}
|
|
287
|
+
if (text.charAt(cursor) === '>') {
|
|
288
|
+
return cursor + 1
|
|
289
|
+
}
|
|
290
|
+
let close = text.indexOf(']', cursor + 1)
|
|
291
|
+
while (close !== -1) {
|
|
292
|
+
const next = skipWhitespace(text, close + 1)
|
|
293
|
+
if (text.charAt(next) === '>') {
|
|
294
|
+
return next + 1
|
|
295
|
+
}
|
|
296
|
+
close = text.indexOf(']', close + 1)
|
|
297
|
+
}
|
|
298
|
+
return -1
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* The index just past the prolog item that starts at `start` (an XML
|
|
303
|
+
* declaration, a comment or a doctype), or -1 when none starts there or the
|
|
304
|
+
* item is unterminated.
|
|
305
|
+
* @param {string} text
|
|
306
|
+
* @param {number} start
|
|
307
|
+
*/
|
|
308
|
+
function endOfPrologItem(text, start) {
|
|
309
|
+
if (hasPrefixIgnoreCase(text, start, '<?xml')) {
|
|
310
|
+
const close = text.indexOf('?>', start + 5)
|
|
311
|
+
return close === -1 ? -1 : close + 2
|
|
312
|
+
}
|
|
313
|
+
if (text.startsWith('<!--', start)) {
|
|
314
|
+
const close = text.indexOf('-->', start + 4)
|
|
315
|
+
return close === -1 ? -1 : close + 3
|
|
316
|
+
}
|
|
317
|
+
if (hasPrefixIgnoreCase(text, start, '<!doctype')) {
|
|
318
|
+
return endOfDoctype(text, start + 9)
|
|
319
|
+
}
|
|
320
|
+
return -1
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Whether the text is an SVG document: an optional BOM and any number of
|
|
325
|
+
* prolog items (each followed by optional whitespace) before an `<svg` root.
|
|
326
|
+
* A single forward scan built on `indexOf`, so the cost is linear in the text.
|
|
327
|
+
* @param {string} text
|
|
328
|
+
*/
|
|
329
|
+
function looksLikeSvg(text) {
|
|
330
|
+
let cursor = skipWhitespace(text, text.startsWith('\uFEFF') ? 1 : 0)
|
|
331
|
+
for (;;) {
|
|
332
|
+
const end = endOfPrologItem(text, cursor)
|
|
333
|
+
if (end === -1) {
|
|
334
|
+
break
|
|
335
|
+
}
|
|
336
|
+
cursor = skipWhitespace(text, end)
|
|
337
|
+
}
|
|
338
|
+
return /^<svg[\s>/]/i.test(text.slice(cursor, cursor + 5))
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Detects the format from magic bytes only, never from the file name.
|
|
343
|
+
* @param {Buffer} bytes
|
|
344
|
+
* @returns {'png' | 'jpeg' | 'gif' | 'webp' | 'svg' | 'unknown'}
|
|
345
|
+
*/
|
|
346
|
+
function detectType(bytes) {
|
|
347
|
+
if (bytes.length >= 8 && bytes.subarray(0, 8).equals(PNG_SIGNATURE)) {
|
|
348
|
+
return 'png'
|
|
349
|
+
}
|
|
350
|
+
if (
|
|
351
|
+
bytes.length >= 3 &&
|
|
352
|
+
bytes[0] === 0xff &&
|
|
353
|
+
bytes[1] === 0xd8 &&
|
|
354
|
+
bytes[2] === 0xff
|
|
355
|
+
) {
|
|
356
|
+
return 'jpeg'
|
|
357
|
+
}
|
|
358
|
+
const head = bytes.toString('latin1', 0, 6)
|
|
359
|
+
if (head === 'GIF87a' || head === 'GIF89a') {
|
|
360
|
+
return 'gif'
|
|
361
|
+
}
|
|
362
|
+
if (
|
|
363
|
+
bytes.length >= 12 &&
|
|
364
|
+
bytes.toString('latin1', 0, 4) === 'RIFF' &&
|
|
365
|
+
bytes.toString('latin1', 8, 12) === 'WEBP'
|
|
366
|
+
) {
|
|
367
|
+
return 'webp'
|
|
368
|
+
}
|
|
369
|
+
if (looksLikeSvg(bytes.toString('utf8', 0, SVG_SNIFF_BYTES))) {
|
|
370
|
+
return 'svg'
|
|
371
|
+
}
|
|
372
|
+
return 'unknown'
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// ── Naming ─────────────────────────────────────────────────────────────────
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* The topic slug: NFKC, lowercase, runs of anything outside [a-z0-9] become
|
|
379
|
+
* one hyphen, trimmed, at most 60 characters. The output holds only
|
|
380
|
+
* [a-z0-9-], never starts or ends with a hyphen, and may be empty.
|
|
381
|
+
* @param {string} value
|
|
382
|
+
*/
|
|
383
|
+
export function slugTopic(value) {
|
|
384
|
+
return value
|
|
385
|
+
.normalize('NFKC')
|
|
386
|
+
.toLowerCase()
|
|
387
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
388
|
+
.replace(/^-+|-+$/g, '')
|
|
389
|
+
.slice(0, MAX_SLUG_LENGTH)
|
|
390
|
+
.replace(/-+$/g, '')
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* The destination file stem: the same normalisation, over [a-z0-9._-].
|
|
395
|
+
* Starts and ends with a letter or digit, and may be empty.
|
|
396
|
+
* @param {string} stem
|
|
397
|
+
*/
|
|
398
|
+
export function slugName(stem) {
|
|
399
|
+
return stem
|
|
400
|
+
.normalize('NFKC')
|
|
401
|
+
.toLowerCase()
|
|
402
|
+
.replace(/[^a-z0-9._-]+/g, '-')
|
|
403
|
+
.replace(/\.{2,}/g, '.')
|
|
404
|
+
.replace(/-{2,}/g, '-')
|
|
405
|
+
.replace(/^[._-]+|[._-]+$/g, '')
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** @param {string} source */
|
|
409
|
+
function sourceStem(source) {
|
|
410
|
+
const base = path.basename(source)
|
|
411
|
+
return base.slice(0, base.length - path.extname(base).length)
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// ── File inspection ────────────────────────────────────────────────────────
|
|
415
|
+
|
|
416
|
+
/** @param {Buffer} bytes */
|
|
417
|
+
function blobSha(bytes) {
|
|
418
|
+
return createHash('sha1')
|
|
419
|
+
.update(`blob ${bytes.length}\0`)
|
|
420
|
+
.update(bytes)
|
|
421
|
+
.digest('hex')
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Opens without blocking, so a FIFO or device path cannot hang the helper.
|
|
426
|
+
* @param {string} source
|
|
427
|
+
* @param {number} maxBytes
|
|
428
|
+
* @returns {{ ok: true, bytes: Buffer } | { ok: false, reason: string }}
|
|
429
|
+
*/
|
|
430
|
+
function readCandidate(source, maxBytes) {
|
|
431
|
+
let fd
|
|
432
|
+
try {
|
|
433
|
+
fd = openSync(
|
|
434
|
+
path.resolve(source),
|
|
435
|
+
fsConstants.O_RDONLY | fsConstants.O_NONBLOCK,
|
|
436
|
+
)
|
|
437
|
+
} catch (error) {
|
|
438
|
+
const code = error && typeof error === 'object' ? error.code : undefined
|
|
439
|
+
return {
|
|
440
|
+
ok: false,
|
|
441
|
+
reason:
|
|
442
|
+
code === 'ENOENT' || code === 'ENOTDIR' ? 'missing' : 'unreadable',
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
try {
|
|
446
|
+
const stats = fstatSync(fd)
|
|
447
|
+
if (!stats.isFile()) {
|
|
448
|
+
return { ok: false, reason: 'not-a-file' }
|
|
449
|
+
}
|
|
450
|
+
if (stats.size > maxBytes) {
|
|
451
|
+
return { ok: false, reason: 'too-large' }
|
|
452
|
+
}
|
|
453
|
+
const bytes = Buffer.alloc(stats.size)
|
|
454
|
+
let offset = 0
|
|
455
|
+
while (offset < bytes.length) {
|
|
456
|
+
const read = readSync(fd, bytes, offset, bytes.length - offset, null)
|
|
457
|
+
if (read === 0) {
|
|
458
|
+
break
|
|
459
|
+
}
|
|
460
|
+
offset += read
|
|
461
|
+
}
|
|
462
|
+
return { ok: true, bytes: bytes.subarray(0, offset) }
|
|
463
|
+
} catch {
|
|
464
|
+
return { ok: false, reason: 'unreadable' }
|
|
465
|
+
} finally {
|
|
466
|
+
closeSync(fd)
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* @param {string} source
|
|
472
|
+
* @param {number} maxBytes
|
|
473
|
+
*/
|
|
474
|
+
function inspectFile(source, maxBytes) {
|
|
475
|
+
const candidate = readCandidate(source, maxBytes)
|
|
476
|
+
if (!candidate.ok) {
|
|
477
|
+
return { source, rejection: candidate.reason }
|
|
478
|
+
}
|
|
479
|
+
const type = detectType(candidate.bytes)
|
|
480
|
+
if (type === 'svg') {
|
|
481
|
+
return { source, rejection: 'svg-not-supported' }
|
|
482
|
+
}
|
|
483
|
+
if (type === 'unknown') {
|
|
484
|
+
return { source, rejection: 'unsupported-type' }
|
|
485
|
+
}
|
|
486
|
+
return {
|
|
487
|
+
source,
|
|
488
|
+
type,
|
|
489
|
+
bytes: candidate.bytes.length,
|
|
490
|
+
blobSha: blobSha(candidate.bytes),
|
|
491
|
+
warnings: fileWarnings(source, type, candidate.bytes),
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* @param {string} source
|
|
497
|
+
* @param {string} type
|
|
498
|
+
* @param {Buffer} bytes
|
|
499
|
+
*/
|
|
500
|
+
function fileWarnings(source, type, bytes) {
|
|
501
|
+
const warnings = []
|
|
502
|
+
const trailing = trailingBytes(type, bytes)
|
|
503
|
+
if (trailing > 0) {
|
|
504
|
+
warnings.push({ source, code: 'trailing-data', bytes: trailing })
|
|
505
|
+
}
|
|
506
|
+
return warnings
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// ── Trailing data after the end of the image ───────────────────────────────
|
|
510
|
+
//
|
|
511
|
+
// Each walk follows the format's own structure to its end marker, so text
|
|
512
|
+
// that merely looks like a marker inside a chunk or segment is not taken for
|
|
513
|
+
// one. A file that cannot be walked to its end reports no trailing data.
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* @param {string} type
|
|
517
|
+
* @param {Buffer} bytes
|
|
518
|
+
*/
|
|
519
|
+
function trailingBytes(type, bytes) {
|
|
520
|
+
switch (type) {
|
|
521
|
+
case 'png':
|
|
522
|
+
return pngTrailing(bytes)
|
|
523
|
+
case 'jpeg':
|
|
524
|
+
return jpegTrailing(bytes)
|
|
525
|
+
case 'gif':
|
|
526
|
+
return gifTrailing(bytes)
|
|
527
|
+
case 'webp':
|
|
528
|
+
return webpTrailing(bytes)
|
|
529
|
+
default:
|
|
530
|
+
return 0
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/** @param {Buffer} bytes */
|
|
535
|
+
function pngTrailing(bytes) {
|
|
536
|
+
let pos = PNG_SIGNATURE.length
|
|
537
|
+
while (pos + 12 <= bytes.length) {
|
|
538
|
+
const end = pos + 12 + bytes.readUInt32BE(pos)
|
|
539
|
+
if (end > bytes.length) {
|
|
540
|
+
return 0
|
|
541
|
+
}
|
|
542
|
+
if (bytes.toString('latin1', pos + 4, pos + 8) === 'IEND') {
|
|
543
|
+
return bytes.length - end
|
|
544
|
+
}
|
|
545
|
+
pos = end
|
|
546
|
+
}
|
|
547
|
+
return 0
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/** @param {Buffer} bytes */
|
|
551
|
+
function jpegTrailing(bytes) {
|
|
552
|
+
let pos = 2
|
|
553
|
+
while (pos + 1 < bytes.length) {
|
|
554
|
+
if (bytes[pos] !== 0xff) {
|
|
555
|
+
return 0
|
|
556
|
+
}
|
|
557
|
+
const marker = bytes[pos + 1]
|
|
558
|
+
if (marker === 0xff) {
|
|
559
|
+
pos += 1
|
|
560
|
+
continue
|
|
561
|
+
}
|
|
562
|
+
if (marker === 0xd9) {
|
|
563
|
+
return bytes.length - (pos + 2)
|
|
564
|
+
}
|
|
565
|
+
if (
|
|
566
|
+
marker === 0xd8 ||
|
|
567
|
+
marker === 0x01 ||
|
|
568
|
+
(marker >= 0xd0 && marker <= 0xd7)
|
|
569
|
+
) {
|
|
570
|
+
pos += 2
|
|
571
|
+
continue
|
|
572
|
+
}
|
|
573
|
+
if (pos + 4 > bytes.length) {
|
|
574
|
+
return 0
|
|
575
|
+
}
|
|
576
|
+
const length = bytes.readUInt16BE(pos + 2)
|
|
577
|
+
if (length < 2) {
|
|
578
|
+
return 0
|
|
579
|
+
}
|
|
580
|
+
pos += 2 + length
|
|
581
|
+
if (marker === 0xda) {
|
|
582
|
+
pos = skipJpegScan(bytes, pos)
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
return 0
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/**
|
|
589
|
+
* Skips entropy-coded data up to the next real marker: stuffed 0xFF00 bytes
|
|
590
|
+
* and restart markers belong to the scan.
|
|
591
|
+
* @param {Buffer} bytes
|
|
592
|
+
* @param {number} start
|
|
593
|
+
*/
|
|
594
|
+
function skipJpegScan(bytes, start) {
|
|
595
|
+
let pos = start
|
|
596
|
+
while (pos + 1 < bytes.length) {
|
|
597
|
+
if (bytes[pos] === 0xff) {
|
|
598
|
+
const next = bytes[pos + 1]
|
|
599
|
+
if (next === 0x00 || (next >= 0xd0 && next <= 0xd7)) {
|
|
600
|
+
pos += 2
|
|
601
|
+
continue
|
|
602
|
+
}
|
|
603
|
+
if (next === 0xff) {
|
|
604
|
+
pos += 1
|
|
605
|
+
continue
|
|
606
|
+
}
|
|
607
|
+
return pos
|
|
608
|
+
}
|
|
609
|
+
pos += 1
|
|
610
|
+
}
|
|
611
|
+
return bytes.length
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** @param {Buffer} bytes */
|
|
615
|
+
function gifTrailing(bytes) {
|
|
616
|
+
if (bytes.length < 13) {
|
|
617
|
+
return 0
|
|
618
|
+
}
|
|
619
|
+
let pos = 13 + colorTableSize(bytes[10])
|
|
620
|
+
while (pos < bytes.length) {
|
|
621
|
+
const block = bytes[pos]
|
|
622
|
+
if (block === 0x3b) {
|
|
623
|
+
return bytes.length - (pos + 1)
|
|
624
|
+
}
|
|
625
|
+
if (block === 0x21) {
|
|
626
|
+
pos = skipSubBlocks(bytes, pos + 2)
|
|
627
|
+
} else if (block === 0x2c && pos + 10 <= bytes.length) {
|
|
628
|
+
// Image descriptor, local color table, LZW code size, then data.
|
|
629
|
+
pos = skipSubBlocks(bytes, pos + 10 + colorTableSize(bytes[pos + 9]) + 1)
|
|
630
|
+
} else {
|
|
631
|
+
return 0
|
|
632
|
+
}
|
|
633
|
+
if (pos < 0) {
|
|
634
|
+
return 0
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
return 0
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** @param {number} flags */
|
|
641
|
+
function colorTableSize(flags) {
|
|
642
|
+
return flags & 0x80 ? 3 * 2 ** ((flags & 0x07) + 1) : 0
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
/**
|
|
646
|
+
* Returns the offset after the terminating empty sub-block, or -1.
|
|
647
|
+
* @param {Buffer} bytes
|
|
648
|
+
* @param {number} start
|
|
649
|
+
*/
|
|
650
|
+
function skipSubBlocks(bytes, start) {
|
|
651
|
+
let pos = start
|
|
652
|
+
while (pos < bytes.length) {
|
|
653
|
+
const size = bytes[pos]
|
|
654
|
+
pos += 1 + size
|
|
655
|
+
if (size === 0) {
|
|
656
|
+
return pos
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
return -1
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/** @param {Buffer} bytes */
|
|
663
|
+
function webpTrailing(bytes) {
|
|
664
|
+
const end = 8 + bytes.readUInt32LE(4)
|
|
665
|
+
return end <= bytes.length ? bytes.length - end : 0
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// ── Classification ─────────────────────────────────────────────────────────
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* @param {{
|
|
672
|
+
* paths: string[],
|
|
673
|
+
* branch?: string,
|
|
674
|
+
* topic?: string,
|
|
675
|
+
* existing: Record<string, string>,
|
|
676
|
+
* maxBytes: number,
|
|
677
|
+
* }} input
|
|
678
|
+
*/
|
|
679
|
+
export function classify(input) {
|
|
680
|
+
const inspected = input.paths.map((source) =>
|
|
681
|
+
inspectFile(source, input.maxBytes),
|
|
682
|
+
)
|
|
683
|
+
const warnings = inspected.flatMap((file) => file.warnings ?? [])
|
|
684
|
+
|
|
685
|
+
const topicSource =
|
|
686
|
+
typeof input.topic === 'string' && input.topic.length > 0
|
|
687
|
+
? 'topic'
|
|
688
|
+
: 'branch'
|
|
689
|
+
const slug = slugTopic(
|
|
690
|
+
(topicSource === 'topic' ? input.topic : input.branch) ?? '',
|
|
691
|
+
)
|
|
692
|
+
const topic = slug.length > 0 ? { slug, source: topicSource } : null
|
|
693
|
+
|
|
694
|
+
const rejections = inspected
|
|
695
|
+
.filter((file) => file.rejection !== undefined)
|
|
696
|
+
.map((file) => ({
|
|
697
|
+
source: file.source,
|
|
698
|
+
reason: file.rejection,
|
|
699
|
+
message: REJECTION_MESSAGES[file.rejection],
|
|
700
|
+
}))
|
|
701
|
+
if (rejections.length > 0) {
|
|
702
|
+
return response('rejected', { topic, rejections, warnings })
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
const asks = collectAsks(inspected, topic, input.existing)
|
|
706
|
+
if (asks.length > 0) {
|
|
707
|
+
return response('needs-input', { topic, asks, warnings })
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
const entries = inspected.map((file) => ({
|
|
711
|
+
source: file.source,
|
|
712
|
+
destination: destinationFor(topic.slug, file),
|
|
713
|
+
type: file.type,
|
|
714
|
+
bytes: file.bytes,
|
|
715
|
+
blob_sha: file.blobSha,
|
|
716
|
+
status:
|
|
717
|
+
input.existing[destinationFor(topic.slug, file)] === undefined
|
|
718
|
+
? 'new'
|
|
719
|
+
: 'reused',
|
|
720
|
+
}))
|
|
721
|
+
return response('accepted', { topic, entries, warnings })
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/**
|
|
725
|
+
* @param {string} slug
|
|
726
|
+
* @param {{ source: string, type: string }} file
|
|
727
|
+
*/
|
|
728
|
+
function destinationFor(slug, file) {
|
|
729
|
+
const stem = slugName(sourceStem(file.source))
|
|
730
|
+
return `${EVIDENCE_ROOT}/${slug}/${stem}.${EXTENSIONS[file.type]}`
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
/**
|
|
734
|
+
* Everything the user must settle before anything is copied: an empty
|
|
735
|
+
* topic, a source name that slugs to nothing, two inputs that map to one
|
|
736
|
+
* destination, and a destination that already holds different bytes.
|
|
737
|
+
* @param {Array<{ source: string, type: string, blobSha: string }>} files
|
|
738
|
+
* @param {{ slug: string } | null} topic
|
|
739
|
+
* @param {Record<string, string>} existing
|
|
740
|
+
*/
|
|
741
|
+
function collectAsks(files, topic, existing) {
|
|
742
|
+
const asks = []
|
|
743
|
+
if (topic === null) {
|
|
744
|
+
asks.push({ code: 'empty-topic' })
|
|
745
|
+
}
|
|
746
|
+
const named = []
|
|
747
|
+
for (const file of files) {
|
|
748
|
+
if (slugName(sourceStem(file.source)).length === 0) {
|
|
749
|
+
asks.push({ code: 'empty-name', source: file.source })
|
|
750
|
+
} else if (topic !== null) {
|
|
751
|
+
named.push({ file, destination: destinationFor(topic.slug, file) })
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
const byDestination = new Map()
|
|
755
|
+
for (const item of named) {
|
|
756
|
+
const group = byDestination.get(item.destination) ?? []
|
|
757
|
+
group.push(item)
|
|
758
|
+
byDestination.set(item.destination, group)
|
|
759
|
+
}
|
|
760
|
+
for (const [destination, group] of byDestination) {
|
|
761
|
+
if (group.length > 1) {
|
|
762
|
+
asks.push({
|
|
763
|
+
code: 'duplicate-destination',
|
|
764
|
+
destination,
|
|
765
|
+
sources: group.map((item) => item.file.source),
|
|
766
|
+
})
|
|
767
|
+
} else if (
|
|
768
|
+
existing[destination] !== undefined &&
|
|
769
|
+
existing[destination] !== group[0].file.blobSha
|
|
770
|
+
) {
|
|
771
|
+
asks.push({
|
|
772
|
+
code: 'destination-conflict',
|
|
773
|
+
destination,
|
|
774
|
+
sources: [group[0].file.source],
|
|
775
|
+
})
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return asks
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* @param {'accepted' | 'rejected' | 'needs-input'} result
|
|
783
|
+
* @param {Record<string, unknown>} parts
|
|
784
|
+
*/
|
|
785
|
+
function response(result, parts) {
|
|
786
|
+
return {
|
|
787
|
+
exitCode: result === 'accepted' ? 0 : 1,
|
|
788
|
+
body: {
|
|
789
|
+
schema_version: SCHEMA_VERSION,
|
|
790
|
+
operation: 'classify',
|
|
791
|
+
result,
|
|
792
|
+
topic: parts.topic ?? null,
|
|
793
|
+
entries: parts.entries ?? [],
|
|
794
|
+
rejections: parts.rejections ?? [],
|
|
795
|
+
asks: parts.asks ?? [],
|
|
796
|
+
warnings: parts.warnings ?? [],
|
|
797
|
+
},
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
// ── Markdown structure ─────────────────────────────────────────────────────
|
|
802
|
+
//
|
|
803
|
+
// Just enough of CommonMark to find H1/H2 boundaries and to know which lines
|
|
804
|
+
// are hidden from headings: fenced code, raw HTML blocks (`<pre>`, `<script>`,
|
|
805
|
+
// `<style>`, `<textarea>` and comments) and block quotes. A fence counts
|
|
806
|
+
// wherever it opens, also inside a list item or a quote, because a literal
|
|
807
|
+
// `## Demo` shown inside a fence must never be taken for the section to
|
|
808
|
+
// replace. When in doubt a line is hidden, never shown: the worst outcome is
|
|
809
|
+
// a refusal, not a deleted line. Anything that is not a heading is left
|
|
810
|
+
// exactly as it is.
|
|
811
|
+
|
|
812
|
+
// The managed heading starts at column 0. An indented one is a heading or a
|
|
813
|
+
// quote of one, but never the section this helper replaces.
|
|
814
|
+
const DEMO_HEADING = /^##[ \t]+Demo(?:[ \t]+#+)?[ \t]*$/
|
|
815
|
+
const SCREENSHOTS_HEADING = /^ {0,3}##[ \t]+Screenshots(?:[ \t]+#+)?[ \t]*$/
|
|
816
|
+
const ATX_HEADING = /^ {0,3}(#{1,6})(?:[ \t]|$)/
|
|
817
|
+
const SETEXT_UNDERLINE = /^ {0,3}(?:=+|-+)[ \t]*$/
|
|
818
|
+
const THEMATIC_BREAK =
|
|
819
|
+
/^ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,})$/
|
|
820
|
+
const LIST_ITEM = /^ {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]|$)/
|
|
821
|
+
// Everything a fence line may start with before its run: whitespace, quote
|
|
822
|
+
// markers, bullets and list numbers, repeated.
|
|
823
|
+
const CONTAINER_LEAD =
|
|
824
|
+
'(?:[ \\t]*(?:>|(?:[-+*]|\\d{1,9}[.)])(?=[ \\t])))*[ \\t]*'
|
|
825
|
+
const CONTAINER_PREFIX = new RegExp(`^${CONTAINER_LEAD}`)
|
|
826
|
+
const PLAIN_PREFIX = /^ {0,3}$/
|
|
827
|
+
const FENCE_RUN = /^(`{3,}|~{3,})(.*)$/
|
|
828
|
+
// Indentation of four columns or more: with no list or quote marker before it,
|
|
829
|
+
// the line is indented code, and never the start of a fence.
|
|
830
|
+
const INDENTED_CODE = /^(?: {4}| {0,3}\t)/
|
|
831
|
+
const COMMENT_OPEN = /^ {0,3}<!--/
|
|
832
|
+
// GitHub Markdown ends a comment block only at a literal `-->`. HTML error
|
|
833
|
+
// recovery in a browser also ends a comment at `--!>`, so a comment holding
|
|
834
|
+
// one reads two ways. The scanner looks for both: `-->` ends the block, and a
|
|
835
|
+
// `--!>` seen before it marks the body ambiguous (see `ambiguousComment`).
|
|
836
|
+
const COMMENT_END = /--!?>/g
|
|
837
|
+
const HTML_BLOCK_OPEN = /^ {0,3}<(pre|script|style|textarea)(?=[ \t>]|$)/i
|
|
838
|
+
const BLOCK_QUOTE = /^ {0,3}>/
|
|
839
|
+
const NOT_INDENTED = /^ {0,3}[^ \t]/
|
|
840
|
+
const BLANK = /^[ \t]*$/
|
|
841
|
+
// Lines of these kinds are not rendered as Markdown, so a URL in one is not an embed.
|
|
842
|
+
const HIDDEN_KINDS = new Set(['fence', 'comment', 'html'])
|
|
843
|
+
const LINE_BREAK = /\r\n|\n|\r/g
|
|
844
|
+
const ENDS_WITH_BREAK = /(?:\r\n|\n|\r)$/
|
|
845
|
+
|
|
846
|
+
/**
|
|
847
|
+
* Reads one line of a comment block. `closed` is true when the line holds a
|
|
848
|
+
* `-->`; `ambiguous` is true when a `--!>` comes before it (or the line holds
|
|
849
|
+
* no `-->`). A `--!>` after the closing `-->` is outside the comment.
|
|
850
|
+
* @param {string} content
|
|
851
|
+
*/
|
|
852
|
+
function readCommentLine(content) {
|
|
853
|
+
let ambiguous = false
|
|
854
|
+
for (const end of content.matchAll(COMMENT_END)) {
|
|
855
|
+
if (end[0] === '-->') {
|
|
856
|
+
return { closed: true, ambiguous }
|
|
857
|
+
}
|
|
858
|
+
ambiguous = true
|
|
859
|
+
}
|
|
860
|
+
return { closed: false, ambiguous }
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* Splits on CRLF, LF or CR. Every line keeps its terminator's position, so
|
|
865
|
+
* the lines tile the text exactly.
|
|
866
|
+
* @param {string} text
|
|
867
|
+
* @returns {Array<{ text: string, start: number, next: number, eol: string }>}
|
|
868
|
+
*/
|
|
869
|
+
function splitLines(text) {
|
|
870
|
+
const lines = []
|
|
871
|
+
let start = 0
|
|
872
|
+
for (const match of text.matchAll(LINE_BREAK)) {
|
|
873
|
+
lines.push({
|
|
874
|
+
text: text.slice(start, match.index),
|
|
875
|
+
start,
|
|
876
|
+
next: match.index + match[0].length,
|
|
877
|
+
eol: match[0],
|
|
878
|
+
})
|
|
879
|
+
start = match.index + match[0].length
|
|
880
|
+
}
|
|
881
|
+
if (start < text.length) {
|
|
882
|
+
lines.push({ text: text.slice(start), start, next: text.length, eol: '' })
|
|
883
|
+
}
|
|
884
|
+
return lines
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
/**
|
|
888
|
+
* The terminator used most often; the latest one wins a tie. Null when the
|
|
889
|
+
* text has no terminator at all.
|
|
890
|
+
* @param {Array<{ eol: string }>} lines
|
|
891
|
+
*/
|
|
892
|
+
function dominantEol(lines) {
|
|
893
|
+
const seen = new Map()
|
|
894
|
+
lines.forEach((line, index) => {
|
|
895
|
+
if (line.eol !== '') {
|
|
896
|
+
const entry = seen.get(line.eol) ?? { count: 0, last: 0 }
|
|
897
|
+
seen.set(line.eol, { count: entry.count + 1, last: index })
|
|
898
|
+
}
|
|
899
|
+
})
|
|
900
|
+
let best = null
|
|
901
|
+
for (const [eol, entry] of seen) {
|
|
902
|
+
if (
|
|
903
|
+
best === null ||
|
|
904
|
+
entry.count > best.count ||
|
|
905
|
+
(entry.count === best.count && entry.last > best.last)
|
|
906
|
+
) {
|
|
907
|
+
best = { eol, ...entry }
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
return best === null ? null : best.eol
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
/**
|
|
914
|
+
* Walks the lines once, tracking which block they are in.
|
|
915
|
+
* `headings` holds H1 and H2 only: `line` is the first line of the heading
|
|
916
|
+
* (the first line of the paragraph, for a setext heading). `endState` is the
|
|
917
|
+
* block still open at the end of the text, and `openLine` the line it began
|
|
918
|
+
* on (-1 when none is open). `ambiguousComment` is true when a comment block
|
|
919
|
+
* holds a `--!>` before its `-->` (or a `--!>` and no `-->` at all): GitHub
|
|
920
|
+
* Markdown and a browser then disagree on where it ends.
|
|
921
|
+
* @param {string} text
|
|
922
|
+
*/
|
|
923
|
+
function scanMarkdown(text) {
|
|
924
|
+
const lines = splitLines(text)
|
|
925
|
+
const kinds = []
|
|
926
|
+
const headings = []
|
|
927
|
+
let mode = 'normal'
|
|
928
|
+
let openLine = -1
|
|
929
|
+
let closer = null
|
|
930
|
+
let paragraph = -1
|
|
931
|
+
let ambiguousComment = false
|
|
932
|
+
|
|
933
|
+
/**
|
|
934
|
+
* @param {'fence' | 'comment' | 'html'} next
|
|
935
|
+
* @param {number} index
|
|
936
|
+
* @param {RegExp | null} end null for a comment, which `readCommentLine` ends
|
|
937
|
+
*/
|
|
938
|
+
const open = (next, index, end) => {
|
|
939
|
+
mode = next
|
|
940
|
+
openLine = index
|
|
941
|
+
closer = end
|
|
942
|
+
paragraph = -1
|
|
943
|
+
}
|
|
944
|
+
const close = () => {
|
|
945
|
+
mode = 'normal'
|
|
946
|
+
openLine = -1
|
|
947
|
+
closer = null
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
lines.forEach((line, index) => {
|
|
951
|
+
const content = line.text
|
|
952
|
+
if (mode !== 'normal') {
|
|
953
|
+
// Inside a block every line is hidden; one that ends it is hidden too.
|
|
954
|
+
kinds.push(mode)
|
|
955
|
+
if (mode === 'comment') {
|
|
956
|
+
const read = readCommentLine(content)
|
|
957
|
+
ambiguousComment ||= read.ambiguous
|
|
958
|
+
if (read.closed) {
|
|
959
|
+
close()
|
|
960
|
+
}
|
|
961
|
+
} else if (closer.test(content)) {
|
|
962
|
+
close()
|
|
963
|
+
}
|
|
964
|
+
return
|
|
965
|
+
}
|
|
966
|
+
if (BLANK.test(content)) {
|
|
967
|
+
kinds.push('text')
|
|
968
|
+
paragraph = -1
|
|
969
|
+
return
|
|
970
|
+
}
|
|
971
|
+
const fence = fenceStart(content)
|
|
972
|
+
if (fence !== null) {
|
|
973
|
+
kinds.push('fence')
|
|
974
|
+
open('fence', index, fence.closer)
|
|
975
|
+
return
|
|
976
|
+
}
|
|
977
|
+
if (BLOCK_QUOTE.test(content)) {
|
|
978
|
+
kinds.push('quote')
|
|
979
|
+
paragraph = -1
|
|
980
|
+
return
|
|
981
|
+
}
|
|
982
|
+
if (!NOT_INDENTED.test(content)) {
|
|
983
|
+
// Indented code, or a paragraph continuation: it changes no structure.
|
|
984
|
+
kinds.push('text')
|
|
985
|
+
return
|
|
986
|
+
}
|
|
987
|
+
if (COMMENT_OPEN.test(content)) {
|
|
988
|
+
kinds.push('comment')
|
|
989
|
+
const read = readCommentLine(content)
|
|
990
|
+
ambiguousComment ||= read.ambiguous
|
|
991
|
+
if (!read.closed) {
|
|
992
|
+
open('comment', index, null)
|
|
993
|
+
}
|
|
994
|
+
paragraph = -1
|
|
995
|
+
return
|
|
996
|
+
}
|
|
997
|
+
const html = HTML_BLOCK_OPEN.exec(content)
|
|
998
|
+
if (html !== null) {
|
|
999
|
+
kinds.push('html')
|
|
1000
|
+
const end = new RegExp(`</${html[1]}>`, 'i')
|
|
1001
|
+
if (!end.test(content.slice(html[0].length))) {
|
|
1002
|
+
open('html', index, end)
|
|
1003
|
+
}
|
|
1004
|
+
paragraph = -1
|
|
1005
|
+
return
|
|
1006
|
+
}
|
|
1007
|
+
kinds.push('text')
|
|
1008
|
+
const atx = ATX_HEADING.exec(content)
|
|
1009
|
+
if (atx !== null) {
|
|
1010
|
+
paragraph = -1
|
|
1011
|
+
if (atx[1].length <= 2) {
|
|
1012
|
+
headings.push({
|
|
1013
|
+
line: index,
|
|
1014
|
+
demo: DEMO_HEADING.test(content),
|
|
1015
|
+
screenshots: SCREENSHOTS_HEADING.test(content),
|
|
1016
|
+
})
|
|
1017
|
+
}
|
|
1018
|
+
return
|
|
1019
|
+
}
|
|
1020
|
+
if (paragraph >= 0 && SETEXT_UNDERLINE.test(content)) {
|
|
1021
|
+
headings.push({ line: paragraph, demo: false, screenshots: false })
|
|
1022
|
+
paragraph = -1
|
|
1023
|
+
return
|
|
1024
|
+
}
|
|
1025
|
+
if (THEMATIC_BREAK.test(content) || LIST_ITEM.test(content)) {
|
|
1026
|
+
paragraph = -1
|
|
1027
|
+
return
|
|
1028
|
+
}
|
|
1029
|
+
if (paragraph < 0) {
|
|
1030
|
+
paragraph = index
|
|
1031
|
+
}
|
|
1032
|
+
})
|
|
1033
|
+
|
|
1034
|
+
return {
|
|
1035
|
+
text,
|
|
1036
|
+
lines,
|
|
1037
|
+
kinds,
|
|
1038
|
+
headings,
|
|
1039
|
+
endState: mode,
|
|
1040
|
+
openLine,
|
|
1041
|
+
ambiguousComment,
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
/**
|
|
1046
|
+
* A line that opens a fence, or null. The run may follow whitespace, quote
|
|
1047
|
+
* markers and list markers, so a fence inside a list item or a quote counts.
|
|
1048
|
+
* A run indented four columns or more with no marker before it is indented
|
|
1049
|
+
* code, not a fence. `closer` matches the line that ends it: the same character, at least as
|
|
1050
|
+
* long, nothing after it. A fence that opened at the top level is closed
|
|
1051
|
+
* only by a run indented at most 3 spaces, so an indented line inside it
|
|
1052
|
+
* cannot end it early; one that opened inside a container may be closed with
|
|
1053
|
+
* any indent and markers.
|
|
1054
|
+
* @param {string} content
|
|
1055
|
+
*/
|
|
1056
|
+
function fenceStart(content) {
|
|
1057
|
+
const prefix = CONTAINER_PREFIX.exec(content)[0]
|
|
1058
|
+
if (BLANK.test(prefix) && INDENTED_CODE.test(prefix)) {
|
|
1059
|
+
return null
|
|
1060
|
+
}
|
|
1061
|
+
const run = FENCE_RUN.exec(content.slice(prefix.length))
|
|
1062
|
+
if (run === null || (run[1][0] === '`' && run[2].includes('`'))) {
|
|
1063
|
+
return null
|
|
1064
|
+
}
|
|
1065
|
+
const lead = PLAIN_PREFIX.test(prefix) ? ' {0,3}' : CONTAINER_LEAD
|
|
1066
|
+
return {
|
|
1067
|
+
closer: new RegExp(`^${lead}${run[1][0]}{${run[1].length},}[ \\t]*$`),
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/**
|
|
1072
|
+
* Every Demo section in the scanned text. `endLine` is where the next H1 or
|
|
1073
|
+
* H2 begins, or the end of the text. `contentEnd` is the first line of the
|
|
1074
|
+
* trailing blank run, so the gap before the next section stays outside.
|
|
1075
|
+
* @param {ReturnType<typeof scanMarkdown>} scan
|
|
1076
|
+
*/
|
|
1077
|
+
function findDemoSections(scan) {
|
|
1078
|
+
const { lines, headings } = scan
|
|
1079
|
+
return headings
|
|
1080
|
+
.filter((heading) => heading.demo)
|
|
1081
|
+
.map((heading) => {
|
|
1082
|
+
const following = headings.find((other) => other.line > heading.line)
|
|
1083
|
+
const endLine = following === undefined ? lines.length : following.line
|
|
1084
|
+
let contentEnd = endLine
|
|
1085
|
+
while (
|
|
1086
|
+
contentEnd > heading.line + 1 &&
|
|
1087
|
+
BLANK.test(lines[contentEnd - 1].text)
|
|
1088
|
+
) {
|
|
1089
|
+
contentEnd -= 1
|
|
1090
|
+
}
|
|
1091
|
+
return { headingLine: heading.line, contentEnd, endLine }
|
|
1092
|
+
})
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
/**
|
|
1096
|
+
* Whether a block that never closes began inside the section. Everything
|
|
1097
|
+
* after its opening line is hidden from the scan, so where the section really
|
|
1098
|
+
* ends is unknown, and replacing or copying it could take in lines that are
|
|
1099
|
+
* not its own.
|
|
1100
|
+
* @param {ReturnType<typeof scanMarkdown>} scan
|
|
1101
|
+
* @param {{ headingLine: number, endLine: number }} section
|
|
1102
|
+
*/
|
|
1103
|
+
function endsInOpenBlock(scan, section) {
|
|
1104
|
+
return (
|
|
1105
|
+
scan.endState !== 'normal' &&
|
|
1106
|
+
scan.openLine > section.headingLine &&
|
|
1107
|
+
scan.openLine < section.endLine
|
|
1108
|
+
)
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
/**
|
|
1112
|
+
* Whether every non-blank line under the section's heading is one the skill
|
|
1113
|
+
* writes itself (see `demoText`): an image embed in the raw or the blob form,
|
|
1114
|
+
* with the file stem as its alt text. Any other line, a heading, a fence, HTML,
|
|
1115
|
+
* a table or free prose, means the section is not the managed one or its end
|
|
1116
|
+
* was misjudged, so it is neither replaced nor copied.
|
|
1117
|
+
* @param {ReturnType<typeof scanMarkdown>} scan
|
|
1118
|
+
* @param {{ headingLine: number, contentEnd: number }} section
|
|
1119
|
+
*/
|
|
1120
|
+
function holdsOnlyWrittenLines(scan, section) {
|
|
1121
|
+
for (
|
|
1122
|
+
let line = section.headingLine + 1;
|
|
1123
|
+
line < section.contentEnd;
|
|
1124
|
+
line += 1
|
|
1125
|
+
) {
|
|
1126
|
+
const { text } = scan.lines[line]
|
|
1127
|
+
if (!BLANK.test(text) && !DEMO_EMBED_LINE.test(text)) {
|
|
1128
|
+
return false
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
return true
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
/**
|
|
1135
|
+
* @param {ReturnType<typeof scanMarkdown>} scan
|
|
1136
|
+
* @param {number} line
|
|
1137
|
+
*/
|
|
1138
|
+
function offsetOfLine(scan, line) {
|
|
1139
|
+
return line < scan.lines.length ? scan.lines[line].start : scan.text.length
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
// ── Splice, extract ────────────────────────────────────────────────────────
|
|
1143
|
+
|
|
1144
|
+
/**
|
|
1145
|
+
* The Demo text must be exactly one Demo section: the heading on the first
|
|
1146
|
+
* line, no other H1 or H2, and no fence or comment left open (it would swallow
|
|
1147
|
+
* whatever follows it in the body). Trailing blank lines are dropped.
|
|
1148
|
+
* @param {string} text
|
|
1149
|
+
* @returns {{ ok: true, section: string, scan: ReturnType<typeof scanMarkdown> } | { ok: false }}
|
|
1150
|
+
*/
|
|
1151
|
+
function parseDemoText(text) {
|
|
1152
|
+
const scan = scanMarkdown(text)
|
|
1153
|
+
const [only] = scan.headings
|
|
1154
|
+
if (
|
|
1155
|
+
scan.headings.length !== 1 ||
|
|
1156
|
+
only.line !== 0 ||
|
|
1157
|
+
!only.demo ||
|
|
1158
|
+
scan.endState !== 'normal' ||
|
|
1159
|
+
scan.ambiguousComment
|
|
1160
|
+
) {
|
|
1161
|
+
return { ok: false }
|
|
1162
|
+
}
|
|
1163
|
+
const [section] = findDemoSections(scan)
|
|
1164
|
+
return {
|
|
1165
|
+
ok: true,
|
|
1166
|
+
section: text.slice(0, offsetOfLine(scan, section.contentEnd)),
|
|
1167
|
+
scan,
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/** @param {string} body */
|
|
1172
|
+
function inspectBody(body) {
|
|
1173
|
+
const scan = scanMarkdown(body)
|
|
1174
|
+
return {
|
|
1175
|
+
scan,
|
|
1176
|
+
sections: findDemoSections(scan),
|
|
1177
|
+
screenshots: scan.headings.some((heading) => heading.screenshots),
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
/**
|
|
1182
|
+
* Splices the Demo section into the body: replaces the content under the one
|
|
1183
|
+
* existing heading (the heading line's bytes are kept), or appends the whole
|
|
1184
|
+
* section after exactly one blank line.
|
|
1185
|
+
* @param {string} body
|
|
1186
|
+
* @param {string} demoText
|
|
1187
|
+
*/
|
|
1188
|
+
export function spliceDemo(body, demoText) {
|
|
1189
|
+
const demo = parseDemoText(demoText)
|
|
1190
|
+
if (!demo.ok) {
|
|
1191
|
+
return { kind: 'invalid-demo' }
|
|
1192
|
+
}
|
|
1193
|
+
const { scan, sections, screenshots } = inspectBody(body)
|
|
1194
|
+
const base = { demoCount: sections.length, screenshots }
|
|
1195
|
+
if (scan.ambiguousComment) {
|
|
1196
|
+
return { kind: 'unterminated-region', region: 'comment', ...base }
|
|
1197
|
+
}
|
|
1198
|
+
if (sections.length > 1) {
|
|
1199
|
+
return { kind: 'multiple-demo-sections', ...base }
|
|
1200
|
+
}
|
|
1201
|
+
const eol = dominantEol(scan.lines) ?? dominantEol(demo.scan.lines) ?? '\n'
|
|
1202
|
+
|
|
1203
|
+
if (sections.length === 1) {
|
|
1204
|
+
const [section] = sections
|
|
1205
|
+
if (endsInOpenBlock(scan, section)) {
|
|
1206
|
+
return { kind: 'unterminated-region', region: scan.endState, ...base }
|
|
1207
|
+
}
|
|
1208
|
+
if (!holdsOnlyWrittenLines(scan, section)) {
|
|
1209
|
+
return { kind: 'unexpected-section-content', ...base }
|
|
1210
|
+
}
|
|
1211
|
+
const heading = scan.lines[section.headingLine]
|
|
1212
|
+
const content = demo.section.slice(demo.scan.lines[0].next)
|
|
1213
|
+
const suffix = body.slice(offsetOfLine(scan, section.contentEnd))
|
|
1214
|
+
const headBreak = heading.eol === '' && content.length > 0 ? eol : ''
|
|
1215
|
+
const tailBreak =
|
|
1216
|
+
suffix.length > 0 && content.length > 0 && !ENDS_WITH_BREAK.test(content)
|
|
1217
|
+
? eol
|
|
1218
|
+
: ''
|
|
1219
|
+
const output = joinPieces([
|
|
1220
|
+
body.slice(0, heading.next),
|
|
1221
|
+
headBreak,
|
|
1222
|
+
content,
|
|
1223
|
+
tailBreak,
|
|
1224
|
+
suffix,
|
|
1225
|
+
])
|
|
1226
|
+
if (output === null) {
|
|
1227
|
+
return { kind: 'line-ending-conflict', ...base }
|
|
1228
|
+
}
|
|
1229
|
+
return {
|
|
1230
|
+
kind: 'spliced',
|
|
1231
|
+
action: output === body ? 'unchanged' : 'replaced',
|
|
1232
|
+
output,
|
|
1233
|
+
...base,
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
if (scan.endState !== 'normal') {
|
|
1238
|
+
return { kind: 'unterminated-region', region: scan.endState, ...base }
|
|
1239
|
+
}
|
|
1240
|
+
const output = joinPieces([
|
|
1241
|
+
body,
|
|
1242
|
+
separatorBeforeAppend(scan.lines, eol),
|
|
1243
|
+
demo.section,
|
|
1244
|
+
])
|
|
1245
|
+
if (output === null) {
|
|
1246
|
+
return { kind: 'line-ending-conflict', ...base }
|
|
1247
|
+
}
|
|
1248
|
+
return { kind: 'spliced', action: 'appended', output, ...base }
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/**
|
|
1252
|
+
* Concatenates the pieces, or returns null when a CR would meet an LF. That
|
|
1253
|
+
* only happens in a body that mixes terminators, and the pair would fuse into
|
|
1254
|
+
* one CRLF and silently drop a line, so the helper stops instead.
|
|
1255
|
+
* @param {string[]} pieces
|
|
1256
|
+
*/
|
|
1257
|
+
function joinPieces(pieces) {
|
|
1258
|
+
const present = pieces.filter((piece) => piece.length > 0)
|
|
1259
|
+
for (let i = 1; i < present.length; i += 1) {
|
|
1260
|
+
if (present[i - 1].endsWith('\r') && present[i].startsWith('\n')) {
|
|
1261
|
+
return null
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
return present.join('')
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
/**
|
|
1268
|
+
* Just enough line breaks to end the last line and leave one blank line.
|
|
1269
|
+
* A body with no non-blank content needs no blank line.
|
|
1270
|
+
* @param {Array<{ text: string, eol: string }>} lines
|
|
1271
|
+
* @param {string} eol
|
|
1272
|
+
*/
|
|
1273
|
+
function separatorBeforeAppend(lines, eol) {
|
|
1274
|
+
if (lines.length === 0) {
|
|
1275
|
+
return ''
|
|
1276
|
+
}
|
|
1277
|
+
const last = lines[lines.length - 1]
|
|
1278
|
+
const endsLine = last.eol === '' ? eol : ''
|
|
1279
|
+
if (lines.every((line) => BLANK.test(line.text))) {
|
|
1280
|
+
return endsLine
|
|
1281
|
+
}
|
|
1282
|
+
return BLANK.test(last.text) ? endsLine : `${endsLine}${eol}`
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
/**
|
|
1286
|
+
* The existing Demo section, heading through its last non-blank line.
|
|
1287
|
+
* @param {string} body
|
|
1288
|
+
*/
|
|
1289
|
+
export function extractDemo(body) {
|
|
1290
|
+
const { scan, sections, screenshots } = inspectBody(body)
|
|
1291
|
+
const base = { demoCount: sections.length, screenshots }
|
|
1292
|
+
if (scan.ambiguousComment) {
|
|
1293
|
+
return { kind: 'unterminated-region', region: 'comment', ...base }
|
|
1294
|
+
}
|
|
1295
|
+
if (sections.length > 1) {
|
|
1296
|
+
return { kind: 'multiple-demo-sections', ...base }
|
|
1297
|
+
}
|
|
1298
|
+
if (sections.length === 0) {
|
|
1299
|
+
return { kind: 'absent', ...base }
|
|
1300
|
+
}
|
|
1301
|
+
const [section] = sections
|
|
1302
|
+
if (endsInOpenBlock(scan, section)) {
|
|
1303
|
+
return { kind: 'unterminated-region', region: scan.endState, ...base }
|
|
1304
|
+
}
|
|
1305
|
+
if (!holdsOnlyWrittenLines(scan, section)) {
|
|
1306
|
+
return { kind: 'unexpected-section-content', ...base }
|
|
1307
|
+
}
|
|
1308
|
+
return {
|
|
1309
|
+
kind: 'extracted',
|
|
1310
|
+
section: body.slice(
|
|
1311
|
+
offsetOfLine(scan, section.headingLine),
|
|
1312
|
+
offsetOfLine(scan, section.contentEnd),
|
|
1313
|
+
),
|
|
1314
|
+
...base,
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
// ── Managed URLs ───────────────────────────────────────────────────────────
|
|
1319
|
+
|
|
1320
|
+
const OWNER_OR_REPO = '[A-Za-z0-9._-]+'
|
|
1321
|
+
const SHA_SEGMENT = '[0-9a-fA-F]{40}'
|
|
1322
|
+
// docs/evidence/<slug>/<name>, as classify writes it: a slug of at most 60
|
|
1323
|
+
// characters over [a-z0-9-], a name over [a-z0-9._-] with no "..", and an
|
|
1324
|
+
// extension that follows the detected image type.
|
|
1325
|
+
const EVIDENCE_PATH =
|
|
1326
|
+
'docs/evidence/[a-z0-9](?:[a-z0-9-]{0,58}[a-z0-9])?/(?!.*\\.\\.)[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?\\.(?:png|jpg|gif|webp)'
|
|
1327
|
+
const MANAGED_PATH = new RegExp(`^${EVIDENCE_PATH}$`)
|
|
1328
|
+
const RAW_URL = new RegExp(
|
|
1329
|
+
`^https://raw\\.githubusercontent\\.com/(${OWNER_OR_REPO})/(${OWNER_OR_REPO})/(${SHA_SEGMENT})/(${EVIDENCE_PATH})$`,
|
|
1330
|
+
)
|
|
1331
|
+
const BLOB_URL = new RegExp(
|
|
1332
|
+
`^https://github\\.com/(${OWNER_OR_REPO})/(${OWNER_OR_REPO})/blob/(${SHA_SEGMENT})/(${EVIDENCE_PATH})(?:\\?raw=true)?$`,
|
|
1333
|
+
)
|
|
1334
|
+
// A line of a Demo section the skill writes: ``, where
|
|
1335
|
+
// the URL is the raw or the blob form `demoText` builds for a managed path.
|
|
1336
|
+
const DEMO_EMBED_LINE = new RegExp(
|
|
1337
|
+
`^!\\[[a-z0-9][a-z0-9._-]*\\]\\((?:https://raw\\.githubusercontent\\.com/${OWNER_OR_REPO}/${OWNER_OR_REPO}/${SHA_SEGMENT}/${EVIDENCE_PATH}|https://github\\.com/${OWNER_OR_REPO}/${OWNER_OR_REPO}/blob/${SHA_SEGMENT}/${EVIDENCE_PATH}\\?raw=true)\\)$`,
|
|
1338
|
+
)
|
|
1339
|
+
const URL_CANDIDATE = /https?:\/\/[^\s<>()"'`[\]]+/g
|
|
1340
|
+
const TRAILING_PUNCTUATION = /[.,;:!?]+$/
|
|
1341
|
+
const HEAD_REPO = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/
|
|
1342
|
+
const FULL_SHA = /^[0-9a-f]{40}$/
|
|
1343
|
+
|
|
1344
|
+
/**
|
|
1345
|
+
* @param {string} url
|
|
1346
|
+
* @param {string} headRepo
|
|
1347
|
+
*/
|
|
1348
|
+
function parseManagedUrl(url, headRepo) {
|
|
1349
|
+
const raw = RAW_URL.exec(url)
|
|
1350
|
+
const blob = raw === null ? BLOB_URL.exec(url) : null
|
|
1351
|
+
const match = raw ?? blob
|
|
1352
|
+
if (match === null) {
|
|
1353
|
+
return null
|
|
1354
|
+
}
|
|
1355
|
+
const [, owner, repo, sha, urlPath] = match
|
|
1356
|
+
if (`${owner}/${repo}`.toLowerCase() !== headRepo.toLowerCase()) {
|
|
1357
|
+
return null
|
|
1358
|
+
}
|
|
1359
|
+
const prefix =
|
|
1360
|
+
raw === null
|
|
1361
|
+
? `https://github.com/${owner}/${repo}/blob/`
|
|
1362
|
+
: `https://raw.githubusercontent.com/${owner}/${repo}/`
|
|
1363
|
+
return {
|
|
1364
|
+
owner,
|
|
1365
|
+
repo,
|
|
1366
|
+
sha,
|
|
1367
|
+
path: urlPath,
|
|
1368
|
+
form: raw === null ? 'blob' : 'raw',
|
|
1369
|
+
shaOffset: prefix.length,
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
/**
|
|
1374
|
+
* Every http(s) URL in lines that render, in order: managed ones with the
|
|
1375
|
+
* absolute offset of their ref, and a count of all the others. URLs in fenced
|
|
1376
|
+
* code and raw HTML blocks are not embeds, so they count as neither.
|
|
1377
|
+
* @param {string} body
|
|
1378
|
+
* @param {string} headRepo
|
|
1379
|
+
*/
|
|
1380
|
+
function scanUrls(body, headRepo) {
|
|
1381
|
+
const scan = scanMarkdown(body)
|
|
1382
|
+
const managed = []
|
|
1383
|
+
let ignored = 0
|
|
1384
|
+
scan.lines.forEach((line, index) => {
|
|
1385
|
+
if (HIDDEN_KINDS.has(scan.kinds[index])) {
|
|
1386
|
+
return
|
|
1387
|
+
}
|
|
1388
|
+
for (const match of line.text.matchAll(URL_CANDIDATE)) {
|
|
1389
|
+
const url = match[0].replace(TRAILING_PUNCTUATION, '')
|
|
1390
|
+
const parsed = parseManagedUrl(url, headRepo)
|
|
1391
|
+
if (parsed === null) {
|
|
1392
|
+
ignored += 1
|
|
1393
|
+
} else {
|
|
1394
|
+
const { shaOffset, ...fields } = parsed
|
|
1395
|
+
managed.push({
|
|
1396
|
+
...fields,
|
|
1397
|
+
offset: line.start + match.index + shaOffset,
|
|
1398
|
+
})
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
})
|
|
1402
|
+
return { managed, ignored }
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
/**
|
|
1406
|
+
* @param {string} body
|
|
1407
|
+
* @param {string} headRepo
|
|
1408
|
+
*/
|
|
1409
|
+
export function findManagedUrls(body, headRepo) {
|
|
1410
|
+
const { managed, ignored } = scanUrls(body, headRepo)
|
|
1411
|
+
return {
|
|
1412
|
+
urls: managed.map(({ offset, ...fields }) => fields),
|
|
1413
|
+
ignored,
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
/**
|
|
1418
|
+
* Swaps the 40-hex ref of each managed URL whose path has a different pin.
|
|
1419
|
+
* @param {string} body
|
|
1420
|
+
* @param {string} headRepo
|
|
1421
|
+
* @param {Record<string, string>} pins
|
|
1422
|
+
*/
|
|
1423
|
+
export function repinBody(body, headRepo, pins) {
|
|
1424
|
+
const { managed, ignored } = scanUrls(body, headRepo)
|
|
1425
|
+
const stale = managed.filter(
|
|
1426
|
+
(url) => Object.hasOwn(pins, url.path) && pins[url.path] !== url.sha,
|
|
1427
|
+
)
|
|
1428
|
+
let output = body
|
|
1429
|
+
for (const url of [...stale].reverse()) {
|
|
1430
|
+
output = `${output.slice(0, url.offset)}${pins[url.path]}${output.slice(url.offset + 40)}`
|
|
1431
|
+
}
|
|
1432
|
+
return {
|
|
1433
|
+
output,
|
|
1434
|
+
rewritten: stale.length,
|
|
1435
|
+
ignored,
|
|
1436
|
+
unmappedPaths: [
|
|
1437
|
+
...new Set(
|
|
1438
|
+
managed
|
|
1439
|
+
.filter((url) => !Object.hasOwn(pins, url.path))
|
|
1440
|
+
.map((url) => url.path),
|
|
1441
|
+
),
|
|
1442
|
+
].sort(),
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
// ── Git ────────────────────────────────────────────────────────────────────
|
|
1447
|
+
//
|
|
1448
|
+
// `pins` and `cleanup-set` run git themselves, from the top of the repository
|
|
1449
|
+
// that holds the working directory. Their input is the caller's own data (the
|
|
1450
|
+
// head commit, the base branch, the managed paths); git output is never an
|
|
1451
|
+
// input, so no SHA is ever transcribed. Every call is an argv array, never a
|
|
1452
|
+
// shell, with a timeout, no stdin and a sanitized environment. Git output is
|
|
1453
|
+
// parsed and never echoed: the report carries fixed codes, paths and SHAs.
|
|
1454
|
+
|
|
1455
|
+
const DEFAULT_GIT_TIMEOUT_MS = 10_000
|
|
1456
|
+
const MAX_GIT_TIMEOUT_MS = 120_000
|
|
1457
|
+
const GIT_MAX_BUFFER = 64 * 1024 * 1024
|
|
1458
|
+
const DIGEST_DOMAIN = 'pr-attachments/cleanup-set/v1'
|
|
1459
|
+
// The only GIT_* variables that reach git: they isolate config, and cannot
|
|
1460
|
+
// redirect the repository, the index or the object store.
|
|
1461
|
+
const GIT_ENV_PASSTHROUGH = new Set([
|
|
1462
|
+
'GIT_CONFIG_GLOBAL',
|
|
1463
|
+
'GIT_CONFIG_NOSYSTEM',
|
|
1464
|
+
])
|
|
1465
|
+
const COMMIT_SHA = /^[0-9a-f]{40}$/
|
|
1466
|
+
|
|
1467
|
+
/**
|
|
1468
|
+
* Every other GIT_* variable is dropped, so the caller's environment cannot
|
|
1469
|
+
* point git at another repository, index or object store.
|
|
1470
|
+
*/
|
|
1471
|
+
function gitEnv() {
|
|
1472
|
+
const env = {}
|
|
1473
|
+
for (const [key, value] of Object.entries(process.env)) {
|
|
1474
|
+
if (
|
|
1475
|
+
value !== undefined &&
|
|
1476
|
+
(!key.startsWith('GIT_') || GIT_ENV_PASSTHROUGH.has(key))
|
|
1477
|
+
) {
|
|
1478
|
+
env[key] = value
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
env.LC_ALL = 'C'
|
|
1482
|
+
env.LANG = 'C'
|
|
1483
|
+
env.GIT_TERMINAL_PROMPT = '0'
|
|
1484
|
+
env.GIT_OPTIONAL_LOCKS = '0'
|
|
1485
|
+
return env
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
/**
|
|
1489
|
+
* @param {'git-timeout' | 'git-failed'} kind
|
|
1490
|
+
* @param {number | null} status
|
|
1491
|
+
*/
|
|
1492
|
+
function gitFailure(kind, status) {
|
|
1493
|
+
return Object.assign(new Error(kind), { gitFailure: { kind, status } })
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
/** @param {unknown} error */
|
|
1497
|
+
function failureOf(error) {
|
|
1498
|
+
return error instanceof Error && 'gitFailure' in error
|
|
1499
|
+
? error.gitFailure
|
|
1500
|
+
: null
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
/**
|
|
1504
|
+
* Runs git and returns its stdout. Only the listed exit statuses are
|
|
1505
|
+
* answers; any other status, a timeout or a launch failure throws. The step
|
|
1506
|
+
* name is recorded first, so a failure reports where it stopped.
|
|
1507
|
+
* @param {{ cwd: string, timeoutMs: number, reached: { step: string } }} ctx
|
|
1508
|
+
* @param {string} step
|
|
1509
|
+
* @param {string[]} args
|
|
1510
|
+
* @param {number[]} okStatuses
|
|
1511
|
+
*/
|
|
1512
|
+
function runGit(ctx, step, args, okStatuses = [0]) {
|
|
1513
|
+
ctx.reached.step = step
|
|
1514
|
+
const result = spawnSync('git', ['--literal-pathspecs', ...args], {
|
|
1515
|
+
cwd: ctx.cwd,
|
|
1516
|
+
env: gitEnv(),
|
|
1517
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1518
|
+
timeout: ctx.timeoutMs,
|
|
1519
|
+
maxBuffer: GIT_MAX_BUFFER,
|
|
1520
|
+
windowsHide: true,
|
|
1521
|
+
})
|
|
1522
|
+
if (result.error) {
|
|
1523
|
+
throw gitFailure(
|
|
1524
|
+
result.error.code === 'ETIMEDOUT' ? 'git-timeout' : 'git-failed',
|
|
1525
|
+
null,
|
|
1526
|
+
)
|
|
1527
|
+
}
|
|
1528
|
+
if (result.status === null || !okStatuses.includes(result.status)) {
|
|
1529
|
+
throw gitFailure('git-failed', result.status)
|
|
1530
|
+
}
|
|
1531
|
+
return { status: result.status, stdout: result.stdout }
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
/** @param {Buffer} output */
|
|
1535
|
+
function firstLine(output) {
|
|
1536
|
+
return output.toString('utf8').split('\n')[0].trim()
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
/**
|
|
1540
|
+
* Runs from the top of the repository, so every path is root-relative no
|
|
1541
|
+
* matter which subdirectory the helper started in.
|
|
1542
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1543
|
+
*/
|
|
1544
|
+
function locateRepo(ctx) {
|
|
1545
|
+
const top = firstLine(
|
|
1546
|
+
runGit(ctx, 'locate-repo', ['rev-parse', '--show-toplevel']).stdout,
|
|
1547
|
+
)
|
|
1548
|
+
if (top.length === 0) {
|
|
1549
|
+
throw gitFailure('git-failed', 0)
|
|
1550
|
+
}
|
|
1551
|
+
ctx.cwd = top
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
/**
|
|
1555
|
+
* The commit a revision names, or null when the repository does not have it.
|
|
1556
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1557
|
+
* @param {string} step
|
|
1558
|
+
* @param {string} rev
|
|
1559
|
+
*/
|
|
1560
|
+
function resolveCommit(ctx, step, rev) {
|
|
1561
|
+
const result = runGit(
|
|
1562
|
+
ctx,
|
|
1563
|
+
step,
|
|
1564
|
+
['rev-parse', '--verify', '--quiet', `${rev}^{commit}`],
|
|
1565
|
+
[0, 1],
|
|
1566
|
+
)
|
|
1567
|
+
return result.status === 0 ? firstLine(result.stdout) : null
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
/**
|
|
1571
|
+
* The object id at `<commit>:<path>`, or null when the path is not there.
|
|
1572
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1573
|
+
* @param {string} step
|
|
1574
|
+
* @param {string} spec
|
|
1575
|
+
*/
|
|
1576
|
+
function resolveObject(ctx, step, spec) {
|
|
1577
|
+
const result = runGit(
|
|
1578
|
+
ctx,
|
|
1579
|
+
step,
|
|
1580
|
+
['rev-parse', '--verify', '--quiet', spec],
|
|
1581
|
+
[0, 1],
|
|
1582
|
+
)
|
|
1583
|
+
return result.status === 0 ? firstLine(result.stdout) : null
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
/**
|
|
1587
|
+
* Splits NUL-terminated output. Names that are not valid UTF-8 cannot make
|
|
1588
|
+
* the round trip through JSON and back to git, so they fail the run.
|
|
1589
|
+
* @param {Buffer} output
|
|
1590
|
+
*/
|
|
1591
|
+
function parseNulList(output) {
|
|
1592
|
+
let text
|
|
1593
|
+
try {
|
|
1594
|
+
text = new TextDecoder('utf-8', { fatal: true }).decode(output)
|
|
1595
|
+
} catch {
|
|
1596
|
+
throw gitFailure('git-failed', 0)
|
|
1597
|
+
}
|
|
1598
|
+
return text.split('\0').filter((entry) => entry.length > 0)
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
// ── pins ───────────────────────────────────────────────────────────────────
|
|
1602
|
+
|
|
1603
|
+
/**
|
|
1604
|
+
* The newest commit reachable from the head that added or modified the path.
|
|
1605
|
+
* Renames count as an add, so the commit that moved a file in is the answer.
|
|
1606
|
+
* `-m` lets a merge that resolved the file to bytes neither parent had count
|
|
1607
|
+
* as a modification; topological order puts a child before its parents.
|
|
1608
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1609
|
+
* @param {string} headOid
|
|
1610
|
+
* @param {string} file
|
|
1611
|
+
*/
|
|
1612
|
+
function newestCommitFor(ctx, headOid, file) {
|
|
1613
|
+
return newestCommitForPaths(ctx, headOid, [file])
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
/**
|
|
1617
|
+
* The newest commit reachable from the head that added or modified any of the
|
|
1618
|
+
* paths, chosen the way `newestCommitFor` chooses for one.
|
|
1619
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1620
|
+
* @param {string} headOid
|
|
1621
|
+
* @param {string[]} files
|
|
1622
|
+
*/
|
|
1623
|
+
function newestCommitForPaths(ctx, headOid, files) {
|
|
1624
|
+
const result = runGit(ctx, 'newest-commit', [
|
|
1625
|
+
'log',
|
|
1626
|
+
'--topo-order',
|
|
1627
|
+
'-m',
|
|
1628
|
+
'--no-renames',
|
|
1629
|
+
'--diff-filter=AM',
|
|
1630
|
+
'--no-show-signature',
|
|
1631
|
+
'--format=%H',
|
|
1632
|
+
headOid,
|
|
1633
|
+
'--',
|
|
1634
|
+
...files,
|
|
1635
|
+
])
|
|
1636
|
+
const first = firstLine(result.stdout)
|
|
1637
|
+
return COMMIT_SHA.test(first) ? first : null
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
/**
|
|
1641
|
+
* A pin is valid when the commit exists, is reachable from the head, and holds
|
|
1642
|
+
* the same blob at the path as the newest add-or-modify commit. Reachability
|
|
1643
|
+
* alone would let a replaced image come back.
|
|
1644
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1645
|
+
* @param {string} headOid
|
|
1646
|
+
* @param {string} file
|
|
1647
|
+
* @param {string} newestBlob
|
|
1648
|
+
* @param {string} sha
|
|
1649
|
+
*/
|
|
1650
|
+
function judgePin(ctx, headOid, file, newestBlob, sha) {
|
|
1651
|
+
if (resolveCommit(ctx, 'pin-commit', sha) === null) {
|
|
1652
|
+
return { sha, valid: false, reason: 'commit-unknown' }
|
|
1653
|
+
}
|
|
1654
|
+
const reachable = runGit(
|
|
1655
|
+
ctx,
|
|
1656
|
+
'pin-reachable',
|
|
1657
|
+
['merge-base', '--is-ancestor', sha, headOid],
|
|
1658
|
+
[0, 1],
|
|
1659
|
+
)
|
|
1660
|
+
if (reachable.status !== 0) {
|
|
1661
|
+
return { sha, valid: false, reason: 'not-reachable' }
|
|
1662
|
+
}
|
|
1663
|
+
const blob = resolveObject(ctx, 'pin-blob', `${sha}:${file}`)
|
|
1664
|
+
if (blob === null) {
|
|
1665
|
+
return { sha, valid: false, reason: 'path-missing' }
|
|
1666
|
+
}
|
|
1667
|
+
if (blob !== newestBlob) {
|
|
1668
|
+
return { sha, valid: false, reason: 'blob-differs' }
|
|
1669
|
+
}
|
|
1670
|
+
return { sha, valid: true, reason: null }
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
/**
|
|
1674
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1675
|
+
* @param {string} headOid
|
|
1676
|
+
* @param {string} file
|
|
1677
|
+
* @param {string[]} shas
|
|
1678
|
+
*/
|
|
1679
|
+
function assessPath(ctx, headOid, file, shas) {
|
|
1680
|
+
const newest = newestCommitFor(ctx, headOid, file)
|
|
1681
|
+
if (newest === null) {
|
|
1682
|
+
return {
|
|
1683
|
+
path: file,
|
|
1684
|
+
state: 'unresolved',
|
|
1685
|
+
newest_commit: null,
|
|
1686
|
+
current: shas.map((sha) => ({
|
|
1687
|
+
sha,
|
|
1688
|
+
valid: false,
|
|
1689
|
+
reason: 'no-newest-commit',
|
|
1690
|
+
})),
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
const newestBlob = resolveObject(ctx, 'newest-blob', `${newest}:${file}`)
|
|
1694
|
+
if (newestBlob === null) {
|
|
1695
|
+
throw gitFailure('git-failed', 1)
|
|
1696
|
+
}
|
|
1697
|
+
const current = shas.map((sha) =>
|
|
1698
|
+
judgePin(ctx, headOid, file, newestBlob, sha),
|
|
1699
|
+
)
|
|
1700
|
+
return {
|
|
1701
|
+
path: file,
|
|
1702
|
+
state: current.every((pin) => pin.valid) ? 'valid' : 'stale',
|
|
1703
|
+
newest_commit: newest,
|
|
1704
|
+
current,
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
/**
|
|
1709
|
+
* @param {Parameters<typeof runGit>[0] & { reached: { step: string, paths_done: number, paths_total: number } }} ctx
|
|
1710
|
+
* @param {{ headOid: string, paths: Map<string, string[]> }} input
|
|
1711
|
+
*/
|
|
1712
|
+
function pinsReport(ctx, input) {
|
|
1713
|
+
locateRepo(ctx)
|
|
1714
|
+
if (resolveCommit(ctx, 'resolve-head', input.headOid) === null) {
|
|
1715
|
+
return outcome('pins', 1, 'head-not-found')
|
|
1716
|
+
}
|
|
1717
|
+
const entries = []
|
|
1718
|
+
for (const file of [...input.paths.keys()].sort()) {
|
|
1719
|
+
entries.push(assessPath(ctx, input.headOid, file, input.paths.get(file)))
|
|
1720
|
+
ctx.reached.paths_done += 1
|
|
1721
|
+
}
|
|
1722
|
+
// The map repin-rewrite takes: a stale path goes to its newest commit, and a
|
|
1723
|
+
// path whose pins are all valid keeps the pin it has.
|
|
1724
|
+
const pins = {}
|
|
1725
|
+
for (const entry of entries) {
|
|
1726
|
+
if (entry.state === 'stale') {
|
|
1727
|
+
pins[entry.path] = entry.newest_commit
|
|
1728
|
+
} else if (entry.state === 'valid') {
|
|
1729
|
+
pins[entry.path] = entry.current[0].sha
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
const unresolved = entries
|
|
1733
|
+
.filter((entry) => entry.state === 'unresolved')
|
|
1734
|
+
.map((entry) => entry.path)
|
|
1735
|
+
const parts = {
|
|
1736
|
+
head: input.headOid,
|
|
1737
|
+
paths: entries,
|
|
1738
|
+
pins,
|
|
1739
|
+
unresolved_paths: unresolved,
|
|
1740
|
+
}
|
|
1741
|
+
if (unresolved.length === entries.length) {
|
|
1742
|
+
return outcome('pins', 1, 'unresolved', parts)
|
|
1743
|
+
}
|
|
1744
|
+
if (unresolved.length > 0) {
|
|
1745
|
+
return outcome('pins', 1, 'partial', parts)
|
|
1746
|
+
}
|
|
1747
|
+
const stale = entries.some((entry) => entry.state === 'stale')
|
|
1748
|
+
return outcome('pins', 0, stale ? 'repin-needed' : 'already-valid', parts)
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
// ── pins: select ─────────────────────────────────────────────────────────
|
|
1752
|
+
|
|
1753
|
+
/**
|
|
1754
|
+
* The Demo text for the selected pins: the heading, then one embed per path
|
|
1755
|
+
* in the order the caller listed them. The caption is the file stem.
|
|
1756
|
+
* @param {'raw' | 'blob'} form
|
|
1757
|
+
* @param {string} headRepo
|
|
1758
|
+
* @param {Array<[string, string]>} chosen path and pin pairs, in order
|
|
1759
|
+
*/
|
|
1760
|
+
function demoText(form, headRepo, chosen) {
|
|
1761
|
+
let text = '## Demo\n'
|
|
1762
|
+
for (const [file, pin] of chosen) {
|
|
1763
|
+
const url =
|
|
1764
|
+
form === 'raw'
|
|
1765
|
+
? `https://raw.githubusercontent.com/${headRepo}/${pin}/${file}`
|
|
1766
|
+
: `https://github.com/${headRepo}/blob/${pin}/${file}?raw=true`
|
|
1767
|
+
text += `\n\n`
|
|
1768
|
+
}
|
|
1769
|
+
return text
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
/**
|
|
1773
|
+
* Picks, for each path, the commit to embed: the newest add-or-modify commit
|
|
1774
|
+
* on the head, chosen as `pins` chooses it. The blob at that commit must be
|
|
1775
|
+
* the blob the caller expects (the `blob_sha` that classify reported), or the
|
|
1776
|
+
* run stops: a pin that serves other bytes would pass a check that looks only
|
|
1777
|
+
* at the content type. On success the Demo text is written to `out_file`.
|
|
1778
|
+
* @param {Parameters<typeof runGit>[0] & { reached: { step: string, paths_done: number, paths_total: number } }} ctx
|
|
1779
|
+
* @param {{ headOid?: string, expected: Map<string, string>, headRepo: string, form: 'raw' | 'blob', outFile: string }} input
|
|
1780
|
+
*/
|
|
1781
|
+
function selectReport(ctx, input) {
|
|
1782
|
+
locateRepo(ctx)
|
|
1783
|
+
const headOid = resolveCommit(ctx, 'resolve-head', input.headOid ?? 'HEAD')
|
|
1784
|
+
if (headOid === null) {
|
|
1785
|
+
return outcome('pins', 1, 'head-not-found')
|
|
1786
|
+
}
|
|
1787
|
+
const chosen = []
|
|
1788
|
+
const unresolved = []
|
|
1789
|
+
const mismatched = []
|
|
1790
|
+
for (const [file, expectedBlob] of input.expected) {
|
|
1791
|
+
const pin = newestCommitFor(ctx, headOid, file)
|
|
1792
|
+
ctx.reached.paths_done += 1
|
|
1793
|
+
if (pin === null) {
|
|
1794
|
+
unresolved.push(file)
|
|
1795
|
+
continue
|
|
1796
|
+
}
|
|
1797
|
+
const blob = resolveObject(ctx, 'pin-blob', `${pin}:${file}`)
|
|
1798
|
+
if (blob === null) {
|
|
1799
|
+
throw gitFailure('git-failed', 1)
|
|
1800
|
+
}
|
|
1801
|
+
chosen.push([file, pin])
|
|
1802
|
+
if (blob !== expectedBlob) {
|
|
1803
|
+
mismatched.push({
|
|
1804
|
+
path: file,
|
|
1805
|
+
pin,
|
|
1806
|
+
blob_sha: blob,
|
|
1807
|
+
expected_blob_sha: expectedBlob,
|
|
1808
|
+
})
|
|
1809
|
+
}
|
|
1810
|
+
}
|
|
1811
|
+
const parts = {
|
|
1812
|
+
head: headOid,
|
|
1813
|
+
pins: Object.fromEntries(chosen),
|
|
1814
|
+
unresolved_paths: unresolved,
|
|
1815
|
+
mismatched,
|
|
1816
|
+
}
|
|
1817
|
+
if (unresolved.length > 0) {
|
|
1818
|
+
return outcome('pins', 1, 'unresolved', parts)
|
|
1819
|
+
}
|
|
1820
|
+
if (mismatched.length > 0) {
|
|
1821
|
+
return outcome('pins', 1, 'pin-mismatch', parts)
|
|
1822
|
+
}
|
|
1823
|
+
const evidence = newestCommitForPaths(ctx, headOid, [
|
|
1824
|
+
...input.expected.keys(),
|
|
1825
|
+
])
|
|
1826
|
+
if (evidence === null) {
|
|
1827
|
+
throw gitFailure('git-failed', 1)
|
|
1828
|
+
}
|
|
1829
|
+
const written = writeBody(
|
|
1830
|
+
'pins',
|
|
1831
|
+
input.outFile,
|
|
1832
|
+
demoText(input.form, input.headRepo, chosen),
|
|
1833
|
+
{ ...parts, evidence_sha: evidence },
|
|
1834
|
+
)
|
|
1835
|
+
return 'exitCode' in written
|
|
1836
|
+
? written
|
|
1837
|
+
: outcome('pins', 0, 'selected', written)
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
// ── cleanup-set ────────────────────────────────────────────────────────────
|
|
1841
|
+
|
|
1842
|
+
/**
|
|
1843
|
+
* A digest of the set that does not depend on the order the paths arrive in.
|
|
1844
|
+
* @param {string[]} paths
|
|
1845
|
+
*/
|
|
1846
|
+
export function cleanupDigest(paths) {
|
|
1847
|
+
const hash = createHash('sha256').update(DIGEST_DOMAIN).update('\0')
|
|
1848
|
+
for (const file of [...paths].sort()) {
|
|
1849
|
+
hash.update(file).update('\0')
|
|
1850
|
+
}
|
|
1851
|
+
return hash.digest('hex')
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
/**
|
|
1855
|
+
* The set: files the PR added under docs/evidence (rename detection off, so a
|
|
1856
|
+
* rename in counts as an add; `git diff` is the porcelain that would detect
|
|
1857
|
+
* renames by default, where plumbing would not), less every path the base tip
|
|
1858
|
+
* already has. The
|
|
1859
|
+
* base tip is the right subtrahend, not the merge-base: the base may have
|
|
1860
|
+
* gained a path since the branch point.
|
|
1861
|
+
* @param {Parameters<typeof runGit>[0]} ctx
|
|
1862
|
+
* @param {string} mergeBase
|
|
1863
|
+
* @param {string} baseTip
|
|
1864
|
+
* @param {string} headOid
|
|
1865
|
+
*/
|
|
1866
|
+
function computeCleanupSet(ctx, mergeBase, baseTip, headOid) {
|
|
1867
|
+
const added = parseNulList(
|
|
1868
|
+
runGit(ctx, 'diff-added', [
|
|
1869
|
+
'diff',
|
|
1870
|
+
'--no-renames',
|
|
1871
|
+
'--no-ext-diff',
|
|
1872
|
+
'--diff-filter=A',
|
|
1873
|
+
'--name-only',
|
|
1874
|
+
'-z',
|
|
1875
|
+
mergeBase,
|
|
1876
|
+
headOid,
|
|
1877
|
+
'--',
|
|
1878
|
+
`${EVIDENCE_ROOT}/`,
|
|
1879
|
+
]).stdout,
|
|
1880
|
+
)
|
|
1881
|
+
const atBase = new Set(
|
|
1882
|
+
parseNulList(
|
|
1883
|
+
runGit(ctx, 'list-base', [
|
|
1884
|
+
'ls-tree',
|
|
1885
|
+
'-r',
|
|
1886
|
+
'-z',
|
|
1887
|
+
'--name-only',
|
|
1888
|
+
baseTip,
|
|
1889
|
+
'--',
|
|
1890
|
+
`${EVIDENCE_ROOT}/`,
|
|
1891
|
+
]).stdout,
|
|
1892
|
+
),
|
|
1893
|
+
)
|
|
1894
|
+
return [...new Set(added)]
|
|
1895
|
+
.filter((file) => file.startsWith(`${EVIDENCE_ROOT}/`) && !atBase.has(file))
|
|
1896
|
+
.sort()
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
/**
|
|
1900
|
+
* @param {Parameters<typeof runGit>[0] & { reached: { step: string, removal_attempted: boolean } }} ctx
|
|
1901
|
+
* @param {{ headOid: string, baseRef: string, removeDigest?: string }} input
|
|
1902
|
+
*/
|
|
1903
|
+
function cleanupReport(ctx, input) {
|
|
1904
|
+
locateRepo(ctx)
|
|
1905
|
+
if (resolveCommit(ctx, 'resolve-head', input.headOid) === null) {
|
|
1906
|
+
return outcome('cleanup-set', 1, 'head-not-found')
|
|
1907
|
+
}
|
|
1908
|
+
const baseTip = resolveCommit(ctx, 'resolve-base', input.baseRef)
|
|
1909
|
+
if (baseTip === null) {
|
|
1910
|
+
return outcome('cleanup-set', 1, 'base-not-found')
|
|
1911
|
+
}
|
|
1912
|
+
const mergeBaseRun = runGit(
|
|
1913
|
+
ctx,
|
|
1914
|
+
'merge-base',
|
|
1915
|
+
['merge-base', baseTip, input.headOid],
|
|
1916
|
+
[0, 1],
|
|
1917
|
+
)
|
|
1918
|
+
if (mergeBaseRun.status !== 0) {
|
|
1919
|
+
return outcome('cleanup-set', 1, 'no-merge-base', { base_tip: baseTip })
|
|
1920
|
+
}
|
|
1921
|
+
const mergeBase = firstLine(mergeBaseRun.stdout)
|
|
1922
|
+
const paths = computeCleanupSet(ctx, mergeBase, baseTip, input.headOid)
|
|
1923
|
+
const parts = {
|
|
1924
|
+
head: input.headOid,
|
|
1925
|
+
base_tip: baseTip,
|
|
1926
|
+
merge_base: mergeBase,
|
|
1927
|
+
paths,
|
|
1928
|
+
count: paths.length,
|
|
1929
|
+
digest: cleanupDigest(paths),
|
|
1930
|
+
}
|
|
1931
|
+
if (input.removeDigest === undefined) {
|
|
1932
|
+
return outcome('cleanup-set', 0, paths.length > 0 ? 'set' : 'empty', parts)
|
|
1933
|
+
}
|
|
1934
|
+
// Removal: the set just computed must be the set that was approved.
|
|
1935
|
+
if (parts.digest !== input.removeDigest) {
|
|
1936
|
+
return outcome('cleanup-set', 1, 'digest-mismatch', parts)
|
|
1937
|
+
}
|
|
1938
|
+
if (paths.length === 0) {
|
|
1939
|
+
return outcome('cleanup-set', 0, 'empty', parts)
|
|
1940
|
+
}
|
|
1941
|
+
const checkedOut = resolveCommit(ctx, 'check-head', 'HEAD')
|
|
1942
|
+
if (checkedOut !== input.headOid) {
|
|
1943
|
+
return outcome('cleanup-set', 1, 'head-not-checked-out', parts)
|
|
1944
|
+
}
|
|
1945
|
+
ctx.reached.removal_attempted = true
|
|
1946
|
+
runGit(ctx, 'remove', ['rm', '-q', '--', ...paths])
|
|
1947
|
+
return outcome('cleanup-set', 0, 'removed', parts)
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
/**
|
|
1951
|
+
* Runs a git-backed operation. A git failure or timeout becomes exit 1 with
|
|
1952
|
+
* the step it reached; anything else is a bug and propagates.
|
|
1953
|
+
* @param {string} operation
|
|
1954
|
+
* @param {number} timeoutMs
|
|
1955
|
+
* @param {Record<string, unknown>} reached
|
|
1956
|
+
* @param {(ctx: any) => { exitCode: number, body: Record<string, unknown> }} work
|
|
1957
|
+
*/
|
|
1958
|
+
function withGit(operation, timeoutMs, reached, work) {
|
|
1959
|
+
const ctx = {
|
|
1960
|
+
cwd: process.cwd(),
|
|
1961
|
+
timeoutMs,
|
|
1962
|
+
reached: { step: 'start', ...reached },
|
|
1963
|
+
}
|
|
1964
|
+
try {
|
|
1965
|
+
return work(ctx)
|
|
1966
|
+
} catch (error) {
|
|
1967
|
+
const failure = failureOf(error)
|
|
1968
|
+
if (failure === null) {
|
|
1969
|
+
throw error
|
|
1970
|
+
}
|
|
1971
|
+
return outcome(operation, 1, failure.kind, {
|
|
1972
|
+
reached: { ...ctx.reached },
|
|
1973
|
+
git_status: failure.status,
|
|
1974
|
+
})
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// ── Verification ───────────────────────────────────────────────────────────
|
|
1979
|
+
//
|
|
1980
|
+
// The one network call in the helper, because retry timing has to be exact.
|
|
1981
|
+
// It sends a bare GET (no credentials, redirects handled here, hop by hop),
|
|
1982
|
+
// looks at the status and headers, and drops the body unread. The clock, the
|
|
1983
|
+
// sleep, the timer, the random source, the fetch and the host rule are
|
|
1984
|
+
// injected, so tests need no network and no real waiting.
|
|
1985
|
+
|
|
1986
|
+
const ALLOWED_HOSTS = new Set([
|
|
1987
|
+
'raw.githubusercontent.com',
|
|
1988
|
+
'media.githubusercontent.com',
|
|
1989
|
+
])
|
|
1990
|
+
const MAX_REDIRECTS = 5
|
|
1991
|
+
const MAX_VERIFY_URLS = 50
|
|
1992
|
+
const BACKOFF_MS = [2000, 4000, 8000, 15_000, 30_000]
|
|
1993
|
+
const JITTER = 0.2
|
|
1994
|
+
const TOTAL_BUDGET_MS = 300_000
|
|
1995
|
+
const DEFAULT_MAX_WAIT_MS = 90_000
|
|
1996
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 10_000
|
|
1997
|
+
// A backstop for odd injected schedules; the default schedule ends by time.
|
|
1998
|
+
const DEFAULT_MAX_ATTEMPTS = 30
|
|
1999
|
+
const IMAGE_TYPE = /^image\/[a-z0-9][a-z0-9.+-]*$/
|
|
2000
|
+
const REDIRECT_CODES = new Set([301, 302, 303, 307, 308])
|
|
2001
|
+
const RETRYABLE_STATUSES = new Set([
|
|
2002
|
+
'not-found',
|
|
2003
|
+
'rate-limited',
|
|
2004
|
+
'server-error',
|
|
2005
|
+
'network-error',
|
|
2006
|
+
'timeout',
|
|
2007
|
+
])
|
|
2008
|
+
const TIMED_OUT = Symbol('timed-out')
|
|
2009
|
+
|
|
2010
|
+
/**
|
|
2011
|
+
* Where a request may go: https, one of the two GitHub content hosts, the
|
|
2012
|
+
* default port, no credentials in the URL.
|
|
2013
|
+
* @param {URL} target
|
|
2014
|
+
*/
|
|
2015
|
+
function isManagedTarget(target) {
|
|
2016
|
+
return (
|
|
2017
|
+
target.protocol === 'https:' &&
|
|
2018
|
+
ALLOWED_HOSTS.has(target.hostname) &&
|
|
2019
|
+
target.port === '' &&
|
|
2020
|
+
target.username === '' &&
|
|
2021
|
+
target.password === ''
|
|
2022
|
+
)
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
/** @param {unknown} response */
|
|
2026
|
+
function dropBody(response) {
|
|
2027
|
+
try {
|
|
2028
|
+
// Not awaited: cancelling must never be able to hold the check up.
|
|
2029
|
+
Promise.resolve(response.body?.cancel()).catch(() => {})
|
|
2030
|
+
} catch {}
|
|
2031
|
+
}
|
|
2032
|
+
|
|
2033
|
+
/**
|
|
2034
|
+
* Reads only the status and headers of a response.
|
|
2035
|
+
* @param {Response} response
|
|
2036
|
+
*/
|
|
2037
|
+
function judgeResponse(response) {
|
|
2038
|
+
const code = response.status
|
|
2039
|
+
const known = (status, extra = {}) => ({ status, httpStatus: code, ...extra })
|
|
2040
|
+
if (REDIRECT_CODES.has(code)) {
|
|
2041
|
+
return known('redirect', { location: response.headers.get('location') })
|
|
2042
|
+
}
|
|
2043
|
+
if (code === 200) {
|
|
2044
|
+
const type = (response.headers.get('content-type') ?? '')
|
|
2045
|
+
.split(';')[0]
|
|
2046
|
+
.trim()
|
|
2047
|
+
.toLowerCase()
|
|
2048
|
+
return known(IMAGE_TYPE.test(type) ? 'ok' : 'not-an-image')
|
|
2049
|
+
}
|
|
2050
|
+
if (code === 404) {
|
|
2051
|
+
return known('not-found')
|
|
2052
|
+
}
|
|
2053
|
+
if (code === 429) {
|
|
2054
|
+
// Delta-seconds only; an HTTP date falls back to the schedule.
|
|
2055
|
+
const header = response.headers.get('retry-after')?.trim() ?? ''
|
|
2056
|
+
return known('rate-limited', {
|
|
2057
|
+
retryAfterMs: /^\d{1,6}$/.test(header) ? Number(header) * 1000 : 0,
|
|
2058
|
+
})
|
|
2059
|
+
}
|
|
2060
|
+
if (code >= 500 && code < 600) {
|
|
2061
|
+
return known('server-error')
|
|
2062
|
+
}
|
|
2063
|
+
if (code === 401 || code === 403) {
|
|
2064
|
+
return known('forbidden')
|
|
2065
|
+
}
|
|
2066
|
+
return known('unexpected-status')
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
/**
|
|
2070
|
+
* What `verifyUrls` builds once per call and hands to `verifyOne`, `attempt`
|
|
2071
|
+
* and `probe`. The function fields are the injectable seams; the rest are the
|
|
2072
|
+
* resolved limits and the start reading.
|
|
2073
|
+
* @typedef {object} VerifyContext
|
|
2074
|
+
* @property {(input: string, init: RequestInit) => Promise<Response>} fetch
|
|
2075
|
+
* @property {(ms: number) => Promise<void>} sleep
|
|
2076
|
+
* @property {(ms: number, fire: () => void) => () => void} startTimer Returns the function that cancels the timer.
|
|
2077
|
+
* @property {() => number} random A value in [0, 1).
|
|
2078
|
+
* @property {(target: URL) => boolean} isAllowedTarget
|
|
2079
|
+
* @property {number} requestTimeoutMs
|
|
2080
|
+
* @property {number} maxAttempts
|
|
2081
|
+
* @property {number} callLimitMs The most this call may spend waiting.
|
|
2082
|
+
* @property {readonly number[]} scheduleMs
|
|
2083
|
+
* @property {number} waitedMs Time earlier calls already waited.
|
|
2084
|
+
* @property {number} totalBudgetMs
|
|
2085
|
+
* @property {number} firstStep Where in `scheduleMs` the first wait falls.
|
|
2086
|
+
* @property {() => number} now
|
|
2087
|
+
* @property {() => number} epochNow Milliseconds since the epoch, for the deadlines that pass between calls.
|
|
2088
|
+
* @property {number} start The `now()` reading when the call began.
|
|
2089
|
+
*/
|
|
2090
|
+
|
|
2091
|
+
/**
|
|
2092
|
+
* One request, bounded by `timeoutMs`. The body is cancelled whatever the
|
|
2093
|
+
* status was, and a stalled request is aborted.
|
|
2094
|
+
* @param {string} url
|
|
2095
|
+
* @param {VerifyContext} ctx
|
|
2096
|
+
* @param {number} timeoutMs
|
|
2097
|
+
*/
|
|
2098
|
+
async function probe(url, ctx, timeoutMs) {
|
|
2099
|
+
const controller = new AbortController()
|
|
2100
|
+
let cancelTimer = () => {}
|
|
2101
|
+
const timedOut = new Promise((resolve) => {
|
|
2102
|
+
cancelTimer = ctx.startTimer(timeoutMs, () => resolve(TIMED_OUT))
|
|
2103
|
+
})
|
|
2104
|
+
const pending = Promise.resolve().then(() =>
|
|
2105
|
+
ctx.fetch(url, {
|
|
2106
|
+
method: 'GET',
|
|
2107
|
+
redirect: 'manual',
|
|
2108
|
+
credentials: 'omit',
|
|
2109
|
+
headers: { accept: 'image/*' },
|
|
2110
|
+
signal: controller.signal,
|
|
2111
|
+
}),
|
|
2112
|
+
)
|
|
2113
|
+
try {
|
|
2114
|
+
const winner = await Promise.race([pending, timedOut])
|
|
2115
|
+
if (winner === TIMED_OUT) {
|
|
2116
|
+
controller.abort()
|
|
2117
|
+
pending.then(dropBody, () => {})
|
|
2118
|
+
return { status: 'timeout', httpStatus: null }
|
|
2119
|
+
}
|
|
2120
|
+
try {
|
|
2121
|
+
return judgeResponse(winner)
|
|
2122
|
+
} finally {
|
|
2123
|
+
dropBody(winner)
|
|
2124
|
+
}
|
|
2125
|
+
} catch {
|
|
2126
|
+
controller.abort()
|
|
2127
|
+
return { status: 'network-error', httpStatus: null }
|
|
2128
|
+
} finally {
|
|
2129
|
+
cancelTimer()
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
/** The time this call may still spend, in ms. @param {VerifyContext} ctx */
|
|
2134
|
+
function timeLeft(ctx) {
|
|
2135
|
+
return ctx.callLimitMs - (ctx.now() - ctx.start)
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
/**
|
|
2139
|
+
* One attempt: the request plus the redirects it leads to, each hop checked
|
|
2140
|
+
* against the host rule before anything is sent to it. Each request is cut
|
|
2141
|
+
* short at the time the call has left, so a call never runs past its limit by
|
|
2142
|
+
* more than the clock's own slack. The first request of a call's first
|
|
2143
|
+
* attempt is always made, with the full request timeout when no time is left.
|
|
2144
|
+
* @param {string} startUrl
|
|
2145
|
+
* @param {VerifyContext} ctx
|
|
2146
|
+
* @param {boolean} first Whether this is the first attempt of the call.
|
|
2147
|
+
*/
|
|
2148
|
+
async function attempt(startUrl, ctx, first) {
|
|
2149
|
+
let current = startUrl
|
|
2150
|
+
for (let hops = 0; ; hops += 1) {
|
|
2151
|
+
const left = timeLeft(ctx)
|
|
2152
|
+
if (left <= 0 && !(first && hops === 0)) {
|
|
2153
|
+
return { status: 'timeout', httpStatus: null }
|
|
2154
|
+
}
|
|
2155
|
+
const seen = await probe(
|
|
2156
|
+
current,
|
|
2157
|
+
ctx,
|
|
2158
|
+
left > 0 ? Math.min(ctx.requestTimeoutMs, left) : ctx.requestTimeoutMs,
|
|
2159
|
+
)
|
|
2160
|
+
if (seen.status !== 'redirect') {
|
|
2161
|
+
return seen
|
|
2162
|
+
}
|
|
2163
|
+
const { httpStatus } = seen
|
|
2164
|
+
if (hops === MAX_REDIRECTS) {
|
|
2165
|
+
return { status: 'too-many-redirects', httpStatus }
|
|
2166
|
+
}
|
|
2167
|
+
let next = null
|
|
2168
|
+
try {
|
|
2169
|
+
next = seen.location ? new URL(seen.location, current) : null
|
|
2170
|
+
} catch {}
|
|
2171
|
+
if (next === null) {
|
|
2172
|
+
return { status: 'bad-redirect', httpStatus }
|
|
2173
|
+
}
|
|
2174
|
+
if (!ctx.isAllowedTarget(next)) {
|
|
2175
|
+
return { status: 'redirect-off-host', httpStatus }
|
|
2176
|
+
}
|
|
2177
|
+
current = next.href
|
|
2178
|
+
}
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
/**
|
|
2182
|
+
* Where in the schedule to start, given the seconds earlier calls already
|
|
2183
|
+
* waited: the delays that fit in that time are behind us.
|
|
2184
|
+
* @param {readonly number[]} schedule
|
|
2185
|
+
* @param {number} waitedMs
|
|
2186
|
+
*/
|
|
2187
|
+
function scheduleStart(schedule, waitedMs) {
|
|
2188
|
+
let index = 0
|
|
2189
|
+
let spent = 0
|
|
2190
|
+
while (index < 1000 && spent + delayAt(schedule, index) <= waitedMs) {
|
|
2191
|
+
spent += delayAt(schedule, index)
|
|
2192
|
+
index += 1
|
|
2193
|
+
}
|
|
2194
|
+
return index
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
/**
|
|
2198
|
+
* @param {readonly number[]} schedule
|
|
2199
|
+
* @param {number} index
|
|
2200
|
+
*/
|
|
2201
|
+
function delayAt(schedule, index) {
|
|
2202
|
+
return schedule[Math.min(index, schedule.length - 1)] ?? 0
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
/**
|
|
2206
|
+
* `resumeAt` is the epoch deadline before which no request may be sent for
|
|
2207
|
+
* this URL, set when a Retry-After outlasts the call. It is reported only on a
|
|
2208
|
+
* failure that can still be retried.
|
|
2209
|
+
* @param {string} url
|
|
2210
|
+
* @param {{ status: string, httpStatus: number | null, retryAfterMs?: number }} seen
|
|
2211
|
+
* @param {number} attempts
|
|
2212
|
+
* @param {boolean} budgetLeft
|
|
2213
|
+
* @param {number} [resumeAt]
|
|
2214
|
+
*/
|
|
2215
|
+
function verdict(url, seen, attempts, budgetLeft, resumeAt) {
|
|
2216
|
+
const retryable = RETRYABLE_STATUSES.has(seen.status) && budgetLeft
|
|
2217
|
+
const report = {
|
|
2218
|
+
url,
|
|
2219
|
+
status: seen.status,
|
|
2220
|
+
http_status: seen.httpStatus,
|
|
2221
|
+
attempts,
|
|
2222
|
+
retryable,
|
|
2223
|
+
}
|
|
2224
|
+
return retryable && resumeAt !== undefined
|
|
2225
|
+
? { ...report, resume_at: resumeAt }
|
|
2226
|
+
: report
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
/**
|
|
2230
|
+
* Retries 404, 429 (never sooner than Retry-After), 5xx, network errors and
|
|
2231
|
+
* timeouts. Everything else, a 200 that is not an image included, is final.
|
|
2232
|
+
* `retryable` on a failure means that waiting longer could still help and the
|
|
2233
|
+
* 300 s total has room for the next wait.
|
|
2234
|
+
* @param {string} url
|
|
2235
|
+
* @param {VerifyContext} ctx
|
|
2236
|
+
*/
|
|
2237
|
+
async function verifyOne(url, ctx) {
|
|
2238
|
+
let attempts = 0
|
|
2239
|
+
let step = ctx.firstStep
|
|
2240
|
+
for (;;) {
|
|
2241
|
+
attempts += 1
|
|
2242
|
+
const seen = await attempt(url, ctx, attempts === 1)
|
|
2243
|
+
if (seen.status === 'ok' || !RETRYABLE_STATUSES.has(seen.status)) {
|
|
2244
|
+
return verdict(url, seen, attempts, false)
|
|
2245
|
+
}
|
|
2246
|
+
const spread = 1 + (ctx.random() * 2 - 1) * JITTER
|
|
2247
|
+
const delay = Math.max(
|
|
2248
|
+
Math.round(delayAt(ctx.scheduleMs, step) * spread),
|
|
2249
|
+
seen.retryAfterMs ?? 0,
|
|
2250
|
+
)
|
|
2251
|
+
const elapsed = ctx.now() - ctx.start
|
|
2252
|
+
if (attempts >= ctx.maxAttempts || elapsed + delay > ctx.callLimitMs) {
|
|
2253
|
+
const budgetLeft = ctx.waitedMs + elapsed + delay <= ctx.totalBudgetMs
|
|
2254
|
+
// The call ends here, so a cooldown the server asked for would not be
|
|
2255
|
+
// served by it. The deadline goes to the next call, which sleeps first.
|
|
2256
|
+
const resumeAt =
|
|
2257
|
+
(seen.retryAfterMs ?? 0) > 0
|
|
2258
|
+
? ctx.epochNow() + seen.retryAfterMs
|
|
2259
|
+
: undefined
|
|
2260
|
+
return verdict(url, seen, attempts, budgetLeft, resumeAt)
|
|
2261
|
+
}
|
|
2262
|
+
await ctx.sleep(delay)
|
|
2263
|
+
step += 1
|
|
2264
|
+
if (timeLeft(ctx) <= 0) {
|
|
2265
|
+
// The wait used the last of the call's time: end here, with no attempt.
|
|
2266
|
+
const spent = ctx.waitedMs + (ctx.now() - ctx.start)
|
|
2267
|
+
const budgetLeft =
|
|
2268
|
+
spent + delayAt(ctx.scheduleMs, step) <= ctx.totalBudgetMs
|
|
2269
|
+
return verdict(url, seen, attempts, budgetLeft)
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
|
|
2274
|
+
/**
|
|
2275
|
+
* Serves the cooldown a previous call was handed: sleeps until `resumeAtMs`
|
|
2276
|
+
* before the first request, within this call's own limit. Returns null when
|
|
2277
|
+
* the first request may go out. Otherwise no request is sent, and the verdicts
|
|
2278
|
+
* say so (`rate-limited`, no attempt): when the deadline lies past the total
|
|
2279
|
+
* budget they cannot be retried, when it lies past this call the same
|
|
2280
|
+
* deadline is handed on, and when the sleep itself used up the call limit or
|
|
2281
|
+
* the total there is no time left for a request, so none is sent.
|
|
2282
|
+
* @param {readonly string[]} urls
|
|
2283
|
+
* @param {VerifyContext} ctx
|
|
2284
|
+
* @param {number} resumeAtMs
|
|
2285
|
+
*/
|
|
2286
|
+
async function serveCooldown(urls, ctx, resumeAtMs) {
|
|
2287
|
+
const wait = Math.max(0, resumeAtMs - ctx.epochNow())
|
|
2288
|
+
const unsent = (budgetLeft, resumeAt) =>
|
|
2289
|
+
urls.map((url) =>
|
|
2290
|
+
verdict(
|
|
2291
|
+
url,
|
|
2292
|
+
{ status: 'rate-limited', httpStatus: null },
|
|
2293
|
+
0,
|
|
2294
|
+
budgetLeft,
|
|
2295
|
+
resumeAt,
|
|
2296
|
+
),
|
|
2297
|
+
)
|
|
2298
|
+
if (ctx.waitedMs + wait > ctx.totalBudgetMs) {
|
|
2299
|
+
return unsent(false)
|
|
2300
|
+
}
|
|
2301
|
+
if (wait > ctx.callLimitMs) {
|
|
2302
|
+
await ctx.sleep(ctx.callLimitMs)
|
|
2303
|
+
return unsent(true, resumeAtMs)
|
|
2304
|
+
}
|
|
2305
|
+
if (wait > 0) {
|
|
2306
|
+
await ctx.sleep(wait)
|
|
2307
|
+
if (timeLeft(ctx) <= 0) {
|
|
2308
|
+
// The sleep used the last of the call limit or of the total. A request
|
|
2309
|
+
// now would run past either, so none goes out. The cooldown is served,
|
|
2310
|
+
// so no deadline is handed on.
|
|
2311
|
+
const spent = ctx.waitedMs + (ctx.now() - ctx.start)
|
|
2312
|
+
return unsent(
|
|
2313
|
+
spent + delayAt(ctx.scheduleMs, ctx.firstStep) <= ctx.totalBudgetMs,
|
|
2314
|
+
)
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
return null
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
/**
|
|
2321
|
+
* Checks each URL, all at once. The time taken is the longest one's, and the
|
|
2322
|
+
* result lists every URL in the order given, duplicates dropped.
|
|
2323
|
+
* @param {readonly string[]} urls
|
|
2324
|
+
* @param {Record<string, any>} [options]
|
|
2325
|
+
*/
|
|
2326
|
+
export async function verifyUrls(urls, options = {}) {
|
|
2327
|
+
const scheduleMs = options.scheduleMs ?? BACKOFF_MS
|
|
2328
|
+
const waitedMs = options.waitedMs ?? 0
|
|
2329
|
+
const totalBudgetMs = options.totalBudgetMs ?? TOTAL_BUDGET_MS
|
|
2330
|
+
const now = options.now ?? (() => performance.now())
|
|
2331
|
+
const ctx = {
|
|
2332
|
+
fetch: options.fetch ?? ((input, init) => globalThis.fetch(input, init)),
|
|
2333
|
+
sleep:
|
|
2334
|
+
options.sleep ??
|
|
2335
|
+
((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
|
2336
|
+
startTimer:
|
|
2337
|
+
options.startTimer ??
|
|
2338
|
+
((ms, fire) => {
|
|
2339
|
+
const timer = setTimeout(fire, ms)
|
|
2340
|
+
return () => clearTimeout(timer)
|
|
2341
|
+
}),
|
|
2342
|
+
random: options.random ?? Math.random,
|
|
2343
|
+
isAllowedTarget: options.isAllowedTarget ?? isManagedTarget,
|
|
2344
|
+
requestTimeoutMs: options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS,
|
|
2345
|
+
maxAttempts: options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
2346
|
+
callLimitMs: Math.min(
|
|
2347
|
+
options.maxWaitMs ?? DEFAULT_MAX_WAIT_MS,
|
|
2348
|
+
totalBudgetMs - waitedMs,
|
|
2349
|
+
),
|
|
2350
|
+
scheduleMs,
|
|
2351
|
+
waitedMs,
|
|
2352
|
+
totalBudgetMs,
|
|
2353
|
+
firstStep: scheduleStart(scheduleMs, waitedMs),
|
|
2354
|
+
now,
|
|
2355
|
+
epochNow: options.epochNow ?? (() => Date.now()),
|
|
2356
|
+
start: now(),
|
|
2357
|
+
}
|
|
2358
|
+
const unique = [...new Set(urls)]
|
|
2359
|
+
const deferred =
|
|
2360
|
+
options.resumeAtMs === undefined
|
|
2361
|
+
? null
|
|
2362
|
+
: await serveCooldown(unique, ctx, options.resumeAtMs)
|
|
2363
|
+
const results =
|
|
2364
|
+
deferred ?? (await Promise.all(unique.map((url) => verifyOne(url, ctx))))
|
|
2365
|
+
return { results, waitedMs: now() - ctx.start }
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
// ── CLI ────────────────────────────────────────────────────────────────────
|
|
2369
|
+
|
|
2370
|
+
/**
|
|
2371
|
+
* Fixed error codes only: the caller's text is never echoed back. `field`
|
|
2372
|
+
* names the input field that failed, always one of the schema's own names
|
|
2373
|
+
* (or `input` for the whole object and `operation` for the command line), so
|
|
2374
|
+
* the caller can fix that one field and retry.
|
|
2375
|
+
* @param {string} operation
|
|
2376
|
+
* @param {string} error
|
|
2377
|
+
* @param {string} field
|
|
2378
|
+
*/
|
|
2379
|
+
function usageError(operation, error, field) {
|
|
2380
|
+
return {
|
|
2381
|
+
exitCode: 2,
|
|
2382
|
+
body: {
|
|
2383
|
+
schema_version: SCHEMA_VERSION,
|
|
2384
|
+
operation,
|
|
2385
|
+
result: 'usage-error',
|
|
2386
|
+
error,
|
|
2387
|
+
field,
|
|
2388
|
+
usage: USAGE,
|
|
2389
|
+
},
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
|
|
2393
|
+
/**
|
|
2394
|
+
* A parse failure: the fixed error code and the schema field behind it.
|
|
2395
|
+
* @param {string} error
|
|
2396
|
+
* @param {string} field
|
|
2397
|
+
*/
|
|
2398
|
+
function invalid(error, field) {
|
|
2399
|
+
return { ok: false, error, field }
|
|
2400
|
+
}
|
|
2401
|
+
|
|
2402
|
+
/** @param {unknown} value */
|
|
2403
|
+
function isPlainObject(value) {
|
|
2404
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
2405
|
+
}
|
|
2406
|
+
|
|
2407
|
+
/** @param {unknown} value */
|
|
2408
|
+
function isPositiveInteger(value) {
|
|
2409
|
+
return Number.isSafeInteger(value) && value > 0
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
/**
|
|
2413
|
+
* @param {unknown} raw
|
|
2414
|
+
* @returns {{ ok: true, input: Parameters<typeof classify>[0] } | { ok: false, error: string, field: string }}
|
|
2415
|
+
*/
|
|
2416
|
+
function parseInput(raw) {
|
|
2417
|
+
if (!isPlainObject(raw)) {
|
|
2418
|
+
return invalid('invalid-input', 'input')
|
|
2419
|
+
}
|
|
2420
|
+
const { paths, branch, topic, existing, max_bytes } = raw
|
|
2421
|
+
if (!Array.isArray(paths) || paths.length === 0) {
|
|
2422
|
+
return invalid('empty-paths', 'paths')
|
|
2423
|
+
}
|
|
2424
|
+
if (
|
|
2425
|
+
!paths.every(
|
|
2426
|
+
(item) =>
|
|
2427
|
+
typeof item === 'string' && item.length > 0 && !item.includes('\0'),
|
|
2428
|
+
)
|
|
2429
|
+
) {
|
|
2430
|
+
return invalid('invalid-input', 'paths')
|
|
2431
|
+
}
|
|
2432
|
+
const optionalString = (value) =>
|
|
2433
|
+
value === undefined || typeof value === 'string'
|
|
2434
|
+
if (!optionalString(branch)) {
|
|
2435
|
+
return invalid('invalid-input', 'branch')
|
|
2436
|
+
}
|
|
2437
|
+
if (!optionalString(topic)) {
|
|
2438
|
+
return invalid('invalid-input', 'topic')
|
|
2439
|
+
}
|
|
2440
|
+
if (branch === undefined && (topic === undefined || topic.length === 0)) {
|
|
2441
|
+
return invalid('missing-topic-source', 'branch')
|
|
2442
|
+
}
|
|
2443
|
+
if (
|
|
2444
|
+
existing !== undefined &&
|
|
2445
|
+
!(
|
|
2446
|
+
isPlainObject(existing) &&
|
|
2447
|
+
Object.values(existing).every((sha) => typeof sha === 'string')
|
|
2448
|
+
)
|
|
2449
|
+
) {
|
|
2450
|
+
return invalid('invalid-input', 'existing')
|
|
2451
|
+
}
|
|
2452
|
+
if (max_bytes !== undefined && !isPositiveInteger(max_bytes)) {
|
|
2453
|
+
return invalid('invalid-input', 'max_bytes')
|
|
2454
|
+
}
|
|
2455
|
+
return {
|
|
2456
|
+
ok: true,
|
|
2457
|
+
input: {
|
|
2458
|
+
paths,
|
|
2459
|
+
branch,
|
|
2460
|
+
topic,
|
|
2461
|
+
existing: existing ?? {},
|
|
2462
|
+
maxBytes: max_bytes ?? DEFAULT_MAX_BYTES,
|
|
2463
|
+
},
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
|
|
2467
|
+
/**
|
|
2468
|
+
* Fixed vocabulary for every subcommand but classify: `result` is one of
|
|
2469
|
+
* spliced, extracted, absent, urls, repinned, needs-input, write-failed,
|
|
2470
|
+
* usage-error, already-valid, repin-needed, partial, unresolved, set, empty,
|
|
2471
|
+
* removed, digest-mismatch, head-not-found, base-not-found, no-merge-base,
|
|
2472
|
+
* head-not-checked-out, git-failed, git-timeout, verified or failed, and
|
|
2473
|
+
* `asks` carries the
|
|
2474
|
+
* question when it is needs-input. `pins` with `select` adds selected and
|
|
2475
|
+
* pin-mismatch.
|
|
2476
|
+
* @param {string} operation
|
|
2477
|
+
* @param {number} exitCode
|
|
2478
|
+
* @param {string} result
|
|
2479
|
+
* @param {Record<string, unknown>} parts
|
|
2480
|
+
*/
|
|
2481
|
+
function outcome(operation, exitCode, result, parts = {}) {
|
|
2482
|
+
return {
|
|
2483
|
+
exitCode,
|
|
2484
|
+
body: { schema_version: SCHEMA_VERSION, operation, result, ...parts },
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
|
|
2488
|
+
const LONE_SURROGATE =
|
|
2489
|
+
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/
|
|
2490
|
+
const MAX_DEMO_BYTES = 1024 * 1024
|
|
2491
|
+
|
|
2492
|
+
/** A body that survives a UTF-8 write unchanged. @param {unknown} value */
|
|
2493
|
+
function isBody(value) {
|
|
2494
|
+
return typeof value === 'string' && !LONE_SURROGATE.test(value)
|
|
2495
|
+
}
|
|
2496
|
+
|
|
2497
|
+
/** @param {unknown} value */
|
|
2498
|
+
function isPathString(value) {
|
|
2499
|
+
return typeof value === 'string' && value.length > 0 && !value.includes('\0')
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
/** @param {unknown} value */
|
|
2503
|
+
function isPins(value) {
|
|
2504
|
+
return (
|
|
2505
|
+
isPlainObject(value) &&
|
|
2506
|
+
Object.entries(value).every(
|
|
2507
|
+
([key, sha]) =>
|
|
2508
|
+
MANAGED_PATH.test(key) && typeof sha === 'string' && FULL_SHA.test(sha),
|
|
2509
|
+
)
|
|
2510
|
+
)
|
|
2511
|
+
}
|
|
2512
|
+
|
|
2513
|
+
/** @param {unknown} value */
|
|
2514
|
+
function isHeadRepo(value) {
|
|
2515
|
+
return typeof value === 'string' && HEAD_REPO.test(value)
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
/**
|
|
2519
|
+
* The first required field that is missing or malformed, by schema name, or
|
|
2520
|
+
* null. The whole input failing is reported as the field `input`.
|
|
2521
|
+
* @param {unknown} raw
|
|
2522
|
+
* @param {ReadonlyArray<readonly [string, (value: unknown) => boolean]>} checks
|
|
2523
|
+
*/
|
|
2524
|
+
function firstInvalid(raw, checks) {
|
|
2525
|
+
if (!isPlainObject(raw)) {
|
|
2526
|
+
return invalid('invalid-input', 'input')
|
|
2527
|
+
}
|
|
2528
|
+
for (const [field, isValid] of checks) {
|
|
2529
|
+
if (!isValid(raw[field])) {
|
|
2530
|
+
return invalid('invalid-input', field)
|
|
2531
|
+
}
|
|
2532
|
+
}
|
|
2533
|
+
return null
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
/**
|
|
2537
|
+
* The output file is the only thing the helper ever writes. A file that
|
|
2538
|
+
* exists keeps its mode; a new one is created 0600.
|
|
2539
|
+
* @param {string} operation
|
|
2540
|
+
* @param {string} outFile
|
|
2541
|
+
* @param {string} text
|
|
2542
|
+
* @param {Record<string, unknown>} parts
|
|
2543
|
+
*/
|
|
2544
|
+
function writeBody(operation, outFile, text, parts) {
|
|
2545
|
+
const bytes = Buffer.from(text, 'utf8')
|
|
2546
|
+
try {
|
|
2547
|
+
writeFileSync(path.resolve(outFile), bytes, { mode: 0o600 })
|
|
2548
|
+
} catch {
|
|
2549
|
+
return outcome(operation, 1, 'write-failed', { error: 'write-failed' })
|
|
2550
|
+
}
|
|
2551
|
+
return { ...parts, out_file: outFile, bytes: bytes.length }
|
|
2552
|
+
}
|
|
2553
|
+
|
|
2554
|
+
/** @param {unknown} raw */
|
|
2555
|
+
function runClassify(raw) {
|
|
2556
|
+
const parsed = parseInput(raw)
|
|
2557
|
+
return parsed.ok
|
|
2558
|
+
? classify(parsed.input)
|
|
2559
|
+
: usageError('classify', parsed.error, parsed.field)
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2562
|
+
/**
|
|
2563
|
+
* @param {string} file
|
|
2564
|
+
* @returns {{ ok: true, text: string } | { ok: false, error: string, field: string }}
|
|
2565
|
+
*/
|
|
2566
|
+
function readDemoFile(file) {
|
|
2567
|
+
const candidate = readCandidate(file, MAX_DEMO_BYTES)
|
|
2568
|
+
if (!candidate.ok) {
|
|
2569
|
+
return invalid('demo-file-unreadable', 'demo_file')
|
|
2570
|
+
}
|
|
2571
|
+
try {
|
|
2572
|
+
// ignoreBOM keeps a BOM in the text instead of silently dropping it.
|
|
2573
|
+
const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true })
|
|
2574
|
+
return { ok: true, text: decoder.decode(candidate.bytes) }
|
|
2575
|
+
} catch {
|
|
2576
|
+
return invalid('demo-file-not-utf8', 'demo_file')
|
|
2577
|
+
}
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
/** @param {unknown} raw */
|
|
2581
|
+
function runSplice(raw) {
|
|
2582
|
+
const failed = firstInvalid(raw, [
|
|
2583
|
+
['body', isBody],
|
|
2584
|
+
['demo_file', isPathString],
|
|
2585
|
+
['out_file', isPathString],
|
|
2586
|
+
])
|
|
2587
|
+
if (failed !== null) {
|
|
2588
|
+
return usageError('splice', failed.error, failed.field)
|
|
2589
|
+
}
|
|
2590
|
+
const demoFile = readDemoFile(raw.demo_file)
|
|
2591
|
+
if (!demoFile.ok) {
|
|
2592
|
+
return usageError('splice', demoFile.error, demoFile.field)
|
|
2593
|
+
}
|
|
2594
|
+
const result = spliceDemo(raw.body, demoFile.text)
|
|
2595
|
+
const report = {
|
|
2596
|
+
demo_count: result.demoCount,
|
|
2597
|
+
screenshots_present: result.screenshots,
|
|
2598
|
+
}
|
|
2599
|
+
switch (result.kind) {
|
|
2600
|
+
case 'invalid-demo':
|
|
2601
|
+
return usageError('splice', 'invalid-demo', 'demo_file')
|
|
2602
|
+
case 'multiple-demo-sections':
|
|
2603
|
+
return outcome('splice', 1, 'needs-input', {
|
|
2604
|
+
...report,
|
|
2605
|
+
asks: [{ code: 'multiple-demo-sections', count: result.demoCount }],
|
|
2606
|
+
})
|
|
2607
|
+
case 'unterminated-region':
|
|
2608
|
+
return outcome('splice', 1, 'needs-input', {
|
|
2609
|
+
...report,
|
|
2610
|
+
asks: [{ code: 'unterminated-region', kind: result.region }],
|
|
2611
|
+
})
|
|
2612
|
+
case 'unexpected-section-content':
|
|
2613
|
+
return outcome('splice', 1, 'needs-input', {
|
|
2614
|
+
...report,
|
|
2615
|
+
asks: [{ code: 'unexpected-section-content' }],
|
|
2616
|
+
})
|
|
2617
|
+
case 'line-ending-conflict':
|
|
2618
|
+
return outcome('splice', 1, 'needs-input', {
|
|
2619
|
+
...report,
|
|
2620
|
+
asks: [{ code: 'line-ending-conflict' }],
|
|
2621
|
+
})
|
|
2622
|
+
default: {
|
|
2623
|
+
const written = writeBody('splice', raw.out_file, result.output, {
|
|
2624
|
+
action: result.action,
|
|
2625
|
+
...report,
|
|
2626
|
+
})
|
|
2627
|
+
return 'exitCode' in written
|
|
2628
|
+
? written
|
|
2629
|
+
: outcome('splice', 0, 'spliced', written)
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2634
|
+
/** @param {unknown} raw */
|
|
2635
|
+
function runExtract(raw) {
|
|
2636
|
+
const failed = firstInvalid(raw, [
|
|
2637
|
+
['body', isBody],
|
|
2638
|
+
['out_file', isPathString],
|
|
2639
|
+
])
|
|
2640
|
+
if (failed !== null) {
|
|
2641
|
+
return usageError('extract', failed.error, failed.field)
|
|
2642
|
+
}
|
|
2643
|
+
const result = extractDemo(raw.body)
|
|
2644
|
+
const report = { demo_count: result.demoCount }
|
|
2645
|
+
switch (result.kind) {
|
|
2646
|
+
case 'multiple-demo-sections':
|
|
2647
|
+
return outcome('extract', 1, 'needs-input', {
|
|
2648
|
+
...report,
|
|
2649
|
+
asks: [{ code: 'multiple-demo-sections', count: result.demoCount }],
|
|
2650
|
+
})
|
|
2651
|
+
case 'unterminated-region':
|
|
2652
|
+
return outcome('extract', 1, 'needs-input', {
|
|
2653
|
+
...report,
|
|
2654
|
+
asks: [{ code: 'unterminated-region', kind: result.region }],
|
|
2655
|
+
})
|
|
2656
|
+
case 'unexpected-section-content':
|
|
2657
|
+
return outcome('extract', 1, 'needs-input', {
|
|
2658
|
+
...report,
|
|
2659
|
+
asks: [{ code: 'unexpected-section-content' }],
|
|
2660
|
+
})
|
|
2661
|
+
case 'absent':
|
|
2662
|
+
return outcome('extract', 0, 'absent', report)
|
|
2663
|
+
default: {
|
|
2664
|
+
const written = writeBody('extract', raw.out_file, result.section, report)
|
|
2665
|
+
return 'exitCode' in written
|
|
2666
|
+
? written
|
|
2667
|
+
: outcome('extract', 0, 'extracted', written)
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
/** @param {unknown} raw */
|
|
2673
|
+
function runUrls(raw) {
|
|
2674
|
+
const failed = firstInvalid(raw, [
|
|
2675
|
+
['body', isBody],
|
|
2676
|
+
['head_repo', isHeadRepo],
|
|
2677
|
+
])
|
|
2678
|
+
if (failed !== null) {
|
|
2679
|
+
return usageError('urls', failed.error, failed.field)
|
|
2680
|
+
}
|
|
2681
|
+
return outcome('urls', 0, 'urls', findManagedUrls(raw.body, raw.head_repo))
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2684
|
+
/** @param {unknown} raw */
|
|
2685
|
+
function runRepin(raw) {
|
|
2686
|
+
const failed = firstInvalid(raw, [
|
|
2687
|
+
['body', isBody],
|
|
2688
|
+
['head_repo', isHeadRepo],
|
|
2689
|
+
['pins', isPins],
|
|
2690
|
+
['out_file', isPathString],
|
|
2691
|
+
])
|
|
2692
|
+
if (failed !== null) {
|
|
2693
|
+
return usageError('repin-rewrite', failed.error, failed.field)
|
|
2694
|
+
}
|
|
2695
|
+
const result = repinBody(raw.body, raw.head_repo, raw.pins)
|
|
2696
|
+
const written = writeBody('repin-rewrite', raw.out_file, result.output, {
|
|
2697
|
+
action: result.rewritten > 0 ? 'rewritten' : 'unchanged',
|
|
2698
|
+
rewritten: result.rewritten,
|
|
2699
|
+
unmapped_paths: result.unmappedPaths,
|
|
2700
|
+
ignored: result.ignored,
|
|
2701
|
+
})
|
|
2702
|
+
return 'exitCode' in written
|
|
2703
|
+
? written
|
|
2704
|
+
: outcome('repin-rewrite', 0, 'repinned', written)
|
|
2705
|
+
}
|
|
2706
|
+
|
|
2707
|
+
const REF_NAME_START = /^[A-Za-z0-9_][A-Za-z0-9._/-]*$/
|
|
2708
|
+
const REMOTE_NAME = /^[A-Za-z0-9_][A-Za-z0-9._-]*$/
|
|
2709
|
+
const DIGEST = /^[0-9a-f]{64}$/
|
|
2710
|
+
|
|
2711
|
+
/**
|
|
2712
|
+
* A conservative subset of git's ref-name rules. It cannot start with a dash,
|
|
2713
|
+
* so a name can never be read as an option.
|
|
2714
|
+
* @param {unknown} value
|
|
2715
|
+
*/
|
|
2716
|
+
function isRefName(value) {
|
|
2717
|
+
return (
|
|
2718
|
+
typeof value === 'string' &&
|
|
2719
|
+
REF_NAME_START.test(value) &&
|
|
2720
|
+
!value.includes('..') &&
|
|
2721
|
+
!value.includes('//') &&
|
|
2722
|
+
!value.includes('/.') &&
|
|
2723
|
+
!value.endsWith('/') &&
|
|
2724
|
+
!value.endsWith('.') &&
|
|
2725
|
+
!value.endsWith('.lock')
|
|
2726
|
+
)
|
|
2727
|
+
}
|
|
2728
|
+
|
|
2729
|
+
/** @param {unknown} value */
|
|
2730
|
+
function isGitTimeout(value) {
|
|
2731
|
+
return (
|
|
2732
|
+
value === undefined ||
|
|
2733
|
+
(Number.isSafeInteger(value) && value > 0 && value <= MAX_GIT_TIMEOUT_MS)
|
|
2734
|
+
)
|
|
2735
|
+
}
|
|
2736
|
+
|
|
2737
|
+
/**
|
|
2738
|
+
* `urls` is the array the `urls` subcommand reports; extra fields are
|
|
2739
|
+
* ignored. Embeds of one path are grouped, with each pin lower-cased.
|
|
2740
|
+
* @param {unknown} raw
|
|
2741
|
+
*/
|
|
2742
|
+
function parsePinsInput(raw) {
|
|
2743
|
+
if (!isPlainObject(raw)) {
|
|
2744
|
+
return invalid('invalid-input', 'input')
|
|
2745
|
+
}
|
|
2746
|
+
if (raw.select !== undefined) {
|
|
2747
|
+
return parseSelectInput(raw)
|
|
2748
|
+
}
|
|
2749
|
+
if (
|
|
2750
|
+
typeof raw.head_ref_oid !== 'string' ||
|
|
2751
|
+
!COMMIT_SHA.test(raw.head_ref_oid)
|
|
2752
|
+
) {
|
|
2753
|
+
return invalid('invalid-input', 'head_ref_oid')
|
|
2754
|
+
}
|
|
2755
|
+
if (!Array.isArray(raw.urls)) {
|
|
2756
|
+
return invalid('invalid-input', 'urls')
|
|
2757
|
+
}
|
|
2758
|
+
if (!isGitTimeout(raw.git_timeout_ms)) {
|
|
2759
|
+
return invalid('invalid-input', 'git_timeout_ms')
|
|
2760
|
+
}
|
|
2761
|
+
if (raw.urls.length === 0) {
|
|
2762
|
+
return invalid('empty-paths', 'urls')
|
|
2763
|
+
}
|
|
2764
|
+
const paths = new Map()
|
|
2765
|
+
for (const item of raw.urls) {
|
|
2766
|
+
if (
|
|
2767
|
+
!isPlainObject(item) ||
|
|
2768
|
+
typeof item.path !== 'string' ||
|
|
2769
|
+
!MANAGED_PATH.test(item.path) ||
|
|
2770
|
+
typeof item.sha !== 'string' ||
|
|
2771
|
+
!/^[0-9a-fA-F]{40}$/.test(item.sha)
|
|
2772
|
+
) {
|
|
2773
|
+
return invalid('invalid-input', 'urls')
|
|
2774
|
+
}
|
|
2775
|
+
const sha = item.sha.toLowerCase()
|
|
2776
|
+
const shas = paths.get(item.path) ?? []
|
|
2777
|
+
if (!shas.includes(sha)) {
|
|
2778
|
+
shas.push(sha)
|
|
2779
|
+
}
|
|
2780
|
+
paths.set(item.path, shas)
|
|
2781
|
+
}
|
|
2782
|
+
return {
|
|
2783
|
+
ok: true,
|
|
2784
|
+
input: {
|
|
2785
|
+
headOid: raw.head_ref_oid,
|
|
2786
|
+
paths,
|
|
2787
|
+
timeoutMs: raw.git_timeout_ms ?? DEFAULT_GIT_TIMEOUT_MS,
|
|
2788
|
+
},
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2791
|
+
|
|
2792
|
+
/**
|
|
2793
|
+
* The `select` form of `pins`: `urls` and `select` are alternatives, and
|
|
2794
|
+
* `head_ref_oid` is optional (the checked-out HEAD is used without it).
|
|
2795
|
+
* @param {Record<string, unknown>} raw
|
|
2796
|
+
*/
|
|
2797
|
+
function parseSelectInput(raw) {
|
|
2798
|
+
if (raw.urls !== undefined) {
|
|
2799
|
+
return invalid('invalid-input', 'select')
|
|
2800
|
+
}
|
|
2801
|
+
if (
|
|
2802
|
+
raw.head_ref_oid !== undefined &&
|
|
2803
|
+
!(typeof raw.head_ref_oid === 'string' && COMMIT_SHA.test(raw.head_ref_oid))
|
|
2804
|
+
) {
|
|
2805
|
+
return invalid('invalid-input', 'head_ref_oid')
|
|
2806
|
+
}
|
|
2807
|
+
if (!isGitTimeout(raw.git_timeout_ms)) {
|
|
2808
|
+
return invalid('invalid-input', 'git_timeout_ms')
|
|
2809
|
+
}
|
|
2810
|
+
if (!isHeadRepo(raw.head_repo)) {
|
|
2811
|
+
return invalid('invalid-input', 'head_repo')
|
|
2812
|
+
}
|
|
2813
|
+
if (raw.form !== 'raw' && raw.form !== 'blob') {
|
|
2814
|
+
return invalid('invalid-input', 'form')
|
|
2815
|
+
}
|
|
2816
|
+
if (!isPathString(raw.out_file)) {
|
|
2817
|
+
return invalid('invalid-input', 'out_file')
|
|
2818
|
+
}
|
|
2819
|
+
if (!Array.isArray(raw.select) || raw.select.length === 0) {
|
|
2820
|
+
return invalid('invalid-input', 'select')
|
|
2821
|
+
}
|
|
2822
|
+
const expected = new Map()
|
|
2823
|
+
for (const item of raw.select) {
|
|
2824
|
+
if (
|
|
2825
|
+
!isPlainObject(item) ||
|
|
2826
|
+
typeof item.path !== 'string' ||
|
|
2827
|
+
!MANAGED_PATH.test(item.path) ||
|
|
2828
|
+
typeof item.blob_sha !== 'string' ||
|
|
2829
|
+
!COMMIT_SHA.test(item.blob_sha) ||
|
|
2830
|
+
expected.has(item.path)
|
|
2831
|
+
) {
|
|
2832
|
+
return invalid('invalid-input', 'select')
|
|
2833
|
+
}
|
|
2834
|
+
expected.set(item.path, item.blob_sha)
|
|
2835
|
+
}
|
|
2836
|
+
return {
|
|
2837
|
+
ok: true,
|
|
2838
|
+
input: {
|
|
2839
|
+
select: true,
|
|
2840
|
+
headOid: raw.head_ref_oid,
|
|
2841
|
+
expected,
|
|
2842
|
+
headRepo: raw.head_repo,
|
|
2843
|
+
form: raw.form,
|
|
2844
|
+
outFile: raw.out_file,
|
|
2845
|
+
timeoutMs: raw.git_timeout_ms ?? DEFAULT_GIT_TIMEOUT_MS,
|
|
2846
|
+
},
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
|
|
2850
|
+
/**
|
|
2851
|
+
* The base is either `base_ref_name` with the `remote` it was fetched from
|
|
2852
|
+
* (read as refs/remotes/<remote>/<name>), or one fully qualified `base_ref`.
|
|
2853
|
+
* @param {unknown} raw
|
|
2854
|
+
*/
|
|
2855
|
+
function parseCleanupInput(raw) {
|
|
2856
|
+
if (!isPlainObject(raw)) {
|
|
2857
|
+
return invalid('invalid-input', 'input')
|
|
2858
|
+
}
|
|
2859
|
+
if (
|
|
2860
|
+
typeof raw.head_ref_oid !== 'string' ||
|
|
2861
|
+
!COMMIT_SHA.test(raw.head_ref_oid)
|
|
2862
|
+
) {
|
|
2863
|
+
return invalid('invalid-input', 'head_ref_oid')
|
|
2864
|
+
}
|
|
2865
|
+
if (!isGitTimeout(raw.git_timeout_ms)) {
|
|
2866
|
+
return invalid('invalid-input', 'git_timeout_ms')
|
|
2867
|
+
}
|
|
2868
|
+
if (
|
|
2869
|
+
raw.remove_digest !== undefined &&
|
|
2870
|
+
!(typeof raw.remove_digest === 'string' && DIGEST.test(raw.remove_digest))
|
|
2871
|
+
) {
|
|
2872
|
+
return invalid('invalid-input', 'remove_digest')
|
|
2873
|
+
}
|
|
2874
|
+
const named = raw.base_ref_name !== undefined || raw.remote !== undefined
|
|
2875
|
+
let baseRef
|
|
2876
|
+
if (named && raw.base_ref === undefined) {
|
|
2877
|
+
if (!isRefName(raw.base_ref_name)) {
|
|
2878
|
+
return invalid('invalid-input', 'base_ref_name')
|
|
2879
|
+
}
|
|
2880
|
+
if (typeof raw.remote !== 'string' || !REMOTE_NAME.test(raw.remote)) {
|
|
2881
|
+
return invalid('invalid-input', 'remote')
|
|
2882
|
+
}
|
|
2883
|
+
baseRef = `refs/remotes/${raw.remote}/${raw.base_ref_name}`
|
|
2884
|
+
} else if (
|
|
2885
|
+
!named &&
|
|
2886
|
+
isRefName(raw.base_ref) &&
|
|
2887
|
+
raw.base_ref.startsWith('refs/')
|
|
2888
|
+
) {
|
|
2889
|
+
baseRef = raw.base_ref
|
|
2890
|
+
} else {
|
|
2891
|
+
// Naming both forms, or neither, or a base_ref that is not fully qualified.
|
|
2892
|
+
return invalid('invalid-input', 'base_ref')
|
|
2893
|
+
}
|
|
2894
|
+
return {
|
|
2895
|
+
ok: true,
|
|
2896
|
+
input: {
|
|
2897
|
+
headOid: raw.head_ref_oid,
|
|
2898
|
+
baseRef,
|
|
2899
|
+
removeDigest: raw.remove_digest,
|
|
2900
|
+
timeoutMs: raw.git_timeout_ms ?? DEFAULT_GIT_TIMEOUT_MS,
|
|
2901
|
+
},
|
|
2902
|
+
}
|
|
2903
|
+
}
|
|
2904
|
+
|
|
2905
|
+
/** @param {unknown} raw */
|
|
2906
|
+
function runPins(raw) {
|
|
2907
|
+
const parsed = parsePinsInput(raw)
|
|
2908
|
+
if (!parsed.ok) {
|
|
2909
|
+
return usageError('pins', parsed.error, parsed.field)
|
|
2910
|
+
}
|
|
2911
|
+
const { input } = parsed
|
|
2912
|
+
const total = input.select ? input.expected.size : input.paths.size
|
|
2913
|
+
return withGit(
|
|
2914
|
+
'pins',
|
|
2915
|
+
input.timeoutMs,
|
|
2916
|
+
{ paths_done: 0, paths_total: total },
|
|
2917
|
+
(ctx) => (input.select ? selectReport(ctx, input) : pinsReport(ctx, input)),
|
|
2918
|
+
)
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
/** @param {unknown} raw */
|
|
2922
|
+
function runCleanup(raw) {
|
|
2923
|
+
const parsed = parseCleanupInput(raw)
|
|
2924
|
+
if (!parsed.ok) {
|
|
2925
|
+
return usageError('cleanup-set', parsed.error, parsed.field)
|
|
2926
|
+
}
|
|
2927
|
+
const { input } = parsed
|
|
2928
|
+
return withGit(
|
|
2929
|
+
'cleanup-set',
|
|
2930
|
+
input.timeoutMs,
|
|
2931
|
+
{ removal_attempted: false },
|
|
2932
|
+
(ctx) => cleanupReport(ctx, input),
|
|
2933
|
+
)
|
|
2934
|
+
}
|
|
2935
|
+
|
|
2936
|
+
const MAX_CALL_WAIT_S = TOTAL_BUDGET_MS / 1000
|
|
2937
|
+
|
|
2938
|
+
/**
|
|
2939
|
+
* @param {unknown} value
|
|
2940
|
+
* @param {number} max
|
|
2941
|
+
*/
|
|
2942
|
+
function isSeconds(value, max) {
|
|
2943
|
+
return (
|
|
2944
|
+
typeof value === 'number' &&
|
|
2945
|
+
Number.isFinite(value) &&
|
|
2946
|
+
value > 0 &&
|
|
2947
|
+
value <= max
|
|
2948
|
+
)
|
|
2949
|
+
}
|
|
2950
|
+
|
|
2951
|
+
/**
|
|
2952
|
+
* Only managed raw URLs are checked: the raw host, a commit ref and a
|
|
2953
|
+
* docs/evidence path, nothing else in the URL. A blob URL has its own code,
|
|
2954
|
+
* because the skill routes those to the authenticated `gh api` check.
|
|
2955
|
+
* @param {unknown} raw
|
|
2956
|
+
*/
|
|
2957
|
+
function parseVerifyInput(raw) {
|
|
2958
|
+
if (!isPlainObject(raw)) {
|
|
2959
|
+
return invalid('invalid-input', 'input')
|
|
2960
|
+
}
|
|
2961
|
+
if (!Array.isArray(raw.urls)) {
|
|
2962
|
+
return invalid('invalid-input', 'urls')
|
|
2963
|
+
}
|
|
2964
|
+
if (
|
|
2965
|
+
raw.max_wait_s !== undefined &&
|
|
2966
|
+
!isSeconds(raw.max_wait_s, MAX_CALL_WAIT_S)
|
|
2967
|
+
) {
|
|
2968
|
+
return invalid('invalid-input', 'max_wait_s')
|
|
2969
|
+
}
|
|
2970
|
+
if (
|
|
2971
|
+
raw.request_timeout_s !== undefined &&
|
|
2972
|
+
!isSeconds(raw.request_timeout_s, MAX_CALL_WAIT_S)
|
|
2973
|
+
) {
|
|
2974
|
+
return invalid('invalid-input', 'request_timeout_s')
|
|
2975
|
+
}
|
|
2976
|
+
if (
|
|
2977
|
+
raw.waited_s !== undefined &&
|
|
2978
|
+
!(
|
|
2979
|
+
typeof raw.waited_s === 'number' &&
|
|
2980
|
+
Number.isFinite(raw.waited_s) &&
|
|
2981
|
+
raw.waited_s >= 0
|
|
2982
|
+
)
|
|
2983
|
+
) {
|
|
2984
|
+
return invalid('invalid-input', 'waited_s')
|
|
2985
|
+
}
|
|
2986
|
+
if (
|
|
2987
|
+
raw.resume_at !== undefined &&
|
|
2988
|
+
!(Number.isSafeInteger(raw.resume_at) && raw.resume_at > 0)
|
|
2989
|
+
) {
|
|
2990
|
+
return invalid('invalid-input', 'resume_at')
|
|
2991
|
+
}
|
|
2992
|
+
if (raw.head_repo !== undefined && !isHeadRepo(raw.head_repo)) {
|
|
2993
|
+
return invalid('invalid-input', 'head_repo')
|
|
2994
|
+
}
|
|
2995
|
+
if (raw.urls.length === 0) {
|
|
2996
|
+
return invalid('empty-urls', 'urls')
|
|
2997
|
+
}
|
|
2998
|
+
if (raw.urls.length > MAX_VERIFY_URLS) {
|
|
2999
|
+
return invalid('too-many-urls', 'urls')
|
|
3000
|
+
}
|
|
3001
|
+
for (const url of raw.urls) {
|
|
3002
|
+
if (typeof url !== 'string') {
|
|
3003
|
+
return invalid('invalid-url', 'urls')
|
|
3004
|
+
}
|
|
3005
|
+
if (BLOB_URL.test(url)) {
|
|
3006
|
+
return invalid('blob-url', 'urls')
|
|
3007
|
+
}
|
|
3008
|
+
const managed = RAW_URL.test(url)
|
|
3009
|
+
if (
|
|
3010
|
+
!managed ||
|
|
3011
|
+
(raw.head_repo !== undefined &&
|
|
3012
|
+
parseManagedUrl(url, raw.head_repo) === null)
|
|
3013
|
+
) {
|
|
3014
|
+
return invalid('invalid-url', 'urls')
|
|
3015
|
+
}
|
|
3016
|
+
}
|
|
3017
|
+
return {
|
|
3018
|
+
ok: true,
|
|
3019
|
+
input: {
|
|
3020
|
+
urls: raw.urls,
|
|
3021
|
+
maxWaitS: raw.max_wait_s,
|
|
3022
|
+
requestTimeoutS: raw.request_timeout_s,
|
|
3023
|
+
waitedS: raw.waited_s,
|
|
3024
|
+
resumeAt: raw.resume_at,
|
|
3025
|
+
},
|
|
3026
|
+
}
|
|
3027
|
+
}
|
|
3028
|
+
|
|
3029
|
+
/**
|
|
3030
|
+
* `verify` reports every URL as { url, status, http_status, attempts,
|
|
3031
|
+
* retryable }. `status` is ok, or the reason it failed: not-found,
|
|
3032
|
+
* rate-limited, server-error, network-error, timeout (all retried),
|
|
3033
|
+
* not-an-image, forbidden, unexpected-status, bad-redirect,
|
|
3034
|
+
* redirect-off-host or too-many-redirects (all final).
|
|
3035
|
+
* @param {unknown} raw
|
|
3036
|
+
* @param {Record<string, any>} [options]
|
|
3037
|
+
*/
|
|
3038
|
+
export async function runVerify(raw, options = {}) {
|
|
3039
|
+
const parsed = parseVerifyInput(raw)
|
|
3040
|
+
if (!parsed.ok) {
|
|
3041
|
+
return usageError('verify', parsed.error, parsed.field)
|
|
3042
|
+
}
|
|
3043
|
+
const { input } = parsed
|
|
3044
|
+
const { results, waitedMs } = await verifyUrls(input.urls, {
|
|
3045
|
+
...options,
|
|
3046
|
+
...(input.maxWaitS === undefined
|
|
3047
|
+
? {}
|
|
3048
|
+
: { maxWaitMs: input.maxWaitS * 1000 }),
|
|
3049
|
+
...(input.requestTimeoutS === undefined
|
|
3050
|
+
? {}
|
|
3051
|
+
: { requestTimeoutMs: input.requestTimeoutS * 1000 }),
|
|
3052
|
+
...(input.waitedS === undefined ? {} : { waitedMs: input.waitedS * 1000 }),
|
|
3053
|
+
...(input.resumeAt === undefined ? {} : { resumeAtMs: input.resumeAt }),
|
|
3054
|
+
})
|
|
3055
|
+
const failed = results.filter((entry) => entry.status !== 'ok')
|
|
3056
|
+
return outcome(
|
|
3057
|
+
'verify',
|
|
3058
|
+
failed.length === 0 ? 0 : 1,
|
|
3059
|
+
failed.length === 0 ? 'verified' : 'failed',
|
|
3060
|
+
{
|
|
3061
|
+
checked: results.length,
|
|
3062
|
+
passed: results.length - failed.length,
|
|
3063
|
+
// Cumulative: the waited_s that came in, plus this call's own time. A
|
|
3064
|
+
// caller passes the latest report's value on unchanged.
|
|
3065
|
+
waited_s: Math.ceil(((input.waitedS ?? 0) * 1000 + waitedMs) / 1000),
|
|
3066
|
+
retryable: failed.some((entry) => entry.retryable),
|
|
3067
|
+
// The latest deadline of any failure that can still be retried: no
|
|
3068
|
+
// request may go out before it. The next call passes it back as is.
|
|
3069
|
+
...resumeField(failed),
|
|
3070
|
+
results,
|
|
3071
|
+
failed,
|
|
3072
|
+
},
|
|
3073
|
+
)
|
|
3074
|
+
}
|
|
3075
|
+
|
|
3076
|
+
/** @param {Array<{ retryable: boolean, resume_at?: number }>} failed */
|
|
3077
|
+
function resumeField(failed) {
|
|
3078
|
+
const deadlines = failed
|
|
3079
|
+
.filter((entry) => entry.retryable && entry.resume_at !== undefined)
|
|
3080
|
+
.map((entry) => entry.resume_at)
|
|
3081
|
+
return deadlines.length === 0 ? {} : { resume_at: Math.max(...deadlines) }
|
|
3082
|
+
}
|
|
3083
|
+
|
|
3084
|
+
const OPERATIONS = {
|
|
3085
|
+
classify: runClassify,
|
|
3086
|
+
splice: runSplice,
|
|
3087
|
+
extract: runExtract,
|
|
3088
|
+
urls: runUrls,
|
|
3089
|
+
'repin-rewrite': runRepin,
|
|
3090
|
+
pins: runPins,
|
|
3091
|
+
'cleanup-set': runCleanup,
|
|
3092
|
+
verify: runVerify,
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
/** @param {string[]} argv */
|
|
3096
|
+
async function run(argv) {
|
|
3097
|
+
const operation = argv.length === 1 ? argv[0] : undefined
|
|
3098
|
+
if (operation === undefined || !Object.hasOwn(OPERATIONS, operation)) {
|
|
3099
|
+
return usageError('unknown', 'unknown-operation', 'operation')
|
|
3100
|
+
}
|
|
3101
|
+
let raw
|
|
3102
|
+
try {
|
|
3103
|
+
raw = JSON.parse(readFileSync(0, 'utf8'))
|
|
3104
|
+
} catch {
|
|
3105
|
+
return usageError(operation, 'invalid-json', 'input')
|
|
3106
|
+
}
|
|
3107
|
+
return OPERATIONS[operation](raw)
|
|
3108
|
+
}
|
|
3109
|
+
|
|
3110
|
+
async function main() {
|
|
3111
|
+
let outcome
|
|
3112
|
+
try {
|
|
3113
|
+
outcome = await run(process.argv.slice(2))
|
|
3114
|
+
} catch {
|
|
3115
|
+
// One fixed object and exit 1, never a stack trace with local paths.
|
|
3116
|
+
outcome = {
|
|
3117
|
+
exitCode: 1,
|
|
3118
|
+
body: {
|
|
3119
|
+
schema_version: SCHEMA_VERSION,
|
|
3120
|
+
operation: 'unknown',
|
|
3121
|
+
result: 'internal-error',
|
|
3122
|
+
error: 'internal-error',
|
|
3123
|
+
usage: USAGE,
|
|
3124
|
+
},
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
process.stdout.write(`${JSON.stringify(outcome.body)}\n`)
|
|
3128
|
+
process.exitCode = outcome.exitCode
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
// Node reports import.meta.url as a real path, but argv[1] is the path as
|
|
3132
|
+
// typed. Compare real paths so a symlinked skill directory or helper file
|
|
3133
|
+
// still counts as a direct run.
|
|
3134
|
+
const isDirectInvocation =
|
|
3135
|
+
process.argv[1] !== undefined &&
|
|
3136
|
+
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
|
|
3137
|
+
|
|
3138
|
+
if (isDirectInvocation) {
|
|
3139
|
+
await main()
|
|
3140
|
+
}
|