@futurelastic/colab-handbook 1.16.1-rc.1 → 1.17.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/templates/publish-downloads.mjs +211 -15
- package/templates/publish-downloads.yml +3 -1
- package/tools/colab +10 -4
- package/tools/lib/tracker-github.js +65 -4
package/package.json
CHANGED
|
@@ -39,6 +39,13 @@
|
|
|
39
39
|
// is stamped into each installer as `slug:min slug:min` at @DOWNLOADS_REQUIRES@ — an installer of an app that requires
|
|
40
40
|
// anything must carry that placeholder. The installer side (order, once per run, opt-out) is docs/downloads.md.
|
|
41
41
|
//
|
|
42
|
+
// WHAT A PORTAL SHOWS (#655) is manifest.json's optional description, icon, screenshots, examples, install.prompt,
|
|
43
|
+
// install.handover and listed (docs/downloads.md). The icon and screenshot files are repo paths relative to the
|
|
44
|
+
// manifest; they are checked (at most 1 MB each, their bytes the type their name says, a square icon) and published
|
|
45
|
+
// INSIDE <X.Y.Z>/ next to the binaries — in its SHA256SUMS, so a version's images are as immutable as its files, but
|
|
46
|
+
// not in latest.json's `files`, which lists what an installer downloads. The manifest the site gets names them as
|
|
47
|
+
// `X.Y.Z/<name>` (relative to the app's folder) and has `<site>` replaced by the site root in every install text.
|
|
48
|
+
//
|
|
42
49
|
// THE ORDER ON THE SITE, which is what makes a half-finished run harmless:
|
|
43
50
|
// 1. <X.Y.Z>/ with every file except SHA256SUMS (rsync --ignore-existing: a published file is never replaced)
|
|
44
51
|
// 2. every file read back over HTTPS against the SHA256SUMS this build computed
|
|
@@ -78,6 +85,13 @@ const VERSION = /^\d+\.\d+\.\d+$/
|
|
|
78
85
|
export const DEFAULT_MANIFEST = '.github/downloads/manifest.json'
|
|
79
86
|
/** Names the script itself writes at the app's root; an installer may not take one. */
|
|
80
87
|
const ROOT_NAMES = new Set(['latest.json', 'manifest.json'])
|
|
88
|
+
/** What the manifest's install texts write for the site root; replaced in the copy the site gets. */
|
|
89
|
+
export const SITE_TOKEN = '<site>'
|
|
90
|
+
const MANIFEST_KEYS = ['schema', 'name', 'summary', 'description', 'icon', 'screenshots', 'examples', 'install', 'requires', 'listed']
|
|
91
|
+
export const LIMITS = { images: 6, examples: 6, imageBytes: 1024 * 1024, descriptionChars: 2000, outputLines: 10 }
|
|
92
|
+
const IMAGE_TYPES = { png: 'png', jpg: 'jpeg', jpeg: 'jpeg', webp: 'webp', svg: 'svg' }
|
|
93
|
+
const SCREENSHOT_EXT = ['png', 'jpg', 'jpeg', 'webp']
|
|
94
|
+
const ICON_EXT = ['png', 'svg']
|
|
81
95
|
|
|
82
96
|
/** The version of a final tag. A candidate (`vX.Y.Z-rc.N`) and anything else is refused. */
|
|
83
97
|
export function finalVersion(tag) {
|
|
@@ -231,25 +245,91 @@ function isText(v, oneLine = false) {
|
|
|
231
245
|
)
|
|
232
246
|
}
|
|
233
247
|
|
|
248
|
+
/** Problems with a `description` text: the Markdown subset docs/downloads.md gives, nothing heavier. */
|
|
249
|
+
function markdownProblems(s, at) {
|
|
250
|
+
const out = []
|
|
251
|
+
if (s.length > LIMITS.descriptionChars) out.push(`${at}: longer than ${LIMITS.descriptionChars} characters`)
|
|
252
|
+
if (/<[A-Za-z/!?]/.test(s)) out.push(`${at}: raw HTML is not shown`)
|
|
253
|
+
if (/^ {0,3}#/m.test(s)) out.push(`${at}: no headings (#) — the portal titles the app itself`)
|
|
254
|
+
if (/!\[/.test(s)) out.push(`${at}: no inline images — use screenshots`)
|
|
255
|
+
if (/^ {0,3}(```|~~~)/m.test(s)) out.push(`${at}: no code blocks — use examples`)
|
|
256
|
+
for (const [, url] of s.matchAll(/\]\(([^)\s]*)/g))
|
|
257
|
+
if (!/^https:\/\/\S+$/.test(url)) out.push(`${at}: the link ${JSON.stringify(url)} is not an https address`)
|
|
258
|
+
return out
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Problems with a manifest image path: relative to the manifest, inside the repo, a plain file name of a known type. */
|
|
262
|
+
function imagePathProblems(p, at, exts) {
|
|
263
|
+
if (typeof p !== 'string' || p === '') return [`${at}: a file path, relative to manifest.json`]
|
|
264
|
+
if (p.startsWith('/') || p.includes('\\') || p.split('/').some((seg) => seg === '..' || seg === ''))
|
|
265
|
+
return [`${at}: ${JSON.stringify(p)} is not a path inside the repo, relative to manifest.json`]
|
|
266
|
+
const name = basename(p)
|
|
267
|
+
const ext = name.includes('.') ? name.slice(name.lastIndexOf('.') + 1).toLowerCase() : ''
|
|
268
|
+
if (name.startsWith('.') || name === 'SHA256SUMS' || !exts.includes(ext))
|
|
269
|
+
return [`${at}: ${JSON.stringify(name)} is not a ${exts.map((e) => `.${e}`).join(' / ')} file`]
|
|
270
|
+
return []
|
|
271
|
+
}
|
|
272
|
+
|
|
234
273
|
/** Throws naming every problem with a manifest object; returns it unchanged when valid. Fields: docs/downloads.md. */
|
|
235
274
|
export function validateManifest(m) {
|
|
236
275
|
const problems = []
|
|
237
276
|
if (m === null || typeof m !== 'object' || Array.isArray(m)) throw new Error('manifest.json is not a JSON object')
|
|
238
|
-
|
|
239
|
-
|
|
277
|
+
const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v)
|
|
278
|
+
for (const k of Object.keys(m)) if (!MANIFEST_KEYS.includes(k)) problems.push(`unknown field "${k}"`)
|
|
240
279
|
if (m.schema !== undefined && m.schema !== 1) problems.push('schema must be 1')
|
|
241
280
|
if (!isText(m.name, true)) problems.push('name: a one-line text is required')
|
|
242
281
|
if (!isText(m.summary, true)) problems.push('summary: a one-line text is required')
|
|
282
|
+
if (m.description !== undefined) {
|
|
283
|
+
if (!isText(m.description)) problems.push('description: a text')
|
|
284
|
+
else
|
|
285
|
+
for (const [lang, s] of typeof m.description === 'string' ? [[null, m.description]] : Object.entries(m.description))
|
|
286
|
+
problems.push(...markdownProblems(s, lang ? `description.${lang}` : 'description'))
|
|
287
|
+
}
|
|
288
|
+
if (m.icon !== undefined) problems.push(...imagePathProblems(m.icon, 'icon', ICON_EXT))
|
|
289
|
+
if (m.screenshots !== undefined) {
|
|
290
|
+
if (!Array.isArray(m.screenshots) || m.screenshots.length > LIMITS.images)
|
|
291
|
+
problems.push(`screenshots: a list of at most ${LIMITS.images} {"file", "alt", "caption", "theme"}`)
|
|
292
|
+
else
|
|
293
|
+
m.screenshots.forEach((s, i) => {
|
|
294
|
+
const at = `screenshots[${i}]`
|
|
295
|
+
if (!isObject(s)) return problems.push(`${at}: an object {"file", "alt", "caption", "theme"}`)
|
|
296
|
+
for (const k of Object.keys(s)) if (!['file', 'alt', 'caption', 'theme'].includes(k)) problems.push(`${at}.${k}: unknown field`)
|
|
297
|
+
problems.push(...imagePathProblems(s.file, `${at}.file`, SCREENSHOT_EXT))
|
|
298
|
+
if (!isText(s.alt, true)) problems.push(`${at}.alt: a one-line text is required (what the image shows)`)
|
|
299
|
+
if (s.caption !== undefined && !isText(s.caption, true)) problems.push(`${at}.caption: a one-line text`)
|
|
300
|
+
if (s.theme !== undefined && !['light', 'dark'].includes(s.theme)) problems.push(`${at}.theme: "light" or "dark"`)
|
|
301
|
+
})
|
|
302
|
+
}
|
|
303
|
+
if (m.examples !== undefined) {
|
|
304
|
+
if (!Array.isArray(m.examples) || m.examples.length > LIMITS.examples)
|
|
305
|
+
problems.push(`examples: a list of at most ${LIMITS.examples} {"caption", "command", "output"}`)
|
|
306
|
+
else
|
|
307
|
+
m.examples.forEach((x, i) => {
|
|
308
|
+
const at = `examples[${i}]`
|
|
309
|
+
if (!isObject(x)) return problems.push(`${at}: an object {"caption", "command", "output"}`)
|
|
310
|
+
for (const k of Object.keys(x)) if (!['caption', 'command', 'output'].includes(k)) problems.push(`${at}.${k}: unknown field`)
|
|
311
|
+
if (!isText(x.caption, true)) problems.push(`${at}.caption: a one-line text is required`)
|
|
312
|
+
if (typeof x.command !== 'string' || x.command.trim() === '' || /[\r\n]/.test(x.command))
|
|
313
|
+
problems.push(`${at}.command: one line, the command as typed`)
|
|
314
|
+
if (x.output !== undefined) {
|
|
315
|
+
if (typeof x.output !== 'string') problems.push(`${at}.output: a plain text`)
|
|
316
|
+
else if (x.output.replace(/\r?\n$/, '').split(/\r?\n/).length > LIMITS.outputLines)
|
|
317
|
+
problems.push(`${at}.output: more than ${LIMITS.outputLines} lines — trim it to what shows the point`)
|
|
318
|
+
}
|
|
319
|
+
})
|
|
320
|
+
}
|
|
243
321
|
const inst = m.install
|
|
244
|
-
|
|
322
|
+
const platforms = isObject(inst) ? Object.keys(inst).filter((k) => !['prompt', 'handover'].includes(k)) : []
|
|
323
|
+
if (!isObject(inst) || platforms.length === 0) {
|
|
245
324
|
problems.push(`install: an object keyed by platform (${PLATFORMS.join(', ')}) is required`)
|
|
246
325
|
} else {
|
|
247
|
-
for (const
|
|
326
|
+
for (const os of platforms) {
|
|
327
|
+
const how = inst[os]
|
|
248
328
|
if (!PLATFORMS.includes(os)) {
|
|
249
|
-
problems.push(`install.${os}: not a platform (${PLATFORMS.join(', ')})`)
|
|
329
|
+
problems.push(`install.${os}: not a platform (${PLATFORMS.join(', ')}) nor prompt / handover`)
|
|
250
330
|
continue
|
|
251
331
|
}
|
|
252
|
-
if (
|
|
332
|
+
if (!isObject(how)) {
|
|
253
333
|
problems.push(`install.${os}: an object with steps (and optional prompts)`)
|
|
254
334
|
continue
|
|
255
335
|
}
|
|
@@ -260,6 +340,9 @@ export function validateManifest(m) {
|
|
|
260
340
|
if (how.prompts !== undefined && (!Array.isArray(how.prompts) || !how.prompts.every((s) => isText(s))))
|
|
261
341
|
problems.push(`install.${os}.prompts: a list of texts`)
|
|
262
342
|
}
|
|
343
|
+
if (inst.prompt !== undefined && !isText(inst.prompt)) problems.push('install.prompt: a text (what a person pastes into an agent)')
|
|
344
|
+
if (inst.handover !== undefined && (!Array.isArray(inst.handover) || inst.handover.length === 0 || !inst.handover.every((s) => isText(s))))
|
|
345
|
+
problems.push('install.handover: a non-empty list of texts')
|
|
263
346
|
}
|
|
264
347
|
if (m.requires !== undefined) {
|
|
265
348
|
if (!Array.isArray(m.requires)) problems.push('requires: a list of {"slug", "min"}')
|
|
@@ -267,7 +350,7 @@ export function validateManifest(m) {
|
|
|
267
350
|
const seen = new Set()
|
|
268
351
|
m.requires.forEach((r, i) => {
|
|
269
352
|
const at = `requires[${i}]`
|
|
270
|
-
if (
|
|
353
|
+
if (!isObject(r)) return problems.push(`${at}: an object {"slug", "min"}`)
|
|
271
354
|
for (const k of Object.keys(r)) if (!['slug', 'min'].includes(k)) problems.push(`${at}.${k}: unknown field`)
|
|
272
355
|
if (typeof r.slug !== 'string' || !SLUG.test(r.slug)) problems.push(`${at}.slug: an app slug, [a-z0-9][a-z0-9-]*`)
|
|
273
356
|
else if (seen.has(r.slug)) problems.push(`${at}.slug: ${r.slug} is listed twice`)
|
|
@@ -276,6 +359,7 @@ export function validateManifest(m) {
|
|
|
276
359
|
})
|
|
277
360
|
}
|
|
278
361
|
}
|
|
362
|
+
if (m.listed !== undefined && typeof m.listed !== 'boolean') problems.push('listed: true or false')
|
|
279
363
|
if (problems.length) throw new Error(`manifest.json: ${problems.join('; ')}`)
|
|
280
364
|
return m
|
|
281
365
|
}
|
|
@@ -286,6 +370,87 @@ export const requiresOf = (manifest) => (Array.isArray(manifest?.requires) ? man
|
|
|
286
370
|
/** What an installer's @DOWNLOADS_REQUIRES@ becomes: `slug:min slug:min`, in the manifest's order. */
|
|
287
371
|
export const requiresStamp = (manifest) => requiresOf(manifest).map((r) => `${r.slug}:${r.min}`).join(' ')
|
|
288
372
|
|
|
373
|
+
/** The manifest's image paths, in order: the icon first, then each screenshot. */
|
|
374
|
+
export const imagesOf = (manifest) => [
|
|
375
|
+
...(typeof manifest?.icon === 'string' ? [{ field: 'icon', path: manifest.icon }] : []),
|
|
376
|
+
...(Array.isArray(manifest?.screenshots) ? manifest.screenshots : []).map((s, i) => ({ field: `screenshots[${i}].file`, path: s.file })),
|
|
377
|
+
]
|
|
378
|
+
|
|
379
|
+
/** The type a file's first bytes say it is (png, jpeg, webp, svg), or null. */
|
|
380
|
+
export function sniffImage(bytes) {
|
|
381
|
+
const head = bytes.subarray(0, 12)
|
|
382
|
+
if (head.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'png'
|
|
383
|
+
if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) return 'jpeg'
|
|
384
|
+
if (head.subarray(0, 4).toString('latin1') === 'RIFF' && head.subarray(8, 12).toString('latin1') === 'WEBP') return 'webp'
|
|
385
|
+
if (/<svg[\s>]/.test(bytes.subarray(0, 4096).toString('utf8'))) return 'svg'
|
|
386
|
+
return null
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/** An image's width and height when the file says so cheaply (a PNG's header, an SVG's viewBox), else null. */
|
|
390
|
+
function imageSize(bytes, type) {
|
|
391
|
+
if (type === 'png' && bytes.length >= 24) return { w: bytes.readUInt32BE(16), h: bytes.readUInt32BE(20) }
|
|
392
|
+
if (type === 'svg') {
|
|
393
|
+
const vb = /<svg[^>]*\sviewBox\s*=\s*["']\s*[-\d.]+[\s,]+[-\d.]+[\s,]+([\d.]+)[\s,]+([\d.]+)\s*["']/.exec(bytes.toString('utf8'))
|
|
394
|
+
if (vb) return { w: Number(vb[1]), h: Number(vb[2]) }
|
|
395
|
+
}
|
|
396
|
+
return null
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Checks the files behind the manifest's icon and screenshots — present, at most LIMITS.imageBytes, their bytes the
|
|
401
|
+
* type their name says, the icon square, no two published under one name — and returns them as `{ name, source }`.
|
|
402
|
+
* Paths are relative to the manifest file (`manifestPath`, itself relative to `root`).
|
|
403
|
+
*/
|
|
404
|
+
export function imageFiles(root, manifestPath, manifest) {
|
|
405
|
+
const dir = join(root, manifestPath, '..')
|
|
406
|
+
const out = []
|
|
407
|
+
const problems = []
|
|
408
|
+
for (const { field, path: p } of imagesOf(manifest)) {
|
|
409
|
+
const source = join(dir, p)
|
|
410
|
+
const name = basename(p)
|
|
411
|
+
const at = `${field} ${p}`
|
|
412
|
+
if (!existsSync(source) || !statSync(source).isFile()) {
|
|
413
|
+
problems.push(`${at}: no such file next to ${manifestPath}`)
|
|
414
|
+
continue
|
|
415
|
+
}
|
|
416
|
+
const size = statSync(source).size
|
|
417
|
+
if (size > LIMITS.imageBytes) problems.push(`${at}: ${size} bytes, over the ${LIMITS.imageBytes}-byte limit`)
|
|
418
|
+
const bytes = readFileSync(source)
|
|
419
|
+
const type = sniffImage(bytes)
|
|
420
|
+
const want = IMAGE_TYPES[name.slice(name.lastIndexOf('.') + 1).toLowerCase()]
|
|
421
|
+
if (type !== want) problems.push(`${at}: its bytes are ${type ? `a ${type}` : 'not an image'}, not the ${want} its name says`)
|
|
422
|
+
else if (field === 'icon') {
|
|
423
|
+
const dim = imageSize(bytes, type)
|
|
424
|
+
if (!dim) problems.push(`${at}: an SVG icon needs a viewBox, to be checked square`)
|
|
425
|
+
else if (dim.w !== dim.h) problems.push(`${at}: ${dim.w}×${dim.h}, an icon is square`)
|
|
426
|
+
}
|
|
427
|
+
const twin = out.find((o) => o.name === name)
|
|
428
|
+
if (twin && twin.source !== source) problems.push(`${at}: another image is also named ${name}, and both would be ${name} on the site`)
|
|
429
|
+
if (!twin) out.push({ name, source })
|
|
430
|
+
}
|
|
431
|
+
if (problems.length) throw new Error(`${manifestPath}: ${problems.join('; ')}`)
|
|
432
|
+
return out
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function stampSite(v, base) {
|
|
436
|
+
if (typeof v === 'string') return v.split(SITE_TOKEN).join(base)
|
|
437
|
+
if (Array.isArray(v)) return v.map((x) => stampSite(x, base))
|
|
438
|
+
if (v !== null && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, stampSite(x, base)]))
|
|
439
|
+
return v
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* The manifest as the site gets it: the icon and screenshots named `X.Y.Z/<name>` (relative to the app's folder,
|
|
444
|
+
* where this version published them) and `<site>` replaced by the site root in every install text.
|
|
445
|
+
*/
|
|
446
|
+
export function publishedManifest(manifest, { version, base }) {
|
|
447
|
+
const m = { ...manifest }
|
|
448
|
+
if (typeof m.icon === 'string') m.icon = `${version}/${basename(m.icon)}`
|
|
449
|
+
if (Array.isArray(m.screenshots)) m.screenshots = m.screenshots.map((s) => ({ ...s, file: `${version}/${basename(s.file)}` }))
|
|
450
|
+
if (m.install !== undefined) m.install = stampSite(m.install, base)
|
|
451
|
+
return m
|
|
452
|
+
}
|
|
453
|
+
|
|
289
454
|
/** Reads and validates the repo's config + manifest. */
|
|
290
455
|
export function loadRepo(root, env = {}) {
|
|
291
456
|
const yml = join(root, '.github', 'project.yml')
|
|
@@ -305,6 +470,7 @@ export function loadRepo(root, env = {}) {
|
|
|
305
470
|
validateManifest(manifest)
|
|
306
471
|
if (requiresOf(manifest).some((r) => r.slug === config.app))
|
|
307
472
|
throw new Error(`${config.manifest}: requires names ${config.app} itself`)
|
|
473
|
+
imageFiles(root, config.manifest, manifest)
|
|
308
474
|
for (const p of config.installers) {
|
|
309
475
|
const f = join(root, p)
|
|
310
476
|
if (!existsSync(f)) throw new Error(`downloads.installers: ${p} does not exist`)
|
|
@@ -360,16 +526,26 @@ export async function stage({ tag, assetsDir, outDir, baseUrl, root, config, man
|
|
|
360
526
|
)
|
|
361
527
|
}
|
|
362
528
|
const names = [...namesFor(config.files, version), ...namesFor(config.optional, version).filter((n) => present.has(n))]
|
|
529
|
+
const images = imageFiles(root, config.manifest, manifest)
|
|
530
|
+
const clash = images.find((i) => names.includes(i.name))
|
|
531
|
+
if (clash) throw new Error(`${config.manifest}: the image ${clash.name} has the name of a file of this version; rename the image`)
|
|
363
532
|
const folder = join(outDir, version)
|
|
364
533
|
mkdirSync(folder, { recursive: true })
|
|
365
534
|
const files = {}
|
|
535
|
+
const inFolder = {}
|
|
366
536
|
for (const name of names) {
|
|
367
537
|
copyFileSync(join(assetsDir, name), join(folder, name))
|
|
368
538
|
files[name] = { sha256: await sha256File(join(folder, name)), size: statSync(join(folder, name)).size }
|
|
539
|
+
inFolder[name] = files[name]
|
|
540
|
+
}
|
|
541
|
+
// The images join the version folder and its SHA256SUMS, not latest.json's `files` (what an installer downloads).
|
|
542
|
+
for (const { name, source } of images) {
|
|
543
|
+
copyFileSync(source, join(folder, name))
|
|
544
|
+
inFolder[name] = { sha256: await sha256File(join(folder, name)), size: statSync(join(folder, name)).size }
|
|
369
545
|
}
|
|
370
|
-
const sums = Object.keys(
|
|
546
|
+
const sums = Object.keys(inFolder)
|
|
371
547
|
.sort()
|
|
372
|
-
.map((n) => `${
|
|
548
|
+
.map((n) => `${inFolder[n].sha256} ${n}\n`)
|
|
373
549
|
.join('')
|
|
374
550
|
writeFileSync(join(folder, 'SHA256SUMS'), sums)
|
|
375
551
|
const rootFiles = []
|
|
@@ -379,10 +555,10 @@ export async function stage({ tag, assetsDir, outDir, baseUrl, root, config, man
|
|
|
379
555
|
writeFileSync(join(outDir, basename(p)), stamped)
|
|
380
556
|
rootFiles.push(basename(p))
|
|
381
557
|
}
|
|
382
|
-
writeFileSync(join(outDir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`)
|
|
558
|
+
writeFileSync(join(outDir, 'manifest.json'), `${JSON.stringify(publishedManifest(manifest, { version, base }), null, 2)}\n`)
|
|
383
559
|
rootFiles.push('manifest.json')
|
|
384
560
|
writeFileSync(join(outDir, 'latest.json'), `${JSON.stringify({ version, files }, null, 2)}\n`)
|
|
385
|
-
return { version, files, sums, rootFiles }
|
|
561
|
+
return { version, files, images: images.map((i) => i.name), sums, rootFiles }
|
|
386
562
|
}
|
|
387
563
|
|
|
388
564
|
// ── the site ─────────────────────────────────────────────────────────────────────────────────────
|
|
@@ -483,6 +659,13 @@ export async function checkRequires({ app, manifest, baseUrl, fetchImpl = fetch
|
|
|
483
659
|
return visited
|
|
484
660
|
}
|
|
485
661
|
|
|
662
|
+
/** A SHA256SUMS file's lines as `{ <name>: { sha256 } }`. */
|
|
663
|
+
export function parseSums(bytes) {
|
|
664
|
+
const out = {}
|
|
665
|
+
for (const line of bytes.toString('utf8').split('\n')) if (line.length > 66) out[line.slice(66)] = { sha256: line.slice(0, 64) }
|
|
666
|
+
return out
|
|
667
|
+
}
|
|
668
|
+
|
|
486
669
|
/**
|
|
487
670
|
* Uploads a staged tree (see `stage`) in the order the header gives. `dest` is the rsync destination (`user@host:`
|
|
488
671
|
* in production — the account's root is the app's folder), `sshArgs` the `-e` command line.
|
|
@@ -500,8 +683,10 @@ export async function publish({
|
|
|
500
683
|
}) {
|
|
501
684
|
const root = `${siteRoot(baseUrl)}/${app}`
|
|
502
685
|
const latestBytes = readFileSync(join(outDir, 'latest.json'))
|
|
503
|
-
const { version
|
|
686
|
+
const { version } = JSON.parse(latestBytes.toString('utf8'))
|
|
504
687
|
const localSums = readFileSync(join(outDir, version, 'SHA256SUMS'))
|
|
688
|
+
// Everything in the version folder — the files and the manifest's images — is what SHA256SUMS lists.
|
|
689
|
+
const files = parseSums(localSums)
|
|
505
690
|
const rootFiles = readdirSync(outDir).filter((n) => n !== 'latest.json' && n !== version && n !== 'manifest.json')
|
|
506
691
|
rootFiles.sort()
|
|
507
692
|
rootFiles.push('manifest.json') // installers first, manifest after them, latest.json last
|
|
@@ -515,8 +700,17 @@ export async function publish({
|
|
|
515
700
|
|
|
516
701
|
const remoteSums = await readBytes(fetchImpl, `${root}/${version}/SHA256SUMS`)
|
|
517
702
|
if (remoteSums !== null && !remoteSums.equals(localSums)) {
|
|
703
|
+
const theirs = parseSums(remoteSums)
|
|
704
|
+
const differ = [...new Set([...Object.keys(files), ...Object.keys(theirs)])]
|
|
705
|
+
.filter((n) => files[n]?.sha256 !== theirs[n]?.sha256)
|
|
706
|
+
.sort()
|
|
707
|
+
const images = new Set(imagesOf(manifest).map((i) => basename(i.path)))
|
|
708
|
+
const hint =
|
|
709
|
+
differ.length > 0 && differ.every((n) => images.has(n))
|
|
710
|
+
? ' — only images differ: publish from the tag\'s own commit (run the workflow from the tag), whose images this version carries'
|
|
711
|
+
: '; publish a new version instead'
|
|
518
712
|
throw new Error(
|
|
519
|
-
`${version} is already published and its SHA256SUMS differs from this build's — versions are immutable, nothing was changed
|
|
713
|
+
`${version} is already published and its SHA256SUMS differs from this build's (${differ.join(', ') || 'its bytes'}) — versions are immutable, nothing was changed${hint}`,
|
|
520
714
|
)
|
|
521
715
|
}
|
|
522
716
|
log(remoteSums === null ? `${version}: not complete on the site yet` : `${version}: complete on the site — verifying only`)
|
|
@@ -619,7 +813,9 @@ async function main(argv, env) {
|
|
|
619
813
|
const { config, manifest } = loadRepo(root, env)
|
|
620
814
|
if (command === 'check') {
|
|
621
815
|
const req = requiresOf(manifest).length ? `, requires ${requiresStamp(manifest)}` : ''
|
|
622
|
-
|
|
816
|
+
const img = imagesOf(manifest).length ? `, ${imagesOf(manifest).length} image(s)` : ''
|
|
817
|
+
const unlisted = manifest.listed === false ? ', unlisted' : ''
|
|
818
|
+
console.log(`downloads: ${config.app} — ${config.files.length} file(s), ${config.optional.length} optional, ${config.installers.length} installer(s), manifest ${config.manifest} valid${img}${req}${unlisted}`)
|
|
623
819
|
return 0
|
|
624
820
|
}
|
|
625
821
|
if (command === 'names') {
|
|
@@ -632,7 +828,7 @@ async function main(argv, env) {
|
|
|
632
828
|
const baseUrl = env.DOWNLOADS_URL
|
|
633
829
|
const outDir = mkdtempSync(join(tmpdir(), 'publish-downloads-'))
|
|
634
830
|
const staged = await stage({ tag, assetsDir, outDir, baseUrl, root, config, manifest })
|
|
635
|
-
console.log(`Staged ${config.app} ${staged.version}: ${Object.keys(staged.files).join(', ')}`)
|
|
831
|
+
console.log(`Staged ${config.app} ${staged.version}: ${[...Object.keys(staged.files), ...staged.images].join(', ')}`)
|
|
636
832
|
await assertResolves(baseUrl)
|
|
637
833
|
const target = dryRun ? { dest: 'DRY-RUN:', sshArgs: undefined } : sshCommand(env)
|
|
638
834
|
const result = await publish({ outDir, app: config.app, ...target, baseUrl, dryRun })
|
|
@@ -37,7 +37,9 @@
|
|
|
37
37
|
# - By hand (Actions → Run workflow) with a tag: retries a failed publish, finishes a partial one (only the
|
|
38
38
|
# missing files are sent), re-checks a complete one. `dry_run` stages and lists — it uploads nothing.
|
|
39
39
|
# The project.yml, manifest, installers and script are read from THIS run's checkout: the tag's commit when
|
|
40
|
-
# chained, the branch you dispatch from when run by hand.
|
|
40
|
+
# chained, the branch you dispatch from when run by hand. So are the manifest's icon and screenshots (#655),
|
|
41
|
+
# which go INTO the version folder: re-running an older tag by hand, run the workflow from that tag, or its
|
|
42
|
+
# images may differ from the published ones and the run is refused (a published version is immutable).
|
|
41
43
|
#
|
|
42
44
|
# THE RUNNER. The read-back goes over HTTPS to the site, which may be reachable only on a private network —
|
|
43
45
|
# a GitHub-hosted runner then cannot read anything back. `runs-on` is an input: a bare label or a JSON label
|
package/tools/colab
CHANGED
|
@@ -13440,13 +13440,19 @@ function readTargetCodeowners(repoAbs, target) {
|
|
|
13440
13440
|
|
|
13441
13441
|
/**
|
|
13442
13442
|
* #483: is this repo a fork, and which org owns it? The fork test is adopt's own (#449: a remote named
|
|
13443
|
-
* `upstream` whose URL differs from origin's), so adopt and ship never disagree about the shape
|
|
13444
|
-
*
|
|
13443
|
+
* `upstream` whose URL differs from origin's), so adopt and ship never disagree about the shape — or,
|
|
13444
|
+
* with no such remote on this checkout, the committed record of that same test (#658): the descriptor's
|
|
13445
|
+
* `instruction-files: upstream` (#602), which adopt writes on a detected fork. A checkout that never
|
|
13446
|
+
* added the remote then still reads as the fork its descriptor says it is. The owner is read from
|
|
13447
|
+
* this repo's remote URL. Either unknown → `{ fork: false }`, nothing is dropped.
|
|
13445
13448
|
*/
|
|
13446
13449
|
function codeownersForkContext(repoAbs) {
|
|
13447
|
-
|
|
13450
|
+
const declared = instructionFile.parseInstructionFiles(projectDoc(repoAbs)).exempt;
|
|
13451
|
+
if (!adoptDetectFork(repoAbs) && !declared) return { fork: false, ownOrg: null };
|
|
13448
13452
|
const name = git.remoteInfo(repoAbs).name || 'origin';
|
|
13449
|
-
|
|
13453
|
+
// The configured url, as the gh pin reads it (#658): `get-url` expands `url.<x>.insteadOf`, which
|
|
13454
|
+
// changes how git reaches the remote, never which org owns it.
|
|
13455
|
+
const u = git.git(['config', '--get', `remote.${name}.url`], repoAbs);
|
|
13450
13456
|
const ownOrg = u.ok ? codeowners.remoteOwner(u.stdout) : null;
|
|
13451
13457
|
return { fork: !!ownOrg, ownOrg };
|
|
13452
13458
|
}
|
|
@@ -15,7 +15,9 @@
|
|
|
15
15
|
* `null` is "could not read", never "none").
|
|
16
16
|
*
|
|
17
17
|
* Behaviour-preserving by construction: every method spawns the exact argv and options the call
|
|
18
|
-
* site it replaced spawned (pinned by tools/lib/tracker-github.test.js's argv table).
|
|
18
|
+
* site it replaced spawned (pinned by tools/lib/tracker-github.test.js's argv table). The one
|
|
19
|
+
* addition is the environment: `exec` pins `GH_REPO` to the resolved remote's repo (#658,
|
|
20
|
+
* `pinnedOpts`), so no call depends on gh's default repo or on which other remotes exist.
|
|
19
21
|
*
|
|
20
22
|
* This module must NOT require ./tracker (tracker requires this one), and git.js must never require
|
|
21
23
|
* this module at its top level: git.js ends in `module.exports = {…}`, which REPLACES the exports
|
|
@@ -26,9 +28,68 @@
|
|
|
26
28
|
|
|
27
29
|
let _runner = null; // test seam only — see withRunner
|
|
28
30
|
|
|
29
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* #658 — `[HOST/]OWNER/REPO` for a remote URL, in the form `GH_REPO` takes, or null.
|
|
33
|
+
*
|
|
34
|
+
* Only a host gh is known to serve is pinned: `github.com`, or the `GH_HOST` the environment names.
|
|
35
|
+
* Any other host — an SSH alias, a self-hosted forge gh was never pointed at — reads as null, and
|
|
36
|
+
* the call keeps gh's own resolution exactly as before, rather than pinning a name gh cannot reach.
|
|
37
|
+
*/
|
|
38
|
+
function ghRepoFromUrl(url, env = process.env) {
|
|
39
|
+
const s = String(url || '').trim();
|
|
40
|
+
let m = /^[a-z][a-z0-9+.-]*:\/\/(?:[^@/]+@)?([^/:]+)(?::\d+)?\/([^/]+)\/([^/]+?)(?:\.git)?\/*$/i.exec(s);
|
|
41
|
+
if (!m) m = /^(?:[^@/\s]+@)?([^:/\s]+):([^/]+)\/([^/]+?)(?:\.git)?\/*$/.exec(s);
|
|
42
|
+
if (!m) return null;
|
|
43
|
+
const host = m[1].toLowerCase();
|
|
44
|
+
const ghHost = String(env.GH_HOST || '').toLowerCase();
|
|
45
|
+
if (host === 'github.com') return `${m[2]}/${m[3]}`;
|
|
46
|
+
if (ghHost && host === ghHost) return `${host}/${m[2]}/${m[3]}`;
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const _pinCache = new Map(); // resolved cwd → GH_REPO value or null; one repo's remote does not move mid-run
|
|
51
|
+
function _resetPinCache() { _pinCache.clear(); }
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* #658 — the repo every `gh` call is pinned to: the one colab's own resolved remote points at
|
|
55
|
+
* (`origin`, or `colab.remote` / the single remote — tools/lib/git.js `remoteInfo`, #301), read
|
|
56
|
+
* for the call's `cwd`. Without it a bare `gh issue|pr|run …` follows gh's own default, which on
|
|
57
|
+
* a fork checkout with an `upstream` remote and no `gh repo set-default` is the UPSTREAM — a ship
|
|
58
|
+
* then read the carried issues as CLOSED and trunk CI as "no run" from the wrong repo.
|
|
59
|
+
* null when nothing resolves (no remote, ambiguous remotes, a host gh does not serve): unpinned.
|
|
60
|
+
*/
|
|
61
|
+
function pinnedRepo(cwd) {
|
|
62
|
+
const key = require('path').resolve(cwd || process.cwd());
|
|
63
|
+
if (_pinCache.has(key)) return _pinCache.get(key);
|
|
64
|
+
let slug = null;
|
|
65
|
+
try {
|
|
66
|
+
// The CONFIGURED url, not `git remote get-url`: get-url expands `url.<x>.insteadOf`, which
|
|
67
|
+
// rewrites how git reaches the remote (a mirror, a local path), never which forge repo it is.
|
|
68
|
+
const g = require('./git');
|
|
69
|
+
const { name } = g.remoteInfo(key);
|
|
70
|
+
const u = name ? g.git(['config', '--get', `remote.${name}.url`], key) : null;
|
|
71
|
+
slug = u && u.ok ? ghRepoFromUrl(u.stdout) : null;
|
|
72
|
+
} catch (_) { slug = null; }
|
|
73
|
+
_pinCache.set(key, slug);
|
|
74
|
+
return slug;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* `opts` with `GH_REPO` set to `pinnedRepo(opts.cwd)` — unless the call already names its repo
|
|
79
|
+
* (`--repo`/`-R`), the caller passed its own `GH_REPO`, or the environment sets one (an explicit
|
|
80
|
+
* choice, honoured as gh honours it). Every other option passes through untouched.
|
|
81
|
+
*/
|
|
82
|
+
function pinnedOpts(args, opts = {}) {
|
|
83
|
+
if ((args || []).some((a) => a === '--repo' || a === '-R' || /^--repo=/.test(String(a)))) return opts;
|
|
84
|
+
const base = opts.env || process.env;
|
|
85
|
+
if (base.GH_REPO) return opts;
|
|
86
|
+
const slug = pinnedRepo(opts.cwd);
|
|
87
|
+
return slug ? { ...opts, env: { ...base, GH_REPO: slug } } : opts;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Spawn `gh <args>` with `opts` (git.run's options: cwd, timeoutMs, maxBuffer, …), pinned to origin's repo (#658). */
|
|
30
91
|
function exec(args, opts = {}) {
|
|
31
|
-
return _runner ? _runner(args, opts) : require('./git').run('gh', args, opts);
|
|
92
|
+
return _runner ? _runner(args, opts) : require('./git').run('gh', args, pinnedOpts(args, opts));
|
|
32
93
|
}
|
|
33
94
|
|
|
34
95
|
/**
|
|
@@ -103,4 +164,4 @@ function create(repoAbs) {
|
|
|
103
164
|
};
|
|
104
165
|
}
|
|
105
166
|
|
|
106
|
-
module.exports = { exec, withRunner, create };
|
|
167
|
+
module.exports = { exec, withRunner, create, ghRepoFromUrl, pinnedRepo, pinnedOpts, _resetPinCache };
|