kaloko 0.4.9 → 0.4.10

package/src/lib/pr.mjs

@@ -52,8 +52,11 @@ export function prCommentBody ({ manifest, url, at = new Date() }) {
  * Creates or updates the scenario's comment on the current branch's PR.
  * @returns {{ ok: boolean, action?: 'created'|'updated', url?: string, reason?: string }}
  */
-export async function upsertPrComment ({ manifest, url, cwd, run = defaultRun }) {
-  const view = await run('gh', ['pr', 'view', '--json', 'number,url,headRepository,headRepositoryOwner'], cwd)
+export async function upsertPrComment ({ manifest, url, cwd, run = defaultRun, vars = process.env, fetchImpl = fetch }) {
+  // GitLab merge request pipelines: a note on the merge request through the API
+  if (vars.GITLAB_CI && vars.CI_MERGE_REQUEST_IID) return upsertMrNote({ manifest, url, vars, fetchImpl })
+  // in CI the checkout is often a detached commit: KALOKO_PR (kaloko ci preview-url sets it) names the pull request
+  const view = await run('gh', ['pr', 'view', ...(vars.KALOKO_PR ? [String(vars.KALOKO_PR)] : []), '--json', 'number,url,headRepository,headRepositoryOwner'], cwd)
   if (view == null) return { ok: false, reason: 'gh is not installed, not signed in, or this branch has no pull request' }
   let pr
   try { pr = JSON.parse(view) } catch { return { ok: false, reason: 'gh pr view returned no JSON' } }
@@ -73,6 +76,25 @@ export async function upsertPrComment ({ manifest, url, cwd, run = defaultRun })
   return r == null ? { ok: false, reason: 'could not create the comment' } : { ok: true, action: 'created', url: r.trim() || pr.url }
 }
 
+/** One note per scenario on a GitLab merge request (KALOKO_GITLAB_TOKEN: a project access token with the api scope). */
+export async function upsertMrNote ({ manifest, url, vars = process.env, fetchImpl = fetch }) {
+  const token = vars.KALOKO_GITLAB_TOKEN || vars.GITLAB_TOKEN
+  if (!token) return { ok: false, reason: 'set KALOKO_GITLAB_TOKEN (a project access token with the api scope) to comment on the merge request' }
+  const notes = `${vars.CI_API_V4_URL}/projects/${encodeURIComponent(vars.CI_PROJECT_ID)}/merge_requests/${vars.CI_MERGE_REQUEST_IID}/notes`
+  const headers = { 'PRIVATE-TOKEN': token, 'Content-Type': 'application/json' }
+  const body = prCommentBody({ manifest, url })
+  const link = (id) => `${vars.CI_MERGE_REQUEST_PROJECT_URL || vars.CI_PROJECT_URL}/-/merge_requests/${vars.CI_MERGE_REQUEST_IID}#note_${id}`
+  try {
+    const list = await fetchImpl(`${notes}?per_page=100&sort=desc`, { headers })
+    if (!list.ok) return { ok: false, reason: `GitLab answered ${list.status} listing notes` }
+    const mine = (await list.json()).find((n) => String(n.body || '').includes(marker(manifest.scenario.id)))
+    const res = await fetchImpl(mine ? `${notes}/${mine.id}` : notes, { method: mine ? 'PUT' : 'POST', headers, body: JSON.stringify({ body }) })
+    if (!res.ok) return { ok: false, reason: `GitLab answered ${res.status}` }
+    const note = await res.json()
+    return { ok: true, action: mine ? 'updated' : 'created', url: link(note.id || mine?.id) }
+  } catch (e) { return { ok: false, reason: e.message } }
+}
+
 async function defaultRun (cmd, args, cwd) {
   try { const { stdout } = await exec(cmd, args, { cwd, timeout: 30000, maxBuffer: 4 * 1024 * 1024 }); return stdout } catch { return null }
 }

package/src/lib/scenario.mjs

@@ -31,7 +31,8 @@ export function validateScenario (data) {
     if (node.kind === 'stack' && node.items && ['urls', 'file', 'sitemap', 'crawl', 'script'].filter((k) => node.items[k]).length !== 1) problems.push(`node "${node.id}": items needs exactly one source of urls, file, sitemap, crawl or script`)
     if (node.kind !== 'stack' && (node.items || node.fields)) problems.push(`node "${node.id}": items and fields belong to stack steps`)
     // app screens are reached by a script or a person as often as by a deep link
-    if (node.kind === 'screen' && !node.path && (!data.platform || data.platform === 'web')) problems.push(`node "${node.id}": screen steps need a path`)
+    // a manual screen may have no path: a person (or an imported walkthrough) shows how to get there
+    if (node.kind === 'screen' && !node.path && (!data.platform || data.platform === 'web') && node.execution?.strategy !== 'manual') problems.push(`node "${node.id}": screen steps need a path`)
     if (node.kind === 'stack' && data.platform && data.platform !== 'web') problems.push(`node "${node.id}": stack steps list web pages; they are not available on platform ${data.platform}`)
     if (node.kind === 'email' && !node.mail) problems.push(`node "${node.id}": email steps need mail.subject`)
   }

package/src/lib/service.mjs

@@ -43,7 +43,13 @@ export function service (config) {
     },
     async upload (path, data, name, extra = {}) {
       for (let attempt = 0; ; attempt++) {
-        const res = await limited(() => fetch(base + path, { method: 'PUT', headers: { ...headers, ...extra }, body: data }))
+        // a dropped connection (flaky Wi-Fi, CI runner hiccup) is tried again like a 5xx; the next share resumes otherwise
+        let res
+        try { res = await limited(() => fetch(base + path, { method: 'PUT', headers: { ...headers, ...extra }, body: data })) } catch (e) {
+          if (attempt >= 3) throw new UserError(`upload ${name}: ${e.cause?.code || e.message}. Run kaloko share again: files already stored are not sent twice.`)
+          await new Promise((r) => setTimeout(r, 1000 * (attempt + 1)))
+          continue
+        }
         if (res.ok || attempt >= 2 || res.status < 500) return check(res, `upload ${name}`)
         await new Promise((r) => setTimeout(r, 1000 * (attempt + 1)))
       }

package/src/lib/unzip.mjs

@@ -0,0 +1,31 @@
+// Minimal ZIP reader for `kaloko import` (Playwright traces and the report embedded in an HTML report): stored and
+// deflated entries from the central directory. No ZIP64; entries are read when asked for.
+import { inflateRawSync } from 'node:zlib'
+
+/** Entries of a zip archive: Map name → () => Buffer. */
+export function readZip (buf) {
+  let eocd = -1
+  for (let i = buf.length - 22; i >= Math.max(0, buf.length - 65557); i--) if (buf.readUInt32LE(i) === 0x06054b50) { eocd = i; break }
+  if (eocd < 0) throw new Error('not a zip archive (no end of central directory)')
+  const count = buf.readUInt16LE(eocd + 10)
+  let at = buf.readUInt32LE(eocd + 16)
+  const out = new Map()
+  for (let n = 0; n < count; n++) {
+    if (buf.readUInt32LE(at) !== 0x02014b50) throw new Error('broken zip central directory')
+    const method = buf.readUInt16LE(at + 10)
+    const size = buf.readUInt32LE(at + 20)
+    const nameLen = buf.readUInt16LE(at + 28); const extraLen = buf.readUInt16LE(at + 30); const commentLen = buf.readUInt16LE(at + 32)
+    const local = buf.readUInt32LE(at + 42)
+    const name = buf.subarray(at + 46, at + 46 + nameLen).toString('utf8')
+    at += 46 + nameLen + extraLen + commentLen
+    if (name.endsWith('/')) continue
+    out.set(name, () => {
+      const start = local + 30 + buf.readUInt16LE(local + 26) + buf.readUInt16LE(local + 28)
+      const data = buf.subarray(start, start + size)
+      if (method === 0) return Buffer.from(data)
+      if (method === 8) return inflateRawSync(data)
+      throw new Error(`${name}: unsupported compression method ${method}`)
+    })
+  }
+  return out
+}

package/src/lib/upload.mjs

@@ -0,0 +1,114 @@
+// Uploads of a run's files (#28): by content hash, so captures that did not change since an earlier run (or an
+// interrupted share) are not sent again, with progress on the terminal. The service stores each file once as an
+// organization blob and the run's file list points at it. An older service without blob references gets the files
+// by path as before; the paths it already took are remembered in the run folder, so a share that broke off resumes.
+import { readFile, writeFile } from 'node:fs/promises'
+import { gzipSync } from 'node:zlib'
+import { createHash } from 'node:crypto'
+import { join } from 'node:path'
+import { readJson } from './util.mjs'
+
+const TEXT = new Set(['html', 'json'])
+const IMAGE = /\.(png|webp|jpe?g)$/i
+export const PROGRESS_FILE = 'share.progress.json'
+
+export const fmtBytes = (n) => { n = Number(n) || 0; const u = ['B', 'kB', 'MB', 'GB']; let i = 0; while (n >= 1024 && i < u.length - 1) { n /= 1024; i++ } return `${n.toFixed(i ? 1 : 0)} ${u[i]}` }
+const extOf = (p) => { const e = p.split('.').pop().toLowerCase(); return e === 'jpeg' ? 'jpg' : e }
+/** Text files travel and rest gzip-compressed above 512 bytes; the hash is always of the raw content. */
+const encode = (p, raw) => { const gz = TEXT.has(extOf(p)) && raw.length > 512; return { data: gz ? gzipSync(raw, { level: 6 }) : raw, gz } }
+
+/** Content hash, size and type of every file to share (paths relative to the run folder). */
+export async function hashFiles (dir, paths) {
+  const out = []
+  for (const path of paths) {
+    const raw = await readFile(join(dir, path))
+    out.push({ path, hash: createHash('sha256').update(raw).digest('hex'), bytes: raw.length, ext: extOf(path) })
+  }
+  return out
+}
+
+/** Hashes the service does not store yet; null when it has no blob endpoint (an older service, or a token without it). */
+export async function missingHashes (client, org, hashes) {
+  const missing = new Set()
+  const list = [...new Set(hashes)]
+  try {
+    for (let i = 0; i < list.length; i += 1000) for (const h of (await client.request('POST', `/orgs/${org}/blobs/missing`, { hashes: list.slice(i, i + 1000) })).missing) missing.add(h)
+  } catch (e) {
+    if (/HTTP (403|404|405)\b/.test(e.message)) return null
+    throw e
+  }
+  return missing
+}
+
+/** Runs `job` over `items` with `n` workers; `tick` after each item. */
+async function pool (items, n, job, tick) {
+  const queue = [...items]
+  const worker = async () => { for (let it = queue.shift(); it; it = queue.shift()) { await job(it); tick?.(it) } }
+  await Promise.all(Array.from({ length: Math.max(1, Math.min(n, queue.length)) }, worker))
+}
+
+function progressLine (write, label) {
+  let last = 0
+  return (done, total, bytes) => {
+    const now = Date.now()
+    if (done !== total && now - last < 250) return
+    last = now
+    write(`\r  ${label} ${done}/${total}${bytes != null ? ` · ${fmtBytes(bytes)}` : ''}${done === total ? '\n' : ''}`)
+  }
+}
+
+/**
+ * Uploads the files of a created run. `entries` from hashFiles; `missing` from missingHashes (null: by path).
+ * Returns { mode: 'blobs'|'paths', uploaded, reused, bytes }.
+ */
+export async function uploadRunFiles ({ client, org, runId, dir, entries, missing, say = console.log, write = (s) => process.stdout.write(s), concurrency }) {
+  const workers = concurrency || (entries.length > 200 ? 12 : 6)
+  if (missing) {
+    // 1. the blobs the service lacks, each once (identical files in a run share one blob)
+    const byHash = new Map()
+    for (const e of entries) if (missing.has(e.hash) && !byHash.has(e.hash)) byHash.set(e.hash, e)
+    let sent = 0; let done = 0
+    const tick = progressLine(write, 'uploaded')
+    await pool([...byHash.values()], workers, async (e) => {
+      const { data, gz } = encode(e.path, await readFile(join(dir, e.path)))
+      await client.upload(`/orgs/${org}/blobs/${e.hash}`, data, e.path, { 'X-QAwalk-Type': e.ext, ...(gz ? { 'Content-Encoding': 'gzip' } : {}) })
+      sent += data.length
+    }, () => tick(++done, byHash.size, sent))
+    // 2. the run's file list points at the blobs; a service that does not know references says so by leaving out `blobs`
+    const plan = await client.request('POST', `/orgs/${org}/runs/${runId}/files`, { files: entries.map((e) => ({ path: e.path, bytes: e.bytes, blob: e.hash, ...(IMAGE.test(e.path) ? { sha256: e.hash } : {}) })) })
+    if (plan && typeof plan.blobs === 'number') {
+      return { mode: 'blobs', uploaded: byHash.size, reused: new Set(entries.map((e) => e.hash)).size - byHash.size, bytes: sent }
+    }
+    say('  The service does not take blob references yet; uploading the files by path.')
+  }
+  return uploadByPath({ client, org, runId, dir, entries, say, write, workers })
+}
+
+/** Per-path uploads (older services): a plan first, then each file; finished paths are remembered for a resume. */
+async function uploadByPath ({ client, org, runId, dir, entries, say, write, workers }) {
+  const progressFile = join(dir, PROGRESS_FILE)
+  const saved = await readJson(progressFile, null).catch(() => null)
+  const done = new Set(saved?.run === runId ? saved.done : [])
+  const todo = entries.filter((e) => !done.has(`${e.path}|${e.hash}`))
+  if (done.size && todo.length < entries.length) say(`  Resuming: ${entries.length - todo.length} of ${entries.length} file(s) were uploaded before the interruption.`)
+  // screenshots name their SHA-256 (sent uncompressed, so the hash of the raw file): the service checks it on
+  // arrival and compares unchanged steps with it (review mode)
+  if (todo.length) {
+    const plan = []
+    for (const e of todo) plan.push({ path: e.path, bytes: encode(e.path, await readFile(join(dir, e.path))).data.length, ...(IMAGE.test(e.path) ? { sha256: e.hash } : {}) })
+    await client.request('POST', `/orgs/${org}/runs/${runId}/files`, { files: plan }).catch((err) => { if (!/HTTP (404|405)/.test(err.message)) throw err })
+  }
+  let sent = 0; let n = 0; let since = 0
+  const remember = () => writeFile(progressFile, JSON.stringify({ run: runId, done: [...done] }) + '\n').catch(() => {})
+  const tick = progressLine(write, 'uploaded')
+  try {
+    await pool(todo, workers, async (e) => {
+      const { data, gz } = encode(e.path, await readFile(join(dir, e.path)))
+      await client.upload(`/orgs/${org}/runs/${runId}/files/${e.path.split('/').map(encodeURIComponent).join('/')}`, data, e.path, gz ? { 'Content-Encoding': 'gzip' } : {})
+      sent += data.length
+      done.add(`${e.path}|${e.hash}`)
+      if (++since >= 25) { since = 0; await remember() }
+    }, () => tick(++n, todo.length, sent))
+  } finally { await remember() }
+  return { mode: 'paths', uploaded: todo.length, reused: entries.length - todo.length, bytes: sent }
+}