browser4-cli 4.14.0-rc.6 → 4.14.0-rc.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -21
- package/bin/browser4-cli-darwin-arm64 +0 -0
- package/bin/browser4-cli-darwin-x64 +0 -0
- package/bin/browser4-cli-linux-arm64 +0 -0
- package/bin/browser4-cli-linux-musl-arm64 +0 -0
- package/bin/browser4-cli-linux-musl-x64 +0 -0
- package/bin/browser4-cli-linux-x64 +0 -0
- package/bin/browser4-cli-win32-x64.exe +0 -0
- package/package.json +1 -1
- package/scripts/README.md +66 -0
- package/scripts/reconcile-release-assets.sh +293 -0
- package/scripts/tests/reconcile-release-assets.tests.sh +421 -0
- package/scripts/tests/wait-for-oss-sync.tests.sh +382 -0
- package/scripts/wait-for-oss-sync.sh +314 -0
package/README.md
CHANGED
|
@@ -209,7 +209,7 @@ browser4-cli fill e16 "Browser4" --submit
|
|
|
209
209
|
# Extract data from the live page
|
|
210
210
|
browser4-cli get text "h1"
|
|
211
211
|
|
|
212
|
-
# Capture
|
|
212
|
+
# Capture the page into the store, then extract from that copy
|
|
213
213
|
browser4-cli htmlsnapshot
|
|
214
214
|
browser4-cli htmlsnapshot get text "#main-content"
|
|
215
215
|
browser4-cli htmlsnapshot query --sql @query.sql
|
|
@@ -235,7 +235,7 @@ These flags can appear before any command.
|
|
|
235
235
|
| `-h`, `--help [command\|category]` | Show top-level help, category help, or detailed command help |
|
|
236
236
|
| `--help-json` | Emit the machine-readable command reference |
|
|
237
237
|
| `-v`, `--version` | Print the CLI version |
|
|
238
|
-
| `-s`, `--session <name>` | Use a named session instead of the default session |
|
|
238
|
+
| `-s`, `--session <name>` | Use a named session instead of the default session. A **global** flag: place it **before** the command (`browser4-cli -s job-42 snapshot`) — after the command it is rejected as a positional argument |
|
|
239
239
|
| `--server <url>` | Override the Browser4 server URL |
|
|
240
240
|
| `--timeout <seconds>` | Override the HTTP timeout for the current command |
|
|
241
241
|
| `--proxy <url>` | Proxy used for runtime downloads/install operations |
|
|
@@ -257,7 +257,7 @@ These flags can appear before any command.
|
|
|
257
257
|
| Tool | Best for | Input model | Output model |
|
|
258
258
|
|---|---|---|---|
|
|
259
259
|
| `snapshot` | clicking, typing, finding interactive elements | live accessibility tree | refs like `e15` |
|
|
260
|
-
| `htmlsnapshot` | DOM inspection, CSS extraction, X-SQL |
|
|
260
|
+
| `htmlsnapshot` | DOM inspection, CSS extraction, X-SQL | a fresh snapshot of the active page (every command captures the tab first, then reads; `--expires <dur>` reads the stored snapshot instead) | CSS selectors and query results |
|
|
261
261
|
|
|
262
262
|
#### LLM configuration
|
|
263
263
|
|
|
@@ -275,6 +275,12 @@ AI-powered commands such as `extract`, `summarize`, `chat`, `agent run`, and X-S
|
|
|
275
275
|
export DEEPSEEK_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxx
|
|
276
276
|
```
|
|
277
277
|
|
|
278
|
+
> Configure **one** provider. When several provider keys are present, the first one in the
|
|
279
|
+
> built-in detection order wins — `openai` comes after the dedicated providers such as
|
|
280
|
+
> `deepseek`, so a leftover `DEEPSEEK_API_KEY` silently wins over `OPENAI_*`.
|
|
281
|
+
> `browser4-cli doctor` prints the key that won (`Selected key`) and the model the backend
|
|
282
|
+
> will request (`Active model`). See [LLM configuration](docs/config.md#which-provider-is-used).
|
|
283
|
+
|
|
278
284
|
### Complete command reference
|
|
279
285
|
|
|
280
286
|
#### Session lifecycle and server administration
|
|
@@ -289,8 +295,8 @@ export DEEPSEEK_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxx
|
|
|
289
295
|
| `close-all` | Close all sessions without stopping the backend. |
|
|
290
296
|
| `kill-all` | Force-stop the backend and Browser4-managed browser processes. |
|
|
291
297
|
| `stop` | Gracefully stop the Browser4 server. |
|
|
292
|
-
| `status` | Show server version, port, health, and the web status panel URL (`http://<server>:
|
|
293
|
-
| `doctor` | Run diagnostics: build info, LLM status, stale daemon cleanup, optional repair. Supports `--verbose` and `--fix
|
|
298
|
+
| `status` | Show server version, port, health, and the web status panel URL (`http://<server>:18182/status`). When a session is active it also prints a current-session block: Name / Session ID / Status / Connection / Next open. |
|
|
299
|
+
| `doctor` | Run diagnostics: build info, LLM status (including the configuration file it reads, the winning provider key and the active model), stale daemon cleanup, optional repair. Supports `--verbose` and `--fix`; `--fix` also writes a commented LLM config template. |
|
|
294
300
|
| `doctor log [name]` | List, view, tail, or grep backend log files. Supports `--tail`, grep-style flags, and `doctor log <name> grep <pattern>`. |
|
|
295
301
|
| `doctor metrics [filter]` | List, filter, or grep backend metrics. Supports `doctor metrics grep <pattern>`. |
|
|
296
302
|
| `doctor status [--section <name>] [--verbose]` | Print the aggregated status panel report in the terminal: summary layer by default, full detail with `--verbose`, one report with `--section` (health, build, runtime, llm, sessions, pulsar-sessions, swarm, url-pool, browsers, drivers, privacy, plugins, skills, metrics, logs), machine-readable JSON with `--json`. |
|
|
@@ -308,7 +314,7 @@ browser4-cli doctor metrics grep request
|
|
|
308
314
|
browser4-cli doctor status --section skills --verbose
|
|
309
315
|
```
|
|
310
316
|
|
|
311
|
-
**Web status panel:** open `http://127.0.0.1:
|
|
317
|
+
**Web status panel:** open `http://127.0.0.1:18182/status` in a browser for a live dashboard
|
|
312
318
|
(health, version, JVM/runtime, LLM config, sessions, **Pulsar sessions** — SDK identity,
|
|
313
319
|
context and main-loop state, **swarm** — swarm session plus task summary, **URL pool** —
|
|
314
320
|
queued/real-time/delay counts per priority cache, browsers & open tabs — per-session
|
|
@@ -322,7 +328,7 @@ is backed by the aggregated `GET /api/system/status` endpoint; the individual en
|
|
|
322
328
|
reports load/enable state and SDK version for every installed plugin; the same reports can be
|
|
323
329
|
read from the terminal with `browser4-cli doctor status`.
|
|
324
330
|
|
|
325
|
-
**Page screenshots:** open `http://127.0.0.1:
|
|
331
|
+
**Page screenshots:** open `http://127.0.0.1:18182/pages.html` for a grid of every open page
|
|
326
332
|
across sessions. The active tab of each session is captured automatically (click a screenshot
|
|
327
333
|
to re-capture it); inactive tabs show a placeholder that captures on click. Swarm sessions only
|
|
328
334
|
show placeholders. Screenshots load **asynchronously** — the backend captures in the background
|
|
@@ -358,7 +364,7 @@ All interaction commands accept a snapshot ref such as `e15` or a CSS selector u
|
|
|
358
364
|
| `wait [target]` | Wait for a selector/ref, duration, text, URL pattern, page-load state, or JavaScript expression. Supports `--timeout`, `--text`, `--url`, `--load`, `--fn`. |
|
|
359
365
|
| `upload <ref> <file> [file...]` | Upload one or more local files to a page file input. The target must be an `<input type="file">` (anything else errors); the paths must be readable by the browser process — local mode: the same machine, and empty/missing files are rejected with a hint; remote backend: paths resolve on the backend host. Supports `--no-snapshot`. |
|
|
360
366
|
|
|
361
|
-
`wait --load` accepts `domcontentloaded`, `load`, and `networkidle`.
|
|
367
|
+
`wait --load` accepts `domcontentloaded`, `load`, and `networkidle`. `networkidle` only proves the network went quiet — it does **not** mean late-rendered results exist; for result pages poll the element instead: `wait "<result-selector>"`.
|
|
362
368
|
|
|
363
369
|
```bash
|
|
364
370
|
browser4-cli click e8 --follow
|
|
@@ -419,25 +425,27 @@ browser4-cli cdp Runtime.evaluate --json '{"expression":"document.title"}'
|
|
|
419
425
|
|
|
420
426
|
#### HTML snapshot and X-SQL extraction
|
|
421
427
|
|
|
422
|
-
`htmlsnapshot` captures a
|
|
428
|
+
`htmlsnapshot` captures a raw DOM snapshot of the page the active tab is showing and is the center of Browser4's structured extraction workflow. **Every `htmlsnapshot` command works on a fresh snapshot of the active page**: it captures the tab first (serializing the document the tab already shows, without navigating) and then operates on that snapshot — `capture` returns the metadata, while the reads consume it, so a read already sees the live document (form results, SPA updates, `eval` mutations). A read can also be told to serve the **stored** snapshot instead: `--expires <dur>` (default `0s`, the live page) serves the stored copy of the active page while it is younger than the window — `htmlsnapshot get text ".price" --expires 1d` reads the previous snapshot version and never touches the tab. A command aimed at another URL (`readability <url>`, `query --url <url>`) reads *that URL's* stored copy, or an independent read-only load when the store has nothing — a URL the tab does not show is never captured, so the tab's document is never filed under it.
|
|
423
429
|
|
|
424
430
|
| Command | Description |
|
|
425
431
|
|---|---|
|
|
426
432
|
| `htmlsnapshot` | Short form of `htmlsnapshot capture`. |
|
|
427
433
|
| `htmlsnapshot capture` | Capture and store a static HTML snapshot with metadata about the page and interactive elements. |
|
|
428
|
-
| `htmlsnapshot get <field> [selector] [name]` | Extract the first matching `text`, `html`, or `attr` from the
|
|
429
|
-
| `htmlsnapshot get all <field> [selector] [name]` | Extract all matching values from the
|
|
430
|
-
| `htmlsnapshot query [url]` | Run X-SQL. Supports `--sql <query\|@file>`, `--sql-stdin`, `--sql-base64`, result pagination, and extraction-focused output flags. |
|
|
431
|
-
| `htmlsnapshot export` | Export
|
|
432
|
-
| `htmlsnapshot summary` | Generate a compressed Web Page Summary Index (WPSI). |
|
|
433
|
-
| `htmlsnapshot
|
|
434
|
-
| `htmlsnapshot
|
|
435
|
-
| `htmlsnapshot
|
|
434
|
+
| `htmlsnapshot get <field> [selector] [name]` | Extract the first matching `text`, `textcontent`, `html`, or `attr` from a fresh snapshot of the active page. Supports `--expires <dur>`. |
|
|
435
|
+
| `htmlsnapshot get all <field> [selector] [name]` | Extract all matching values from a fresh snapshot of the active page. Supports `--offset`, `--limit`, and `--expires <dur>`. |
|
|
436
|
+
| `htmlsnapshot query [url]` | Run X-SQL against a fresh snapshot of the active page, or against an explicit URL's stored page. Supports `--sql <query\|@file>`, `--sql-stdin`, `--sql-base64`, `--expires <dur>`, result pagination, and extraction-focused output flags. |
|
|
437
|
+
| `htmlsnapshot export` | Export a fresh snapshot's HTML to a file. Supports positional file path or `--file <path>` plus `--clean` and `--expires <dur>`. |
|
|
438
|
+
| `htmlsnapshot summary [--algorithm <id>]` | Generate a compressed Web Page Summary Index (WPSI) from a fresh snapshot of the active page; plugins may contribute additional algorithm ids. Supports `--expires <dur>`. |
|
|
439
|
+
| `htmlsnapshot algorithms` | List installed summary algorithms (built-in `wpsi` plus plugin-contributed ids). No page required. |
|
|
440
|
+
| `htmlsnapshot grep <pattern>` | Search a fresh snapshot's HTML with grep-style flags. Supports `--expires <dur>`. |
|
|
441
|
+
| `htmlsnapshot inspect [selector]` | Discover recurring DOM patterns and selector candidates in a fresh snapshot of the active page. Supports `--max`, `--depth`, `--stdin`, `--selector-base64`, `--expires <dur>`. |
|
|
442
|
+
| `htmlsnapshot readability [url]` | Extract the main article content with a Readability-style heuristic — no LLM, no tokens. Without a URL: the active page; with one: that URL's own stored copy. Supports `--text-only`, `--expires <dur>`, and pagination. |
|
|
436
443
|
|
|
437
444
|
Important rules:
|
|
438
445
|
|
|
439
446
|
- use `snapshot` when you need refs and interaction
|
|
440
|
-
- use `htmlsnapshot` when you need repeated DOM extraction
|
|
447
|
+
- use `htmlsnapshot` when you need repeated DOM extraction — every command captures the tab first, so simply rerunning it sees the page as it is now
|
|
448
|
+
- use `--expires <dur>` on a read when you want the snapshot already in the store (`--expires 1d`) instead of the live page (`--expires 0s`, the default) — the stored copy is left untouched and the tab is not serialized
|
|
441
449
|
- `htmlsnapshot query --sql @query.sql` is the recommended way to avoid shell quoting issues
|
|
442
450
|
- for correlated list extraction, prefer `htmlsnapshot query` over repeated `get all`
|
|
443
451
|
- for one-step article extraction (no selectors needed), use `htmlsnapshot readability`
|
|
@@ -581,15 +589,17 @@ The `co` prefix is accepted as an alias for `swarm`.
|
|
|
581
589
|
| `crawl [url]` | Crawl from a URL or seed file. Supports `--seed-file`, `--sql`, `--sql-stdin`, `--sql-base64`, `--format`, `--output`, `-d/--depth`, `-ol/--out-link-selector`, `-olp/--out-link-pattern`, `-tl/--top-links`, `-a/--args`, `--refresh`, `--parse`, `--expires`, `-p/--priority`, `--page-load-timeout`, `--ignore-url-query`, `--no-norm`, `--readonly`, `-bg/--background`. |
|
|
582
590
|
| `crawl status <id>` | Check crawl task status. |
|
|
583
591
|
| `crawl result <id>` | Fetch crawl results. |
|
|
584
|
-
| `crawl cancel <id>` | Cancel a running crawl. |
|
|
585
|
-
| `crawl
|
|
586
|
-
| `crawl
|
|
592
|
+
| `crawl cancel <id>` | Cancel a running crawl (the checkpoint is kept, so it can be resumed). |
|
|
593
|
+
| `crawl resume <id>` | Continue an interrupted crawl from its checkpoint: same task id, no repeat requests for URLs that already succeeded. `--retry-failed` re-fetches terminal failures; automatic resume at startup is off by default (`crawl.autoResume`). See [Crawl checkpoint & resume](docs/crawl-checkpoint-resume.md). |
|
|
594
|
+
| `crawl clear` | Remove terminal-state crawl tasks; supports force-style cleanup options. Resumable checkpoints are kept until `crawl clear --all`. |
|
|
595
|
+
| `crawl list` | List tracked crawl tasks (`--status interrupted` shows the resumable ones). |
|
|
587
596
|
|
|
588
597
|
```bash
|
|
589
598
|
browser4-cli swarm create --max-open-tabs 12 --display-mode HEADLESS
|
|
590
599
|
browser4-cli swarm query --seed-file urls.txt --sql @query.sql --refresh
|
|
591
600
|
browser4-cli crawl "https://example.com" --depth 2 --out-link-selector "a[href]"
|
|
592
601
|
browser4-cli crawl list
|
|
602
|
+
browser4-cli crawl resume <task-id> # continue an interrupted crawl
|
|
593
603
|
```
|
|
594
604
|
|
|
595
605
|
#### Bundled skill files vs installed runtime skills
|
|
@@ -714,6 +724,13 @@ The runtime bundle is stored separately in a platform-conventional application-d
|
|
|
714
724
|
that auto-build from source. Use `./b4w.ps1 <command>` (PowerShell),
|
|
715
725
|
`./b4w.sh <command>` (Git Bash / Linux / macOS), or `./b4w.bat <command>`
|
|
716
726
|
(CMD) — all accept the same arguments as the installed `browser4-cli` binary.
|
|
727
|
+
Dev mode only ever serves the checked-out code: a local runtime bundle built
|
|
728
|
+
from a **different project version** makes the command refuse to start (naming
|
|
729
|
+
the reason, the bundle's versions and build time, and the rebuild command)
|
|
730
|
+
rather than silently testing an outdated backend; a checkout whose sources
|
|
731
|
+
merely look newer than the bundle jars is rebuilt before the server starts —
|
|
732
|
+
see
|
|
733
|
+
[CLI install & upgrade](docs/cli-install-upgrade.md#dev-mode-the-backend-must-match-the-checkout).
|
|
717
734
|
|
|
718
735
|
---
|
|
719
736
|
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
package/scripts/README.md
CHANGED
|
@@ -85,6 +85,8 @@ Runs a full CLI smoke test suitable for CI and local development:
|
|
|
85
85
|
| `tests/install-browser4-cli.tests.sh` | Unit tests for the Unix install script |
|
|
86
86
|
| `tests/install-browser4-cli.tests.ps1` | Unit tests for the Windows install script (Pester) |
|
|
87
87
|
| `tests/wait-for-npm-version.tests.sh` | Unit tests for `wait-for-npm-version.sh` (stubbed `npm`, no network) |
|
|
88
|
+
| `tests/reconcile-release-assets.tests.sh` | Unit tests for `reconcile-release-assets.sh` (stubbed `gh`, no network) |
|
|
89
|
+
| `tests/wait-for-oss-sync.tests.sh` | Unit tests for `wait-for-oss-sync.sh` (stubbed `gh`, no network) |
|
|
88
90
|
|
|
89
91
|
## Publish (npm)
|
|
90
92
|
|
|
@@ -153,6 +155,70 @@ This makes the npm package README match the repository root README without leavi
|
|
|
153
155
|
|
|
154
156
|
`postinstall.js` runs automatically after `npm install browser4-cli`. It detects the platform and downloads the matching native binary to `bin/`. On global installs, it also patches npm's bin shims to invoke the native binary directly.
|
|
155
157
|
|
|
158
|
+
## Release
|
|
159
|
+
|
|
160
|
+
| Script | Purpose |
|
|
161
|
+
|--------|---------|
|
|
162
|
+
| `reconcile-release-assets.sh` | Makes a GitHub release carry exactly the assets a job produced, re-uploading what is missing or truncated; called by `release.yml` after the publishing action |
|
|
163
|
+
|
|
164
|
+
### Recovering from a failed release upload
|
|
165
|
+
|
|
166
|
+
GitHub's release upload endpoint answers with a transient 5xx a few times a month
|
|
167
|
+
(*"Error creating asset temp dir"*, *"Error saving asset"*, *"Unicorn!"*). v4.14.0-rc.6
|
|
168
|
+
(`release.yml` run 35265014949) lost its whole release pipeline that way: the publishing
|
|
169
|
+
action uploaded 6 of 11 assets and then failed, which skipped the artifact attestation, the
|
|
170
|
+
release verification and the OSS sync.
|
|
171
|
+
|
|
172
|
+
`reconcile-release-assets.sh` treats the release as the source of truth: it lists the assets
|
|
173
|
+
the release carries, uploads only the files that are missing or whose size differs
|
|
174
|
+
(`gh release upload --clobber`), and re-checks until the release matches or the budget
|
|
175
|
+
expires. An already complete release costs one API call and no upload.
|
|
176
|
+
|
|
177
|
+
```shell
|
|
178
|
+
bash scripts/reconcile-release-assets.sh --tag v4.14.0 --repo platonai/Browser4 \
|
|
179
|
+
release-assets/Browser4.jar release-assets/browser4-cli-linux-x64
|
|
180
|
+
bash scripts/reconcile-release-assets.sh --tag v4.14.0 --files-from release-assets.txt
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Exit code 0 = the release carries every given file at the local size; 1 = still missing or
|
|
184
|
+
mismatched when the budget (600 s by default) expired, the release does not exist, or bad
|
|
185
|
+
usage. `release.yml` runs it in its own `Reconcile release assets` step, which decides
|
|
186
|
+
whether that job passed — the publishing action runs with `continue-on-error: true` for
|
|
187
|
+
exactly that reason.
|
|
188
|
+
|
|
189
|
+
## Publish (release workflows)
|
|
190
|
+
|
|
191
|
+
| Script | Platform | Purpose |
|
|
192
|
+
|--------|----------|---------|
|
|
193
|
+
| `wait-for-oss-sync.sh` | Linux/macOS | Triggers the `sync-to-oss.yml` workflow for a release tag and waits for **that** run; called by `release.yml` and `release-cli.yml` |
|
|
194
|
+
|
|
195
|
+
### Verifying the OSS CDN sync
|
|
196
|
+
|
|
197
|
+
GitHub Releases and the Alibaba Cloud OSS mirror are separate distribution channels: the OSS
|
|
198
|
+
mirror is what `install-browser4-cli.sh` prefers in the China mainland, and its `latest`
|
|
199
|
+
symlinks are what a plain "install the latest" resolves to. So the release should not report
|
|
200
|
+
success while the mirror is still on the previous version — and it should not report failure
|
|
201
|
+
just because the mirror is slow.
|
|
202
|
+
|
|
203
|
+
`wait-for-oss-sync.sh` dispatches `sync-to-oss.yml` for the tag and waits for the run it
|
|
204
|
+
**caused**, under a budget of its own:
|
|
205
|
+
|
|
206
|
+
```shell
|
|
207
|
+
bash scripts/wait-for-oss-sync.sh --tag v4.13.21 # 45 min budget, 15 s interval
|
|
208
|
+
bash scripts/wait-for-oss-sync.sh --tag v4.13.21 --timeout 600 --interval 5
|
|
209
|
+
bash scripts/wait-for-oss-sync.sh --tag v4.13.21 --repo platonai/Browser4
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Exit code 0 = the run it triggered concluded `success`; 1 = that run failed, was cancelled,
|
|
213
|
+
never appeared, or was still running when the budget expired. On failure and on timeout the
|
|
214
|
+
message names the run, its URL, the step it is stuck on, and the commands to re-check
|
|
215
|
+
(`gh run view <id>`) and re-trigger (`gh workflow run sync-to-oss.yml -f tag_name=<tag>`)
|
|
216
|
+
it. It does **not** retry the sync itself: a re-triggered sync re-uploads every asset.
|
|
217
|
+
|
|
218
|
+
Runs that already existed before the dispatch are excluded by id, so a previous release's
|
|
219
|
+
successful sync can never be mistaken for the current one — the failure mode the old inline
|
|
220
|
+
`gh run list --limit 1` loop had (see `docs-dev/copilot/ci-stabilization-4.13.x.md` §33).
|
|
221
|
+
|
|
156
222
|
## Documentation
|
|
157
223
|
|
|
158
224
|
| Script | Platform | Purpose |
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
# ---------------------------------------------------------------------------
|
|
3
|
+
# reconcile-release-assets.sh -- make a GitHub release carry exactly the
|
|
4
|
+
# assets the build produced, retrying GitHub's transient upload failures
|
|
5
|
+
# ---------------------------------------------------------------------------
|
|
6
|
+
# Usage:
|
|
7
|
+
# reconcile-release-assets.sh --tag <tag> [--repo <owner/name>] \
|
|
8
|
+
# [--timeout <seconds>] [--interval <seconds>] [--settle <seconds>] \
|
|
9
|
+
# <file>...
|
|
10
|
+
# reconcile-release-assets.sh --tag <tag> [options] --files-from <path>
|
|
11
|
+
#
|
|
12
|
+
# Defaults: timeout 600 s, interval 30 s, settle 5 s.
|
|
13
|
+
#
|
|
14
|
+
# GitHub's release upload endpoint answers with a transient 5xx a few times a
|
|
15
|
+
# month: "Error creating asset temp dir", "Error saving asset", "Unicorn!".
|
|
16
|
+
# v4.14.0-rc.6 (release.yml run 35265014949) failed exactly this way -- the
|
|
17
|
+
# "Publish GitHub release" job uploaded 6 of 11 assets and then died on
|
|
18
|
+
# ##[error]Error creating asset temp dir
|
|
19
|
+
# which skipped the artifact attestation, the release verification and the
|
|
20
|
+
# OSS sync: the whole release pipeline was lost to a server-side blip, with a
|
|
21
|
+
# half-populated GitHub release left behind.
|
|
22
|
+
#
|
|
23
|
+
# GitHub does not say which upload failed, and a failed upload can leave a
|
|
24
|
+
# partial asset behind, so re-running the whole upload is both slow and not
|
|
25
|
+
# obviously correct. This script instead treats the release as the source of
|
|
26
|
+
# truth and converges on it:
|
|
27
|
+
#
|
|
28
|
+
# 1. list the assets the release already carries (name and size),
|
|
29
|
+
# 2. upload every local file that is missing or whose size differs, using
|
|
30
|
+
# `gh release upload --clobber` (which also replaces a partial asset),
|
|
31
|
+
# 3. re-check, and repeat until the release matches the local files or the
|
|
32
|
+
# time budget runs out.
|
|
33
|
+
#
|
|
34
|
+
# It is idempotent: a release that already matches its files costs one API
|
|
35
|
+
# call and no upload. Assets are matched by file name, which is the name the
|
|
36
|
+
# publishing action uses for them. A release that does not exist is reported
|
|
37
|
+
# as a failure rather than recreated here -- that is the release-creation
|
|
38
|
+
# path, which the publishing action creates (and retries) with the workflow's
|
|
39
|
+
# metadata.
|
|
40
|
+
#
|
|
41
|
+
# Exit codes:
|
|
42
|
+
# 0 the release carries every given file with the local byte size
|
|
43
|
+
# 1 still missing/mismatched when the budget expired, the release does not
|
|
44
|
+
# exist, a given file does not exist, or bad usage
|
|
45
|
+
#
|
|
46
|
+
# Designed for CI (GitHub Actions) but also runs on developer machines.
|
|
47
|
+
# Requires `gh`, authenticated via GH_TOKEN/GITHUB_TOKEN or `gh auth login`.
|
|
48
|
+
# ---------------------------------------------------------------------------
|
|
49
|
+
set -euo pipefail
|
|
50
|
+
|
|
51
|
+
TAG=''
|
|
52
|
+
REPO=''
|
|
53
|
+
TIMEOUT_SECONDS=600
|
|
54
|
+
POLL_INTERVAL_SECONDS=30
|
|
55
|
+
SETTLE_SECONDS=5
|
|
56
|
+
FILES_FROM=''
|
|
57
|
+
FILES=()
|
|
58
|
+
|
|
59
|
+
usage() {
|
|
60
|
+
local code="${1:-1}"
|
|
61
|
+
{
|
|
62
|
+
echo "Usage: $0 --tag <tag> [--repo <owner/name>] [--timeout <seconds>]"
|
|
63
|
+
echo " [--interval <seconds>] [--settle <seconds>]"
|
|
64
|
+
echo " [--files-from <path>] <file>..."
|
|
65
|
+
} >&2
|
|
66
|
+
exit "$code"
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
while [ $# -gt 0 ]; do
|
|
70
|
+
case "${1:-}" in
|
|
71
|
+
--tag)
|
|
72
|
+
TAG="${2:-}"
|
|
73
|
+
shift 2 || usage
|
|
74
|
+
;;
|
|
75
|
+
--repo)
|
|
76
|
+
REPO="${2:-}"
|
|
77
|
+
shift 2 || usage
|
|
78
|
+
;;
|
|
79
|
+
--timeout)
|
|
80
|
+
TIMEOUT_SECONDS="${2:-}"
|
|
81
|
+
shift 2 || usage
|
|
82
|
+
;;
|
|
83
|
+
--interval)
|
|
84
|
+
POLL_INTERVAL_SECONDS="${2:-}"
|
|
85
|
+
shift 2 || usage
|
|
86
|
+
;;
|
|
87
|
+
--settle)
|
|
88
|
+
SETTLE_SECONDS="${2:-}"
|
|
89
|
+
shift 2 || usage
|
|
90
|
+
;;
|
|
91
|
+
--files-from)
|
|
92
|
+
FILES_FROM="${2:-}"
|
|
93
|
+
shift 2 || usage
|
|
94
|
+
;;
|
|
95
|
+
-h|--help)
|
|
96
|
+
usage 0
|
|
97
|
+
;;
|
|
98
|
+
--)
|
|
99
|
+
shift
|
|
100
|
+
while [ $# -gt 0 ]; do
|
|
101
|
+
FILES+=("$1")
|
|
102
|
+
shift
|
|
103
|
+
done
|
|
104
|
+
;;
|
|
105
|
+
-*)
|
|
106
|
+
echo "ERROR: unknown option '$1'" >&2
|
|
107
|
+
usage
|
|
108
|
+
;;
|
|
109
|
+
*)
|
|
110
|
+
FILES+=("$1")
|
|
111
|
+
shift
|
|
112
|
+
;;
|
|
113
|
+
esac
|
|
114
|
+
done
|
|
115
|
+
|
|
116
|
+
[ -n "$TAG" ] || { echo "ERROR: --tag is required" >&2; usage; }
|
|
117
|
+
case "$TIMEOUT_SECONDS" in ''|*[!0-9]*) echo "ERROR: --timeout must be a whole number of seconds" >&2; usage ;; esac
|
|
118
|
+
case "$POLL_INTERVAL_SECONDS" in ''|*[!0-9]*) echo "ERROR: --interval must be a whole number of seconds" >&2; usage ;; esac
|
|
119
|
+
case "$SETTLE_SECONDS" in ''|*[!0-9]*) echo "ERROR: --settle must be a whole number of seconds" >&2; usage ;; esac
|
|
120
|
+
[ "$POLL_INTERVAL_SECONDS" -gt 0 ] || { echo "ERROR: --interval must be at least 1 s" >&2; usage; }
|
|
121
|
+
|
|
122
|
+
if [ -n "$FILES_FROM" ]; then
|
|
123
|
+
if [ ! -f "$FILES_FROM" ]; then
|
|
124
|
+
echo "ERROR: --files-from '$FILES_FROM' does not exist" >&2
|
|
125
|
+
exit 1
|
|
126
|
+
fi
|
|
127
|
+
# Newline-separated paths: the shape the publishing action's `files` input
|
|
128
|
+
# and this repository's release asset manifest already use.
|
|
129
|
+
while IFS= read -r manifest_line || [ -n "$manifest_line" ]; do
|
|
130
|
+
manifest_line="${manifest_line%$'\r'}"
|
|
131
|
+
[ -n "$manifest_line" ] || continue
|
|
132
|
+
FILES+=("$manifest_line")
|
|
133
|
+
done < "$FILES_FROM"
|
|
134
|
+
fi
|
|
135
|
+
|
|
136
|
+
if [ "${#FILES[@]}" -eq 0 ]; then
|
|
137
|
+
echo "ERROR: no asset files given" >&2
|
|
138
|
+
usage
|
|
139
|
+
fi
|
|
140
|
+
|
|
141
|
+
for f in "${FILES[@]}"; do
|
|
142
|
+
if [ -z "$f" ]; then
|
|
143
|
+
echo "ERROR: empty asset file argument (is the asset list a real path list?)" >&2
|
|
144
|
+
exit 1
|
|
145
|
+
fi
|
|
146
|
+
if [ ! -f "$f" ]; then
|
|
147
|
+
echo "ERROR: asset file '$f' does not exist" >&2
|
|
148
|
+
exit 1
|
|
149
|
+
fi
|
|
150
|
+
done
|
|
151
|
+
|
|
152
|
+
if ! command -v gh >/dev/null 2>&1; then
|
|
153
|
+
echo "ERROR: gh is not installed -- https://cli.github.com" >&2
|
|
154
|
+
exit 1
|
|
155
|
+
fi
|
|
156
|
+
|
|
157
|
+
STDERR_FILE="$(mktemp)"
|
|
158
|
+
trap 'rm -f "$STDERR_FILE"' EXIT
|
|
159
|
+
|
|
160
|
+
# --repo is appended to the subcommand (gh declares it per command), so the two
|
|
161
|
+
# call sites below branch instead of building an argument array that `set -u`
|
|
162
|
+
# and old bash versions disagree about while it is empty.
|
|
163
|
+
list_release_assets() {
|
|
164
|
+
if [ -n "$REPO" ]; then
|
|
165
|
+
gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[] | "\(.name) \(.size)"'
|
|
166
|
+
else
|
|
167
|
+
gh release view "$TAG" --json assets --jq '.assets[] | "\(.name) \(.size)"'
|
|
168
|
+
fi
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
upload_assets() {
|
|
172
|
+
if [ -n "$REPO" ]; then
|
|
173
|
+
gh release upload "$TAG" "$@" --repo "$REPO" --clobber
|
|
174
|
+
else
|
|
175
|
+
gh release upload "$TAG" "$@" --clobber
|
|
176
|
+
fi
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
local_size_of() {
|
|
180
|
+
wc -c < "$1" | tr -d '[:space:]'
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
# remote_size_of <name> -- size the release reports for <name>; returns 1 when
|
|
184
|
+
# the release does not carry that asset. Asset names may contain spaces, so the
|
|
185
|
+
# size is read as the last field of "name size".
|
|
186
|
+
remote_size_of() {
|
|
187
|
+
local wanted="$1" line name size
|
|
188
|
+
while IFS= read -r line; do
|
|
189
|
+
[ -n "$line" ] || continue
|
|
190
|
+
name="${line% *}"
|
|
191
|
+
size="${line##* }"
|
|
192
|
+
if [ "$name" = "$wanted" ]; then
|
|
193
|
+
printf '%s' "$size"
|
|
194
|
+
return 0
|
|
195
|
+
fi
|
|
196
|
+
done <<< "$remote_listing"
|
|
197
|
+
return 1
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
started=$SECONDS
|
|
201
|
+
attempt=0
|
|
202
|
+
remote_listing=''
|
|
203
|
+
gh_error=''
|
|
204
|
+
|
|
205
|
+
while :; do
|
|
206
|
+
attempt=$((attempt + 1))
|
|
207
|
+
upload_status=1
|
|
208
|
+
|
|
209
|
+
set +e
|
|
210
|
+
remote_listing="$(list_release_assets 2>"$STDERR_FILE")"
|
|
211
|
+
status=$?
|
|
212
|
+
set -e
|
|
213
|
+
|
|
214
|
+
gh_error="$(head -n 1 "$STDERR_FILE" 2>/dev/null || true)"
|
|
215
|
+
elapsed=$((SECONDS - started))
|
|
216
|
+
|
|
217
|
+
if [ "$status" -eq 0 ]; then
|
|
218
|
+
NEEDED=()
|
|
219
|
+
REASONS=()
|
|
220
|
+
for f in "${FILES[@]}"; do
|
|
221
|
+
name="${f##*/}"
|
|
222
|
+
local_size="$(local_size_of "$f")"
|
|
223
|
+
if remote_size="$(remote_size_of "$name")"; then
|
|
224
|
+
if [ "$remote_size" = "$local_size" ]; then
|
|
225
|
+
continue
|
|
226
|
+
fi
|
|
227
|
+
reason="size mismatch: the release has $remote_size bytes, the file has $local_size"
|
|
228
|
+
else
|
|
229
|
+
reason="missing from the release"
|
|
230
|
+
fi
|
|
231
|
+
NEEDED+=("$f")
|
|
232
|
+
REASONS+=("$reason")
|
|
233
|
+
done
|
|
234
|
+
|
|
235
|
+
if [ "${#NEEDED[@]}" -eq 0 ]; then
|
|
236
|
+
echo "Verified ${#FILES[@]} asset(s) on release $TAG after ${elapsed}s (attempt $attempt)"
|
|
237
|
+
exit 0
|
|
238
|
+
fi
|
|
239
|
+
|
|
240
|
+
echo "Uploading ${#NEEDED[@]} of ${#FILES[@]} asset(s) to release $TAG (attempt $attempt, ${elapsed}s elapsed):"
|
|
241
|
+
for i in "${!NEEDED[@]}"; do
|
|
242
|
+
echo " - ${NEEDED[$i]##*/}: ${REASONS[$i]}"
|
|
243
|
+
done
|
|
244
|
+
|
|
245
|
+
set +e
|
|
246
|
+
upload_assets "${NEEDED[@]}" 2>&1 | sed 's/^/ /'
|
|
247
|
+
upload_status="${PIPESTATUS[0]}"
|
|
248
|
+
set -e
|
|
249
|
+
|
|
250
|
+
if [ "$upload_status" -ne 0 ]; then
|
|
251
|
+
echo "WARN: gh release upload failed (exit $upload_status) -- GitHub's upload endpoint returns a transient 5xx a few times a month ('Error creating asset temp dir', 'Error saving asset', 'Unicorn!'); retrying"
|
|
252
|
+
fi
|
|
253
|
+
else
|
|
254
|
+
if printf '%s' "$gh_error" | grep -qiE 'not found|HTTP 404'; then
|
|
255
|
+
echo "ERROR: release $TAG does not exist -- the publishing action failed before it could create the release" >&2
|
|
256
|
+
echo "ERROR: last GitHub answer: ${gh_error:-<no output>}" >&2
|
|
257
|
+
exit 1
|
|
258
|
+
fi
|
|
259
|
+
echo "WARN: could not list the assets of release $TAG (attempt $attempt, ${elapsed}s elapsed): ${gh_error:-<no output>}"
|
|
260
|
+
fi
|
|
261
|
+
|
|
262
|
+
remaining=$((TIMEOUT_SECONDS - elapsed))
|
|
263
|
+
if [ "$remaining" -le 0 ]; then
|
|
264
|
+
break
|
|
265
|
+
fi
|
|
266
|
+
|
|
267
|
+
# A finished upload round can stay invisible to the listing API for a
|
|
268
|
+
# moment; a short settle avoids re-uploading a file the release already
|
|
269
|
+
# has. A failed round waits the full interval before trying again.
|
|
270
|
+
if [ "$status" -eq 0 ] && [ "$upload_status" -eq 0 ]; then
|
|
271
|
+
delay="$SETTLE_SECONDS"
|
|
272
|
+
else
|
|
273
|
+
delay="$POLL_INTERVAL_SECONDS"
|
|
274
|
+
fi
|
|
275
|
+
# never sleep past the deadline: the total wait stays within the budget
|
|
276
|
+
if [ "$delay" -gt "$remaining" ]; then
|
|
277
|
+
sleep "$remaining"
|
|
278
|
+
elif [ "$delay" -gt 0 ]; then
|
|
279
|
+
sleep "$delay"
|
|
280
|
+
fi
|
|
281
|
+
done
|
|
282
|
+
|
|
283
|
+
echo "ERROR: release $TAG still does not match the build output after ${TIMEOUT_SECONDS}s" >&2
|
|
284
|
+
for i in "${!NEEDED[@]}"; do
|
|
285
|
+
echo "ERROR: - ${NEEDED[$i]##*/}: ${REASONS[$i]}" >&2
|
|
286
|
+
done
|
|
287
|
+
if [ -n "$gh_error" ]; then
|
|
288
|
+
echo "ERROR: last GitHub answer: ${gh_error}" >&2
|
|
289
|
+
fi
|
|
290
|
+
echo "ERROR: GitHub's release upload endpoint returns transient 5xx responses" >&2
|
|
291
|
+
echo "ERROR: ('Error creating asset temp dir', 'Error saving asset', 'Unicorn!')." >&2
|
|
292
|
+
echo "ERROR: Re-run the failed jobs of this workflow to retry the upload." >&2
|
|
293
|
+
exit 1
|