@parity/product-lab 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/adapters/environment/engine/README.md +22 -3
- package/adapters/environment/engine/src/engine/engine-config.d.ts +1 -0
- package/adapters/environment/engine/src/engine/fork-bundle.d.ts +1 -1
- package/adapters/environment/engine/src/engine/fork-bundle.js +2 -2
- package/adapters/environment/engine/src/engine/fork-refresh.d.ts +4 -7
- package/adapters/environment/engine/src/engine/fork-refresh.js +52 -23
- package/adapters/environment/engine/src/engine/published-bundle.d.ts +19 -15
- package/adapters/environment/engine/src/engine/published-bundle.js +44 -51
- package/adapters/environment/engine/src/index.d.ts +3 -2
- package/adapters/environment/engine/src/index.js +3 -2
- package/adapters/environment/engine/src/saved-fork/saved-fork-tooling.d.ts +1 -1
- package/adapters/environment/engine/src/saved-fork/saved-fork-tooling.js +5 -2
- package/adapters/environment/engine/src/saved-fork/saved-fork.js +1 -1
- package/adapters/environment/engine/src/setup/engine-build.d.ts +4 -4
- package/adapters/environment/engine/src/setup/engine-build.js +14 -4
- package/adapters/environment/engine/src/setup/engine-tooling.d.ts +33 -0
- package/adapters/environment/engine/src/setup/engine-tooling.js +59 -0
- package/adapters/environment/engine/src/setup/node-workspace.patch +403 -0
- package/adapters/environment/engine/src/setup/patch-catalogue.d.ts +23 -0
- package/adapters/environment/engine/src/setup/patch-catalogue.js +8 -0
- package/adapters/environment/engine/src/setup/patches.d.ts +15 -1
- package/adapters/environment/engine/src/setup/patches.js +17 -2
- package/adapters/environment/engine/src/setup/prepare-engine.d.ts +6 -5
- package/adapters/environment/engine/src/setup/prepare-engine.js +27 -10
- package/composition/README.md +6 -4
- package/composition/src/engine-setup.js +9 -3
- package/composition/src/index.d.ts +2 -1
- package/composition/src/index.js +2 -1
- package/composition/src/people-dev/onboarding.d.ts +17 -3
- package/composition/src/people-dev/onboarding.js +25 -9
- package/composition/src/previewnet-genesis.d.ts +62 -0
- package/composition/src/previewnet-genesis.js +127 -0
- package/core/application/src/runs/failure-rules.js +5 -2
- package/entrypoints/cli/src/setup-commands.d.ts +1 -1
- package/entrypoints/cli/src/setup-commands.js +3 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# @parity/product-lab
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- b2abd7c: Add `previewnetGenesis()` to `@parity/product-lab/composition`: one line of a lab's config, `environments: { ...previewnetGenesis() }`, gives it "Previewnet from genesis" (`previewnet-genesis`), Previewnet's People, Asset Hub and Bulletin started from genesis on your machine by Engine, a new network every start. Once it is ready, and after every reset, People's rings are prepared for new people, so a new person can be onboarded on it like on the local copies. `product-lab fetch --environment previewnet-genesis` prepares its Engine and `product-lab doctor` checks it; it keeps its own folder in the cache, so your local copies' Engine is not changed. It binds Engine's default ports (relay 10000 to 10005, People 10010, Asset Hub 10020, Bulletin 10030, dashboard 8090, dub 8092, dub's database 5433, and Engine's other defaults in its config/ports.env: eth-rpc 8545, IPFS 8080, 5001 and 4001, TURN 3478, 3479 and its relay range 49152 to 49407, p2p 30333 to 30336), not the lab's port block: one Engine network per machine at a time; a port option is a planned follow-up.
|
|
8
|
+
|
|
9
|
+
The package now ships Engine's node-workspace patch, so `EngineSource.patch: "node-workspace"` works from an installed package instead of failing with `EnginePatchesNotInstalled`. The Engine setup script takes `--engine-patch node-workspace`. People's ring preparation is available on its own as `PeopleAttesterChain.prepareRings(collections)`, with `PEOPLE_LITE_COLLECTION`.
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- b2abd7c: A failed step is no longer explained by the folders or Node modules in its stack trace. The failure explanation's rules now read only the error's message, not its stack frames, so a check that fails in a checkout or product folder named like `previewnet-genesis` or `truapi`, or with frames in a `websocket.js` file or `node:events`, keeps the explanation its message gives, instead of "Infrastructure, not the product", a chain mismatch or a host refusal. The environment rule's bare words "genesis" and "baseline" are replaced by the environment errors they were meant for: endpoints that share a genesis, a fast baseline that is missing, for another machine or changed, an attached chain or Chopsticks upstream that is another chain, and a People whose genesis is not the network's. An assertion that mentions genesis, such as "account funded at genesis", is explained as the check's failure.
|
|
14
|
+
|
|
15
|
+
## 0.2.0
|
|
16
|
+
|
|
17
|
+
### Minor Changes
|
|
18
|
+
|
|
19
|
+
- 36bf358: Breaking change: `publishedBundle` now reads only the release asset record, requires a valid SHA-256 digest, and returns no snapshot. `PublishedBundle.digest` is required and `snapshot` is optional. Read bite time, blocks and runtimes from the verified installed or staged manifest after downloading. This removes the dependency on where `manifest.json` appears in the archive.
|
|
20
|
+
|
|
21
|
+
Refresh compares verified bite times before changing saved state. Background checks and explicit refresh download only when the published digest is new or unknown; an already staged matching bundle boots without another download. Older or equal downloads are not staged beside the installed bundle. The digest is a cache key, not an integrity check; existing manifest, snapshot-size, decompression and marker checks still apply. `PRODUCT_LAB_FORK_BUNDLE=keep` still downloads nothing.
|
|
22
|
+
|
|
23
|
+
An installed bundle with a missing or stale completion marker learns the digest after one download when its manifest matches and its snapshot sizes and decompression pass verification. Incomplete bundles remain unknown and continue downloading.
|
|
24
|
+
|
|
25
|
+
### Patch Changes
|
|
26
|
+
|
|
27
|
+
- 759dc0b: Allow repository users to prepare Engine with the optional node-workspace fix, so previewnet collators find binaries in a custom run workspace. Keep stock Engine as the default and verify the selected patch before reuse and launch, including saved forks. Limit node base-path recovery to node launchers.
|
|
28
|
+
|
|
3
29
|
## 0.1.2
|
|
4
30
|
|
|
5
31
|
### Patch Changes
|
|
@@ -33,7 +33,7 @@ use to reach an Engine network. Imported from the original lab.
|
|
|
33
33
|
| `engine/network-reset.ts` | `resetPersistentNetwork(directory, { stop, sessionFile, commands })` (29 September 2026): stops the saved network, removes its data and network.json, and restores the first boot's template as a new generation (`method: "template"`, with `stopMs` and the copy); a workspace without a template gets `method: "fork-again"` with the reason, and the caller's next `persistentNetwork` forks again (the old path). Notes' `paseo-copy` reset uses it. The persistent network's `stop()` (`persistent-network.ts`) now stops every saved process at once (`stopTogether`; on Notes' fork 3.8 s and 4.2 s against 4.5 s one after another), each with its own stop policy, every failure reported together (`tests/stop-together.test.ts`) |
|
|
34
34
|
| `engine/runtime-upgrade.ts` | An in-place runtime upgrade of one chain of a running network (relay, People, Asset Hub, Bulletin): `upgradeRuntime(request)` runs Engine's own `ppn upgrade <chain> <wasm> --ws <loopback> --enact-timeout <min>` (with `--allow-same-spec` for the same spec or `withoutChecks`; the caller decides whether Engine's authorization path applies to its network) as a recorded tree of the network's registry through `runEnginePrepare`, writes its code through the injected `FileSystem.writePrivateBytes`, and judges the result on the chain: `observeRuntime(rpc, ws)` before and after (finalized `:code` hash, spec, block, and `System.LastRuntimeUpgrade`, decoded by `decodeLastRuntimeUpgrade`, which says the new runtime's migrations ran). While Engine waits for enactment, the chain is read every 5 s: an authorization dropped with nothing scheduled and the code unchanged (`System.RejectedInvalidAuthorizedUpgrade`: another chain's runtime, a lower spec), which Engine's tool does not notice until its enact timeout, stops the wait at once as a failure. Outcomes: `already-running` (nothing submitted), `upgraded` (the chain runs exactly the code, `migrated` true or false), `failed` (with `unchanged`: the chain still runs its old code). `recordRuntimeUpgrade(files, …)` rewrites a persistent network's saved session (`network.json` and the session file) on purpose, since `checkEngineSession` refuses any runtime change it was not told about, and appends `runtime-upgrades.json` |
|
|
35
35
|
| `engine/fork-bundle.ts` | A fork bundle is reused only once verified complete. `verifyForkBundle(dir)`: `manifest.json` parses, every snapshot it names is there at the size it records (`snapshotBytes`, written by every bite since Engine v0.8.0; else one snapshot per chain) and decompresses to its end (3.5 s for the 642 MB Paseo bundle). `markForkBundle` writes `.product-lab-complete` (the manifest's sha256, each snapshot's size); `markedComplete` holds while both match. `forkBundleState(bundle)`: complete, absent, or incomplete; an unmarked bundle (fetched before this check, or copied in) is verified once and marked. A start's fetch or bite writes into `<bundle>.incoming` (cleared first; `EngineCommand.installs`), and `installIncomingBundle` verifies it, marks it and renames it into place after the command exits 0, so a fetch or bite stopped part way never leaves a bundle that is reused, and a failed one leaves the bundle in place unchanged. `acquireForkBundleLock` (`<bundle>.lock`, through the injected `LockManager`) is held from the bundle's check to the end of the preparatory commands: a second start from one workspace waits, then finds it complete. A start without isolation, where Engine's own `ppn start` fetches a missing bundle, is not covered |
|
|
36
|
-
| `engine/published-bundle.ts` | Engine's published fork bundles (the rolling `PPN_REPO@PPN_BITE_TAG` pre-release of the checkout's `config/versions.env`, `engineReleaseSettings`): `publishedBundle({ files, checkout, network, fetch })` (bounded by `timeoutMs`, default 180 000 ms, with the caller's optional `signal`) reads the asset record from GitHub's API and the
|
|
36
|
+
| `engine/published-bundle.ts` | Engine's published fork bundles (the rolling `PPN_REPO@PPN_BITE_TAG` pre-release of the checkout's `config/versions.env`, `engineReleaseSettings`): `publishedBundle({ files, checkout, network, fetch })` (bounded by `timeoutMs`, default 180 000 ms, with the caller's optional `signal`) reads only the release asset record from GitHub's API and requires a `sha256:` digest with 64 lowercase hex digits; it never requests the archive. `PublishedBundle.digest` is required and `snapshot` is optional and absent from lookup. Read bite time, blocks and runtimes from `currentBundle` or `stagedBundle` after a verified download. `manifestFromTarHead`, `TarManifestScanner` and `MANIFEST_SEARCH_BYTES` remain exported for existing callers but are unused by lookup. Under `PRODUCT_LAB_FORK_BUNDLE=keep` the background check uses its separate release-record-only report; the caller passes `fetch` (the public-endpoint opt-in) and the `FileSystem` system port for the checkout settings and staged bundles. `startPrefetch` downloads it with Engine's own `ppn fork fetch-bundle` into `fork-bundle-<network>.next.partial` at background priority (`backgroundPriority`: macOS `taskpolicy -c utility`, else `nice -n 19`) as a recorded process of the caller's registry; `finishPrefetch` verifies it complete and marks it (`engine/fork-bundle.ts`; an incomplete download is removed), then `completePrefetch` stages it (`.next`, `stagedBundle`, which offers only a verified one); `installStagedBundle` makes it the bundle new forks start from, keeping the old one until `discardPrevious` or `restorePrevious`. `currentBundle`, `bittenAfter`, `prefetchActive`, `bundleLocations` |
|
|
37
37
|
| `engine/fork-freshness.ts`, `engine/fork-refresh.ts`, `engine/fork-updates.ts` | Freshness, `--update-runtime` and `--refresh` for any saved Engine fork (fixture sharing, 30 September 2026, backlog 17c). They were lifted from an example, which now wraps them. State: `forkState` reads what a saved fork runs: its saved session, `runtimes.json` and its bite. `liveEndpoints` reads the public RPCs from Engine's network file; a relative one (previewnet's `relay/alice`, `people`) resolves under the file's `bite.source` as `wss://` (`resolveNetworkRpc`, as Engine's bite does). Freshness line: `freshness` is pure: it gives the age, the blocks behind live, and each chain whose spec or code changed, ending with the update command. The live read is injected as `LiveRuntimeReader`; the composition root's is `readLiveRuntimes`. `--update-runtime`: `updateForkRuntimes` applies each changed live runtime in place through `upgradeRuntime`. The code comes from an injected `LiveCodeFetcher`, checked against its hash. It then records the upgrade and waits `settleBlocks`. It refuses without a live read or a sudo key. Other pieces: `recordForkRuntimes`. `prefetchNewerState` fetches a newer published state in the background and never rejects. `PRODUCT_LAB_FORK_BUNDLE` (29 October 2026, read through the `EnvironmentVariables` port, so `savedForkUpdates` and a lab's own wrapper get it too): `fresh` or unset as described; `keep` reads only the release record from GitHub's API (no asset bytes, no range, nothing staged, no `ppn fork fetch-bundle`) and logs one line with the kept bundle's bite, its age, a newer state already downloaded, and the asset's `updated_at` when more than 6 hours after that bite (an upload within that is the bite's own); a record or local manifest that cannot be read is said, and the lab carries on. Surrounding white space is ignored; any other value logs a warning with the fix and checks nothing. `--refresh` and commands run by hand are the same under both. `tests/fork-bundle-keep.test.ts`. `prepareRefresh` sets the old fork aside for the newest published state; a lab record is carried across with `carry`. `commit` or `rollback` ends it. For a lab's start: `savedForkUpdates(options)` combines these into three calls. Before the start it runs `--refresh` when asked, and runs the live read and line beside the start. `started(target)` runs `--update-runtime` before anything connects, records the runtimes, commits a refresh and prefetches. `failed()` rolls a refresh back. `liveRpc` (3 October 2026): the lab's own RPC per chain for the live reads, over the network file's. `upgradeToLive(target, signal)` serves `networkSnapshots({ upgrade })` for `fixture refresh`. `tests/fork-freshness.test.ts` |
|
|
38
38
|
| `engine/owned-environment.ts` | `ownedEngineEnvironment`: owned Engine + consumer app sessions as one resettable environment; its local `engineSessionEnvironment` forwards optional app surfaces across reset and closes the current sessions and Engine once |
|
|
39
39
|
| `engine/network-plan.ts`, `engine/persisted-data.ts`, `engine/eth-rpc-probe.ts` | The persistent network's phases around `persistentNetwork`, split out of `persistent-network.ts`, which still exports their public names: the saved plan (`NetworkPlan`, `resumePlan`, `isolatedPlan`, `completePlan`, `planFacts`, `ENGINE_PORTS_FILE`); the resume's preconditions (`hasChainDatabase`, `missingPersistedData`, `requirePersistedData`, `restateEngineLocalEnv`, `engineCheckout`, `restateEnginePorts`, `portNames`, `requireFreePorts`; and, in `engine/saved-tooling.ts`, `missingSavedTooling` and `requireSavedTooling`: every command, what the lab's launchers in `<network>.isolation` exec, and the Engine checkout are on this machine); eth-rpc's health after a start (`probeEthRpc`, `EthRpcStatus`) |
|
|
@@ -45,9 +45,10 @@ use to reach an Engine network. Imported from the original lab.
|
|
|
45
45
|
| `topology/engine-plan.ts` | `engineLaunchPlan`: a resolved relay-backed `EngineTopology` as `ppn` commands (fork, `--fresh-bite`, `--upgrade`, `--runtime`, `--binary`, genesis subsets through `genesis-toml --parachains` then `start --toml`) and `config/ports.env` keys |
|
|
46
46
|
| `topology/engine-topology.ts` | `resolveEngineTopology`, `engineOptionsSchema`, `engineProfiles` (full-fork, genesis-subset), `engineFlagOptions`, `ENGINE_FLAGS` (`--engine-arg`, `--engine-env`, `--engine-toml`), `ENGINE_KIND`, `DEFAULT_ENGINE_PORTS`: the adapter's own topology options, defaults, flags and issues (refactor step 5) |
|
|
47
47
|
| `topology/engine-environment-factory.ts` | `EngineEnvironmentFactory`: the adapter's `TopologyAdapterFactory` (kind `engine`, section `engine`), constructed by the composition root when a lab selects `defineLab({ environmentAdapters: { engine: { engineConfig } } })` (`composition/src/adapter-catalogue.ts`), or registered directly with `defineLab({ environmentFactories: [new EngineEnvironmentFactory({ engineConfig })] })` |
|
|
48
|
-
| `setup/prepare-engine.ts`, `setup/engine-build.ts`, `setup/patches.ts`, `setup/patch-catalogue.ts` | Stock Engine v0.11.0 preparation and shipped release pin; `preparedEngine` checks the pin, stock record, checkout HEAD, built CLI and dot CLI. Retired
|
|
48
|
+
| `setup/prepare-engine.ts`, `setup/engine-build.ts`, `setup/patches.ts`, `setup/patch-catalogue.ts`, `setup/node-workspace.patch` | Stock Engine v0.11.0 preparation and shipped release pin; `preparedEngine` checks the pin, stock record, checkout HEAD, built CLI and dot CLI. The optional `EngineSource.patch: "node-workspace"` is read from the copy the package ships beside the catalogue (`setup/node-workspace.patch`, `shippedEnginePatch`, digest-checked against `NODE_WORKSPACE_PATCH`, its manifest entry in the package; since 12 November 2026, so a product prepares it too: `tests/patch-catalogue.test.ts` holds it byte for byte the repository's `patches/previewnet-engine/node-workspace.patch`), applied before build, keyed by digest and reverse-checked before reuse. Retired owned-processes records and requests are refused. Descriptor overlays remain digest-checked through the repository manifest. |
|
|
49
|
+
| `setup/engine-tooling.ts` | An Engine checkout as an environment's tooling in a state directory of its own (12 November 2026, for composition's `previewnetGenesis`): `engineCheckoutTooling({ state, network, patch, environment })` (id `engine:<state>`; `check` is ready only for the build `patch` names; `prepare` runs `prepareEngine` with it, and refuses `--no-engine-patch` for a patched one), `preparedEngineCheckout` (the start's check, its refusal naming the fix) and `engineFetchCommand(environment)` (`product-lab fetch --environment <id>`, through the checkout's CLI script or npx). `tests/engine-checkout-tooling.test.ts` |
|
|
49
50
|
| `setup/setup-command.ts` | The command every Engine refusal names, for where this adapter runs (`engineSetupCommand(environment?)`): `bun run setup` in product-lab's checkout, `npx --no-install product-lab fetch --environment <id>` in a product, which has no `bun run setup` (`insideProductLabCheckout`, this adapter's own copy of the CLI's check) |
|
|
50
|
-
| `saved-fork/saved-fork-tooling.ts` | The Engine checkout a saved fork starts from, `<cacheDir>/engine` (16 October 2026): `savedForkEngine` (the start's check, `as-prepared`, its refusal naming `SavedForkSettings.setupCommand`, default `engineSetupCommand`), and `savedForkTooling`, the fork's `EnvironmentProvider.tooling` (id `engine:<cacheDir>/engine`, shared by every saved fork with that cache; kind `engine`): `check` is `savedForkEngine`'s, `prepare` runs `prepareEngine` there for the fork's network with the request's source (`repository`, `ref`, `patched: false` for stock, also the default; `patched: true` is refused) and returns the check of exactly what it asked for. `product-lab fetch` prepares it and `product-lab doctor` reports it in a product. A saved fork started from a stock build writes `enginePatch: "none"` in its own engine.json |
|
|
51
|
+
| `saved-fork/saved-fork-tooling.ts` | The Engine checkout a saved fork starts from, `<cacheDir>/engine` (16 October 2026): `savedForkEngine` (the start's check, `as-prepared`, its refusal naming `SavedForkSettings.setupCommand`, default `engineSetupCommand`), and `savedForkTooling`, the fork's `EnvironmentProvider.tooling` (id `engine:<cacheDir>/engine`, shared by every saved fork with that cache; kind `engine`): `check` is `savedForkEngine`'s, `prepare` runs `prepareEngine` there for the fork's network with the request's source (`repository`, `ref`, `patched: false` for stock, also the default; `patched: true` is refused) and returns the check of exactly what it asked for. `product-lab fetch` prepares it and `product-lab doctor` reports it in a product. A saved fork can reuse a node-workspace build prepared through `prepareEngine`; its status names that patch and its launch config records `enginePatch`, so the launch-time reverse check is retained. A saved fork started from a stock build writes `enginePatch: "none"` in its own engine.json |
|
|
51
52
|
| `testing` subpath | Fake Engine dashboard and chain RPCs for contract tests (`codeHashOffset`, `specVersion` change the served runtime) |
|
|
52
53
|
| `versions/engine-versions.ts` | Phase 6: `engineVersionPlan(topology, resolution, { snapshot, now })` (what Engine honours: runtimes read from the network it forks; refused as unsupported: released or git runtimes, resolved binaries, a runtime override on the same chain, a network runtime on genesis, a published bundle that `SnapshotFreshness` finds stale or on another runtime), `engineVersionCheck(session, plan, snapshot)` (each chain's observed `:code` hash, spec, genesis and bite block against the resolution), `VersionedEngineDependency` (checks every start and reset, stops and throws `EngineVersionMismatch` on other code), `engineVersionsRecord`, `workspaceVersionPlan` (persistent networks). Engine cannot fork at a chosen block: a `deployed` resolution is honoured by runtime, and the bite blocks are recorded beside the resolved block |
|
|
53
54
|
|
|
@@ -121,3 +122,21 @@ a taken port named, and the readiness wait's reasons). With the Node
|
|
|
121
122
|
system adapters: the lock on disk, `tools/verify/tests/scenario-lock.test.ts`; an owned Engine started, awaited and
|
|
122
123
|
stopped against a stand-in checkout, and a stock build passing the pre-start check while legacy patched configurations are refused, `tools/verify/tests/engine-owned-start.test.ts`; an Engine session routing the
|
|
123
124
|
truAPI CLI host, `tools/verify/tests/host-routes.test.ts`.
|
|
125
|
+
|
|
126
|
+
Native-node workspace regression: `node --import tsx --test adapters/environment/engine/tests/node-workspace.test.ts`
|
|
127
|
+
from the repository root. This runs the actual before/after shell launcher extracted
|
|
128
|
+
from the full-context manifest patch, under `env -i` with temporary stub binaries,
|
|
129
|
+
and checks the actual TOML writer. No chain, user workspace or Engine cache is used.
|
|
130
|
+
|
|
131
|
+
Lookup cannot compare bite times before download. Under `fresh`, background checks and explicit `--refresh` download
|
|
132
|
+
only when the published digest is new or unknown, then compare verified bite times before touching saved state.
|
|
133
|
+
A matching installed or staged bundle skips the download; `--refresh` boots a matching staged newer bite. A download
|
|
134
|
+
no newer than the installed bundle is not staged. Markers from older versions remain readable; an identical bundle
|
|
135
|
+
needs one download to record its digest. Missing or stale markers are rebuilt only after the installed manifest,
|
|
136
|
+
snapshot sizes and decompression pass verification. When keeping an equal bite, the digest transfers only if manifest hashes
|
|
137
|
+
match. A changed manifest at the same bite stays unknown and downloads on each start until Engine publishes a newer bite.
|
|
138
|
+
|
|
139
|
+
The marker's optional `assetDigest` is a cache key, not an integrity check. Engine owns and removes the tarball;
|
|
140
|
+
the manifest, snapshot-size, decompression and `manifestSha256` checks still protect bundle completeness. The release
|
|
141
|
+
digest is checked again after downloading, and is recorded only if unchanged; a racing release replacement or a failed
|
|
142
|
+
confirmation costs another download rather than skipping a needed one. `keep` still downloads nothing.
|
|
@@ -66,6 +66,7 @@ export declare const engineConfiguration: z.ZodObject<{
|
|
|
66
66
|
enginePatch: z.ZodDefault<z.ZodEnum<{
|
|
67
67
|
none: "none";
|
|
68
68
|
"owned-processes": "owned-processes";
|
|
69
|
+
"node-workspace": "node-workspace";
|
|
69
70
|
}>>;
|
|
70
71
|
recordProgress: z.ZodDefault<z.ZodBoolean>;
|
|
71
72
|
nodeSupervision: z.ZodDefault<z.ZodBoolean>;
|
|
@@ -17,7 +17,7 @@ export type BundleCheck = {
|
|
|
17
17
|
*/
|
|
18
18
|
export declare function verifyForkBundle(directory: string, signal?: AbortSignal): Promise<BundleCheck>;
|
|
19
19
|
/** Record that `directory` was verified complete (written to a temporary name, then renamed: never half a marker). */
|
|
20
|
-
export declare function markForkBundle(directory: string, snapshots: Record<string, number>, source: string): void;
|
|
20
|
+
export declare function markForkBundle(directory: string, snapshots: Record<string, number>, source: string, assetDigest?: string): void;
|
|
21
21
|
/**
|
|
22
22
|
* True when `directory` carries the marker and still matches it: the same manifest, every snapshot at its size. A
|
|
23
23
|
* missing or unparsable marker, manifest or snapshot is "not marked": the caller verifies the bundle again. Any other
|
|
@@ -67,8 +67,8 @@ export async function verifyForkBundle(directory, signal) {
|
|
|
67
67
|
}
|
|
68
68
|
return { complete: true, snapshots };
|
|
69
69
|
}
|
|
70
|
-
export function markForkBundle(directory, snapshots, source) {
|
|
71
|
-
const marker = { manifestSha256: sha256(join(directory, "manifest.json")), snapshots, source, verifiedAt: new Date().toISOString() };
|
|
70
|
+
export function markForkBundle(directory, snapshots, source, assetDigest) {
|
|
71
|
+
const marker = { ...(assetDigest ? { assetDigest } : {}), manifestSha256: sha256(join(directory, "manifest.json")), snapshots, source, verifiedAt: new Date().toISOString() };
|
|
72
72
|
const file = join(directory, BUNDLE_MARKER);
|
|
73
73
|
writeFileSync(`${file}.tmp`, `${JSON.stringify(marker, null, 2)}\n`, { mode: 0o600 });
|
|
74
74
|
renameSync(`${file}.tmp`, file);
|
|
@@ -2,10 +2,7 @@ import type { ForkSnapshot } from "@parity/product-lab/domain/versions";
|
|
|
2
2
|
import type { EnvironmentVariables, FileSystem, Logger, OwnedProcesses } from "@parity/product-lab/ports/system";
|
|
3
3
|
import { type ForkUpdateCommands } from "./fork-freshness.js";
|
|
4
4
|
import { type PublishedBundle } from "./published-bundle.js";
|
|
5
|
-
/**
|
|
6
|
-
* How long the check for a newer published state may take: it streams the bundle until its manifest (the 27
|
|
7
|
-
* September bundle: 8.8 MB read, 67 s at the 180 KB/s one machine had then). It runs in the background.
|
|
8
|
-
*/
|
|
5
|
+
/** How long the background release-record lookup may take. Archive bytes are read only by the download. */
|
|
9
6
|
export declare const PUBLISHED_CHECK_MS = 300000;
|
|
10
7
|
/** What a lab start tells about the published state (the CLI line and the lab window's notice). */
|
|
11
8
|
export declare function newerStateNotice(snapshot: Pick<ForkSnapshot, "bittenAt">, ready: boolean, commands?: ForkUpdateCommands): string;
|
|
@@ -29,8 +26,8 @@ export interface PrefetchDeps {
|
|
|
29
26
|
commands?: ForkUpdateCommands;
|
|
30
27
|
}
|
|
31
28
|
/**
|
|
32
|
-
* While the lab runs:
|
|
33
|
-
*
|
|
29
|
+
* While the lab runs: offer a newer local bite for refresh, or download the published asset in the background to
|
|
30
|
+
* compare its verified bite. Never throws; returns when it has decided (the
|
|
34
31
|
* download itself continues).
|
|
35
32
|
*/
|
|
36
33
|
export declare function prefetchNewerState(forkBite: Pick<ForkSnapshot, "bittenAt"> | undefined, deps: PrefetchDeps): Promise<PublishedBundle | undefined>;
|
|
@@ -68,7 +65,7 @@ export interface RefreshDeps<T = unknown> {
|
|
|
68
65
|
}
|
|
69
66
|
/**
|
|
70
67
|
* `--refresh`: set the saved fork aside and install the newest published state as the bundle a first start boots,
|
|
71
|
-
* downloading
|
|
68
|
+
* downloading a new or unknown digest to read its verified bite (offline: use a newer local bundle). Returns undefined when the fork already
|
|
72
69
|
* runs the newest published state.
|
|
73
70
|
*/
|
|
74
71
|
export declare function prepareRefresh<T>(directory: string, deps: RefreshDeps<T>): Promise<RefreshHandle | undefined>;
|
|
@@ -3,13 +3,13 @@
|
|
|
3
3
|
import { existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { forkState, refreshCommand, shortError } from "./fork-freshness.js";
|
|
6
|
-
import { bittenAfter, bundleLocations, finishPrefetch, currentBundle, discardPrevious, engineReleaseSettings, installStagedBundle, prefetchActive, publishedBundle, restorePrevious, stagedBundle, startPrefetch, } from "./published-bundle.js";
|
|
6
|
+
import { bittenAfter, bundleLocations, bundleAssetDigest, finishPrefetch, currentBundle, discardPrevious, engineReleaseSettings, installStagedBundle, prefetchActive, publishedBundle, restorePrevious, stagedBundle, startPrefetch, } from "./published-bundle.js";
|
|
7
7
|
export const PUBLISHED_CHECK_MS = 300_000;
|
|
8
8
|
export function newerStateNotice(snapshot, ready, commands) {
|
|
9
9
|
const bitten = snapshot.bittenAt.slice(0, 16).replace("T", " ");
|
|
10
10
|
return ready
|
|
11
|
-
? `A newer published state of the network (bitten ${bitten} UTC) is downloaded
|
|
12
|
-
: `A newer published state of the network (bitten ${bitten} UTC) exists; downloading it in the background at low priority so ${refreshCommand(commands)}
|
|
11
|
+
? `A newer published state of the network (bitten ${bitten} UTC) is downloaded; ${refreshCommand(commands)} checks the newest published state before making a new local fork (this fork's identities are discarded).`
|
|
12
|
+
: `A newer published state of the network (bitten ${bitten} UTC) exists; downloading it in the background at low priority so its verified bite is available for ${refreshCommand(commands)}.`;
|
|
13
13
|
}
|
|
14
14
|
export async function prefetchNewerState(forkBite, deps) {
|
|
15
15
|
try {
|
|
@@ -62,7 +62,7 @@ async function reportKeptBundle(forkBite, deps) {
|
|
|
62
62
|
const kept = installed ?? forkBite;
|
|
63
63
|
const parts = [kept ? `bitten ${kept.bittenAt.slice(0, 16).replace("T", " ")} UTC, ${age(kept.bittenAt, Date.now())} old` : "no bundle installed yet"];
|
|
64
64
|
if (staged && (!kept || bittenAfter(staged, kept)))
|
|
65
|
-
parts.push(`a newer state (bitten ${staged.bittenAt.slice(0, 16).replace("T", " ")} UTC) is downloaded
|
|
65
|
+
parts.push(`a newer state (bitten ${staged.bittenAt.slice(0, 16).replace("T", " ")} UTC) is downloaded; ${refreshCommand(deps.commands)} checks the newest published state before making a new fork`);
|
|
66
66
|
parts.push(...problems);
|
|
67
67
|
const line = `Keeping the fork bundle of ${network} on this machine (${FORK_BUNDLE_SETTING}=keep; ${parts.join("; ")})`;
|
|
68
68
|
const asset = `fork-bundle-${network}.tar.gz`;
|
|
@@ -92,6 +92,21 @@ async function reportKeptBundle(forkBite, deps) {
|
|
|
92
92
|
else
|
|
93
93
|
logger.info(`${line}.`);
|
|
94
94
|
}
|
|
95
|
+
function publishedIsLocal(deps, published, staged, installed) {
|
|
96
|
+
const { next, current } = bundleLocations(deps.engine.workspace, deps.network);
|
|
97
|
+
return !!((staged && bundleAssetDigest(deps.files, next) === published.digest) || (installed && bundleAssetDigest(deps.files, current) === published.digest));
|
|
98
|
+
}
|
|
99
|
+
async function confirmedAssetDigest(deps, before) {
|
|
100
|
+
try {
|
|
101
|
+
const after = await publishedBundle({ files: deps.files, checkout: deps.engine.checkout, network: deps.network,
|
|
102
|
+
fetch: deps.fetch ?? fetch, env: deps.environment.all(), token: deps.environment.get("GITHUB_TOKEN"), signal: "signal" in deps ? deps.signal : undefined, timeoutMs: PUBLISHED_CHECK_MS });
|
|
103
|
+
return after.digest === before.digest ? before.digest : undefined;
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
deps.logger.warn(`Could not confirm the downloaded release digest (${shortError(error)}); a later check may download again.`);
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
95
110
|
async function prefetchWhenNewer(forkBite, deps) {
|
|
96
111
|
const { engine, network, logger } = deps;
|
|
97
112
|
let published;
|
|
@@ -110,23 +125,25 @@ async function prefetchWhenNewer(forkBite, deps) {
|
|
|
110
125
|
logger.info(`Could not check for a newer published state of ${network} (${shortError(error)}); a later start checks again.`);
|
|
111
126
|
return undefined;
|
|
112
127
|
}
|
|
113
|
-
if (!
|
|
128
|
+
if (!forkBite)
|
|
114
129
|
return published;
|
|
115
130
|
const staged = stagedBundle(deps.files, engine.workspace, network);
|
|
116
131
|
const installed = currentBundle(deps.files, engine.workspace, network);
|
|
117
|
-
|
|
118
|
-
|
|
132
|
+
const ready = [staged, installed].filter((entry) => !!entry && bittenAfter(entry, forkBite))
|
|
133
|
+
.sort((a, b) => Date.parse(b.bittenAt) - Date.parse(a.bittenAt))[0];
|
|
134
|
+
if (ready) {
|
|
119
135
|
const text = newerStateNotice(ready, true, deps.commands);
|
|
120
136
|
logger.info(text);
|
|
121
137
|
deps.notice(text);
|
|
122
|
-
return published;
|
|
123
138
|
}
|
|
139
|
+
if (publishedIsLocal(deps, published, staged, installed))
|
|
140
|
+
return published;
|
|
124
141
|
const { partial } = bundleLocations(engine.workspace, network);
|
|
125
142
|
if (prefetchActive(deps.files, partial)) {
|
|
126
|
-
logger.info(`
|
|
143
|
+
logger.info(`The published state of ${network} is being downloaded by another lab on this machine.`);
|
|
127
144
|
return published;
|
|
128
145
|
}
|
|
129
|
-
const text =
|
|
146
|
+
const text = `Checking the published state of ${network}: downloading it in the background at low priority to read its bite time.`;
|
|
130
147
|
logger.info(`${text} (${Math.round(published.size / 1_000_000)} MB)`);
|
|
131
148
|
deps.notice(text);
|
|
132
149
|
try {
|
|
@@ -147,10 +164,17 @@ async function prefetchWhenNewer(forkBite, deps) {
|
|
|
147
164
|
return;
|
|
148
165
|
}
|
|
149
166
|
try {
|
|
150
|
-
const
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
167
|
+
const digest = await confirmedAssetDigest(deps, published);
|
|
168
|
+
const snapshot = await finishPrefetch(deps.files, engine.workspace, network, undefined, digest);
|
|
169
|
+
if (bittenAfter(snapshot, forkBite)) {
|
|
170
|
+
const done = newerStateNotice(snapshot, true, deps.commands);
|
|
171
|
+
logger.info(done);
|
|
172
|
+
deps.notice(done);
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
logger.info(`The downloaded state of ${network} (bitten ${snapshot.bittenAt}) is not newer than this fork.`);
|
|
176
|
+
deps.notice(undefined);
|
|
177
|
+
}
|
|
154
178
|
}
|
|
155
179
|
catch (error) {
|
|
156
180
|
logger.warn(`The background download of ${network}'s newer state is unusable (${shortError(error)}); a later start tries again.`);
|
|
@@ -172,7 +196,7 @@ export async function prepareRefresh(directory, deps) {
|
|
|
172
196
|
const saved = fork?.kind === "saved";
|
|
173
197
|
const forkBite = saved && fork.bittenAt ? { bittenAt: fork.bittenAt } : undefined;
|
|
174
198
|
let staged = stagedBundle(deps.files, engine.workspace, network);
|
|
175
|
-
|
|
199
|
+
let installed = currentBundle(deps.files, engine.workspace, network);
|
|
176
200
|
let published;
|
|
177
201
|
try {
|
|
178
202
|
const token = deps.environment.get("GITHUB_TOKEN");
|
|
@@ -184,15 +208,10 @@ export async function prepareRefresh(directory, deps) {
|
|
|
184
208
|
throw new Error(`--refresh could not read the newest published state (${shortError(error)}), and nothing newer than this fork is downloaded. Nothing was changed. Fix: connect to the internet and run it again.`);
|
|
185
209
|
logger.warn(`Could not check GitHub for a newer published state (${shortError(error)}); refreshing from the one downloaded (bitten ${local.bittenAt}).`);
|
|
186
210
|
}
|
|
187
|
-
|
|
188
|
-
if (saved && forkBite && newest && !bittenAfter(newest, forkBite)) {
|
|
189
|
-
logger.info(`--refresh: the fork already runs the newest published state (bitten ${forkBite.bittenAt}); nothing to refresh. Resuming it as it is.`);
|
|
190
|
-
return undefined;
|
|
191
|
-
}
|
|
192
|
-
if (published && (!staged || bittenAfter(published.snapshot, staged)) && (!installed || bittenAfter(published.snapshot, installed))) {
|
|
211
|
+
if (published && !publishedIsLocal(deps, published, staged, installed)) {
|
|
193
212
|
const registry = deps.processes.open(join(existsSync(directory) ? directory : dirname(directory), existsSync(directory) ? "processes" : "refresh-processes"));
|
|
194
213
|
try {
|
|
195
|
-
logger.info(`--refresh: downloading the newest published state (
|
|
214
|
+
logger.info(`--refresh: downloading the newest published state (${Math.round(published.size / 1_000_000)} MB; bite time is read after verification) now to check the rolling release before changing the saved fork.`);
|
|
196
215
|
const child = await startPrefetch({ engine, network, registry, environment: deps.environment, files: deps.files });
|
|
197
216
|
const onAbort = () => {
|
|
198
217
|
registry.stop("bundle-prefetch").catch((error) => logger.warn(`Could not stop the interrupted download: ${shortError(error)}`));
|
|
@@ -204,13 +223,23 @@ export async function prepareRefresh(directory, deps) {
|
|
|
204
223
|
deps.signal?.throwIfAborted();
|
|
205
224
|
if (code !== 0)
|
|
206
225
|
throw new Error(`--refresh: the download failed (exit ${code}; ${registry.logFile("bundle-prefetch")}). Nothing was changed.`);
|
|
207
|
-
await
|
|
226
|
+
const digest = await confirmedAssetDigest(deps, published);
|
|
227
|
+
await finishPrefetch(deps.files, engine.workspace, network, deps.signal, digest);
|
|
208
228
|
staged = stagedBundle(deps.files, engine.workspace, network);
|
|
229
|
+
installed = currentBundle(deps.files, engine.workspace, network);
|
|
209
230
|
}
|
|
210
231
|
finally {
|
|
211
232
|
registry.close();
|
|
212
233
|
}
|
|
213
234
|
}
|
|
235
|
+
const newest = [staged, installed].filter((entry) => !!entry)
|
|
236
|
+
.sort((a, b) => Date.parse(b.bittenAt) - Date.parse(a.bittenAt))[0];
|
|
237
|
+
if (saved && forkBite && newest && !bittenAfter(newest, forkBite)) {
|
|
238
|
+
logger.info(`--refresh: the fork already runs the newest published state (bitten ${forkBite.bittenAt}); nothing to refresh. Resuming it as it is.`);
|
|
239
|
+
return undefined;
|
|
240
|
+
}
|
|
241
|
+
if (!newest)
|
|
242
|
+
throw new Error("--refresh: no bundle is installed after the download; nothing to boot.");
|
|
214
243
|
if (saved) {
|
|
215
244
|
const registry = deps.processes.open(join(directory, "processes"));
|
|
216
245
|
try {
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import type { ForkSnapshot } from "@parity/product-lab/domain";
|
|
2
2
|
import type { EnvironmentVariables, FileSystem, OwnedProcess, OwnedProcesses } from "@parity/product-lab/ports/system";
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 27 September after 15.5 MB of chain specs (about 4 MB compressed); the snapshots (hundreds of MB) come later.
|
|
4
|
+
* Legacy head-reading budget, retained for callers of the tar helpers. Lookup never reads archive bytes.
|
|
5
|
+
* @deprecated Use publishedBundle for release identity and verifyForkBundle for extracted bundles.
|
|
7
6
|
*/
|
|
8
7
|
export declare const MANIFEST_SEARCH_BYTES: number;
|
|
9
8
|
/** `KEY="${KEY:-default}"` lines of Engine's config/versions.env, resolved against `env` as Engine resolves them. */
|
|
@@ -18,28 +17,31 @@ export interface PublishedBundle {
|
|
|
18
17
|
/** The asset's download URL. */
|
|
19
18
|
url: string;
|
|
20
19
|
size: number;
|
|
21
|
-
/** GitHub's digest of the asset (`sha256:<hex>`)
|
|
22
|
-
digest
|
|
20
|
+
/** GitHub's validated SHA-256 digest of the complete asset (`sha256:<64 lowercase hex digits>`). */
|
|
21
|
+
digest: string;
|
|
23
22
|
/** When the asset was last uploaded. */
|
|
24
23
|
updatedAt: string;
|
|
25
|
-
/**
|
|
26
|
-
snapshot
|
|
24
|
+
/** Lookup does not read the archive. Read its snapshot from the verified installed or staged manifest instead. */
|
|
25
|
+
snapshot?: ForkSnapshot;
|
|
27
26
|
}
|
|
28
27
|
/**
|
|
29
28
|
* Scans a tar stream chunk by chunk for `manifest.json` (ustar headers), keeping only the bytes it still needs: the
|
|
30
29
|
* other entries' contents are skipped as they go by. `push` returns the parsed manifest once it is complete.
|
|
30
|
+
* @deprecated Use publishedBundle for release identity and verifyForkBundle for extracted bundles.
|
|
31
31
|
*/
|
|
32
32
|
export declare class TarManifestScanner {
|
|
33
33
|
#private;
|
|
34
34
|
push(chunk: Uint8Array): unknown;
|
|
35
35
|
}
|
|
36
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Find `manifest.json` among a tar stream's first entries (ustar headers).
|
|
38
|
+
* @deprecated Use publishedBundle for release identity and verifyForkBundle for extracted bundles.
|
|
39
|
+
*/
|
|
37
40
|
export declare function manifestFromTarHead(tar: Uint8Array): unknown;
|
|
38
41
|
/**
|
|
39
|
-
* What Engine publishes for `network` now: the asset's record and
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* bundle for the network.
|
|
42
|
+
* What Engine publishes for `network` now: the release asset's record and SHA-256 identity, with no archive request.
|
|
43
|
+
* Rejects when GitHub cannot be reached, the release has no bundle for the network, or its digest is not SHA-256.
|
|
44
|
+
* The archive's entry order is irrelevant; read bite time, blocks and runtimes after downloading and verifying it.
|
|
43
45
|
*/
|
|
44
46
|
export declare function publishedBundle(input: {
|
|
45
47
|
files: Pick<FileSystem, "readText">;
|
|
@@ -49,7 +51,7 @@ export declare function publishedBundle(input: {
|
|
|
49
51
|
env?: Readonly<Record<string, string | undefined>>;
|
|
50
52
|
token?: string;
|
|
51
53
|
signal?: AbortSignal;
|
|
52
|
-
/** Bound of the
|
|
54
|
+
/** Bound of the release-record request (default 180 000 ms); on expiry it rejects. */
|
|
53
55
|
timeoutMs?: number;
|
|
54
56
|
}): Promise<PublishedBundle>;
|
|
55
57
|
/** Where a network's bundle lives in an Engine workspace, the staged newer one, and a download in progress. */
|
|
@@ -63,6 +65,8 @@ type BundleFiles = Pick<FileSystem, "exists" | "readJson" | "listDirectory" | "r
|
|
|
63
65
|
export declare function stagedBundle(files: BundleFiles, workspace: string, network: string): ForkSnapshot | undefined;
|
|
64
66
|
/** The bundle a new fork of this workspace would start from now. */
|
|
65
67
|
export declare function currentBundle(files: BundleFiles, workspace: string, network: string): ForkSnapshot | undefined;
|
|
68
|
+
/** The recorded release cache key; an old or damaged marker has an unknown digest. */
|
|
69
|
+
export declare function bundleAssetDigest(files: Pick<FileSystem, "readJson">, directory: string): string | undefined;
|
|
66
70
|
/** True when `a` was bitten after `b` (either missing: false). */
|
|
67
71
|
export declare function bittenAfter(a: Pick<ForkSnapshot, "bittenAt"> | undefined, b: Pick<ForkSnapshot, "bittenAt"> | undefined): boolean;
|
|
68
72
|
/** A download in progress in `partial`: a file there written within the last minute (another lab's prefetch). */
|
|
@@ -97,9 +101,9 @@ export declare function startPrefetch(input: {
|
|
|
97
101
|
* After the download exited 0: verify it complete and mark it (fork-bundle.ts), then stage it (`completePrefetch`). An
|
|
98
102
|
* incomplete download is removed and rejects, naming what is wrong; the staged and installed bundles are unchanged.
|
|
99
103
|
*/
|
|
100
|
-
export declare function finishPrefetch(files: BundleFiles, workspace: string, network: string, signal?: AbortSignal): Promise<ForkSnapshot>;
|
|
104
|
+
export declare function finishPrefetch(files: BundleFiles & Pick<FileSystem, "writePrivateJson">, workspace: string, network: string, signal?: AbortSignal, assetDigest?: string): Promise<ForkSnapshot>;
|
|
101
105
|
/** Move a finished, verified download into place as the staged bundle (replacing an older staged one). */
|
|
102
|
-
export declare function completePrefetch(files: BundleFiles, workspace: string, network: string): ForkSnapshot;
|
|
106
|
+
export declare function completePrefetch(files: BundleFiles & Pick<FileSystem, "writePrivateJson">, workspace: string, network: string): ForkSnapshot;
|
|
103
107
|
/**
|
|
104
108
|
* Make the staged bundle the one new forks start from. The current bundle is
|
|
105
109
|
* kept beside it (`<current>.previous`) until `discardPrevious`, so a failed
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// Copyright 2026 Parity Technologies (UK) Ltd.
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { readFileSync } from "node:fs";
|
|
3
5
|
import { join } from "node:path";
|
|
4
|
-
import { createGunzip, constants as zlib } from "node:zlib";
|
|
5
6
|
import { forkBundleManifestPath, forkSnapshotFrom } from "./fork-snapshot.js";
|
|
6
7
|
import { BUNDLE_MARKER, markForkBundle, verifyForkBundle } from "./fork-bundle.js";
|
|
7
8
|
export const MANIFEST_SEARCH_BYTES = 48 * 1024 * 1024;
|
|
@@ -92,65 +93,23 @@ export async function publishedBundle(input) {
|
|
|
92
93
|
if (!response.ok)
|
|
93
94
|
throw new Error(`GitHub answered ${response.status} for ${repository}@${tag}`);
|
|
94
95
|
const release = (await response.json());
|
|
96
|
+
if (!release || typeof release !== "object")
|
|
97
|
+
throw new Error(`${repository}@${tag} release response is not an object`);
|
|
95
98
|
const found = release.assets?.find((entry) => entry.name === asset);
|
|
96
99
|
if (!found)
|
|
97
100
|
throw new Error(`${repository}@${tag} publishes no ${asset}`);
|
|
98
|
-
|
|
101
|
+
if (typeof found.digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(found.digest))
|
|
102
|
+
throw new Error(`${repository}@${tag} ${asset} requires a sha256: digest with 64 lowercase hex digits from GitHub`);
|
|
99
103
|
return {
|
|
100
104
|
repository,
|
|
101
105
|
tag,
|
|
102
106
|
asset,
|
|
103
107
|
url: found.browser_download_url,
|
|
104
108
|
size: found.size,
|
|
105
|
-
|
|
109
|
+
digest: found.digest,
|
|
106
110
|
updatedAt: found.updated_at,
|
|
107
|
-
snapshot: forkSnapshotFrom(manifest),
|
|
108
111
|
};
|
|
109
112
|
}
|
|
110
|
-
async function manifestFromStream(url, fetchImpl, signal) {
|
|
111
|
-
const stop = new AbortController();
|
|
112
|
-
const response = await fetchImpl(url, {
|
|
113
|
-
headers: { range: `bytes=0-${MANIFEST_SEARCH_BYTES - 1}`, "user-agent": "product-lab" },
|
|
114
|
-
signal: signal ? AbortSignal.any([signal, stop.signal]) : stop.signal,
|
|
115
|
-
});
|
|
116
|
-
if (!response.ok || !response.body)
|
|
117
|
-
throw new Error(`the bundle download answered ${response.status}`);
|
|
118
|
-
const gunzip = createGunzip({ finishFlush: zlib.Z_SYNC_FLUSH });
|
|
119
|
-
const scanner = new TarManifestScanner();
|
|
120
|
-
let found;
|
|
121
|
-
let failure;
|
|
122
|
-
gunzip.on("data", (chunk) => {
|
|
123
|
-
if (found !== undefined || failure !== undefined)
|
|
124
|
-
return;
|
|
125
|
-
try {
|
|
126
|
-
found = scanner.push(chunk);
|
|
127
|
-
}
|
|
128
|
-
catch (error) {
|
|
129
|
-
failure = error;
|
|
130
|
-
}
|
|
131
|
-
});
|
|
132
|
-
const reader = response.body.getReader();
|
|
133
|
-
try {
|
|
134
|
-
while (found === undefined && failure === undefined) {
|
|
135
|
-
const { done, value } = await reader.read();
|
|
136
|
-
if (value)
|
|
137
|
-
await new Promise((resolve, reject) => gunzip.write(value, (error) => (error ? reject(error) : resolve())));
|
|
138
|
-
await new Promise((resolve) => gunzip.flush(zlib.Z_SYNC_FLUSH, () => resolve()));
|
|
139
|
-
if (done)
|
|
140
|
-
break;
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
finally {
|
|
144
|
-
stop.abort();
|
|
145
|
-
gunzip.destroy();
|
|
146
|
-
await reader.cancel().catch(() => undefined);
|
|
147
|
-
}
|
|
148
|
-
if (failure !== undefined)
|
|
149
|
-
throw failure;
|
|
150
|
-
if (found === undefined)
|
|
151
|
-
throw new Error(`no manifest.json in the bundle's first ${MANIFEST_SEARCH_BYTES / 1024 / 1024} MB`);
|
|
152
|
-
return found;
|
|
153
|
-
}
|
|
154
113
|
export function bundleLocations(workspace, network) {
|
|
155
114
|
const current = join(workspace, `fork-bundle-${network}`);
|
|
156
115
|
return { current, next: `${current}.next`, partial: `${current}.next.partial` };
|
|
@@ -174,6 +133,17 @@ export function currentBundle(files, workspace, network) {
|
|
|
174
133
|
const directory = bundleLocations(workspace, network).current;
|
|
175
134
|
return usableBundle(files, directory) ? forkSnapshotFrom(files.readJson(forkBundleManifestPath(workspace, network)), forkBundleManifestPath(workspace, network)) : undefined;
|
|
176
135
|
}
|
|
136
|
+
export function bundleAssetDigest(files, directory) {
|
|
137
|
+
try {
|
|
138
|
+
const marker = files.readJson(join(directory, BUNDLE_MARKER));
|
|
139
|
+
return typeof marker?.assetDigest === "string" && /^sha256:[0-9a-f]{64}$/.test(marker.assetDigest) ? marker.assetDigest : undefined;
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
if (error instanceof SyntaxError)
|
|
143
|
+
return undefined;
|
|
144
|
+
throw error;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
177
147
|
export function bittenAfter(a, b) {
|
|
178
148
|
if (!a || !b)
|
|
179
149
|
return false;
|
|
@@ -200,14 +170,29 @@ export function startPrefetch(input) {
|
|
|
200
170
|
env: { ...input.environment.all(), PPN_HOME: input.engine.workspace, PPN_NETWORK: input.network, PATH: path },
|
|
201
171
|
});
|
|
202
172
|
}
|
|
203
|
-
export async function finishPrefetch(files, workspace, network, signal) {
|
|
204
|
-
const { partial } = bundleLocations(workspace, network);
|
|
173
|
+
export async function finishPrefetch(files, workspace, network, signal, assetDigest) {
|
|
174
|
+
const { partial, next, current } = bundleLocations(workspace, network);
|
|
205
175
|
const check = await verifyForkBundle(partial, signal);
|
|
206
176
|
if (!check.complete) {
|
|
207
177
|
files.remove(partial);
|
|
208
178
|
throw new Error(`the download in ${partial} is not a complete bundle (${check.problem}); it was removed`);
|
|
209
179
|
}
|
|
210
|
-
markForkBundle(partial, check.snapshots, "download");
|
|
180
|
+
markForkBundle(partial, check.snapshots, "download", assetDigest);
|
|
181
|
+
if (assetDigest) {
|
|
182
|
+
const downloaded = files.readJson(join(partial, BUNDLE_MARKER));
|
|
183
|
+
for (const directory of [next, current]) {
|
|
184
|
+
if (!usableBundle(files, directory))
|
|
185
|
+
continue;
|
|
186
|
+
if (bundleAssetDigest(files, directory) === assetDigest)
|
|
187
|
+
continue;
|
|
188
|
+
const manifestSha256 = createHash("sha256").update(readFileSync(join(directory, "manifest.json"))).digest("hex");
|
|
189
|
+
if (manifestSha256 !== downloaded.manifestSha256)
|
|
190
|
+
continue;
|
|
191
|
+
const local = await verifyForkBundle(directory, signal);
|
|
192
|
+
if (local.complete)
|
|
193
|
+
markForkBundle(directory, local.snapshots, "verified in place", assetDigest);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
211
196
|
return completePrefetch(files, workspace, network);
|
|
212
197
|
}
|
|
213
198
|
export function completePrefetch(files, workspace, network) {
|
|
@@ -215,6 +200,14 @@ export function completePrefetch(files, workspace, network) {
|
|
|
215
200
|
if (!verifiedBundle(files, partial))
|
|
216
201
|
throw new Error(`the download in ${partial} is not a verified bundle (no manifest.json, no snapshots or not verified complete)`);
|
|
217
202
|
const snapshot = forkSnapshotFrom(files.readJson(join(partial, "manifest.json")));
|
|
203
|
+
const staged = stagedBundle(files, workspace, network);
|
|
204
|
+
const installed = currentBundle(files, workspace, network);
|
|
205
|
+
const newest = [staged, installed].filter((entry) => !!entry)
|
|
206
|
+
.sort((a, b) => Date.parse(b.bittenAt) - Date.parse(a.bittenAt))[0];
|
|
207
|
+
if (newest && !bittenAfter(snapshot, newest)) {
|
|
208
|
+
files.remove(partial);
|
|
209
|
+
return newest;
|
|
210
|
+
}
|
|
218
211
|
files.remove(next);
|
|
219
212
|
if (!files.rename(partial, next))
|
|
220
213
|
throw new Error(`the completed bundle disappeared from ${partial}`);
|
|
@@ -38,8 +38,9 @@ export { defaultEngineRepository, defaultStateDirectory, enginePatch, EngineNotP
|
|
|
38
38
|
export { ENGINE_PATCHES, engineBuildIssue, type EnginePatch } from "./setup/engine-build.js";
|
|
39
39
|
export { CHECKOUT_ENGINE_SETUP, engineSetupCommand } from "./setup/setup-command.js";
|
|
40
40
|
export { engineBuildWords, savedForkEngine, savedForkEngineState, savedForkSetupCommand, savedForkTooling } from "./saved-fork/saved-fork-tooling.js";
|
|
41
|
-
export {
|
|
42
|
-
export {
|
|
41
|
+
export { engineCheckoutTooling, engineFetchCommand, preparedEngineCheckout, type EngineCheckoutSettings } from "./setup/engine-tooling.js";
|
|
42
|
+
export { ENGINE_PATCHES_REPOSITORY_ONLY, EnginePatchesNotInstalled, patchesDirectory, patchesInstalled, readPatchManifest, resolvePatch, shippedEnginePatch, shippedPatchesDirectory, type PatchEntry, } from "./setup/patches.js";
|
|
43
|
+
export { DESCRIPTOR_OVERLAYS, LIGHT_TOPOLOGY_OVERLAY, NODE_WORKSPACE_PATCH, OWNED_PROCESSES_PATCH, SHIPPED_ENGINE_PATCHES } from "./setup/patch-catalogue.js";
|
|
43
44
|
export { engineLaunchPlan, type EngineCommand, type EngineLaunchPlan, } from "./topology/engine-plan.js";
|
|
44
45
|
export { DEFAULT_ENGINE_DASHBOARD, enginePortOverrides, engineRun, type EngineIsolationInput, type EngineRun, type EngineRunIsolation, } from "./topology/engine-run.js";
|
|
45
46
|
export { ENGINE_TIME_UNSUPPORTED, describeEngine, engineEnvironment, type EngineEnvironmentSettings, } from "./topology/engine-provider.js";
|
|
@@ -40,8 +40,9 @@ export { defaultEngineRepository, defaultStateDirectory, enginePatch, EngineNotP
|
|
|
40
40
|
export { ENGINE_PATCHES, engineBuildIssue } from "./setup/engine-build.js";
|
|
41
41
|
export { CHECKOUT_ENGINE_SETUP, engineSetupCommand } from "./setup/setup-command.js";
|
|
42
42
|
export { engineBuildWords, savedForkEngine, savedForkEngineState, savedForkSetupCommand, savedForkTooling } from "./saved-fork/saved-fork-tooling.js";
|
|
43
|
-
export {
|
|
44
|
-
export {
|
|
43
|
+
export { engineCheckoutTooling, engineFetchCommand, preparedEngineCheckout } from "./setup/engine-tooling.js";
|
|
44
|
+
export { ENGINE_PATCHES_REPOSITORY_ONLY, EnginePatchesNotInstalled, patchesDirectory, patchesInstalled, readPatchManifest, resolvePatch, shippedEnginePatch, shippedPatchesDirectory, } from "./setup/patches.js";
|
|
45
|
+
export { DESCRIPTOR_OVERLAYS, LIGHT_TOPOLOGY_OVERLAY, NODE_WORKSPACE_PATCH, OWNED_PROCESSES_PATCH, SHIPPED_ENGINE_PATCHES } from "./setup/patch-catalogue.js";
|
|
45
46
|
export { engineLaunchPlan, } from "./topology/engine-plan.js";
|
|
46
47
|
export { DEFAULT_ENGINE_DASHBOARD, enginePortOverrides, engineRun, } from "./topology/engine-run.js";
|
|
47
48
|
export { ENGINE_TIME_UNSUPPORTED, describeEngine, engineEnvironment, } from "./topology/engine-provider.js";
|
|
@@ -9,7 +9,7 @@ export declare const savedForkEngineState: (cacheDir: string) => string;
|
|
|
9
9
|
export declare const savedForkSetupCommand: (fork: SavedFork, settings: Pick<SavedForkSettings, "setupCommand">) => string;
|
|
10
10
|
/**
|
|
11
11
|
* The prepared Engine a saved fork starts from, or `EngineNotPrepared` naming the command that prepares it. A start
|
|
12
|
-
* takes the recorded stock build (`as-prepared`), including an explicitly selected ref. Retired
|
|
12
|
+
* takes the recorded stock or node-workspace build (`as-prepared`), including an explicitly selected ref. Retired owned-processes records
|
|
13
13
|
* are refused. `ref`: when supplied, also check that commit.
|
|
14
14
|
*/
|
|
15
15
|
export declare function savedForkEngine(fork: SavedFork, settings: EngineSettings, commands: SyncCommandRunner, patch?: PreparedEnginePatch, ref?: string): PreparedEngine;
|
|
@@ -13,9 +13,12 @@ export function engineBuildWords(engine) {
|
|
|
13
13
|
const ref = build.resolvedRef.slice(0, 12);
|
|
14
14
|
const from = build.repository && build.repository !== defaultEngineRepository ? ` from ${build.repository}` : "";
|
|
15
15
|
const tested = build.resolvedRef === testedEngineRef ? "" : ` (not the tested ${testedEngineRef.slice(0, 12)}: \`product-lab fetch\` without --engine-ref prepares it)`;
|
|
16
|
-
|
|
16
|
+
const patch = recordedPatch(build);
|
|
17
|
+
return patch === "none"
|
|
17
18
|
? `stock Engine ${ref}${from}${tested}, ${engine.checkout}`
|
|
18
|
-
:
|
|
19
|
+
: patch === "node-workspace"
|
|
20
|
+
? `Engine ${ref}${from} with node-workspace patch, ${engine.checkout}`
|
|
21
|
+
: `retired patched Engine ${ref}${from}; prepare stock Engine again, ${engine.checkout}`;
|
|
19
22
|
}
|
|
20
23
|
function syncCommands(system) {
|
|
21
24
|
if (!system.syncCommands)
|