@wildshard/engine 0.1.18 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/budgets/shard-size-baseline.json +10 -4
- package/index.html +6 -4
- package/lint/engine-sides.json +14 -0
- package/lint/ratchet.json +0 -68
- package/lint/ratchet.mjs +1 -4
- package/package.json +17 -15
- package/scripts/agent-sync.mjs +27 -8
- package/scripts/app-build/appCook.ts +8 -2
- package/scripts/app-build/catalogue.ts +10 -7
- package/scripts/app-build/gen.ts +22 -2
- package/scripts/app-build/isolation.d.mts +10 -0
- package/scripts/app-build/isolation.mjs +47 -0
- package/scripts/app-build/outputProducts.mjs +4 -0
- package/scripts/app-build/previousDeploy.d.mts +2 -0
- package/scripts/app-build/previousDeploy.mjs +10 -1
- package/scripts/app-build/serviceWorker.ts +10 -3
- package/scripts/app-build/splitOutput.mjs +4 -2
- package/scripts/app-build/standaloneOutput.d.mts +1 -1
- package/scripts/app-build/standaloneOutput.mjs +5 -3
- package/scripts/app-build/titlePreloads.d.mts +8 -0
- package/scripts/app-build/titlePreloads.mjs +69 -0
- package/scripts/app-build/worldMapAtlas.ts +10 -6
- package/scripts/app-build-pipeline.mjs +31 -3
- package/scripts/built-product.mjs +1 -1
- package/scripts/cell-mem-bench.mjs +81 -23
- package/scripts/check-custom-sections-zero.mjs +1 -1
- package/scripts/check-devserver.mjs +2 -0
- package/scripts/check-models.mjs +2 -0
- package/scripts/check-paths.allow.json +0 -115
- package/scripts/check-paths.mjs +2 -2
- package/scripts/check-platform-ratchets.mjs +2 -3
- package/scripts/chromium-keychain.d.mts +1 -1
- package/scripts/ci-static.sh +1 -1
- package/scripts/committed-fast-checks.mjs +0 -2
- package/scripts/content-sw-proof.mjs +2 -1
- package/scripts/cook-worker.mjs +9 -3
- package/scripts/cook.mjs +18 -5
- package/scripts/docs-inputs-proof.mjs +6 -6
- package/scripts/e7-voice-staging-proof.mjs +1 -1
- package/scripts/gate-related-tests.mjs +1 -1
- package/scripts/git-source-view.mjs +1 -1
- package/scripts/goldens/capture.mjs +17 -5
- package/scripts/grid-crossing-nightly.mjs +59 -0
- package/scripts/grid-crossing-qa.mjs +44 -0
- package/scripts/grid-crossing-ratchet.d.mts +6 -0
- package/scripts/grid-crossing-ratchet.mjs +52 -0
- package/scripts/grid-crossing-report.mjs +122 -0
- package/scripts/grid-crossing-stages.mjs +47 -0
- package/scripts/grid-crossing-tour.mjs +277 -0
- package/scripts/grid-crossing-walk-proof.mjs +124 -17
- package/scripts/host-ruler/roots.mjs +2 -2
- package/scripts/host-ruler.d.mts +91 -0
- package/scripts/host-ruler.mjs +565 -0
- package/scripts/live-prepush.py +3 -5
- package/scripts/loading-flow/benchmax.mjs +3 -1
- package/scripts/look-loop/test_look_loop.py +1 -38
- package/scripts/memory-proof.mjs +61 -6
- package/scripts/mmo-bots.mjs +8 -2
- package/scripts/net-lag-proof.mjs +219 -0
- package/scripts/night-market-kit-budget.mjs +2 -2
- package/scripts/offline-proof-rules.d.mts +2 -2
- package/scripts/offline-proof-rules.mjs +15 -4
- package/scripts/offline-proof.mjs +74 -13
- package/scripts/{shard-platform-output.mjs → output-reader.mjs} +1 -1
- package/scripts/page-release/content.d.mts +9 -1
- package/scripts/page-release/content.mjs +131 -17
- package/scripts/page-release/flow.mjs +36 -10
- package/scripts/page-release/stamp-content.mjs +24 -0
- package/scripts/page-release.mjs +28 -6
- package/scripts/page-size.mjs +2 -1
- package/scripts/pin-packs.ts +1 -1
- package/scripts/pins.ts +4 -4
- package/scripts/precommit-checks.mjs +1 -7
- package/scripts/precommit-jobs.mjs +0 -1
- package/scripts/precommit-source.py +1 -9
- package/scripts/program-discovery.mjs +1 -1
- package/scripts/ring-cook.mjs +2 -1
- package/scripts/ring-step.ts +1 -1
- package/scripts/script-scenario-metrics.mjs +1 -1
- package/scripts/shard-output-content.d.mts +1 -1
- package/scripts/shard-output-content.mjs +2 -2
- package/scripts/shard-outputs.mjs +1 -1
- package/scripts/shard-public.mjs +1 -1
- package/scripts/shard-wire-meter.mjs +24 -9
- package/scripts/side-parity.mjs +4 -2
- package/scripts/standalone-smoke.mjs +8 -3
- package/scripts/tier-textures.mjs +22 -4
- package/scripts/tile-bake-report.mjs +1 -1
- package/scripts/tile-bakes.mjs +1 -1
- package/scripts/title-assets.generated.json +2 -0
- package/scripts/tsconfig.json +0 -2
- package/scripts/vendor/basisu/LICENSE +201 -0
- package/scripts/vendor/basisu/basis_encoder.cjs +2 -0
- package/scripts/vendor/basisu/basis_encoder.wasm +0 -0
- package/scripts/vitest-push-guards.config.ts +1 -1
- package/scripts/vps/harden-apply.sh +4 -1
- package/scripts/vps/harden-prometheus.sh +176 -0
- package/scripts/vps/harden.sh +16 -0
- package/scripts/vps/rebuild.sh +18 -0
- package/scripts/vps/wildshard-config +6 -2
- package/scripts/worker-client-play.mjs +6 -2
- package/scripts/worktree-reap.mjs +5 -5
- package/src/.oxlintrc.json +3 -182
- package/src/client/app/entry.ts +10 -2
- package/src/client/engine/boot/bootCache.ts +2 -2
- package/src/client/engine/boot/bootTrace.ts +12 -4
- package/src/client/engine/boot/browserChunkRecovery.ts +4 -0
- package/src/client/engine/boot/clearDownloads.ts +5 -2
- package/src/client/engine/boot/contentRead.ts +11 -0
- package/src/client/engine/boot/gpuFiles.ts +4 -2
- package/src/client/engine/boot/lastEnd.ts +99 -12
- package/src/client/engine/boot/texturePack.ts +2 -2
- package/src/client/engine/net/netPlayer.ts +19 -2
- package/src/client/engine/net/otherPlayers.ts +55 -8
- package/src/client/engine/net/predictor.ts +33 -9
- package/src/client/engine/net/redial.ts +29 -6
- package/src/client/engine/pwa/cacheNames.ts +58 -1
- package/src/client/engine/pwa/sw.js +41 -11
- package/src/client/engine/render/dataLook.ts +152 -40
- package/src/client/engine/render/families/detail.ts +89 -0
- package/src/client/engine/render/families/pbr.ts +7 -1
- package/src/client/engine/render/look.ts +6 -0
- package/src/client/engine/render/lookLight.ts +103 -11
- package/src/client/engine/render/lookRender.ts +105 -0
- package/src/client/engine/saves/indexed.ts +2 -2
- package/src/client/engine/telemetry/browserErrors.ts +24 -3
- package/src/client/engine/telemetry/staleBuildReport.ts +22 -1
- package/src/client/engine/ui/debugOptions.ts +1 -1
- package/src/client/engine/ui/lookLightOption.ts +10 -6
- package/src/client/engine/ui/namePrompt.ts +7 -0
- package/src/client/game/boot/audio.generated.ts +114 -114
- package/src/client/game/boot/bytes.generated.ts +118 -220
- package/src/client/game/boot/placedSpawn.ts +35 -0
- package/src/client/game/boot/spawnDownload.ts +25 -6
- package/src/client/game/boot/versions.generated.ts +7 -5
- package/src/client/game/devScene/hook.ts +291 -0
- package/src/client/game/grid/cellLoading.ts +12 -2
- package/src/client/game/grid/live.ts +112 -16
- package/src/client/game/grid/liveSession.ts +33 -2
- package/src/client/game/grid/mapAtlas.ts +18 -4
- package/src/client/game/grid/minimapBlend.ts +15 -5
- package/src/client/game/grid/oneWayIn.ts +5 -1
- package/src/client/game/grid/placedCells.ts +114 -12
- package/src/client/game/grid/placedContent.ts +85 -0
- package/src/client/game/grid/prefetch.ts +9 -1
- package/src/client/game/grid/productInputs.ts +3 -1
- package/src/client/game/grid/productLeases.ts +4 -1
- package/src/client/game/grid/products.ts +8 -4
- package/src/client/game/grid/roadLook.ts +24 -15
- package/src/client/game/grid/roadLookPlatform.ts +4 -2
- package/src/client/game/grid/roadResources.ts +24 -0
- package/src/client/game/grid/roadSign.worker.ts +4 -3
- package/src/client/game/grid/roadSignPaint.ts +16 -5
- package/src/client/game/grid/roadSignPreparation.ts +4 -3
- package/src/client/game/grid/roadSignScale.ts +8 -0
- package/src/client/game/grid/roadTileCapture.worker.ts +3 -1
- package/src/client/game/grid/roadTileCaptureWorker.ts +2 -1
- package/src/client/game/grid/roadTileWrite.worker.ts +4 -2
- package/src/client/game/grid/roadTileWriter.ts +2 -1
- package/src/client/game/grid/screenReports.ts +39 -0
- package/src/client/game/grid/session.ts +20 -4
- package/src/client/game/grid/softWallLook.ts +25 -8
- package/src/client/game/mainMenu.ts +6 -2
- package/src/client/game/plugin/context.ts +2 -1
- package/src/client/game/plugin/followedMovers.ts +7 -1
- package/src/client/game/quest/figures.ts +2 -0
- package/src/client/game/session/finish.ts +6 -0
- package/src/client/game/shardfile/assetCache.ts +4 -4
- package/src/client/game/shardfile/client.ts +4 -1
- package/src/client/game/shardfile/clientAssets.ts +9 -5
- package/src/client/game/shardfile/clientLibrary.ts +1 -1
- package/src/client/game/shardfile/clientWater.ts +51 -9
- package/src/client/game/shardfile/clientWorker.ts +19 -7
- package/src/client/game/shardfile/look.ts +8 -3
- package/src/client/game/shardfile/product.ts +34 -7
- package/src/client/game/shardfile/productChecks.ts +2 -2
- package/src/client/game/shardfile/productCost.ts +2 -2
- package/src/client/game/shardfile/readAsset.ts +2 -2
- package/src/client/game/shardfile/reflectingWater.ts +73 -0
- package/src/client/game/shardfile/seatGrace.ts +9 -0
- package/src/client/game/shardfile/skinPlayback.ts +1 -1
- package/src/client/game/systems/npc/figureRows.ts +9 -0
- package/src/client/host/online/doctor.ts +2 -0
- package/src/client/host/online/linkScreens.ts +30 -4
- package/src/client/host/standalone/shell.ts +4 -0
- package/src/cook/game/astc6Encode.ts +55 -0
- package/src/cook/game/astc6Wasm.ts +176 -0
- package/src/cook/game/cookJob.ts +30 -6
- package/src/cook/game/cookWorker.ts +38 -11
- package/src/cook/game/entryTotals.ts +51 -24
- package/src/cook/game/shardCook.ts +35 -7
- package/src/cook/game/spawnParts.ts +33 -8
- package/src/cook/game/tierTextures.ts +87 -14
- package/src/server/engine/ai/bossContenders.ts +27 -3
- package/src/server/engine/ai/phases.ts +1 -1
- package/src/server/engine/entities/creatureRefill.ts +9 -2
- package/src/server/engine/net/ledgerBook.ts +2 -0
- package/src/server/engine/net/room.ts +15 -0
- package/src/server/engine/sim/respawn.ts +8 -0
- package/src/server/game/boss/arenaRoomEntry.ts +5 -1
- package/src/server/game/boss/arenaRoomNative.ts +2 -0
- package/src/server/game/boss/native.ts +2 -1
- package/src/server/game/boss/nativeWake.ts +9 -5
- package/src/server/game/boss/openArenaNative.ts +2 -2
- package/src/server/game/boss/openArenaRoom.ts +3 -1
- package/src/server/game/grid/room.ts +44 -3
- package/src/server/game/grid/simulation.ts +23 -0
- package/src/server/game/population/mixedRoster.ts +11 -3
- package/src/server/game/population/nativeBake.ts +8 -3
- package/src/server/game/population/nativeReturns.ts +6 -3
- package/src/server/game/population/roster.ts +5 -3
- package/src/server/game/quest/eliteDrops.ts +15 -4
- package/src/server/game/quest/nativeChapter.ts +1 -1
- package/src/server/game/quest/nativeComposition.ts +5 -1
- package/src/server/game/riding/nativeTaming.ts +10 -6
- package/src/server/game/riding/wildRefill.ts +3 -2
- package/src/server/game/shardfile/bossRow.ts +4 -2
- package/src/server/game/shardfile/homeKeeper.ts +4 -2
- package/src/server/host/bake/headlessWorker.ts +1 -1
- package/src/server/host/dedicated/cellHost.ts +98 -16
- package/src/server/host/dedicated/cellProtocol.ts +16 -1
- package/src/server/host/dedicated/cellWorker.ts +8 -5
- package/src/server/host/dedicated/dedicatedRoom.ts +16 -6
- package/src/server/host/dedicated/gridCells.ts +15 -6
- package/src/server/host/dedicated/loopLag.ts +21 -0
- package/src/server/host/dedicated/main.ts +13 -5
- package/src/server/host/dedicated/packReader.ts +3 -3
- package/src/server/host/dedicated/placementFile.ts +3 -2
- package/src/server/host/dedicated/roadRoom.ts +34 -4
- package/src/server/host/dedicated/serverMetrics.ts +126 -0
- package/src/server/host/dedicated/socketPump.ts +21 -4
- package/src/server/host/dedicated/workerTable.ts +15 -4
- package/src/server/host/dedicated/world.ts +65 -7
- package/src/server/host/headless.ts +1 -1
- package/src/server/host/headlessRuntime.ts +1 -1
- package/src/server/host/standalone/roadWorld.ts +15 -3
- package/src/server/host/standalone/serverHost.ts +4 -1
- package/src/server/host/standalone/serverProduct.ts +41 -11
- package/src/shared/engine/ai/BossBrain.ts +8 -0
- package/src/shared/game/boss/wake.ts +6 -0
- package/src/shared/game/devColliders.ts +47 -0
- package/src/shared/game/eliteSystem.ts +8 -0
- package/src/shared/game/grid/allocator.ts +3 -0
- package/src/shared/game/grid/placementRows.ts +5 -2
- package/src/shared/game/plugin/bossRecord.ts +3 -2
- package/src/shared/game/shardfile/ktx2Targets.ts +23 -3
- package/src/shared/game/shardfile/outputDescriptor.ts +9 -5
- package/src/shared/game/shardfile/packReads.ts +28 -3
- package/src/shared/game/shardfile/serverPack.ts +9 -0
- package/src/shared/game/systems/looks/trampleField.ts +2 -0
- package/src/shared/host/formatJsonSchema.ts +2 -2
- package/src/tsconfig.json +1 -27
- package/lint/host-reviews.json +0 -14
- package/lint/listed-view-code.json +0 -13
- package/lint/platform-pending-users.json +0 -2520
- package/lint/platform-pending-users.md +0 -18
- package/scripts/cell-mem/metrics-preload.mjs +0 -65
- package/scripts/check-closed-components.d.mts +0 -7
- package/scripts/check-closed-components.mjs +0 -87
- package/scripts/g571-hosts.d.mts +0 -3
- package/scripts/g571-hosts.mjs +0 -54
- package/scripts/host-ruler/register-migrate.mjs +0 -242
- package/scripts/listed-view-code.d.mts +0 -7
- package/scripts/listed-view-code.mjs +0 -93
- package/scripts/look-loop/fit-lut.py +0 -142
- package/scripts/look-loop/palette-delta.py +0 -183
- package/scripts/look-loop/regions/driftwood-isle.json +0 -25
- package/scripts/look-loop/regions/far-reach.json +0 -225
- package/scripts/look-loop/regions/open-plot-night-market.json +0 -855
- package/scripts/look-loop/regions/pine-hollow.json +0 -22
- package/scripts/look-loop/regions/sunscar-dunes.json +0 -281
- package/scripts/platform-chargebacks.d.mts +0 -9
- package/scripts/platform-chargebacks.mjs +0 -135
- package/scripts/shard-component-spans.d.mts +0 -15
- package/scripts/shard-component-spans.mjs +0 -18
- package/scripts/shard-components.d.mts +0 -16
- package/scripts/shard-components.mjs +0 -363
- package/scripts/shard-platform-hosts.d.mts +0 -2
- package/scripts/shard-platform-hosts.mjs +0 -93
- package/scripts/shard-platform-sections.d.mts +0 -2
- package/scripts/shard-platform-sections.mjs +0 -9
- package/scripts/shard-platform.d.mts +0 -76
- package/scripts/shard-platform.mjs +0 -1072
- package/src/.oxfmtrc.json +0 -48
- package/src/scripts/ask-new.sh +0 -29
- package/src/scripts/asks.mjs +0 -142
- package/src/scripts/herdr-prompt.sh +0 -34
- package/src/scripts/herdr-question-open.sh +0 -67
- package/src/scripts/md-wrap.mjs +0 -229
- /package/scripts/{shard-platform-format.d.mts → node-format.d.mts} +0 -0
- /package/scripts/{shard-platform-format.mjs → node-format.mjs} +0 -0
- /package/scripts/{shard-platform-output.d.mts → output-reader.d.mts} +0 -0
|
@@ -169,7 +169,7 @@ export function linkScreens(scope: Scope, hooks: LinkScreenHooks, parent: HTMLEl
|
|
|
169
169
|
parent.append(banner, deploy);
|
|
170
170
|
|
|
171
171
|
let state: LinkState = 'live', reason: string | undefined, wasDown = false, quietUntil = 0, deployLeft = 0, deployTicking = false;
|
|
172
|
-
let open: { scope: Scope; root: HTMLElement; kind: 'hold' | 'stop'; paint: () => void; what?: string; chat?: boolean; key?: string } | null = null;
|
|
172
|
+
let open: { scope: Scope; root: HTMLElement; kind: 'hold' | 'stop' | 'offline'; paint: () => void; what?: string; chat?: boolean; key?: string } | null = null;
|
|
173
173
|
const held: { text: string; send: (text: string) => void; drop: () => void }[] = [];
|
|
174
174
|
const down = (): boolean => state === 'reconnecting' || state === 'resuming';
|
|
175
175
|
|
|
@@ -180,10 +180,11 @@ export function linkScreens(scope: Scope, hooks: LinkScreenHooks, parent: HTMLEl
|
|
|
180
180
|
// a card takes the pointer back from the game (desktop pointer lock), so its buttons can be clicked
|
|
181
181
|
const freePointer = (): void => { if ('exitPointerLock' in document && document.pointerLockElement instanceof Element) document.exitPointerLock(); };
|
|
182
182
|
const paintBanner = (): void => {
|
|
183
|
-
|
|
183
|
+
// offline, the card is the screen (FM4b mode 2): no banner under it
|
|
184
|
+
const shown = down() && hooks.why() !== 'offline';
|
|
184
185
|
banner.hidden = !shown;
|
|
185
186
|
// a link that is down takes the top band from the deploy countdown; the countdown comes back with the link
|
|
186
|
-
deploy.hidden =
|
|
187
|
+
deploy.hidden = down() || deployLeft <= 0;
|
|
187
188
|
if (!shown) return;
|
|
188
189
|
const said = bannerFor(state, hooks.why(), hooks.progress().dials);
|
|
189
190
|
banner.dataset['failmode'] = said.ids; banner.dataset['tone'] = said.tone; banner.dataset['state'] = state;
|
|
@@ -201,7 +202,8 @@ export function linkScreens(scope: Scope, hooks: LinkScreenHooks, parent: HTMLEl
|
|
|
201
202
|
};
|
|
202
203
|
/** The hold card: the banner opened up, on a tap it can't carry. It repaints each second from the dial's progress. */
|
|
203
204
|
const hold = (what: string, chat: boolean): void => {
|
|
204
|
-
|
|
205
|
+
// the offline card already says why nothing goes out; a held chat line still waits for the link
|
|
206
|
+
if (open?.kind === 'stop' || open?.kind === 'offline') return;
|
|
205
207
|
remember(); open?.scope.dispose(); open = null;
|
|
206
208
|
const own = scope.child('online.hold'), said = bannerFor(state, hooks.why(), hooks.progress().dials);
|
|
207
209
|
const title = chat ? 'Your message will send when you\'re back' : state === 'resuming' ? 'Still reconnecting' : hooks.why() === 'restart' ? 'The world is restarting' : 'Not connected yet';
|
|
@@ -248,6 +250,28 @@ export function linkScreens(scope: Scope, hooks: LinkScreenHooks, parent: HTMLEl
|
|
|
248
250
|
root.querySelector('button')?.focus();
|
|
249
251
|
};
|
|
250
252
|
const doctor = (from: string): { label: string; busy: string; act: () => void }[] => hooks.doctor === undefined ? [] : [{ label: 'Check connection', busy: 'Opening…', act: () => { hooks.doctor?.(from); } }];
|
|
253
|
+
/**
|
|
254
|
+
* FM4b mode 2 (`offline-play`): the device says it has no network. Play waits under the card, the dial waits for the network
|
|
255
|
+
* (Redial's `offline`), and the card closes by itself when the network is back (the banner takes over) or the link is.
|
|
256
|
+
* J229: no seat-hold or save time is claimed, only what this page measured.
|
|
257
|
+
*/
|
|
258
|
+
const offlineCard = (): void => {
|
|
259
|
+
if (open?.kind === 'offline') return;
|
|
260
|
+
remember(); open?.scope.dispose(); open = null;
|
|
261
|
+
const own = scope.child('online.offline'), { root, box } = card(own, 'offline-play', 'warn', 'Connection dropped', 'You\'re offline');
|
|
262
|
+
box.append(make('p', 'ws-link-body', 'This device lost its internet connection. The world needs it to carry on.'));
|
|
263
|
+
const status = make('ul', 'ws-link-facts'), waiting = make('li');
|
|
264
|
+
status.setAttribute('role', 'status'); status.append(waiting);
|
|
265
|
+
const paint = (): void => { waiting.replaceChildren(ring(), `Down for ${String(Math.floor(hooks.progress().downMs / 1000))} s`); };
|
|
266
|
+
paint(); own.interval(1000, paint);
|
|
267
|
+
box.append(status, make('p', 'ws-link-auto', 'We\'ll connect again by ourselves as soon as the network is back. Nothing is sent while you\'re offline.'));
|
|
268
|
+
const actions = doctor('offline-play');
|
|
269
|
+
if (actions.length > 0) box.append(buttons(own, actions));
|
|
270
|
+
const details = make('details'); details.append(make('summary', '', 'Details'), make('p', '', 'navigator.onLine is false · dialling waits for "online"')); box.append(details);
|
|
271
|
+
parent.append(root); freePointer();
|
|
272
|
+
open = { scope: own, root, kind: 'offline', paint };
|
|
273
|
+
root.querySelector('button')?.focus();
|
|
274
|
+
};
|
|
251
275
|
|
|
252
276
|
const link = (next: LinkState, why?: string): void => {
|
|
253
277
|
state = next; reason = why;
|
|
@@ -256,6 +280,8 @@ export function linkScreens(scope: Scope, hooks: LinkScreenHooks, parent: HTMLEl
|
|
|
256
280
|
if (down()) {
|
|
257
281
|
wasDown = true; unnotice();
|
|
258
282
|
if (open?.kind === 'stop') close();
|
|
283
|
+
if (hooks.why() === 'offline') { offlineCard(); return; }
|
|
284
|
+
if (open?.kind === 'offline') close();
|
|
259
285
|
// a hold card follows its banner: a new cause (silent to restart) or 35 s down redraws it, else it just ticks
|
|
260
286
|
if (open?.kind === 'hold' && open.key !== `${state}:${hooks.why()}` && open.what !== undefined) hold(open.what, open.chat === true); else open?.paint();
|
|
261
287
|
return;
|
|
@@ -12,6 +12,7 @@ import type { SaveStorage } from '@wildshard/engine/saves/store';
|
|
|
12
12
|
import { ServerMessageSchema, ShellMessageSchema, type ProbedRow, type ServerMessage, type ShellMessage } from '@wildshard/format/standaloneControl';
|
|
13
13
|
import { answerHeldReads, type HeldRead } from '../../../shared/host/standalone/heldReads';
|
|
14
14
|
import type { HarnessFlag, HarnessGod, HarnessHits, HarnessJoin, HarnessMove, HarnessPlace } from '../../../shared/host/standalone/harnessPlace';
|
|
15
|
+
import type { DevBox, DevCollidersMessage } from '@wildshard/engine/game/devColliders';
|
|
15
16
|
import type { ServerWorkerStart } from '../../../server/host/standalone/serverRuntime';
|
|
16
17
|
import { WorkerLoopback } from '../../../shared/host/standalone/workerLoopback';
|
|
17
18
|
|
|
@@ -66,6 +67,8 @@ export interface StandaloneShell {
|
|
|
66
67
|
raise: (flag: string) => void;
|
|
67
68
|
/** A proof's second client, a real one on its own link: it stands at this point in this cell, and `move` puts its body elsewhere. */
|
|
68
69
|
join: (id: string, x: number, y: number, z: number) => { readonly client: NetClient; readonly move: (x: number, y: number, z: number) => void };
|
|
70
|
+
/** SHARD-SOURCE AP5: a watched part's solid boxes for the server's world (DEVSERVER pages only; elsewhere it does nothing). */
|
|
71
|
+
devColliders: (name: string, boxes: readonly DevBox[]) => void;
|
|
69
72
|
dispose: () => void;
|
|
70
73
|
}
|
|
71
74
|
|
|
@@ -136,6 +139,7 @@ export function connectStandalone(config: StandaloneConfig, control: MessagePort
|
|
|
136
139
|
hardHits: by => { harness?.postMessage({ hits: by } satisfies HarnessHits, []); },
|
|
137
140
|
bossGod: () => { harness?.postMessage({ god: true } satisfies HarnessGod, []); },
|
|
138
141
|
raise: flag => { harness?.postMessage({ flag } satisfies HarnessFlag, []); },
|
|
142
|
+
devColliders: (name, boxes) => { if (typeof __DEVSERVER__ === 'boolean' && __DEVSERVER__) harness?.postMessage({ wildshardDevColliders: name, boxes } satisfies DevCollidersMessage, []); },
|
|
139
143
|
join: (id, x, y, z) => {
|
|
140
144
|
const line = new MessageChannel(), second = new NetClient(new WorkerLoopback(line.port1));
|
|
141
145
|
harness?.postMessage({ join: id, x, y, z } satisfies HarnessJoin, [line.port2]);
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** S12c: the cook's ASTC 6×6 encoder (`Astc6Encode`), S4's hand-made phone sets done by the cook. S4 baked them with
|
|
2
|
+
* `basisu -ldr_6x6 -effort 4` from source images (sharddev `scripts/bake-astc6.mjs`); a cook has no source images, only
|
|
3
|
+
* the shard's UASTC masters, so it decodes the master (already trimmed to its tier size, every level the master's own)
|
|
4
|
+
* to RGBA with basisu's own transcoder (`-export_dds RGBA32`) and encodes those levels as 6×6 with S4's flags. That
|
|
5
|
+
* re-encodes lossy data, so goldens judge it (G998).
|
|
6
|
+
*
|
|
7
|
+
* Cached by input hash (C4): the key is the input bytes, the role, the flags and basisu's version, and a hit reads the
|
|
8
|
+
* file back without running basisu. `stats` counts both, so a warm recook can prove it encoded nothing. */
|
|
9
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
10
|
+
import {execFileSync} from 'node:child_process';
|
|
11
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
12
|
+
import {createHash} from 'node:crypto';
|
|
13
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
14
|
+
import {existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync} from 'node:fs';
|
|
15
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
16
|
+
import {join} from 'node:path';
|
|
17
|
+
import type {Astc6Encode, TextureRole} from './tierTextures';
|
|
18
|
+
|
|
19
|
+
/** S4's 6×6 classes (`bake-astc6.mjs` CLASS), effort 4 of 10. */
|
|
20
|
+
const ROLE_FLAGS: Readonly<Record<TextureRole, readonly string[]>> = {srgb: ['-srgb'], linear: ['-linear'], normal: ['-normal_map']};
|
|
21
|
+
const ENCODE_FLAGS = ['-ktx2', '-ldr_6x6', '-effort', '4'];
|
|
22
|
+
|
|
23
|
+
export interface Astc6Cache {
|
|
24
|
+
readonly encode: Astc6Encode;
|
|
25
|
+
readonly stats: {encoded: number; reused: number; failed: number};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A basisu-backed `Astc6Encode` caching its files in `cacheDir`. Throws at once when basisu isn't on the PATH. */
|
|
29
|
+
export function basisuAstc6(cacheDir: string, basisu = 'basisu'): Astc6Cache {
|
|
30
|
+
const version = /v[\d.]+/u.exec(execFileSync(basisu, ['-version'], {encoding: 'utf8'}))?.[0] ?? '?';
|
|
31
|
+
mkdirSync(cacheDir, {recursive: true});
|
|
32
|
+
const stats = {encoded: 0, reused: 0, failed: 0};
|
|
33
|
+
const encode: Astc6Encode = (ktx2, role) => {
|
|
34
|
+
const key = createHash('sha256').update(`astc6/1|${version}|${ENCODE_FLAGS.join(' ')}|${ROLE_FLAGS[role].join(' ')}|`).update(ktx2).digest('hex');
|
|
35
|
+
const cached = join(cacheDir, `${key}.ktx2`);
|
|
36
|
+
if (existsSync(cached)) { stats.reused++; return new Uint8Array(readFileSync(cached)); }
|
|
37
|
+
const work = mkdtempSync(join(cacheDir, 'work-'));
|
|
38
|
+
try {
|
|
39
|
+
const input = join(work, 'in.ktx2'), levels = join(work, 'in.dds'), output = join(work, 'out.ktx2');
|
|
40
|
+
writeFileSync(input, ktx2);
|
|
41
|
+
execFileSync(basisu, ['-export_dds', 'RGBA32', input, '-output_file', levels], {stdio: 'ignore'});
|
|
42
|
+
execFileSync(basisu, [...ENCODE_FLAGS, ...ROLE_FLAGS[role], levels, '-output_file', output], {stdio: 'ignore'});
|
|
43
|
+
const bytes = new Uint8Array(readFileSync(output));
|
|
44
|
+
renameSync(output, cached);
|
|
45
|
+
stats.encoded++;
|
|
46
|
+
return bytes;
|
|
47
|
+
} catch {
|
|
48
|
+
stats.failed++;
|
|
49
|
+
return undefined;
|
|
50
|
+
} finally {
|
|
51
|
+
rmSync(work, {recursive: true, force: true});
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
return {encode, stats};
|
|
55
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/** S12d: the cook's ASTC 6×6 encoder with no child process: basisu's own WebAssembly build (`scripts/vendor/basisu/`, Binomial's
|
|
2
|
+
* `webgl/encoder/build` at 32c7b74, Apache-2.0), so the box's cook job (`--permission`, no child processes, no basisu
|
|
3
|
+
* installed) and CI's bake encode what `cook --phone-astc6` encoded with the CLI (S12c). Same steps and flags: the
|
|
4
|
+
* trimmed master's top level decoded to RGBA, encoded as ASTC LDR 6×6 at effort 4 with the role's preset (`-srgb`,
|
|
5
|
+
* `-linear`, `-normal_map`), zstd in a KTX2. The CLI took the master's own lower levels; this build regenerates them from
|
|
6
|
+
* the top one. Probed on Pine's `wood_planks_dirt` at 1024: top level 66–71 dB from the CLI's, the same PSNR against the
|
|
7
|
+
* master, sizes within 0.2% (colour, normal) and 5% (ARM).
|
|
8
|
+
*
|
|
9
|
+
* `encode` is synchronous, as `phoneTier` calls it. `record` and `flush` let a cook run the same encodes on worker
|
|
10
|
+
* threads first: a recording pass collects every input, `flush` encodes them `lanes` at a time, and the real pass then
|
|
11
|
+
* reads them back. A cache folder, when given, keeps encodes across cooks by input hash (C4); the box's job passes none,
|
|
12
|
+
* because a cook job's files can't be trusted by the next one. */
|
|
13
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
14
|
+
import {createHash} from 'node:crypto';
|
|
15
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
16
|
+
import {existsSync, mkdirSync, readFileSync, renameSync, utimesSync, writeFileSync} from 'node:fs';
|
|
17
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
18
|
+
import {createRequire} from 'node:module';
|
|
19
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
20
|
+
import {availableParallelism} from 'node:os';
|
|
21
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
22
|
+
import {join} from 'node:path';
|
|
23
|
+
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
24
|
+
import {Worker} from 'node:worker_threads';
|
|
25
|
+
import type {Astc6Encode, TextureRole} from './tierTextures';
|
|
26
|
+
|
|
27
|
+
/** The vendored build; a new build is a new cache key. */
|
|
28
|
+
export const BASISU_WASM_BUILD = '32c7b74';
|
|
29
|
+
const KEY_PREFIX = `astc6-wasm/1|${BASISU_WASM_BUILD}|ASTC_LDR_6x6 effort 4|`;
|
|
30
|
+
const MAX_LANES = 6;
|
|
31
|
+
|
|
32
|
+
interface Ktx2File {
|
|
33
|
+
readonly isValid: () => boolean; readonly startTranscoding: () => boolean; readonly getLevels: () => number; readonly close: () => void; readonly delete: () => void;
|
|
34
|
+
readonly getImageLevelInfo: (level: number, layer: number, face: number) => {origWidth: number; origHeight: number};
|
|
35
|
+
readonly getImageTranscodedSizeInBytes: (level: number, layer: number, face: number, format: number) => number;
|
|
36
|
+
readonly transcodeImage: (out: Uint8Array, level: number, layer: number, face: number, format: number, alpha: number, channel0: number, channel1: number) => number;
|
|
37
|
+
}
|
|
38
|
+
interface BasisEncoder {
|
|
39
|
+
readonly setCreateKTX2File: (on: boolean) => void; readonly setKTX2UASTCSupercompression: (on: boolean) => void; readonly setDebug: (on: boolean) => void; readonly setStatusOutput: (on: boolean) => void;
|
|
40
|
+
readonly setSliceSourceImage: (slice: number, rgba: Uint8Array, width: number, height: number, type: number) => boolean;
|
|
41
|
+
readonly setFormatMode: (format: number) => void; readonly setFormatModeAndQualityEffort: (format: number, quality: number, effort: number, defaults: boolean) => void;
|
|
42
|
+
readonly setSRGBPreset: () => void; readonly setLinearPreset: () => void; readonly setNormalMapPreset: () => void; readonly setMipGen: (on: boolean) => void; readonly setMipSRGB: (on: boolean) => void;
|
|
43
|
+
readonly encode: (out: Uint8Array) => number; readonly delete: () => void;
|
|
44
|
+
}
|
|
45
|
+
interface BasisModule {
|
|
46
|
+
readonly initializeBasis: () => void;
|
|
47
|
+
KTX2File: new (bytes: Uint8Array) => Ktx2File;
|
|
48
|
+
BasisEncoder: new () => BasisEncoder;
|
|
49
|
+
transcoder_texture_format: {cTFRGBA32: {value: number}};
|
|
50
|
+
basis_tex_format: {cASTC_LDR_6x6: {value: number}};
|
|
51
|
+
ldr_image_type: {cRGBA32: {value: number}};
|
|
52
|
+
}
|
|
53
|
+
type BasisFactory = (options: {wasmBinary: Uint8Array; print: () => void; printErr: () => void}) => Promise<BasisModule>;
|
|
54
|
+
|
|
55
|
+
const PRESET = {srgb: 'setSRGBPreset', linear: 'setLinearPreset', normal: 'setNormalMapPreset'} as const satisfies Record<TextureRole, keyof BasisEncoder>;
|
|
56
|
+
|
|
57
|
+
/** One encoder on this thread: the master's top level to RGBA, then ASTC 6×6. Undefined when basisu refuses the file. */
|
|
58
|
+
export async function loadBasisuAstc6(): Promise<Astc6Encode> {
|
|
59
|
+
const require = createRequire(import.meta.url);
|
|
60
|
+
const factory = require('../../../scripts/vendor/basisu/basis_encoder.cjs') as BasisFactory;
|
|
61
|
+
const wasm = new Uint8Array(readFileSync(new URL('../../../scripts/vendor/basisu/basis_encoder.wasm', import.meta.url)));
|
|
62
|
+
const basis = await factory({wasmBinary: wasm, print: () => undefined, printErr: () => undefined});
|
|
63
|
+
basis.initializeBasis();
|
|
64
|
+
const rgba32 = basis.transcoder_texture_format.cTFRGBA32.value, astc6 = basis.basis_tex_format.cASTC_LDR_6x6.value;
|
|
65
|
+
return (ktx2, role) => {
|
|
66
|
+
const file = new basis.KTX2File(ktx2);
|
|
67
|
+
let rgba: Uint8Array, width: number, height: number, levels: number;
|
|
68
|
+
try {
|
|
69
|
+
if (!file.isValid() || !file.startTranscoding()) return undefined;
|
|
70
|
+
({origWidth: width, origHeight: height} = file.getImageLevelInfo(0, 0, 0));
|
|
71
|
+
levels = file.getLevels();
|
|
72
|
+
rgba = new Uint8Array(file.getImageTranscodedSizeInBytes(0, 0, 0, rgba32));
|
|
73
|
+
if (file.transcodeImage(rgba, 0, 0, 0, rgba32, 0, -1, -1) === 0) return undefined;
|
|
74
|
+
} finally { file.close(); file.delete(); }
|
|
75
|
+
const encoder = new basis.BasisEncoder();
|
|
76
|
+
try {
|
|
77
|
+
encoder.setDebug(false); encoder.setStatusOutput(false);
|
|
78
|
+
encoder.setCreateKTX2File(true); encoder.setKTX2UASTCSupercompression(true);
|
|
79
|
+
if (!encoder.setSliceSourceImage(0, rgba, width, height, basis.ldr_image_type.cRGBA32.value)) return undefined;
|
|
80
|
+
// The CLI's order: the format flag, the role's preset, then the effort over both.
|
|
81
|
+
encoder.setFormatMode(astc6);
|
|
82
|
+
encoder[PRESET[role]]();
|
|
83
|
+
encoder.setFormatModeAndQualityEffort(astc6, -1, 4, false);
|
|
84
|
+
encoder.setMipGen(levels > 1); encoder.setMipSRGB(role === 'srgb');
|
|
85
|
+
const out = new Uint8Array(width * height * 4 + (1 << 20)), length = encoder.encode(out);
|
|
86
|
+
return length > 0 ? out.slice(0, length) : undefined;
|
|
87
|
+
} finally { encoder.delete(); }
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A worker thread's loop around `loadBasisuAstc6`; workers inherit the cook's `--import` loader. */
|
|
92
|
+
const WORKER = `const {parentPort, workerData} = process.getBuiltinModule('node:worker_threads');
|
|
93
|
+
import(workerData.module).then(async module => {
|
|
94
|
+
const encode = await module.loadBasisuAstc6();
|
|
95
|
+
parentPort.on('message', ({id, bytes, role}) => { let out; try { out = encode(bytes, role); } catch { out = undefined; } parentPort.postMessage({id, out}); });
|
|
96
|
+
parentPort.postMessage({ready: true});
|
|
97
|
+
});`;
|
|
98
|
+
|
|
99
|
+
export interface WasmAstc6 {
|
|
100
|
+
readonly encode: Astc6Encode;
|
|
101
|
+
/** Collects an input for `flush` and answers as an encode would, so a recording pass follows the real pass's path. */
|
|
102
|
+
readonly record: Astc6Encode;
|
|
103
|
+
readonly flush: () => Promise<void>;
|
|
104
|
+
readonly stats: {encoded: number; reused: number; failed: number};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The cook's 6×6 encoder: `lanes` worker threads for `flush`, this thread for anything `flush` didn't see. */
|
|
108
|
+
export async function wasmAstc6(options: {readonly cacheDir?: string; readonly lanes?: number} = {}): Promise<WasmAstc6> {
|
|
109
|
+
// Each lane holds its own WebAssembly heap; six keep a cook well inside the job's resident limit.
|
|
110
|
+
const {cacheDir} = options, lanes = Math.max(1, Math.min(MAX_LANES, options.lanes ?? availableParallelism() - 2));
|
|
111
|
+
if (cacheDir !== undefined) mkdirSync(cacheDir, {recursive: true});
|
|
112
|
+
const stats = {encoded: 0, reused: 0, failed: 0};
|
|
113
|
+
const keyOf = (ktx2: Uint8Array, role: TextureRole): string => createHash('sha256').update(`${KEY_PREFIX}${role}|`).update(ktx2).digest('hex');
|
|
114
|
+
const done = new Map<string, Uint8Array | null>(), pending = new Map<string, {bytes: Uint8Array; role: TextureRole}>();
|
|
115
|
+
const stored = (key: string): Uint8Array | undefined => {
|
|
116
|
+
if (cacheDir === undefined) return undefined;
|
|
117
|
+
const path = join(cacheDir, `${key}.ktx2`);
|
|
118
|
+
if (!existsSync(path)) return undefined;
|
|
119
|
+
// A hit marks the file in use, so CI's cache keeps only what its builds read (ci.yml prunes by age).
|
|
120
|
+
try {
|
|
121
|
+
const now = new Date();
|
|
122
|
+
utimesSync(path, now, now);
|
|
123
|
+
} catch {
|
|
124
|
+
// A read-only cache still serves.
|
|
125
|
+
}
|
|
126
|
+
return new Uint8Array(readFileSync(path));
|
|
127
|
+
};
|
|
128
|
+
const keep = (key: string, out: Uint8Array | undefined): void => {
|
|
129
|
+
done.set(key, out ?? null);
|
|
130
|
+
if (out === undefined) { stats.failed++; return; }
|
|
131
|
+
stats.encoded++;
|
|
132
|
+
if (cacheDir !== undefined) { const path = join(cacheDir, `${key}.ktx2`), temp = `${path}.${process.pid}.tmp`; writeFileSync(temp, out); renameSync(temp, path); }
|
|
133
|
+
};
|
|
134
|
+
const here = await loadBasisuAstc6();
|
|
135
|
+
const encode: Astc6Encode = (ktx2, role) => {
|
|
136
|
+
const key = keyOf(ktx2, role), kept = done.get(key);
|
|
137
|
+
if (kept !== undefined) return kept ?? undefined;
|
|
138
|
+
const hit = stored(key);
|
|
139
|
+
if (hit !== undefined) { stats.reused++; done.set(key, hit); return hit; }
|
|
140
|
+
let out: Uint8Array | undefined;
|
|
141
|
+
try { out = here(ktx2, role); } catch { out = undefined; }
|
|
142
|
+
keep(key, out);
|
|
143
|
+
return out;
|
|
144
|
+
};
|
|
145
|
+
const record: Astc6Encode = (ktx2, role) => {
|
|
146
|
+
const key = keyOf(ktx2, role);
|
|
147
|
+
if (!done.has(key) && stored(key) === undefined) pending.set(key, {bytes: new Uint8Array(ktx2), role});
|
|
148
|
+
return ktx2;
|
|
149
|
+
};
|
|
150
|
+
const flush = async (): Promise<void> => {
|
|
151
|
+
const jobs = [...pending]; pending.clear();
|
|
152
|
+
if (jobs.length === 0) return;
|
|
153
|
+
const count = Math.min(lanes, jobs.length);
|
|
154
|
+
const module = new URL('astc6Wasm.ts', import.meta.url).href;
|
|
155
|
+
const workers = Array.from({length: count}, () => new Worker(WORKER, {eval: true, workerData: {module}}));
|
|
156
|
+
try {
|
|
157
|
+
await Promise.all(workers.map(worker => new Promise<void>((resolve, reject) => {
|
|
158
|
+
let id = 0;
|
|
159
|
+
const next = (): void => {
|
|
160
|
+
const job = jobs.shift();
|
|
161
|
+
if (job === undefined) { resolve(); return; }
|
|
162
|
+
id++;
|
|
163
|
+
// oxlint-disable-next-line unicorn/require-post-message-target-origin -- a worker_threads port has no origin.
|
|
164
|
+
worker.postMessage({id, bytes: job[1].bytes, role: job[1].role});
|
|
165
|
+
// A lane's refusal isn't kept: the real pass encodes that input again on this thread, and counts a failure there.
|
|
166
|
+
worker.once('message', ({out}: {out?: Uint8Array}) => { if (out !== undefined) keep(job[0], new Uint8Array(out)); next(); });
|
|
167
|
+
};
|
|
168
|
+
worker.once('message', () => { next(); });
|
|
169
|
+
worker.once('error', reject);
|
|
170
|
+
// A lane that dies leaves its inputs to this thread.
|
|
171
|
+
worker.once('exit', () => { resolve(); });
|
|
172
|
+
})));
|
|
173
|
+
} finally { await Promise.all(workers.map(worker => worker.terminate())); }
|
|
174
|
+
};
|
|
175
|
+
return {encode, record, flush, stats};
|
|
176
|
+
}
|
package/src/cook/game/cookJob.ts
CHANGED
|
@@ -11,8 +11,8 @@ import {createHash} from 'node:crypto';
|
|
|
11
11
|
import {mkdirSync, writeFileSync} from 'node:fs';
|
|
12
12
|
// oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
|
|
13
13
|
import {dirname, join} from 'node:path';
|
|
14
|
-
import {prepareAssetDecoders} from '@wildshard/format/
|
|
15
|
-
import {
|
|
14
|
+
import {prepareAssetDecoders} from '@wildshard/format/shardfileAssets';
|
|
15
|
+
import {validateShardfileFolder, type ValidatedShardfile as ValidatedProduct} from '@wildshard/format/shardfileValidation';
|
|
16
16
|
import type {Shardfile} from '@wildshard/format/shardSchema';
|
|
17
17
|
import {readTar} from '@wildshard/format/tar';
|
|
18
18
|
import {cookShard, openShardfile, serverRows, type CookReport, type OpenShardfile, type WireBytes} from './shardCook';
|
|
@@ -48,20 +48,43 @@ export interface CookJobReport {
|
|
|
48
48
|
readonly header: string;
|
|
49
49
|
readonly server: ServerPackReport;
|
|
50
50
|
readonly seconds: number;
|
|
51
|
+
/** SA48: what a page's admission row needs of these bytes, from the receipt the validator just held (`cookReceipt`). */
|
|
52
|
+
readonly receipt?: CookReceipt;
|
|
51
53
|
}
|
|
54
|
+
/** SHARD-AUTHORING SA48: the facts a bake's `productVerdict` stamps on a page, read here from a product the format's validator has
|
|
55
|
+
* just admitted, so a page whose build never saw the shard can admit it from the cook's index. The worker stamps its validator
|
|
56
|
+
* revision beside them (`CookIndex.verdict`), and a page believes them only under its own. */
|
|
57
|
+
export interface CookReceipt {readonly sourceHash: string; readonly indexHash: string; readonly residentBytes: number; readonly postCost: unknown; readonly worst?: unknown}
|
|
52
58
|
|
|
53
59
|
const sha256 = (bytes: Uint8Array): string => createHash('sha256').update(bytes).digest('hex');
|
|
54
60
|
|
|
55
|
-
function admit(tarball: Uint8Array, cap: number): ValidatedProduct {
|
|
61
|
+
function admit(tarball: Uint8Array, cap: number): {readonly product: ValidatedProduct; readonly receipt?: CookReceipt} {
|
|
56
62
|
if (tarball.length > cap) throw new ShardfileRefusedError(`Shardfile is ${tarball.length} bytes, past the ${cap} cap`);
|
|
57
63
|
try {
|
|
58
64
|
const files = readTar(tarball);
|
|
59
|
-
|
|
65
|
+
const product = validateShardfileFolder({
|
|
60
66
|
read: (path, maximum) => { const bytes = files.get(path); if (bytes === undefined || bytes.length > maximum) throw new Error(`No ${path} within ${maximum} bytes`); return bytes; },
|
|
61
67
|
paths: () => files.keys(),
|
|
62
68
|
}, {hash: sha256});
|
|
69
|
+
const receipt = cookReceipt(files);
|
|
70
|
+
return {product, ...(receipt === undefined ? {} : {receipt})};
|
|
63
71
|
} catch (error) { throw new ShardfileRefusedError(error instanceof Error ? error.message : 'Shardfile refused', {cause: error}); }
|
|
64
72
|
}
|
|
73
|
+
const object = (value: unknown): value is Readonly<Record<string, unknown>> => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
74
|
+
/** The receipt's facts, held as `buildValidationReceipts` holds a build's own: its index hash is this header's, its source hash the
|
|
75
|
+
* header's, its residency and post cost present. Anything else is no receipt: the cook still runs, and a page leaves the shard out. */
|
|
76
|
+
export function cookReceipt(files: ReadonlyMap<string, Uint8Array>): CookReceipt | undefined {
|
|
77
|
+
const receipt = files.get('validation.json'), header = files.get('shard.json');
|
|
78
|
+
if (receipt === undefined || header === undefined) return undefined;
|
|
79
|
+
try {
|
|
80
|
+
const row: unknown = JSON.parse(new TextDecoder().decode(receipt)), index: unknown = JSON.parse(new TextDecoder().decode(header));
|
|
81
|
+
const indexHash = sha256(header), build = object(index) ? index['build'] : undefined, sourceHash = object(build) ? build['sourceHash'] : undefined;
|
|
82
|
+
if (!object(row) || typeof sourceHash !== 'string' || row['sourceHash'] !== sourceHash || row['indexHash'] !== indexHash) return undefined;
|
|
83
|
+
const {residentBytes, postCost, worst} = row;
|
|
84
|
+
if (typeof residentBytes !== 'number' || !Number.isSafeInteger(residentBytes) || residentBytes <= 0 || !object(postCost)) return undefined;
|
|
85
|
+
return {sourceHash, indexHash, residentBytes, postCost, ...(worst === undefined ? {} : {worst})};
|
|
86
|
+
} catch { return undefined; }
|
|
87
|
+
}
|
|
65
88
|
|
|
66
89
|
/** A failed proof is a verdict on these bytes: another try walks the same lanes. A dead child never reaches this. */
|
|
67
90
|
async function proved(product: ValidatedProduct, ports: CookJobPorts): Promise<SimulationProof | 'unmeasured'> {
|
|
@@ -87,7 +110,7 @@ function writeServerPack(shard: OpenShardfile, out: string): ServerPackReport {
|
|
|
87
110
|
export async function runCookJob(tarball: Uint8Array, out: string, ports: CookJobPorts): Promise<CookJobReport> {
|
|
88
111
|
const started = ports.now();
|
|
89
112
|
await prepareAssetDecoders();
|
|
90
|
-
const product = admit(tarball, ports.maxBytes ?? MAX_SHARDFILE_BYTES);
|
|
113
|
+
const {product, receipt} = admit(tarball, ports.maxBytes ?? MAX_SHARDFILE_BYTES);
|
|
91
114
|
// Match the transport's generic identity allowlist, without shard behavior branches.
|
|
92
115
|
if (ports.expectedSlug !== undefined && !new Set([ports.expectedSlug]).has(product.index.identity.slug)) {
|
|
93
116
|
throw new ShardfileRefusedError(`A shardfile for ${product.index.identity.slug} under ${ports.expectedSlug}`);
|
|
@@ -99,5 +122,6 @@ export async function runCookJob(tarball: Uint8Array, out: string, ports: CookJo
|
|
|
99
122
|
// Each manifest names the header at its product URL, and no part holds it (DL18); its `?v=` is what caches it.
|
|
100
123
|
const shard = openShardfile(tarball, renderer), header = `shardfiles/${shard.slug}/shard.json`, file = join(client, header);
|
|
101
124
|
mkdirSync(dirname(file), {recursive: true}); writeFileSync(file, shard.header);
|
|
102
|
-
return {slug: product.index.identity.slug, hash: sha256(tarball), proof, cook, header, server: writeServerPack(shard, join(out, 'server')), seconds: (ports.now() - started) / 1000
|
|
125
|
+
return {slug: product.index.identity.slug, hash: sha256(tarball), proof, cook, header, server: writeServerPack(shard, join(out, 'server')), seconds: (ports.now() - started) / 1000,
|
|
126
|
+
...(receipt === undefined ? {} : {receipt})};
|
|
103
127
|
}
|
|
@@ -28,9 +28,12 @@ import {MAX_SHARDFILE_BYTES, type CookJobReport} from './cookJob';
|
|
|
28
28
|
import {stalePreparationLine} from './shardCook';
|
|
29
29
|
import type {ObjectHeaders, PackStore, StoredObject} from './packStore';
|
|
30
30
|
import {installedRuntimeReads} from './installedRuntimeReads';
|
|
31
|
-
import {readServerPackBook, serverPackBookPath} from '../../shared/game/shardfile/serverPack';
|
|
31
|
+
import {cookIndexPath, readServerPackBook, serverPackBookPath} from '../../shared/game/shardfile/serverPack';
|
|
32
32
|
|
|
33
|
-
export interface CatalogueRow {readonly slug: string; readonly revision
|
|
33
|
+
export interface CatalogueRow {readonly slug: string; readonly revision?: number; readonly hash: string}
|
|
34
|
+
/** A head the cook can be asked for. `live` is production and `preview` the preview channel (the old two stages); MS2.4:
|
|
35
|
+
* `staging` is the staging channel's pointer, which is where `upload --channel staging` puts a new author's shard. */
|
|
36
|
+
export type CookStage = 'live' | 'preview' | 'staging';
|
|
34
37
|
/** The shape `api-server` POSTs (its `ShardfileNoticeSchema`, J30). */
|
|
35
38
|
export interface CookNotice {
|
|
36
39
|
readonly event: 'shardfile.new' | 'catalogue.flip';
|
|
@@ -49,7 +52,10 @@ const NoticeSchema: v.GenericSchema<CookNotice> = v.strictObject({
|
|
|
49
52
|
interface RevisionRef {readonly revision: number; readonly hash: string}
|
|
50
53
|
const RevisionSchema: v.GenericSchema<RevisionRef> = v.object({revision: RevisionNumberSchema, hash: ShardfileHashSchema});
|
|
51
54
|
/** `GET /api/catalogue`: it answers before a grid is set, unlike the `catalogue.json` a build bakes. */
|
|
52
|
-
|
|
55
|
+
/** J275 (AC17): a channel's pointer names the raw upload (`raw`); the cook downloads and migrates that, as for `live`. The API
|
|
56
|
+
* sends every channel's block for every slug and leaves `raw` out where the channel has no pointer. */
|
|
57
|
+
const ChannelHeadSchema: v.GenericSchema<{readonly raw?: string | undefined}> = v.object({raw: v.optional(ShardfileHashSchema)});
|
|
58
|
+
const CatalogueSchema: v.GenericSchema<{readonly shards: readonly {readonly slug: string; readonly live?: RevisionRef | undefined; readonly preview?: RevisionRef | undefined; readonly channels?: {readonly staging?: {readonly raw?: string | undefined} | undefined} | undefined}[]}> = v.object({shards: v.array(v.object({slug: ShardSlugSchema, live: v.optional(RevisionSchema), preview: v.optional(RevisionSchema), channels: v.optional(v.object({staging: v.optional(ChannelHeadSchema)}))}))});
|
|
53
59
|
|
|
54
60
|
/** What a request needs from a cook: one shard of one build, by the two things packs are keyed by. */
|
|
55
61
|
export interface CookIndex {
|
|
@@ -68,17 +74,27 @@ export interface CookIndex {
|
|
|
68
74
|
readonly seconds: number;
|
|
69
75
|
/** J262 (SP1a): which author preparation the cook went by; an index from before it has none. */
|
|
70
76
|
readonly preparation?: CookJobReport['cook']['preparation'];
|
|
77
|
+
/** SA48: where this upload's loose files are again (`looseFolder`); an index from before SA48 names none. */
|
|
78
|
+
readonly loose?: string;
|
|
79
|
+
/** SA48: the page's admission row for these bytes (`productVerdict`'s, but for its path), under the validator revision the worker
|
|
80
|
+
* cooks with (`CookWorkerOptions.validator`). A page whose validator differs, or an index with none, leaves the shard out. */
|
|
81
|
+
readonly verdict?: NonNullable<CookJobReport['receipt']> & {readonly revision: string};
|
|
71
82
|
}
|
|
72
83
|
|
|
73
|
-
|
|
84
|
+
/** SA47: a box reads the same key (`serverProduct.ts`), so it lives with the server pack's paths. */
|
|
85
|
+
export const indexKey: (build: string, hash: string) => string = cookIndexPath;
|
|
74
86
|
const IMMUTABLE = 'public, max-age=31536000, immutable';
|
|
75
87
|
const CONTENT_NAMED = /(?:^|\/)[a-f0-9]{16,64}\.[a-z0-9.]+$|-[a-f0-9]{16}\.json$/u;
|
|
88
|
+
/** SA48: a cook's loose files (its header, its programs, its mounts' loose rows) again by upload, `loose/<hash>/<key>`: the slug's own
|
|
89
|
+
* keys are the last cook's, whichever upload that was, so a page that never baked the shard reads these (`CookIndex.loose`). */
|
|
90
|
+
const LOOSE = 'loose/';
|
|
91
|
+
export const looseFolder = (hash: string): string => `${LOOSE}${hash}/`;
|
|
76
92
|
|
|
77
93
|
/** J84: a part is q11 Brotli once and a browser decodes it; a store must say so. */
|
|
78
94
|
export function objectHeaders(key: string): ObjectHeaders {
|
|
79
95
|
if (key.endsWith('.pack.br')) return {type: 'application/octet-stream', encoding: 'br', cache: IMMUTABLE};
|
|
80
96
|
const type = key.endsWith('.json') ? 'application/json' : key.endsWith('.webp') ? 'image/webp' : key.endsWith('.jpg') ? 'image/jpeg' : key.endsWith('.png') ? 'image/png' : 'application/octet-stream';
|
|
81
|
-
return {type, cache: CONTENT_NAMED.test(key) ? IMMUTABLE : 'public, max-age=300'};
|
|
97
|
+
return {type, cache: CONTENT_NAMED.test(key) || key.startsWith(LOOSE) ? IMMUTABLE : 'public, max-age=300'};
|
|
82
98
|
}
|
|
83
99
|
|
|
84
100
|
export interface CookLimits {
|
|
@@ -188,7 +204,7 @@ export async function readCapped(response: Response, maximum: number): Promise<U
|
|
|
188
204
|
}
|
|
189
205
|
|
|
190
206
|
export interface ShardfileSource {
|
|
191
|
-
readonly heads: (stage:
|
|
207
|
+
readonly heads: (stage: CookStage) => Promise<readonly CatalogueRow[]>;
|
|
192
208
|
readonly download: (hash: string) => Promise<Uint8Array>;
|
|
193
209
|
}
|
|
194
210
|
|
|
@@ -198,7 +214,10 @@ export function apiSource(base: string, read: typeof fetch = fetch, maxBytes: nu
|
|
|
198
214
|
heads: async stage => {
|
|
199
215
|
const response = await read(`${base}/api/catalogue`, {signal: AbortSignal.timeout(30_000)});
|
|
200
216
|
if (!response.ok) throw new Error(`Catalogue answered ${response.status}`);
|
|
201
|
-
return v.parse(CatalogueSchema, await response.json()).shards.flatMap(row => {
|
|
217
|
+
return v.parse(CatalogueSchema, await response.json()).shards.flatMap(row => {
|
|
218
|
+
if (stage === 'staging') { const raw = row.channels?.staging?.raw; return raw === undefined ? [] : [{slug: row.slug, hash: raw}]; }
|
|
219
|
+
const head = row[stage]; return head === undefined ? [] : [{slug: row.slug, ...head}];
|
|
220
|
+
});
|
|
202
221
|
},
|
|
203
222
|
download: async hash => {
|
|
204
223
|
const response = await read(`${base}/api/shardfiles/${hash}`, {signal: AbortSignal.timeout(300_000)});
|
|
@@ -276,6 +295,9 @@ export interface CookWorkerOptions {
|
|
|
276
295
|
readonly budgetBytes?: number;
|
|
277
296
|
readonly log?: (line: string) => void;
|
|
278
297
|
readonly now?: () => number;
|
|
298
|
+
/** SA48: this engine's shardfile validator revision (`shardfileValidationRevision`), the one each job validates with; with it an
|
|
299
|
+
* index carries the page's admission row (`CookIndex.verdict`). */
|
|
300
|
+
readonly validator?: string;
|
|
279
301
|
}
|
|
280
302
|
export interface CookRow {
|
|
281
303
|
readonly slug: string; readonly hash: string; readonly state: 'cooked' | 'present' | 'refused' | 'failed'; readonly detail?: string; readonly seconds?: number; readonly peakMB?: number;
|
|
@@ -290,7 +312,7 @@ export interface CookWorker {
|
|
|
290
312
|
/** Cook one head unless this build already has it. Never throws: the row says what happened. */
|
|
291
313
|
readonly cook: (head: {readonly slug: string; readonly hash: string}) => Promise<CookRow>;
|
|
292
314
|
/** J29, and J30's list: every head of the catalogue at the stages asked for. */
|
|
293
|
-
readonly recook: (stages: readonly
|
|
315
|
+
readonly recook: (stages: readonly CookStage[]) => Promise<readonly CookRow[]>;
|
|
294
316
|
readonly enqueue: (notice: CookNotice) => void;
|
|
295
317
|
readonly idle: () => Promise<void>;
|
|
296
318
|
readonly rows: () => readonly CookRow[];
|
|
@@ -304,8 +326,11 @@ export function createCookWorker(options: CookWorkerOptions): CookWorker {
|
|
|
304
326
|
const verdicts = new Map<string, CookRow>();
|
|
305
327
|
const publish = async (work: string, report: CookJobReport, hash: string): Promise<void> => {
|
|
306
328
|
const client = join(work, 'out/client'), keys = filesUnder(client);
|
|
329
|
+
// SA48: the loose files again under this upload's own folder (`looseFolder`)
|
|
330
|
+
const folder = looseFolder(hash), loose = keys.filter(key => key.startsWith('shardfiles/') || key.startsWith('programs/'));
|
|
307
331
|
const rows = await store.list(''), held = new Map(rows.map(row => [row.key, row.modified]));
|
|
308
|
-
const fresh = keys.reduce((sum, key) => sum + (held.has(key) ? 0 : statSync(join(client, key)).size), 0)
|
|
332
|
+
const fresh = keys.reduce((sum, key) => sum + (held.has(key) ? 0 : statSync(join(client, key)).size), 0)
|
|
333
|
+
+ loose.reduce((sum, key) => sum + (held.has(folder + key) ? 0 : statSync(join(client, key)).size), 0);
|
|
309
334
|
const used = usageOf(rows, budget).bytes;
|
|
310
335
|
if (used + fresh > budget) throw new Error(`Over the pack budget: ${used} + ${fresh} of ${budget}`);
|
|
311
336
|
// CF-8: skipping a PUT keeps a part's old date, so GC could take it before this index lands. A part past half the grace
|
|
@@ -315,11 +340,13 @@ export function createCookWorker(options: CookWorkerOptions): CookWorker {
|
|
|
315
340
|
for (const key of [...keys].sort((a, b) => order(a) - order(b))) {
|
|
316
341
|
if (!CONTENT_NAMED.test(key) || !current(key)) await store.put(key, readFileSync(join(client, key)), objectHeaders(key));
|
|
317
342
|
}
|
|
343
|
+
for (const key of loose) if (!current(folder + key)) await store.put(folder + key, readFileSync(join(client, key)), {...objectHeaders(key), cache: IMMUTABLE});
|
|
318
344
|
const target = join(serverPacks, report.slug, hash), parked = `${target}.${process.pid}.tmp`;
|
|
319
345
|
rmSync(parked, {recursive: true, force: true}); mkdirSync(dirname(target), {recursive: true});
|
|
320
346
|
cpSync(join(work, 'out/server'), parked, {recursive: true}); rmSync(target, {recursive: true, force: true}); renameSync(parked, target);
|
|
321
|
-
const index: CookIndex = {version: 1, build, slug: report.slug, hash, header: report.header, manifests: report.cook.manifests, keys, server: report.server, serverPack: report.cook.server, proof: report.proof === 'unmeasured' ? 'unmeasured' : 'proved', seconds: report.seconds,
|
|
322
|
-
...(report.cook.preparation === undefined ? {} : {preparation: report.cook.preparation})
|
|
347
|
+
const index: CookIndex = {version: 1, build, slug: report.slug, hash, header: report.header, manifests: report.cook.manifests, keys: [...keys, ...loose.map(key => folder + key)], loose: folder, server: report.server, serverPack: report.cook.server, proof: report.proof === 'unmeasured' ? 'unmeasured' : 'proved', seconds: report.seconds,
|
|
348
|
+
...(report.cook.preparation === undefined ? {} : {preparation: report.cook.preparation}),
|
|
349
|
+
...(report.receipt === undefined || options.validator === undefined ? {} : {verdict: {...report.receipt, revision: options.validator}})};
|
|
323
350
|
await store.put(indexKey(build, hash), new TextEncoder().encode(JSON.stringify(index)), {type: 'application/json', cache: 'public, max-age=300'});
|
|
324
351
|
};
|
|
325
352
|
const attempt = async (head: {readonly slug: string; readonly hash: string}, tarball: Uint8Array, work: string): Promise<CookRow> => {
|