@wildshard/engine 0.1.17 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/budgets/shard-size-baseline.json +10 -4
  3. package/index.html +6 -4
  4. package/lint/engine-sides.json +21 -0
  5. package/lint/host-callback-syntax.mjs +2 -1
  6. package/lint/host-callbacks.mjs +3 -3
  7. package/lint/host-files.mjs +5 -4
  8. package/lint/no-base64.mjs +3 -1
  9. package/lint/ratchet.json +0 -68
  10. package/lint/ratchet.mjs +1 -4
  11. package/lint/shard-imports.mjs +6 -7
  12. package/lint/shard-source.d.mts +9 -0
  13. package/lint/shard-source.mjs +91 -0
  14. package/lint/shardfile-boundaries.mjs +10 -8
  15. package/lint/sim-closure.mjs +6 -3
  16. package/lint/wildshard-plugin.d.ts +4 -0
  17. package/lint/wildshard-plugin.js +22 -23
  18. package/package.json +25 -16
  19. package/scripts/agent-sync.mjs +27 -8
  20. package/scripts/app-build/appCook.ts +8 -2
  21. package/scripts/app-build/catalogue.ts +10 -7
  22. package/scripts/app-build/gen.ts +22 -2
  23. package/scripts/app-build/isolation.d.mts +10 -0
  24. package/scripts/app-build/isolation.mjs +47 -0
  25. package/scripts/app-build/outputProducts.d.mts +3 -0
  26. package/scripts/app-build/outputProducts.mjs +35 -0
  27. package/scripts/app-build/previousDeploy.d.mts +2 -0
  28. package/scripts/app-build/previousDeploy.mjs +10 -1
  29. package/scripts/app-build/serviceWorker.ts +10 -3
  30. package/scripts/app-build/splitOutput.mjs +4 -2
  31. package/scripts/app-build/standaloneOutput.d.mts +1 -1
  32. package/scripts/app-build/standaloneOutput.mjs +5 -3
  33. package/scripts/app-build/titlePreloads.d.mts +8 -0
  34. package/scripts/app-build/titlePreloads.mjs +69 -0
  35. package/scripts/app-build/worldMapAtlas.ts +10 -6
  36. package/scripts/app-build-pipeline.mjs +34 -3
  37. package/scripts/built-product.mjs +1 -1
  38. package/scripts/cell-mem-bench.mjs +81 -23
  39. package/scripts/check-custom-sections-zero.mjs +1 -1
  40. package/scripts/check-devserver.mjs +3 -1
  41. package/scripts/check-models.mjs +2 -0
  42. package/scripts/check-paths.allow.json +0 -115
  43. package/scripts/check-paths.mjs +2 -2
  44. package/scripts/check-platform-ratchets.mjs +2 -3
  45. package/scripts/chromium-keychain-check.mjs +4 -2
  46. package/scripts/chromium-keychain.d.mts +1 -1
  47. package/scripts/chromium-keychain.mjs +3 -1
  48. package/scripts/ci-static.sh +1 -1
  49. package/scripts/committed-fast-checks.mjs +0 -2
  50. package/scripts/content-sw-proof.mjs +2 -1
  51. package/scripts/cook-worker.mjs +9 -3
  52. package/scripts/cook.mjs +20 -3
  53. package/scripts/devices.d.mts +4 -0
  54. package/scripts/devices.mjs +39 -0
  55. package/scripts/devserver-client.d.mts +4 -2
  56. package/scripts/devserver-client.mjs +25 -8
  57. package/scripts/docs-inputs-proof.mjs +6 -6
  58. package/scripts/doctor-proof.mjs +1 -1
  59. package/scripts/e2e-faults.mjs +159 -0
  60. package/scripts/e7-voice-staging-proof.mjs +1 -1
  61. package/scripts/gate-related-tests.mjs +1 -1
  62. package/scripts/git-source-view.mjs +1 -1
  63. package/scripts/goldens/capture.mjs +17 -5
  64. package/scripts/grid-crossing-nightly.mjs +59 -0
  65. package/scripts/grid-crossing-qa.mjs +44 -0
  66. package/scripts/grid-crossing-ratchet.d.mts +6 -0
  67. package/scripts/grid-crossing-ratchet.mjs +52 -0
  68. package/scripts/grid-crossing-report.mjs +122 -0
  69. package/scripts/grid-crossing-stages.mjs +47 -0
  70. package/scripts/grid-crossing-tour.mjs +277 -0
  71. package/scripts/grid-crossing-walk-proof.mjs +136 -18
  72. package/scripts/host-ruler/roots.mjs +2 -2
  73. package/scripts/host-ruler.d.mts +91 -0
  74. package/scripts/host-ruler.mjs +565 -0
  75. package/scripts/live-prepush.py +3 -5
  76. package/scripts/loading-flow/benchmax.mjs +3 -1
  77. package/scripts/look-loop/test_look_loop.py +1 -38
  78. package/scripts/memory-proof.mjs +213 -0
  79. package/scripts/mmo-bots.mjs +8 -2
  80. package/scripts/net-lag-proof.mjs +219 -0
  81. package/scripts/night-market-kit-budget.mjs +2 -2
  82. package/scripts/offline-proof-rules.d.mts +5 -3
  83. package/scripts/offline-proof-rules.mjs +34 -7
  84. package/scripts/offline-proof.mjs +88 -18
  85. package/scripts/{shard-platform-output.mjs → output-reader.mjs} +1 -1
  86. package/scripts/page-release/content.d.mts +9 -1
  87. package/scripts/page-release/content.mjs +131 -17
  88. package/scripts/page-release/flow.mjs +36 -10
  89. package/scripts/page-release/stamp-content.mjs +24 -0
  90. package/scripts/page-release.mjs +28 -6
  91. package/scripts/page-size.mjs +2 -1
  92. package/scripts/pin-packs.ts +1 -1
  93. package/scripts/pins.ts +4 -4
  94. package/scripts/precommit-checks.mjs +1 -7
  95. package/scripts/precommit-jobs.mjs +0 -1
  96. package/scripts/precommit-source.py +1 -9
  97. package/scripts/program-discovery.mjs +1 -1
  98. package/scripts/ring-cook.mjs +2 -1
  99. package/scripts/ring-step.ts +1 -1
  100. package/scripts/script-scenario-metrics.mjs +1 -1
  101. package/scripts/shard-output-content.d.mts +1 -1
  102. package/scripts/shard-output-content.mjs +2 -2
  103. package/scripts/shard-outputs.mjs +1 -1
  104. package/scripts/shard-public.mjs +1 -1
  105. package/scripts/shard-wire-meter.mjs +24 -9
  106. package/scripts/side-parity.mjs +4 -2
  107. package/scripts/standalone-smoke.mjs +8 -3
  108. package/scripts/tier-textures-proof.mjs +101 -0
  109. package/scripts/tier-textures.mjs +78 -0
  110. package/scripts/tile-bake-report.mjs +1 -1
  111. package/scripts/tile-bakes.mjs +1 -1
  112. package/scripts/title-assets.generated.json +2 -0
  113. package/scripts/tsconfig.json +1 -2
  114. package/scripts/vendor/basisu/LICENSE +201 -0
  115. package/scripts/vendor/basisu/basis_encoder.cjs +2 -0
  116. package/scripts/vendor/basisu/basis_encoder.wasm +0 -0
  117. package/scripts/vitest-push-guards.config.ts +1 -1
  118. package/scripts/vps/harden-apply.sh +4 -1
  119. package/scripts/vps/harden-prometheus.sh +176 -0
  120. package/scripts/vps/harden.sh +16 -0
  121. package/scripts/vps/rebuild.sh +18 -0
  122. package/scripts/vps/wildshard-config +84 -24
  123. package/scripts/worker-client-play.mjs +7 -2
  124. package/scripts/worker-client-proof.mjs +1 -1
  125. package/scripts/worker-client-slug.mjs +44 -0
  126. package/scripts/worktree-reap.mjs +5 -5
  127. package/src/.oxlintrc.json +3 -182
  128. package/src/client/app/entry.ts +47 -12
  129. package/src/client/engine/boot/bootCache.ts +2 -2
  130. package/src/client/engine/boot/bootTrace.ts +12 -4
  131. package/src/client/engine/boot/browserChunkRecovery.ts +4 -0
  132. package/src/client/engine/boot/clearDownloads.ts +5 -2
  133. package/src/client/engine/boot/contentRead.ts +11 -0
  134. package/src/client/engine/boot/gpuFiles.ts +4 -2
  135. package/src/client/engine/boot/lastEnd.ts +99 -12
  136. package/src/client/engine/boot/parsedPages.ts +127 -58
  137. package/src/client/engine/boot/texturePack.ts +2 -2
  138. package/src/client/engine/build.d.ts +2 -0
  139. package/src/client/engine/core/Game.ts +18 -0
  140. package/src/client/engine/core/bootstrap.ts +4 -1
  141. package/src/client/engine/debug/probe.ts +18 -1
  142. package/src/client/engine/fx/LightPool.ts +41 -1
  143. package/src/client/engine/net/netPlayer.ts +19 -2
  144. package/src/client/engine/net/otherPlayers.ts +55 -8
  145. package/src/client/engine/net/predictor.ts +33 -9
  146. package/src/client/engine/net/redial.ts +29 -6
  147. package/src/client/engine/pwa/cacheNames.ts +58 -1
  148. package/src/client/engine/pwa/sw.js +53 -14
  149. package/src/client/engine/render/dataLook.ts +152 -40
  150. package/src/client/engine/render/families/detail.ts +89 -0
  151. package/src/client/engine/render/families/pbr.ts +7 -1
  152. package/src/client/engine/render/look.ts +6 -0
  153. package/src/client/engine/render/lookLight.ts +103 -11
  154. package/src/client/engine/render/lookRender.ts +105 -0
  155. package/src/client/engine/render/precompile.ts +20 -14
  156. package/src/client/engine/saves/indexed.ts +2 -2
  157. package/src/client/engine/telemetry/browserErrors.ts +28 -4
  158. package/src/client/engine/telemetry/deviceSnapshot.ts +155 -0
  159. package/src/client/engine/telemetry/staleBuildReport.ts +22 -1
  160. package/src/client/engine/ui/debugOptions.ts +1 -1
  161. package/src/client/engine/ui/lookLightOption.ts +10 -6
  162. package/src/client/engine/ui/namePrompt.ts +7 -0
  163. package/src/client/game/boot/audio.generated.ts +114 -216
  164. package/src/client/game/boot/bytes.generated.ts +118 -238
  165. package/src/client/game/boot/placedSpawn.ts +35 -0
  166. package/src/client/game/boot/spawnDownload.ts +25 -6
  167. package/src/client/game/boot/versions.generated.ts +7 -7
  168. package/src/client/game/devScene/hook.ts +291 -0
  169. package/src/client/game/grid/area.ts +38 -0
  170. package/src/client/game/grid/cellLoading.ts +12 -2
  171. package/src/client/game/grid/live.ts +134 -12
  172. package/src/client/game/grid/liveSession.ts +47 -5
  173. package/src/client/game/grid/mapAtlas.ts +18 -4
  174. package/src/client/game/grid/menu.ts +4 -0
  175. package/src/client/game/grid/minimapBlend.ts +15 -5
  176. package/src/client/game/grid/neighbourPanels.ts +10 -1
  177. package/src/client/game/grid/oneWayIn.ts +5 -1
  178. package/src/client/game/grid/placedCards.ts +55 -0
  179. package/src/client/game/grid/placedCells.ts +233 -0
  180. package/src/client/game/grid/placedContent.ts +85 -0
  181. package/src/client/game/grid/prefetch.ts +9 -1
  182. package/src/client/game/grid/productInputs.ts +3 -1
  183. package/src/client/game/grid/productLeases.ts +4 -1
  184. package/src/client/game/grid/products.ts +8 -4
  185. package/src/client/game/grid/regionalRuntime.ts +6 -0
  186. package/src/client/game/grid/regionalWorld.ts +15 -2
  187. package/src/client/game/grid/roadCull.ts +44 -14
  188. package/src/client/game/grid/roadLook.ts +24 -15
  189. package/src/client/game/grid/roadLookPlatform.ts +4 -2
  190. package/src/client/game/grid/roadResources.ts +24 -0
  191. package/src/client/game/grid/roadSign.worker.ts +4 -3
  192. package/src/client/game/grid/roadSignPaint.ts +16 -5
  193. package/src/client/game/grid/roadSignPreparation.ts +4 -3
  194. package/src/client/game/grid/roadSignScale.ts +8 -0
  195. package/src/client/game/grid/roadTileBacking.ts +13 -3
  196. package/src/client/game/grid/roadTileCapture.worker.ts +3 -1
  197. package/src/client/game/grid/roadTileCaptureWorker.ts +2 -1
  198. package/src/client/game/grid/roadTileWrite.worker.ts +4 -2
  199. package/src/client/game/grid/roadTileWriter.ts +2 -1
  200. package/src/client/game/grid/screenReports.ts +39 -0
  201. package/src/client/game/grid/session.ts +29 -4
  202. package/src/client/game/grid/shardfileFar.ts +9 -3
  203. package/src/client/game/grid/softWallLook.ts +25 -8
  204. package/src/client/game/mainMenu.ts +6 -2
  205. package/src/client/game/plugin/context.ts +2 -1
  206. package/src/client/game/plugin/followedMovers.ts +7 -1
  207. package/src/client/game/quest/figures.ts +2 -0
  208. package/src/client/game/session/finish.ts +11 -1
  209. package/src/client/game/shard/titleList.ts +6 -2
  210. package/src/client/game/shardfile/assetCache.ts +4 -4
  211. package/src/client/game/shardfile/client.ts +4 -1
  212. package/src/client/game/shardfile/clientAssets.ts +9 -5
  213. package/src/client/game/shardfile/clientLibrary.ts +1 -1
  214. package/src/client/game/shardfile/clientWater.ts +51 -9
  215. package/src/client/game/shardfile/clientWorker.ts +21 -8
  216. package/src/client/game/shardfile/look.ts +8 -3
  217. package/src/client/game/shardfile/product.ts +34 -7
  218. package/src/client/game/shardfile/productChecks.ts +2 -2
  219. package/src/client/game/shardfile/productCost.ts +2 -2
  220. package/src/client/game/shardfile/readAsset.ts +2 -2
  221. package/src/client/game/shardfile/reflectingWater.ts +73 -0
  222. package/src/client/game/shardfile/seatGrace.ts +9 -0
  223. package/src/client/game/shardfile/skinPlayback.ts +1 -1
  224. package/src/client/game/systems/npc/figureRows.ts +9 -0
  225. package/src/client/game/titleDeck.ts +4 -3
  226. package/src/client/host/online/doctor.ts +2 -0
  227. package/src/client/host/online/linkScreens.ts +30 -4
  228. package/src/client/host/online/shell.ts +14 -0
  229. package/src/client/host/standalone/shell.ts +4 -0
  230. package/src/cook/game/astc6Encode.ts +55 -0
  231. package/src/cook/game/astc6Wasm.ts +176 -0
  232. package/src/cook/game/cookJob.ts +30 -6
  233. package/src/cook/game/cookWorker.ts +38 -11
  234. package/src/cook/game/entryTotals.ts +51 -24
  235. package/src/cook/game/shardCook.ts +67 -9
  236. package/src/cook/game/spawnParts.ts +33 -8
  237. package/src/cook/game/tierTextures.ts +349 -0
  238. package/src/e2e/smoke/game.ts +19 -14
  239. package/src/e2e/smoke/page-globals.d.ts +9 -8
  240. package/src/e2e/smoke/template2.spec.ts +8 -7
  241. package/src/server/engine/ai/bossContenders.ts +27 -3
  242. package/src/server/engine/ai/phases.ts +1 -1
  243. package/src/server/engine/entities/creatureRefill.ts +9 -2
  244. package/src/server/engine/net/ledgerBook.ts +2 -0
  245. package/src/server/engine/net/room.ts +15 -0
  246. package/src/server/engine/sim/respawn.ts +8 -0
  247. package/src/server/game/boss/arenaRoomEntry.ts +5 -1
  248. package/src/server/game/boss/arenaRoomNative.ts +2 -0
  249. package/src/server/game/boss/native.ts +2 -1
  250. package/src/server/game/boss/nativeWake.ts +9 -5
  251. package/src/server/game/boss/openArenaNative.ts +2 -2
  252. package/src/server/game/boss/openArenaRoom.ts +3 -1
  253. package/src/server/game/grid/room.ts +44 -3
  254. package/src/server/game/grid/simulation.ts +23 -0
  255. package/src/server/game/population/mixedRoster.ts +11 -3
  256. package/src/server/game/population/nativeBake.ts +8 -3
  257. package/src/server/game/population/nativeReturns.ts +6 -3
  258. package/src/server/game/population/roster.ts +5 -3
  259. package/src/server/game/quest/eliteDrops.ts +15 -4
  260. package/src/server/game/quest/nativeChapter.ts +1 -1
  261. package/src/server/game/quest/nativeComposition.ts +5 -1
  262. package/src/server/game/riding/nativeTaming.ts +10 -6
  263. package/src/server/game/riding/wildRefill.ts +3 -2
  264. package/src/server/game/shardfile/bossRow.ts +4 -2
  265. package/src/server/game/shardfile/homeKeeper.ts +4 -2
  266. package/src/server/host/bake/headlessWorker.ts +1 -1
  267. package/src/server/host/dedicated/cellHost.ts +98 -16
  268. package/src/server/host/dedicated/cellProtocol.ts +16 -1
  269. package/src/server/host/dedicated/cellWorker.ts +8 -5
  270. package/src/server/host/dedicated/dedicatedRoom.ts +16 -6
  271. package/src/server/host/dedicated/gridCells.ts +15 -6
  272. package/src/server/host/dedicated/loopLag.ts +21 -0
  273. package/src/server/host/dedicated/main.ts +13 -5
  274. package/src/server/host/dedicated/packReader.ts +3 -3
  275. package/src/server/host/dedicated/placementFile.ts +4 -19
  276. package/src/server/host/dedicated/roadRoom.ts +34 -4
  277. package/src/server/host/dedicated/serverMetrics.ts +126 -0
  278. package/src/server/host/dedicated/socketPump.ts +21 -4
  279. package/src/server/host/dedicated/socketTransport.ts +2 -1
  280. package/src/server/host/dedicated/workerTable.ts +15 -4
  281. package/src/server/host/dedicated/world.ts +95 -8
  282. package/src/server/host/headless.ts +1 -1
  283. package/src/server/host/headlessRuntime.ts +1 -1
  284. package/src/server/host/standalone/roadWorld.ts +15 -3
  285. package/src/server/host/standalone/serverHost.ts +4 -1
  286. package/src/server/host/standalone/serverProduct.ts +41 -11
  287. package/src/shared/engine/ai/BossBrain.ts +8 -0
  288. package/src/shared/engine/net/deviceWord.ts +138 -0
  289. package/src/shared/engine/net/socketFrames.ts +7 -4
  290. package/src/shared/game/boss/wake.ts +6 -0
  291. package/src/shared/game/devColliders.ts +47 -0
  292. package/src/shared/game/eliteSystem.ts +8 -0
  293. package/src/shared/game/grid/allocator.ts +3 -0
  294. package/src/shared/game/grid/catalogueSlot.ts +7 -0
  295. package/src/shared/game/grid/placementRows.ts +36 -0
  296. package/src/shared/game/plugin/bossRecord.ts +3 -2
  297. package/src/shared/game/shardfile/ktx2Targets.ts +23 -3
  298. package/src/shared/game/shardfile/outputDescriptor.ts +9 -5
  299. package/src/shared/game/shardfile/packReads.ts +28 -3
  300. package/src/shared/game/shardfile/serverPack.ts +9 -0
  301. package/src/shared/game/systems/looks/trampleField.ts +2 -0
  302. package/src/shared/host/formatJsonSchema.ts +2 -2
  303. package/src/tsconfig.json +1 -27
  304. package/lint/host-reviews.json +0 -14
  305. package/lint/listed-view-code.json +0 -13
  306. package/lint/platform-pending-users.json +0 -2520
  307. package/lint/platform-pending-users.md +0 -18
  308. package/scripts/cell-mem/metrics-preload.mjs +0 -65
  309. package/scripts/check-closed-components.d.mts +0 -7
  310. package/scripts/check-closed-components.mjs +0 -87
  311. package/scripts/g571-hosts.d.mts +0 -3
  312. package/scripts/g571-hosts.mjs +0 -54
  313. package/scripts/host-ruler/register-migrate.mjs +0 -242
  314. package/scripts/listed-view-code.d.mts +0 -7
  315. package/scripts/listed-view-code.mjs +0 -93
  316. package/scripts/look-loop/fit-lut.py +0 -142
  317. package/scripts/look-loop/palette-delta.py +0 -183
  318. package/scripts/look-loop/regions/driftwood-isle.json +0 -25
  319. package/scripts/look-loop/regions/far-reach.json +0 -225
  320. package/scripts/look-loop/regions/open-plot-night-market.json +0 -855
  321. package/scripts/look-loop/regions/pine-hollow.json +0 -22
  322. package/scripts/look-loop/regions/sunscar-dunes.json +0 -281
  323. package/scripts/platform-chargebacks.d.mts +0 -9
  324. package/scripts/platform-chargebacks.mjs +0 -135
  325. package/scripts/shard-component-spans.d.mts +0 -15
  326. package/scripts/shard-component-spans.mjs +0 -18
  327. package/scripts/shard-components.d.mts +0 -16
  328. package/scripts/shard-components.mjs +0 -363
  329. package/scripts/shard-platform-hosts.d.mts +0 -2
  330. package/scripts/shard-platform-hosts.mjs +0 -93
  331. package/scripts/shard-platform-sections.d.mts +0 -2
  332. package/scripts/shard-platform-sections.mjs +0 -9
  333. package/scripts/shard-platform.d.mts +0 -76
  334. package/scripts/shard-platform.mjs +0 -1072
  335. package/src/.oxfmtrc.json +0 -48
  336. package/src/scripts/ask-new.sh +0 -29
  337. package/src/scripts/asks.mjs +0 -142
  338. package/src/scripts/herdr-prompt.sh +0 -34
  339. package/src/scripts/herdr-question-open.sh +0 -67
  340. package/src/scripts/md-wrap.mjs +0 -229
  341. /package/scripts/{shard-platform-format.d.mts → node-format.d.mts} +0 -0
  342. /package/scripts/{shard-platform-format.mjs → node-format.mjs} +0 -0
  343. /package/scripts/{shard-platform-output.d.mts → output-reader.d.mts} +0 -0
@@ -0,0 +1,55 @@
1
+ /** S12c: the cook's ASTC 6×6 encoder (`Astc6Encode`), S4's hand-made phone sets done by the cook. S4 baked them with
2
+ * `basisu -ldr_6x6 -effort 4` from source images (sharddev `scripts/bake-astc6.mjs`); a cook has no source images, only
3
+ * the shard's UASTC masters, so it decodes the master (already trimmed to its tier size, every level the master's own)
4
+ * to RGBA with basisu's own transcoder (`-export_dds RGBA32`) and encodes those levels as 6×6 with S4's flags. That
5
+ * re-encodes lossy data, so goldens judge it (G998).
6
+ *
7
+ * Cached by input hash (C4): the key is the input bytes, the role, the flags and basisu's version, and a hit reads the
8
+ * file back without running basisu. `stats` counts both, so a warm recook can prove it encoded nothing. */
9
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
10
+ import {execFileSync} from 'node:child_process';
11
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
12
+ import {createHash} from 'node:crypto';
13
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
14
+ import {existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync} from 'node:fs';
15
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
16
+ import {join} from 'node:path';
17
+ import type {Astc6Encode, TextureRole} from './tierTextures';
18
+
19
+ /** S4's 6×6 classes (`bake-astc6.mjs` CLASS), effort 4 of 10. */
20
+ const ROLE_FLAGS: Readonly<Record<TextureRole, readonly string[]>> = {srgb: ['-srgb'], linear: ['-linear'], normal: ['-normal_map']};
21
+ const ENCODE_FLAGS = ['-ktx2', '-ldr_6x6', '-effort', '4'];
22
+
23
+ export interface Astc6Cache {
24
+ readonly encode: Astc6Encode;
25
+ readonly stats: {encoded: number; reused: number; failed: number};
26
+ }
27
+
28
+ /** A basisu-backed `Astc6Encode` caching its files in `cacheDir`. Throws at once when basisu isn't on the PATH. */
29
+ export function basisuAstc6(cacheDir: string, basisu = 'basisu'): Astc6Cache {
30
+ const version = /v[\d.]+/u.exec(execFileSync(basisu, ['-version'], {encoding: 'utf8'}))?.[0] ?? '?';
31
+ mkdirSync(cacheDir, {recursive: true});
32
+ const stats = {encoded: 0, reused: 0, failed: 0};
33
+ const encode: Astc6Encode = (ktx2, role) => {
34
+ const key = createHash('sha256').update(`astc6/1|${version}|${ENCODE_FLAGS.join(' ')}|${ROLE_FLAGS[role].join(' ')}|`).update(ktx2).digest('hex');
35
+ const cached = join(cacheDir, `${key}.ktx2`);
36
+ if (existsSync(cached)) { stats.reused++; return new Uint8Array(readFileSync(cached)); }
37
+ const work = mkdtempSync(join(cacheDir, 'work-'));
38
+ try {
39
+ const input = join(work, 'in.ktx2'), levels = join(work, 'in.dds'), output = join(work, 'out.ktx2');
40
+ writeFileSync(input, ktx2);
41
+ execFileSync(basisu, ['-export_dds', 'RGBA32', input, '-output_file', levels], {stdio: 'ignore'});
42
+ execFileSync(basisu, [...ENCODE_FLAGS, ...ROLE_FLAGS[role], levels, '-output_file', output], {stdio: 'ignore'});
43
+ const bytes = new Uint8Array(readFileSync(output));
44
+ renameSync(output, cached);
45
+ stats.encoded++;
46
+ return bytes;
47
+ } catch {
48
+ stats.failed++;
49
+ return undefined;
50
+ } finally {
51
+ rmSync(work, {recursive: true, force: true});
52
+ }
53
+ };
54
+ return {encode, stats};
55
+ }
@@ -0,0 +1,176 @@
1
+ /** S12d: the cook's ASTC 6×6 encoder with no child process: basisu's own WebAssembly build (`scripts/vendor/basisu/`, Binomial's
2
+ * `webgl/encoder/build` at 32c7b74, Apache-2.0), so the box's cook job (`--permission`, no child processes, no basisu
3
+ * installed) and CI's bake encode what `cook --phone-astc6` encoded with the CLI (S12c). Same steps and flags: the
4
+ * trimmed master's top level decoded to RGBA, encoded as ASTC LDR 6×6 at effort 4 with the role's preset (`-srgb`,
5
+ * `-linear`, `-normal_map`), zstd in a KTX2. The CLI took the master's own lower levels; this build regenerates them from
6
+ * the top one. Probed on Pine's `wood_planks_dirt` at 1024: top level 66–71 dB from the CLI's, the same PSNR against the
7
+ * master, sizes within 0.2% (colour, normal) and 5% (ARM).
8
+ *
9
+ * `encode` is synchronous, as `phoneTier` calls it. `record` and `flush` let a cook run the same encodes on worker
10
+ * threads first: a recording pass collects every input, `flush` encodes them `lanes` at a time, and the real pass then
11
+ * reads them back. A cache folder, when given, keeps encodes across cooks by input hash (C4); the box's job passes none,
12
+ * because a cook job's files can't be trusted by the next one. */
13
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
14
+ import {createHash} from 'node:crypto';
15
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
16
+ import {existsSync, mkdirSync, readFileSync, renameSync, utimesSync, writeFileSync} from 'node:fs';
17
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
18
+ import {createRequire} from 'node:module';
19
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
20
+ import {availableParallelism} from 'node:os';
21
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
22
+ import {join} from 'node:path';
23
+ // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
24
+ import {Worker} from 'node:worker_threads';
25
+ import type {Astc6Encode, TextureRole} from './tierTextures';
26
+
27
+ /** The vendored build; a new build is a new cache key. */
28
+ export const BASISU_WASM_BUILD = '32c7b74';
29
+ const KEY_PREFIX = `astc6-wasm/1|${BASISU_WASM_BUILD}|ASTC_LDR_6x6 effort 4|`;
30
+ const MAX_LANES = 6;
31
+
32
+ interface Ktx2File {
33
+ readonly isValid: () => boolean; readonly startTranscoding: () => boolean; readonly getLevels: () => number; readonly close: () => void; readonly delete: () => void;
34
+ readonly getImageLevelInfo: (level: number, layer: number, face: number) => {origWidth: number; origHeight: number};
35
+ readonly getImageTranscodedSizeInBytes: (level: number, layer: number, face: number, format: number) => number;
36
+ readonly transcodeImage: (out: Uint8Array, level: number, layer: number, face: number, format: number, alpha: number, channel0: number, channel1: number) => number;
37
+ }
38
+ interface BasisEncoder {
39
+ readonly setCreateKTX2File: (on: boolean) => void; readonly setKTX2UASTCSupercompression: (on: boolean) => void; readonly setDebug: (on: boolean) => void; readonly setStatusOutput: (on: boolean) => void;
40
+ readonly setSliceSourceImage: (slice: number, rgba: Uint8Array, width: number, height: number, type: number) => boolean;
41
+ readonly setFormatMode: (format: number) => void; readonly setFormatModeAndQualityEffort: (format: number, quality: number, effort: number, defaults: boolean) => void;
42
+ readonly setSRGBPreset: () => void; readonly setLinearPreset: () => void; readonly setNormalMapPreset: () => void; readonly setMipGen: (on: boolean) => void; readonly setMipSRGB: (on: boolean) => void;
43
+ readonly encode: (out: Uint8Array) => number; readonly delete: () => void;
44
+ }
45
+ interface BasisModule {
46
+ readonly initializeBasis: () => void;
47
+ KTX2File: new (bytes: Uint8Array) => Ktx2File;
48
+ BasisEncoder: new () => BasisEncoder;
49
+ transcoder_texture_format: {cTFRGBA32: {value: number}};
50
+ basis_tex_format: {cASTC_LDR_6x6: {value: number}};
51
+ ldr_image_type: {cRGBA32: {value: number}};
52
+ }
53
+ type BasisFactory = (options: {wasmBinary: Uint8Array; print: () => void; printErr: () => void}) => Promise<BasisModule>;
54
+
55
+ const PRESET = {srgb: 'setSRGBPreset', linear: 'setLinearPreset', normal: 'setNormalMapPreset'} as const satisfies Record<TextureRole, keyof BasisEncoder>;
56
+
57
+ /** One encoder on this thread: the master's top level to RGBA, then ASTC 6×6. Undefined when basisu refuses the file. */
58
+ export async function loadBasisuAstc6(): Promise<Astc6Encode> {
59
+ const require = createRequire(import.meta.url);
60
+ const factory = require('../../../scripts/vendor/basisu/basis_encoder.cjs') as BasisFactory;
61
+ const wasm = new Uint8Array(readFileSync(new URL('../../../scripts/vendor/basisu/basis_encoder.wasm', import.meta.url)));
62
+ const basis = await factory({wasmBinary: wasm, print: () => undefined, printErr: () => undefined});
63
+ basis.initializeBasis();
64
+ const rgba32 = basis.transcoder_texture_format.cTFRGBA32.value, astc6 = basis.basis_tex_format.cASTC_LDR_6x6.value;
65
+ return (ktx2, role) => {
66
+ const file = new basis.KTX2File(ktx2);
67
+ let rgba: Uint8Array, width: number, height: number, levels: number;
68
+ try {
69
+ if (!file.isValid() || !file.startTranscoding()) return undefined;
70
+ ({origWidth: width, origHeight: height} = file.getImageLevelInfo(0, 0, 0));
71
+ levels = file.getLevels();
72
+ rgba = new Uint8Array(file.getImageTranscodedSizeInBytes(0, 0, 0, rgba32));
73
+ if (file.transcodeImage(rgba, 0, 0, 0, rgba32, 0, -1, -1) === 0) return undefined;
74
+ } finally { file.close(); file.delete(); }
75
+ const encoder = new basis.BasisEncoder();
76
+ try {
77
+ encoder.setDebug(false); encoder.setStatusOutput(false);
78
+ encoder.setCreateKTX2File(true); encoder.setKTX2UASTCSupercompression(true);
79
+ if (!encoder.setSliceSourceImage(0, rgba, width, height, basis.ldr_image_type.cRGBA32.value)) return undefined;
80
+ // The CLI's order: the format flag, the role's preset, then the effort over both.
81
+ encoder.setFormatMode(astc6);
82
+ encoder[PRESET[role]]();
83
+ encoder.setFormatModeAndQualityEffort(astc6, -1, 4, false);
84
+ encoder.setMipGen(levels > 1); encoder.setMipSRGB(role === 'srgb');
85
+ const out = new Uint8Array(width * height * 4 + (1 << 20)), length = encoder.encode(out);
86
+ return length > 0 ? out.slice(0, length) : undefined;
87
+ } finally { encoder.delete(); }
88
+ };
89
+ }
90
+
91
+ /** A worker thread's loop around `loadBasisuAstc6`; workers inherit the cook's `--import` loader. */
92
+ const WORKER = `const {parentPort, workerData} = process.getBuiltinModule('node:worker_threads');
93
+ import(workerData.module).then(async module => {
94
+ const encode = await module.loadBasisuAstc6();
95
+ parentPort.on('message', ({id, bytes, role}) => { let out; try { out = encode(bytes, role); } catch { out = undefined; } parentPort.postMessage({id, out}); });
96
+ parentPort.postMessage({ready: true});
97
+ });`;
98
+
99
+ export interface WasmAstc6 {
100
+ readonly encode: Astc6Encode;
101
+ /** Collects an input for `flush` and answers as an encode would, so a recording pass follows the real pass's path. */
102
+ readonly record: Astc6Encode;
103
+ readonly flush: () => Promise<void>;
104
+ readonly stats: {encoded: number; reused: number; failed: number};
105
+ }
106
+
107
+ /** The cook's 6×6 encoder: `lanes` worker threads for `flush`, this thread for anything `flush` didn't see. */
108
+ export async function wasmAstc6(options: {readonly cacheDir?: string; readonly lanes?: number} = {}): Promise<WasmAstc6> {
109
+ // Each lane holds its own WebAssembly heap; six keep a cook well inside the job's resident limit.
110
+ const {cacheDir} = options, lanes = Math.max(1, Math.min(MAX_LANES, options.lanes ?? availableParallelism() - 2));
111
+ if (cacheDir !== undefined) mkdirSync(cacheDir, {recursive: true});
112
+ const stats = {encoded: 0, reused: 0, failed: 0};
113
+ const keyOf = (ktx2: Uint8Array, role: TextureRole): string => createHash('sha256').update(`${KEY_PREFIX}${role}|`).update(ktx2).digest('hex');
114
+ const done = new Map<string, Uint8Array | null>(), pending = new Map<string, {bytes: Uint8Array; role: TextureRole}>();
115
+ const stored = (key: string): Uint8Array | undefined => {
116
+ if (cacheDir === undefined) return undefined;
117
+ const path = join(cacheDir, `${key}.ktx2`);
118
+ if (!existsSync(path)) return undefined;
119
+ // A hit marks the file in use, so CI's cache keeps only what its builds read (ci.yml prunes by age).
120
+ try {
121
+ const now = new Date();
122
+ utimesSync(path, now, now);
123
+ } catch {
124
+ // A read-only cache still serves.
125
+ }
126
+ return new Uint8Array(readFileSync(path));
127
+ };
128
+ const keep = (key: string, out: Uint8Array | undefined): void => {
129
+ done.set(key, out ?? null);
130
+ if (out === undefined) { stats.failed++; return; }
131
+ stats.encoded++;
132
+ if (cacheDir !== undefined) { const path = join(cacheDir, `${key}.ktx2`), temp = `${path}.${process.pid}.tmp`; writeFileSync(temp, out); renameSync(temp, path); }
133
+ };
134
+ const here = await loadBasisuAstc6();
135
+ const encode: Astc6Encode = (ktx2, role) => {
136
+ const key = keyOf(ktx2, role), kept = done.get(key);
137
+ if (kept !== undefined) return kept ?? undefined;
138
+ const hit = stored(key);
139
+ if (hit !== undefined) { stats.reused++; done.set(key, hit); return hit; }
140
+ let out: Uint8Array | undefined;
141
+ try { out = here(ktx2, role); } catch { out = undefined; }
142
+ keep(key, out);
143
+ return out;
144
+ };
145
+ const record: Astc6Encode = (ktx2, role) => {
146
+ const key = keyOf(ktx2, role);
147
+ if (!done.has(key) && stored(key) === undefined) pending.set(key, {bytes: new Uint8Array(ktx2), role});
148
+ return ktx2;
149
+ };
150
+ const flush = async (): Promise<void> => {
151
+ const jobs = [...pending]; pending.clear();
152
+ if (jobs.length === 0) return;
153
+ const count = Math.min(lanes, jobs.length);
154
+ const module = new URL('astc6Wasm.ts', import.meta.url).href;
155
+ const workers = Array.from({length: count}, () => new Worker(WORKER, {eval: true, workerData: {module}}));
156
+ try {
157
+ await Promise.all(workers.map(worker => new Promise<void>((resolve, reject) => {
158
+ let id = 0;
159
+ const next = (): void => {
160
+ const job = jobs.shift();
161
+ if (job === undefined) { resolve(); return; }
162
+ id++;
163
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- a worker_threads port has no origin.
164
+ worker.postMessage({id, bytes: job[1].bytes, role: job[1].role});
165
+ // A lane's refusal isn't kept: the real pass encodes that input again on this thread, and counts a failure there.
166
+ worker.once('message', ({out}: {out?: Uint8Array}) => { if (out !== undefined) keep(job[0], new Uint8Array(out)); next(); });
167
+ };
168
+ worker.once('message', () => { next(); });
169
+ worker.once('error', reject);
170
+ // A lane that dies leaves its inputs to this thread.
171
+ worker.once('exit', () => { resolve(); });
172
+ })));
173
+ } finally { await Promise.all(workers.map(worker => worker.terminate())); }
174
+ };
175
+ return {encode, record, flush, stats};
176
+ }
@@ -11,8 +11,8 @@ import {createHash} from 'node:crypto';
11
11
  import {mkdirSync, writeFileSync} from 'node:fs';
12
12
  // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
13
13
  import {dirname, join} from 'node:path';
14
- import {prepareAssetDecoders} from '@wildshard/format/product';
15
- import {validateShardfileProduct, type ValidatedProduct} from '@wildshard/format/productValidation';
14
+ import {prepareAssetDecoders} from '@wildshard/format/shardfileAssets';
15
+ import {validateShardfileFolder, type ValidatedShardfile as ValidatedProduct} from '@wildshard/format/shardfileValidation';
16
16
  import type {Shardfile} from '@wildshard/format/shardSchema';
17
17
  import {readTar} from '@wildshard/format/tar';
18
18
  import {cookShard, openShardfile, serverRows, type CookReport, type OpenShardfile, type WireBytes} from './shardCook';
@@ -48,20 +48,43 @@ export interface CookJobReport {
48
48
  readonly header: string;
49
49
  readonly server: ServerPackReport;
50
50
  readonly seconds: number;
51
+ /** SA48: what a page's admission row needs of these bytes, from the receipt the validator just held (`cookReceipt`). */
52
+ readonly receipt?: CookReceipt;
51
53
  }
54
+ /** SHARD-AUTHORING SA48: the facts a bake's `productVerdict` stamps on a page, read here from a product the format's validator has
55
+ * just admitted, so a page whose build never saw the shard can admit it from the cook's index. The worker stamps its validator
56
+ * revision beside them (`CookIndex.verdict`), and a page believes them only under its own. */
57
+ export interface CookReceipt {readonly sourceHash: string; readonly indexHash: string; readonly residentBytes: number; readonly postCost: unknown; readonly worst?: unknown}
52
58
 
53
59
  const sha256 = (bytes: Uint8Array): string => createHash('sha256').update(bytes).digest('hex');
54
60
 
55
- function admit(tarball: Uint8Array, cap: number): ValidatedProduct {
61
+ function admit(tarball: Uint8Array, cap: number): {readonly product: ValidatedProduct; readonly receipt?: CookReceipt} {
56
62
  if (tarball.length > cap) throw new ShardfileRefusedError(`Shardfile is ${tarball.length} bytes, past the ${cap} cap`);
57
63
  try {
58
64
  const files = readTar(tarball);
59
- return validateShardfileProduct({
65
+ const product = validateShardfileFolder({
60
66
  read: (path, maximum) => { const bytes = files.get(path); if (bytes === undefined || bytes.length > maximum) throw new Error(`No ${path} within ${maximum} bytes`); return bytes; },
61
67
  paths: () => files.keys(),
62
68
  }, {hash: sha256});
69
+ const receipt = cookReceipt(files);
70
+ return {product, ...(receipt === undefined ? {} : {receipt})};
63
71
  } catch (error) { throw new ShardfileRefusedError(error instanceof Error ? error.message : 'Shardfile refused', {cause: error}); }
64
72
  }
73
+ const object = (value: unknown): value is Readonly<Record<string, unknown>> => typeof value === 'object' && value !== null && !Array.isArray(value);
74
+ /** The receipt's facts, held as `buildValidationReceipts` holds a build's own: its index hash is this header's, its source hash the
75
+ * header's, its residency and post cost present. Anything else is no receipt: the cook still runs, and a page leaves the shard out. */
76
+ export function cookReceipt(files: ReadonlyMap<string, Uint8Array>): CookReceipt | undefined {
77
+ const receipt = files.get('validation.json'), header = files.get('shard.json');
78
+ if (receipt === undefined || header === undefined) return undefined;
79
+ try {
80
+ const row: unknown = JSON.parse(new TextDecoder().decode(receipt)), index: unknown = JSON.parse(new TextDecoder().decode(header));
81
+ const indexHash = sha256(header), build = object(index) ? index['build'] : undefined, sourceHash = object(build) ? build['sourceHash'] : undefined;
82
+ if (!object(row) || typeof sourceHash !== 'string' || row['sourceHash'] !== sourceHash || row['indexHash'] !== indexHash) return undefined;
83
+ const {residentBytes, postCost, worst} = row;
84
+ if (typeof residentBytes !== 'number' || !Number.isSafeInteger(residentBytes) || residentBytes <= 0 || !object(postCost)) return undefined;
85
+ return {sourceHash, indexHash, residentBytes, postCost, ...(worst === undefined ? {} : {worst})};
86
+ } catch { return undefined; }
87
+ }
65
88
 
66
89
  /** A failed proof is a verdict on these bytes: another try walks the same lanes. A dead child never reaches this. */
67
90
  async function proved(product: ValidatedProduct, ports: CookJobPorts): Promise<SimulationProof | 'unmeasured'> {
@@ -87,7 +110,7 @@ function writeServerPack(shard: OpenShardfile, out: string): ServerPackReport {
87
110
  export async function runCookJob(tarball: Uint8Array, out: string, ports: CookJobPorts): Promise<CookJobReport> {
88
111
  const started = ports.now();
89
112
  await prepareAssetDecoders();
90
- const product = admit(tarball, ports.maxBytes ?? MAX_SHARDFILE_BYTES);
113
+ const {product, receipt} = admit(tarball, ports.maxBytes ?? MAX_SHARDFILE_BYTES);
91
114
  // Match the transport's generic identity allowlist, without shard behavior branches.
92
115
  if (ports.expectedSlug !== undefined && !new Set([ports.expectedSlug]).has(product.index.identity.slug)) {
93
116
  throw new ShardfileRefusedError(`A shardfile for ${product.index.identity.slug} under ${ports.expectedSlug}`);
@@ -99,5 +122,6 @@ export async function runCookJob(tarball: Uint8Array, out: string, ports: CookJo
99
122
  // Each manifest names the header at its product URL, and no part holds it (DL18); its `?v=` is what caches it.
100
123
  const shard = openShardfile(tarball, renderer), header = `shardfiles/${shard.slug}/shard.json`, file = join(client, header);
101
124
  mkdirSync(dirname(file), {recursive: true}); writeFileSync(file, shard.header);
102
- return {slug: product.index.identity.slug, hash: sha256(tarball), proof, cook, header, server: writeServerPack(shard, join(out, 'server')), seconds: (ports.now() - started) / 1000};
125
+ return {slug: product.index.identity.slug, hash: sha256(tarball), proof, cook, header, server: writeServerPack(shard, join(out, 'server')), seconds: (ports.now() - started) / 1000,
126
+ ...(receipt === undefined ? {} : {receipt})};
103
127
  }
@@ -28,9 +28,12 @@ import {MAX_SHARDFILE_BYTES, type CookJobReport} from './cookJob';
28
28
  import {stalePreparationLine} from './shardCook';
29
29
  import type {ObjectHeaders, PackStore, StoredObject} from './packStore';
30
30
  import {installedRuntimeReads} from './installedRuntimeReads';
31
- import {readServerPackBook, serverPackBookPath} from '../../shared/game/shardfile/serverPack';
31
+ import {cookIndexPath, readServerPackBook, serverPackBookPath} from '../../shared/game/shardfile/serverPack';
32
32
 
33
- export interface CatalogueRow {readonly slug: string; readonly revision: number; readonly hash: string}
33
+ export interface CatalogueRow {readonly slug: string; readonly revision?: number; readonly hash: string}
34
+ /** A head the cook can be asked for. `live` is production and `preview` the preview channel (the old two stages); MS2.4:
35
+ * `staging` is the staging channel's pointer, which is where `upload --channel staging` puts a new author's shard. */
36
+ export type CookStage = 'live' | 'preview' | 'staging';
34
37
  /** The shape `api-server` POSTs (its `ShardfileNoticeSchema`, J30). */
35
38
  export interface CookNotice {
36
39
  readonly event: 'shardfile.new' | 'catalogue.flip';
@@ -49,7 +52,10 @@ const NoticeSchema: v.GenericSchema<CookNotice> = v.strictObject({
49
52
  interface RevisionRef {readonly revision: number; readonly hash: string}
50
53
  const RevisionSchema: v.GenericSchema<RevisionRef> = v.object({revision: RevisionNumberSchema, hash: ShardfileHashSchema});
51
54
  /** `GET /api/catalogue`: it answers before a grid is set, unlike the `catalogue.json` a build bakes. */
52
- const CatalogueSchema: v.GenericSchema<{readonly shards: readonly {readonly slug: string; readonly live?: RevisionRef | undefined; readonly preview?: RevisionRef | undefined}[]}> = v.object({shards: v.array(v.object({slug: ShardSlugSchema, live: v.optional(RevisionSchema), preview: v.optional(RevisionSchema)}))});
55
+ /** J275 (AC17): a channel's pointer names the raw upload (`raw`); the cook downloads and migrates that, as for `live`. The API
56
+ * sends every channel's block for every slug and leaves `raw` out where the channel has no pointer. */
57
+ const ChannelHeadSchema: v.GenericSchema<{readonly raw?: string | undefined}> = v.object({raw: v.optional(ShardfileHashSchema)});
58
+ const CatalogueSchema: v.GenericSchema<{readonly shards: readonly {readonly slug: string; readonly live?: RevisionRef | undefined; readonly preview?: RevisionRef | undefined; readonly channels?: {readonly staging?: {readonly raw?: string | undefined} | undefined} | undefined}[]}> = v.object({shards: v.array(v.object({slug: ShardSlugSchema, live: v.optional(RevisionSchema), preview: v.optional(RevisionSchema), channels: v.optional(v.object({staging: v.optional(ChannelHeadSchema)}))}))});
53
59
 
54
60
  /** What a request needs from a cook: one shard of one build, by the two things packs are keyed by. */
55
61
  export interface CookIndex {
@@ -68,17 +74,27 @@ export interface CookIndex {
68
74
  readonly seconds: number;
69
75
  /** J262 (SP1a): which author preparation the cook went by; an index from before it has none. */
70
76
  readonly preparation?: CookJobReport['cook']['preparation'];
77
+ /** SA48: where this upload's loose files are again (`looseFolder`); an index from before SA48 names none. */
78
+ readonly loose?: string;
79
+ /** SA48: the page's admission row for these bytes (`productVerdict`'s, but for its path), under the validator revision the worker
80
+ * cooks with (`CookWorkerOptions.validator`). A page whose validator differs, or an index with none, leaves the shard out. */
81
+ readonly verdict?: NonNullable<CookJobReport['receipt']> & {readonly revision: string};
71
82
  }
72
83
 
73
- export const indexKey = (build: string, hash: string): string => `index/${build}/${hash}.json`;
84
+ /** SA47: a box reads the same key (`serverProduct.ts`), so it lives with the server pack's paths. */
85
+ export const indexKey: (build: string, hash: string) => string = cookIndexPath;
74
86
  const IMMUTABLE = 'public, max-age=31536000, immutable';
75
87
  const CONTENT_NAMED = /(?:^|\/)[a-f0-9]{16,64}\.[a-z0-9.]+$|-[a-f0-9]{16}\.json$/u;
88
+ /** SA48: a cook's loose files (its header, its programs, its mounts' loose rows) again by upload, `loose/<hash>/<key>`: the slug's own
89
+ * keys are the last cook's, whichever upload that was, so a page that never baked the shard reads these (`CookIndex.loose`). */
90
+ const LOOSE = 'loose/';
91
+ export const looseFolder = (hash: string): string => `${LOOSE}${hash}/`;
76
92
 
77
93
  /** J84: a part is q11 Brotli once and a browser decodes it; a store must say so. */
78
94
  export function objectHeaders(key: string): ObjectHeaders {
79
95
  if (key.endsWith('.pack.br')) return {type: 'application/octet-stream', encoding: 'br', cache: IMMUTABLE};
80
96
  const type = key.endsWith('.json') ? 'application/json' : key.endsWith('.webp') ? 'image/webp' : key.endsWith('.jpg') ? 'image/jpeg' : key.endsWith('.png') ? 'image/png' : 'application/octet-stream';
81
- return {type, cache: CONTENT_NAMED.test(key) ? IMMUTABLE : 'public, max-age=300'};
97
+ return {type, cache: CONTENT_NAMED.test(key) || key.startsWith(LOOSE) ? IMMUTABLE : 'public, max-age=300'};
82
98
  }
83
99
 
84
100
  export interface CookLimits {
@@ -188,7 +204,7 @@ export async function readCapped(response: Response, maximum: number): Promise<U
188
204
  }
189
205
 
190
206
  export interface ShardfileSource {
191
- readonly heads: (stage: 'live' | 'preview') => Promise<readonly CatalogueRow[]>;
207
+ readonly heads: (stage: CookStage) => Promise<readonly CatalogueRow[]>;
192
208
  readonly download: (hash: string) => Promise<Uint8Array>;
193
209
  }
194
210
 
@@ -198,7 +214,10 @@ export function apiSource(base: string, read: typeof fetch = fetch, maxBytes: nu
198
214
  heads: async stage => {
199
215
  const response = await read(`${base}/api/catalogue`, {signal: AbortSignal.timeout(30_000)});
200
216
  if (!response.ok) throw new Error(`Catalogue answered ${response.status}`);
201
- return v.parse(CatalogueSchema, await response.json()).shards.flatMap(row => { const head = row[stage]; return head === undefined ? [] : [{slug: row.slug, ...head}]; });
217
+ return v.parse(CatalogueSchema, await response.json()).shards.flatMap(row => {
218
+ if (stage === 'staging') { const raw = row.channels?.staging?.raw; return raw === undefined ? [] : [{slug: row.slug, hash: raw}]; }
219
+ const head = row[stage]; return head === undefined ? [] : [{slug: row.slug, ...head}];
220
+ });
202
221
  },
203
222
  download: async hash => {
204
223
  const response = await read(`${base}/api/shardfiles/${hash}`, {signal: AbortSignal.timeout(300_000)});
@@ -276,6 +295,9 @@ export interface CookWorkerOptions {
276
295
  readonly budgetBytes?: number;
277
296
  readonly log?: (line: string) => void;
278
297
  readonly now?: () => number;
298
+ /** SA48: this engine's shardfile validator revision (`shardfileValidationRevision`), the one each job validates with; with it an
299
+ * index carries the page's admission row (`CookIndex.verdict`). */
300
+ readonly validator?: string;
279
301
  }
280
302
  export interface CookRow {
281
303
  readonly slug: string; readonly hash: string; readonly state: 'cooked' | 'present' | 'refused' | 'failed'; readonly detail?: string; readonly seconds?: number; readonly peakMB?: number;
@@ -290,7 +312,7 @@ export interface CookWorker {
290
312
  /** Cook one head unless this build already has it. Never throws: the row says what happened. */
291
313
  readonly cook: (head: {readonly slug: string; readonly hash: string}) => Promise<CookRow>;
292
314
  /** J29, and J30's list: every head of the catalogue at the stages asked for. */
293
- readonly recook: (stages: readonly ('live' | 'preview')[]) => Promise<readonly CookRow[]>;
315
+ readonly recook: (stages: readonly CookStage[]) => Promise<readonly CookRow[]>;
294
316
  readonly enqueue: (notice: CookNotice) => void;
295
317
  readonly idle: () => Promise<void>;
296
318
  readonly rows: () => readonly CookRow[];
@@ -304,8 +326,11 @@ export function createCookWorker(options: CookWorkerOptions): CookWorker {
304
326
  const verdicts = new Map<string, CookRow>();
305
327
  const publish = async (work: string, report: CookJobReport, hash: string): Promise<void> => {
306
328
  const client = join(work, 'out/client'), keys = filesUnder(client);
329
+ // SA48: the loose files again under this upload's own folder (`looseFolder`)
330
+ const folder = looseFolder(hash), loose = keys.filter(key => key.startsWith('shardfiles/') || key.startsWith('programs/'));
307
331
  const rows = await store.list(''), held = new Map(rows.map(row => [row.key, row.modified]));
308
- const fresh = keys.reduce((sum, key) => sum + (held.has(key) ? 0 : statSync(join(client, key)).size), 0);
332
+ const fresh = keys.reduce((sum, key) => sum + (held.has(key) ? 0 : statSync(join(client, key)).size), 0)
333
+ + loose.reduce((sum, key) => sum + (held.has(folder + key) ? 0 : statSync(join(client, key)).size), 0);
309
334
  const used = usageOf(rows, budget).bytes;
310
335
  if (used + fresh > budget) throw new Error(`Over the pack budget: ${used} + ${fresh} of ${budget}`);
311
336
  // CF-8: skipping a PUT keeps a part's old date, so GC could take it before this index lands. A part past half the grace
@@ -315,11 +340,13 @@ export function createCookWorker(options: CookWorkerOptions): CookWorker {
315
340
  for (const key of [...keys].sort((a, b) => order(a) - order(b))) {
316
341
  if (!CONTENT_NAMED.test(key) || !current(key)) await store.put(key, readFileSync(join(client, key)), objectHeaders(key));
317
342
  }
343
+ for (const key of loose) if (!current(folder + key)) await store.put(folder + key, readFileSync(join(client, key)), {...objectHeaders(key), cache: IMMUTABLE});
318
344
  const target = join(serverPacks, report.slug, hash), parked = `${target}.${process.pid}.tmp`;
319
345
  rmSync(parked, {recursive: true, force: true}); mkdirSync(dirname(target), {recursive: true});
320
346
  cpSync(join(work, 'out/server'), parked, {recursive: true}); rmSync(target, {recursive: true, force: true}); renameSync(parked, target);
321
- const index: CookIndex = {version: 1, build, slug: report.slug, hash, header: report.header, manifests: report.cook.manifests, keys, server: report.server, serverPack: report.cook.server, proof: report.proof === 'unmeasured' ? 'unmeasured' : 'proved', seconds: report.seconds,
322
- ...(report.cook.preparation === undefined ? {} : {preparation: report.cook.preparation})};
347
+ const index: CookIndex = {version: 1, build, slug: report.slug, hash, header: report.header, manifests: report.cook.manifests, keys: [...keys, ...loose.map(key => folder + key)], loose: folder, server: report.server, serverPack: report.cook.server, proof: report.proof === 'unmeasured' ? 'unmeasured' : 'proved', seconds: report.seconds,
348
+ ...(report.cook.preparation === undefined ? {} : {preparation: report.cook.preparation}),
349
+ ...(report.receipt === undefined || options.validator === undefined ? {} : {verdict: {...report.receipt, revision: options.validator}})};
323
350
  await store.put(indexKey(build, hash), new TextEncoder().encode(JSON.stringify(index)), {type: 'application/json', cache: 'public, max-age=300'});
324
351
  };
325
352
  const attempt = async (head: {readonly slug: string; readonly hash: string}, tarball: Uint8Array, work: string): Promise<CookRow> => {
@@ -1,9 +1,9 @@
1
1
  /// <reference types="node" />
2
2
  import * as v from 'valibot';
3
3
  // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
4
- import {existsSync, readFileSync, mkdirSync, readdirSync} from 'node:fs';
4
+ import {existsSync, readFileSync, mkdirSync, readdirSync, rmSync, writeFileSync} from 'node:fs';
5
5
  // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
6
- import {join, relative, resolve, sep} from 'node:path';
6
+ import {dirname, join, relative, resolve, sep} from 'node:path';
7
7
  import {PickupCatalogueSchema} from '@wildshard/format/specimenCatalogue';
8
8
  import {parseOutputIndex} from '@wildshard/format/outputIndex';
9
9
  // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
@@ -11,7 +11,7 @@ import {createHash} from 'node:crypto';
11
11
  import {readSpawnManifest, type SpawnFile, type SpawnManifest, type SpawnManifestAddress} from '../../shared/game/shardfile/spawnManifest';
12
12
  import {assetIdentities, audioTableRefs, bundlerSources as manifestSources, identityFiles, joinRefs, standInRefs, packMembers, readFenceReads, resolveFenceReads, sectionRefs, type PackPart} from './fenceReads';
13
13
  import {createPartEncoder, writeSpawnManifestFiles, writeSpawnParts, type CookedMember, type SpawnPartEncoding} from './spawnParts';
14
- import {cookParts, cookShardServerPack, openShardfile, servedAsIs, shardHalf, stalePreparationLine, type PreparationState} from './shardCook';
14
+ import {cookParts, cookShardServerPack, openShardfile, preparedShardHalf, servedAsIs, stalePreparationLine, type CookTiers, type PreparationState, type ShardHalf} from './shardCook';
15
15
  import type {ServerPack} from '../../shared/game/shardfile/serverPack';
16
16
  import {contentNamed} from './contentNamed';
17
17
  // oxlint-disable-next-line import/no-nodejs-modules -- The cook is a Node process (J28).
@@ -64,6 +64,9 @@ export interface BuildHalf {
64
64
  readonly rendererRevision?: string;
65
65
  readonly cacheName: string;
66
66
  readonly gpuFiles: Parameters<typeof standInRefs>[0];
67
+ /** S12d: the cook's per-tier textures (`defaultCookTiers`), as the box's cook makes them. None: every tier reads the
68
+ * product's own table. A preparation-only build never takes them: a fence records the product's own URLs. */
69
+ readonly tiers?: () => Promise<CookTiers>;
67
70
  /** The boot tables' common and pack sources, as `assetTables.mjs` bakes them. */
68
71
  readonly tables: (root: string) => {readonly common: {readonly source: string}; readonly all: {readonly source: string}; readonly packs: Readonly<Record<string, {readonly source: string}>>};
69
72
  readonly tableAddresses: (root: string) => {readonly common: Address; readonly all: Address; readonly packs: Readonly<Record<string, Address>>};
@@ -80,17 +83,21 @@ export function desktopFence(reads: readonly string[], urls: readonly string[]):
80
83
  const desktop = new Map(urls.filter(url => !phoneOnly(url)).sort((a, b) => named(a) - named(b)).map(url => [plain(url), url]));
81
84
  return [...new Set(reads.flatMap(url => { if (!phoneOnly(url)) return [url]; const own = desktop.get(plain(url)); return own === undefined ? [] : [own]; }))];
82
85
  }
83
- /** S4: a shard's phone stand-ins laid over the engine's table (`data/astc6.generated.json`). A phone reads the overlay's
84
- * file under the same served URL, so its fence read keeps the identity it had. */
85
- function phoneOverlay(root: string, directory: string): Readonly<Record<string, string>> {
86
- const path = join(root, 'src/shards', directory, 'data/astc6.generated.json');
87
- if (existsSync(path)) return v.parse(v.record(v.string(), v.string()), JSON.parse(readFileSync(path, 'utf8')));
88
- // Any other shard's overlay is in its product: the discovery's phone table, the rows that name a 6×6 file.
89
- const discovery = join(root, 'out', directory, 'mounts/shardfiles', directory, 'discovery.json');
90
- if (!existsSync(discovery)) return {};
91
- const table = v.safeParse(v.looseObject({native: v.looseObject({ktx2: v.looseObject({GPU_FILES: v.looseObject({phone: v.record(v.string(), v.string())})})})}), JSON.parse(readFileSync(discovery, 'utf8')));
92
- return table.success ? Object.fromEntries(Object.entries(table.output.native.ktx2.GPU_FILES.phone).filter(([, url]) => url.includes('/astc6/'))) : {};
86
+ /** A phone file in place of the one its source names on a desktop: S4's hand-made 6×6 overlay (`/assets/<shard>/astc6/`)
87
+ * or the cook's own phone tier (`<name>.t<size>-<hash8>.<ext>`, S12b). */
88
+ // oxlint-disable-next-line typescript/no-inferrable-types -- isolatedDeclarations wants the type written.
89
+ export const PHONE_STAND_IN: RegExp = /\/astc6\/|\.t[0-9]+-[a-f0-9]{8}\.(?:ktx2|glb)$/u;
90
+ /** S4, S12d: a shard's phone stand-ins, read off the discovery document the phone reads (the cook's tiered table): the
91
+ * phone rows that name a stand-in. A phone reads that file under the source's URL, so its fence read keeps its identity. */
92
+ export function phoneOverlay(discovery: Uint8Array | undefined): Readonly<Record<string, string>> {
93
+ if (discovery === undefined) return {};
94
+ const table = v.safeParse(v.looseObject({native: v.looseObject({ktx2: v.looseObject({GPU_FILES: v.looseObject({phone: v.record(v.string(), v.string())})})})}), JSON.parse(new TextDecoder().decode(discovery)));
95
+ return table.success ? Object.fromEntries(Object.entries(table.output.native.ktx2.GPU_FILES.phone).filter(([, url]) => PHONE_STAND_IN.test(url))) : {};
93
96
  }
97
+ /** S12d: where a build keeps each shard's discovery document as its cook serves it, inside that build's own output, until
98
+ * `copyOutputProducts` lays it over the product's (and removes the folder); the catalogue hashes it there. */
99
+ export const SERVED_DISCOVERY = '.discovery';
100
+ export function servedDiscoveryPath(outDir: string, directory: string): string { return join(outDir, SERVED_DISCOVERY, directory, 'discovery.json'); }
94
101
  /** DL5: a shard's packs set, `src/shards/<slug>/packs-set.json` (`{"version": 1, "set": "fence"}`): 'fence' packs the
95
102
  * default set (what the boot reads before control), absent packs the whole level. Anything else fails the build. */
96
103
  export function readPacksSet(path: string): 'whole' | 'fence' {
@@ -201,22 +208,42 @@ export async function writeSpawnManifests(outDir: string, input: unknown,
201
208
  const clientAt: Record<string, {phone: number; desktop: number}> = {}, gridAt: Record<string, Record<string, {phone: number; desktop: number}>> = {};
202
209
  const packsClientAt: typeof clientAt = {}, packsGridAt: typeof gridAt = {};
203
210
  const encoder = createPartEncoder(), cooked = new Map<string, CookedMember>(), serverPacks: Record<string, ServerPack> = {};
211
+ // C1: the shard half comes from the shardfile alone, one tarball per product, and its parts from the shard's own cook.
212
+ // S12d: with the cook's tiers, the phone reads the tiered table and its own files, as from the box's cook.
213
+ const tiers = build.preparationOnly === true || build.tiers === undefined ? {} : await build.tiers();
214
+ rmSync(join(output, SERVED_DISCOVERY), {recursive: true, force: true});
215
+ // Only what the manifests' loop needs: an opened product stays open no longer than its own pass (no 27 at once).
216
+ const served = new Map<string, NonNullable<ShardHalf['discovery']>>(), open = new Map<string, () => Promise<ShardHalf>>();
217
+ for (const prefix of Object.keys(bank.products)) {
218
+ const directory = prefix.split('/')[2]; if (directory === undefined || !/^[a-z0-9_-]+$/u.test(directory)) throw new Error('Invalid spawn shard');
219
+ const product = join(root, 'out', directory), index = parseOutputIndex(JSON.parse(readFileSync(join(product, 'shard.json'), 'utf8')));
220
+ if (index.runtime !== null && (!('kind' in index.runtime) || index.runtime.kind !== 'client-plan')) continue;
221
+ const reopen = (): Promise<ShardHalf> => preparedShardHalf(openShardfile(productTarball(product), {preparationOnly: build.preparationOnly === true,
222
+ ...(build.rendererRevision === undefined ? {} : {renderer: build.rendererRevision})}), wireBytes, undefined, directory, tiers);
223
+ const half = await reopen();
224
+ open.set(directory, reopen);
225
+ // What no part answers serves at its URL: the tiered discovery document (beside its product) and the phone's own files.
226
+ if (half.discovery !== undefined) {
227
+ served.set(directory, half.discovery);
228
+ const file = servedDiscoveryPath(output, directory); mkdirSync(dirname(file), {recursive: true}); writeFileSync(file, half.discovery.bytes);
229
+ }
230
+ for (const row of half.tierFiles) { const file = join(output, row.url.slice(1)); mkdirSync(dirname(file), {recursive: true}); writeFileSync(file, half.read(row.url)); }
231
+ }
204
232
  // C3: a grid page reads every primary neighbour's discovery document before it enters.
205
233
  const discoveries = Object.keys(bank.products).flatMap(prefix => {
206
234
  const directory = prefix.split('/')[2] ?? '', url = `/shardfiles/${directory}/discovery.json`, file = join(root, 'out', directory, 'mounts', url);
207
- if (!existsSync(file)) return [];
208
- const bytes = readFileSync(file), hash = createHash('sha256').update(bytes).digest('hex'), row = {url: `${url}?v=${hash.slice(0, 8)}`, bytes: bytes.length, hash, wireBytes: wireBytes(url, bytes, hash)};
235
+ const own = served.get(directory);
236
+ if (own === undefined && !existsSync(file)) return [];
237
+ const bytes = own?.bytes ?? readFileSync(file), hash = createHash('sha256').update(bytes).digest('hex'), row = {url: `${url}?v=${hash.slice(0, 8)}`, bytes: bytes.length, hash, wireBytes: wireBytes(url, bytes, hash)};
209
238
  sources.set(row.url, () => bytes);
210
239
  return [row];
211
240
  });
212
241
  const preparations: Partial<Record<PreparationState, string[]>> = {};
213
242
  for (const prefix of Object.keys(bank.products)) {
214
- const directory = prefix.split('/')[2]; if (directory === undefined || !/^[a-z0-9_-]+$/u.test(directory)) throw new Error('Invalid spawn shard');
215
- const product = join(root, 'out', directory), index = parseOutputIndex(JSON.parse(readFileSync(join(product, 'shard.json'), 'utf8')));
216
- if (index.runtime !== null && (!('kind' in index.runtime) || index.runtime.kind !== 'client-plan')) continue;
217
- // C1: the shard half comes from the shardfile alone, one tarball per product, and its parts from the shard's own cook.
218
- const half = shardHalf(openShardfile(productTarball(product), {preparationOnly: build.preparationOnly === true,
219
- ...(build.rendererRevision === undefined ? {} : {renderer: build.rendererRevision})}), wireBytes, undefined, directory), {sections} = half.shard;
243
+ // The second opening's encodes come straight back from the first's (the encoder keeps them).
244
+ const directory = prefix.split('/')[2] ?? '', reopen = open.get(directory); if (reopen === undefined) continue;
245
+ const half = await reopen();
246
+ const index = parseOutputIndex(JSON.parse(readFileSync(join(root, 'out', directory, 'shard.json'), 'utf8'))), {sections} = half.shard, overlay = phoneOverlay(half.discovery?.bytes);
220
247
  // J262 (SP1a): a preparation from another renderer cooks as it is, and the build says which shards took that path.
221
248
  (preparations[half.shard.preparation] ??= []).push(directory);
222
249
  if (half.shard.preparation === 'stale') console.warn(stalePreparationLine(directory));
@@ -242,7 +269,7 @@ export async function writeSpawnManifests(outDir: string, input: unknown,
242
269
  const urls = publicAssets(), members = [...packParts().values()].flatMap(part => part.members);
243
270
  // Section references, then the boot tables' audio slots (Pine's colliding sfx revisions).
244
271
  const refs = joinRefs(sectionRefs(sections, new Set([...urls, ...members].map(url => url.split('?')[0] ?? url))),
245
- audioTableRefs(JSON.parse(build.tables(root).common.source) as {sfx?: unknown; music?: unknown}), standInRefs({...build.gpuFiles, phone: {...build.gpuFiles['phone'], ...phoneOverlay(root, directory)}}));
272
+ audioTableRefs(JSON.parse(build.tables(root).common.source) as {sfx?: unknown; music?: unknown}), standInRefs({...build.gpuFiles, phone: {...build.gpuFiles['phone'], ...overlay}}));
246
273
  // An identity is one file or none: one naming two distinct files (two vendor versions, sp-x9) refuses, never the first.
247
274
  const owners = identityFiles(assetIdentities(urls, packParts(), refs, {digests: assetDigests, sources: bundlerSources()}));
248
275
  const ambiguous = shell.filter(identity => (owners.get(identity)?.length ?? 0) > 1);
@@ -292,8 +319,8 @@ export async function writeSpawnManifests(outDir: string, input: unknown,
292
319
  if (inventory.length !== extra[1] || createHash('sha256').update(inventory).digest('hex') !== extra[3]) throw new Error('Spawn public inventory differs');
293
320
  // S4 on Sky: this inventory names a page's image sources on every tier. A phone with an overlay reads the 6×6 file in
294
321
  // their place, and one with no ASTC reads them loose, so they stay off the phone's list.
295
- const overlay = tier === 'phone' ? phoneOverlay(root, directory) : {};
296
- for (const [url, bytes] of v.parse(files, JSON.parse(new TextDecoder().decode(inventory)))) if (!Object.hasOwn(overlay, url.split('?')[0] ?? url)) add(asset(url, bytes));
322
+ const standIns = tier === 'phone' ? overlay : {};
323
+ for (const [url, bytes] of v.parse(files, JSON.parse(new TextDecoder().decode(inventory)))) if (!Object.hasOwn(standIns, url.split('?')[0] ?? url)) add(asset(url, bytes));
297
324
  }
298
325
  // Required code can resolve while bulk content downloads. Keep each
299
326
  // partition's original order, every row and the full download barrier.