@wildshard/engine 0.1.17 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/budgets/shard-size-baseline.json +10 -4
  3. package/index.html +6 -4
  4. package/lint/engine-sides.json +21 -0
  5. package/lint/host-callback-syntax.mjs +2 -1
  6. package/lint/host-callbacks.mjs +3 -3
  7. package/lint/host-files.mjs +5 -4
  8. package/lint/no-base64.mjs +3 -1
  9. package/lint/ratchet.json +0 -68
  10. package/lint/ratchet.mjs +1 -4
  11. package/lint/shard-imports.mjs +6 -7
  12. package/lint/shard-source.d.mts +9 -0
  13. package/lint/shard-source.mjs +91 -0
  14. package/lint/shardfile-boundaries.mjs +10 -8
  15. package/lint/sim-closure.mjs +6 -3
  16. package/lint/wildshard-plugin.d.ts +4 -0
  17. package/lint/wildshard-plugin.js +22 -23
  18. package/package.json +25 -16
  19. package/scripts/agent-sync.mjs +27 -8
  20. package/scripts/app-build/appCook.ts +8 -2
  21. package/scripts/app-build/catalogue.ts +10 -7
  22. package/scripts/app-build/gen.ts +22 -2
  23. package/scripts/app-build/isolation.d.mts +10 -0
  24. package/scripts/app-build/isolation.mjs +47 -0
  25. package/scripts/app-build/outputProducts.d.mts +3 -0
  26. package/scripts/app-build/outputProducts.mjs +35 -0
  27. package/scripts/app-build/previousDeploy.d.mts +2 -0
  28. package/scripts/app-build/previousDeploy.mjs +10 -1
  29. package/scripts/app-build/serviceWorker.ts +10 -3
  30. package/scripts/app-build/splitOutput.mjs +4 -2
  31. package/scripts/app-build/standaloneOutput.d.mts +1 -1
  32. package/scripts/app-build/standaloneOutput.mjs +5 -3
  33. package/scripts/app-build/titlePreloads.d.mts +8 -0
  34. package/scripts/app-build/titlePreloads.mjs +69 -0
  35. package/scripts/app-build/worldMapAtlas.ts +10 -6
  36. package/scripts/app-build-pipeline.mjs +34 -3
  37. package/scripts/built-product.mjs +1 -1
  38. package/scripts/cell-mem-bench.mjs +81 -23
  39. package/scripts/check-custom-sections-zero.mjs +1 -1
  40. package/scripts/check-devserver.mjs +3 -1
  41. package/scripts/check-models.mjs +2 -0
  42. package/scripts/check-paths.allow.json +0 -115
  43. package/scripts/check-paths.mjs +2 -2
  44. package/scripts/check-platform-ratchets.mjs +2 -3
  45. package/scripts/chromium-keychain-check.mjs +4 -2
  46. package/scripts/chromium-keychain.d.mts +1 -1
  47. package/scripts/chromium-keychain.mjs +3 -1
  48. package/scripts/ci-static.sh +1 -1
  49. package/scripts/committed-fast-checks.mjs +0 -2
  50. package/scripts/content-sw-proof.mjs +2 -1
  51. package/scripts/cook-worker.mjs +9 -3
  52. package/scripts/cook.mjs +20 -3
  53. package/scripts/devices.d.mts +4 -0
  54. package/scripts/devices.mjs +39 -0
  55. package/scripts/devserver-client.d.mts +4 -2
  56. package/scripts/devserver-client.mjs +25 -8
  57. package/scripts/docs-inputs-proof.mjs +6 -6
  58. package/scripts/doctor-proof.mjs +1 -1
  59. package/scripts/e2e-faults.mjs +159 -0
  60. package/scripts/e7-voice-staging-proof.mjs +1 -1
  61. package/scripts/gate-related-tests.mjs +1 -1
  62. package/scripts/git-source-view.mjs +1 -1
  63. package/scripts/goldens/capture.mjs +17 -5
  64. package/scripts/grid-crossing-nightly.mjs +59 -0
  65. package/scripts/grid-crossing-qa.mjs +44 -0
  66. package/scripts/grid-crossing-ratchet.d.mts +6 -0
  67. package/scripts/grid-crossing-ratchet.mjs +52 -0
  68. package/scripts/grid-crossing-report.mjs +122 -0
  69. package/scripts/grid-crossing-stages.mjs +47 -0
  70. package/scripts/grid-crossing-tour.mjs +277 -0
  71. package/scripts/grid-crossing-walk-proof.mjs +136 -18
  72. package/scripts/host-ruler/roots.mjs +2 -2
  73. package/scripts/host-ruler.d.mts +91 -0
  74. package/scripts/host-ruler.mjs +565 -0
  75. package/scripts/live-prepush.py +3 -5
  76. package/scripts/loading-flow/benchmax.mjs +3 -1
  77. package/scripts/look-loop/test_look_loop.py +1 -38
  78. package/scripts/memory-proof.mjs +213 -0
  79. package/scripts/mmo-bots.mjs +8 -2
  80. package/scripts/net-lag-proof.mjs +219 -0
  81. package/scripts/night-market-kit-budget.mjs +2 -2
  82. package/scripts/offline-proof-rules.d.mts +5 -3
  83. package/scripts/offline-proof-rules.mjs +34 -7
  84. package/scripts/offline-proof.mjs +88 -18
  85. package/scripts/{shard-platform-output.mjs → output-reader.mjs} +1 -1
  86. package/scripts/page-release/content.d.mts +9 -1
  87. package/scripts/page-release/content.mjs +131 -17
  88. package/scripts/page-release/flow.mjs +36 -10
  89. package/scripts/page-release/stamp-content.mjs +24 -0
  90. package/scripts/page-release.mjs +28 -6
  91. package/scripts/page-size.mjs +2 -1
  92. package/scripts/pin-packs.ts +1 -1
  93. package/scripts/pins.ts +4 -4
  94. package/scripts/precommit-checks.mjs +1 -7
  95. package/scripts/precommit-jobs.mjs +0 -1
  96. package/scripts/precommit-source.py +1 -9
  97. package/scripts/program-discovery.mjs +1 -1
  98. package/scripts/ring-cook.mjs +2 -1
  99. package/scripts/ring-step.ts +1 -1
  100. package/scripts/script-scenario-metrics.mjs +1 -1
  101. package/scripts/shard-output-content.d.mts +1 -1
  102. package/scripts/shard-output-content.mjs +2 -2
  103. package/scripts/shard-outputs.mjs +1 -1
  104. package/scripts/shard-public.mjs +1 -1
  105. package/scripts/shard-wire-meter.mjs +24 -9
  106. package/scripts/side-parity.mjs +4 -2
  107. package/scripts/standalone-smoke.mjs +8 -3
  108. package/scripts/tier-textures-proof.mjs +101 -0
  109. package/scripts/tier-textures.mjs +78 -0
  110. package/scripts/tile-bake-report.mjs +1 -1
  111. package/scripts/tile-bakes.mjs +1 -1
  112. package/scripts/title-assets.generated.json +2 -0
  113. package/scripts/tsconfig.json +1 -2
  114. package/scripts/vendor/basisu/LICENSE +201 -0
  115. package/scripts/vendor/basisu/basis_encoder.cjs +2 -0
  116. package/scripts/vendor/basisu/basis_encoder.wasm +0 -0
  117. package/scripts/vitest-push-guards.config.ts +1 -1
  118. package/scripts/vps/harden-apply.sh +4 -1
  119. package/scripts/vps/harden-prometheus.sh +176 -0
  120. package/scripts/vps/harden.sh +16 -0
  121. package/scripts/vps/rebuild.sh +18 -0
  122. package/scripts/vps/wildshard-config +84 -24
  123. package/scripts/worker-client-play.mjs +7 -2
  124. package/scripts/worker-client-proof.mjs +1 -1
  125. package/scripts/worker-client-slug.mjs +44 -0
  126. package/scripts/worktree-reap.mjs +5 -5
  127. package/src/.oxlintrc.json +3 -182
  128. package/src/client/app/entry.ts +47 -12
  129. package/src/client/engine/boot/bootCache.ts +2 -2
  130. package/src/client/engine/boot/bootTrace.ts +12 -4
  131. package/src/client/engine/boot/browserChunkRecovery.ts +4 -0
  132. package/src/client/engine/boot/clearDownloads.ts +5 -2
  133. package/src/client/engine/boot/contentRead.ts +11 -0
  134. package/src/client/engine/boot/gpuFiles.ts +4 -2
  135. package/src/client/engine/boot/lastEnd.ts +99 -12
  136. package/src/client/engine/boot/parsedPages.ts +127 -58
  137. package/src/client/engine/boot/texturePack.ts +2 -2
  138. package/src/client/engine/build.d.ts +2 -0
  139. package/src/client/engine/core/Game.ts +18 -0
  140. package/src/client/engine/core/bootstrap.ts +4 -1
  141. package/src/client/engine/debug/probe.ts +18 -1
  142. package/src/client/engine/fx/LightPool.ts +41 -1
  143. package/src/client/engine/net/netPlayer.ts +19 -2
  144. package/src/client/engine/net/otherPlayers.ts +55 -8
  145. package/src/client/engine/net/predictor.ts +33 -9
  146. package/src/client/engine/net/redial.ts +29 -6
  147. package/src/client/engine/pwa/cacheNames.ts +58 -1
  148. package/src/client/engine/pwa/sw.js +53 -14
  149. package/src/client/engine/render/dataLook.ts +152 -40
  150. package/src/client/engine/render/families/detail.ts +89 -0
  151. package/src/client/engine/render/families/pbr.ts +7 -1
  152. package/src/client/engine/render/look.ts +6 -0
  153. package/src/client/engine/render/lookLight.ts +103 -11
  154. package/src/client/engine/render/lookRender.ts +105 -0
  155. package/src/client/engine/render/precompile.ts +20 -14
  156. package/src/client/engine/saves/indexed.ts +2 -2
  157. package/src/client/engine/telemetry/browserErrors.ts +28 -4
  158. package/src/client/engine/telemetry/deviceSnapshot.ts +155 -0
  159. package/src/client/engine/telemetry/staleBuildReport.ts +22 -1
  160. package/src/client/engine/ui/debugOptions.ts +1 -1
  161. package/src/client/engine/ui/lookLightOption.ts +10 -6
  162. package/src/client/engine/ui/namePrompt.ts +7 -0
  163. package/src/client/game/boot/audio.generated.ts +114 -216
  164. package/src/client/game/boot/bytes.generated.ts +118 -238
  165. package/src/client/game/boot/placedSpawn.ts +35 -0
  166. package/src/client/game/boot/spawnDownload.ts +25 -6
  167. package/src/client/game/boot/versions.generated.ts +7 -7
  168. package/src/client/game/devScene/hook.ts +291 -0
  169. package/src/client/game/grid/area.ts +38 -0
  170. package/src/client/game/grid/cellLoading.ts +12 -2
  171. package/src/client/game/grid/live.ts +134 -12
  172. package/src/client/game/grid/liveSession.ts +47 -5
  173. package/src/client/game/grid/mapAtlas.ts +18 -4
  174. package/src/client/game/grid/menu.ts +4 -0
  175. package/src/client/game/grid/minimapBlend.ts +15 -5
  176. package/src/client/game/grid/neighbourPanels.ts +10 -1
  177. package/src/client/game/grid/oneWayIn.ts +5 -1
  178. package/src/client/game/grid/placedCards.ts +55 -0
  179. package/src/client/game/grid/placedCells.ts +233 -0
  180. package/src/client/game/grid/placedContent.ts +85 -0
  181. package/src/client/game/grid/prefetch.ts +9 -1
  182. package/src/client/game/grid/productInputs.ts +3 -1
  183. package/src/client/game/grid/productLeases.ts +4 -1
  184. package/src/client/game/grid/products.ts +8 -4
  185. package/src/client/game/grid/regionalRuntime.ts +6 -0
  186. package/src/client/game/grid/regionalWorld.ts +15 -2
  187. package/src/client/game/grid/roadCull.ts +44 -14
  188. package/src/client/game/grid/roadLook.ts +24 -15
  189. package/src/client/game/grid/roadLookPlatform.ts +4 -2
  190. package/src/client/game/grid/roadResources.ts +24 -0
  191. package/src/client/game/grid/roadSign.worker.ts +4 -3
  192. package/src/client/game/grid/roadSignPaint.ts +16 -5
  193. package/src/client/game/grid/roadSignPreparation.ts +4 -3
  194. package/src/client/game/grid/roadSignScale.ts +8 -0
  195. package/src/client/game/grid/roadTileBacking.ts +13 -3
  196. package/src/client/game/grid/roadTileCapture.worker.ts +3 -1
  197. package/src/client/game/grid/roadTileCaptureWorker.ts +2 -1
  198. package/src/client/game/grid/roadTileWrite.worker.ts +4 -2
  199. package/src/client/game/grid/roadTileWriter.ts +2 -1
  200. package/src/client/game/grid/screenReports.ts +39 -0
  201. package/src/client/game/grid/session.ts +29 -4
  202. package/src/client/game/grid/shardfileFar.ts +9 -3
  203. package/src/client/game/grid/softWallLook.ts +25 -8
  204. package/src/client/game/mainMenu.ts +6 -2
  205. package/src/client/game/plugin/context.ts +2 -1
  206. package/src/client/game/plugin/followedMovers.ts +7 -1
  207. package/src/client/game/quest/figures.ts +2 -0
  208. package/src/client/game/session/finish.ts +11 -1
  209. package/src/client/game/shard/titleList.ts +6 -2
  210. package/src/client/game/shardfile/assetCache.ts +4 -4
  211. package/src/client/game/shardfile/client.ts +4 -1
  212. package/src/client/game/shardfile/clientAssets.ts +9 -5
  213. package/src/client/game/shardfile/clientLibrary.ts +1 -1
  214. package/src/client/game/shardfile/clientWater.ts +51 -9
  215. package/src/client/game/shardfile/clientWorker.ts +21 -8
  216. package/src/client/game/shardfile/look.ts +8 -3
  217. package/src/client/game/shardfile/product.ts +34 -7
  218. package/src/client/game/shardfile/productChecks.ts +2 -2
  219. package/src/client/game/shardfile/productCost.ts +2 -2
  220. package/src/client/game/shardfile/readAsset.ts +2 -2
  221. package/src/client/game/shardfile/reflectingWater.ts +73 -0
  222. package/src/client/game/shardfile/seatGrace.ts +9 -0
  223. package/src/client/game/shardfile/skinPlayback.ts +1 -1
  224. package/src/client/game/systems/npc/figureRows.ts +9 -0
  225. package/src/client/game/titleDeck.ts +4 -3
  226. package/src/client/host/online/doctor.ts +2 -0
  227. package/src/client/host/online/linkScreens.ts +30 -4
  228. package/src/client/host/online/shell.ts +14 -0
  229. package/src/client/host/standalone/shell.ts +4 -0
  230. package/src/cook/game/astc6Encode.ts +55 -0
  231. package/src/cook/game/astc6Wasm.ts +176 -0
  232. package/src/cook/game/cookJob.ts +30 -6
  233. package/src/cook/game/cookWorker.ts +38 -11
  234. package/src/cook/game/entryTotals.ts +51 -24
  235. package/src/cook/game/shardCook.ts +67 -9
  236. package/src/cook/game/spawnParts.ts +33 -8
  237. package/src/cook/game/tierTextures.ts +349 -0
  238. package/src/e2e/smoke/game.ts +19 -14
  239. package/src/e2e/smoke/page-globals.d.ts +9 -8
  240. package/src/e2e/smoke/template2.spec.ts +8 -7
  241. package/src/server/engine/ai/bossContenders.ts +27 -3
  242. package/src/server/engine/ai/phases.ts +1 -1
  243. package/src/server/engine/entities/creatureRefill.ts +9 -2
  244. package/src/server/engine/net/ledgerBook.ts +2 -0
  245. package/src/server/engine/net/room.ts +15 -0
  246. package/src/server/engine/sim/respawn.ts +8 -0
  247. package/src/server/game/boss/arenaRoomEntry.ts +5 -1
  248. package/src/server/game/boss/arenaRoomNative.ts +2 -0
  249. package/src/server/game/boss/native.ts +2 -1
  250. package/src/server/game/boss/nativeWake.ts +9 -5
  251. package/src/server/game/boss/openArenaNative.ts +2 -2
  252. package/src/server/game/boss/openArenaRoom.ts +3 -1
  253. package/src/server/game/grid/room.ts +44 -3
  254. package/src/server/game/grid/simulation.ts +23 -0
  255. package/src/server/game/population/mixedRoster.ts +11 -3
  256. package/src/server/game/population/nativeBake.ts +8 -3
  257. package/src/server/game/population/nativeReturns.ts +6 -3
  258. package/src/server/game/population/roster.ts +5 -3
  259. package/src/server/game/quest/eliteDrops.ts +15 -4
  260. package/src/server/game/quest/nativeChapter.ts +1 -1
  261. package/src/server/game/quest/nativeComposition.ts +5 -1
  262. package/src/server/game/riding/nativeTaming.ts +10 -6
  263. package/src/server/game/riding/wildRefill.ts +3 -2
  264. package/src/server/game/shardfile/bossRow.ts +4 -2
  265. package/src/server/game/shardfile/homeKeeper.ts +4 -2
  266. package/src/server/host/bake/headlessWorker.ts +1 -1
  267. package/src/server/host/dedicated/cellHost.ts +98 -16
  268. package/src/server/host/dedicated/cellProtocol.ts +16 -1
  269. package/src/server/host/dedicated/cellWorker.ts +8 -5
  270. package/src/server/host/dedicated/dedicatedRoom.ts +16 -6
  271. package/src/server/host/dedicated/gridCells.ts +15 -6
  272. package/src/server/host/dedicated/loopLag.ts +21 -0
  273. package/src/server/host/dedicated/main.ts +13 -5
  274. package/src/server/host/dedicated/packReader.ts +3 -3
  275. package/src/server/host/dedicated/placementFile.ts +4 -19
  276. package/src/server/host/dedicated/roadRoom.ts +34 -4
  277. package/src/server/host/dedicated/serverMetrics.ts +126 -0
  278. package/src/server/host/dedicated/socketPump.ts +21 -4
  279. package/src/server/host/dedicated/socketTransport.ts +2 -1
  280. package/src/server/host/dedicated/workerTable.ts +15 -4
  281. package/src/server/host/dedicated/world.ts +95 -8
  282. package/src/server/host/headless.ts +1 -1
  283. package/src/server/host/headlessRuntime.ts +1 -1
  284. package/src/server/host/standalone/roadWorld.ts +15 -3
  285. package/src/server/host/standalone/serverHost.ts +4 -1
  286. package/src/server/host/standalone/serverProduct.ts +41 -11
  287. package/src/shared/engine/ai/BossBrain.ts +8 -0
  288. package/src/shared/engine/net/deviceWord.ts +138 -0
  289. package/src/shared/engine/net/socketFrames.ts +7 -4
  290. package/src/shared/game/boss/wake.ts +6 -0
  291. package/src/shared/game/devColliders.ts +47 -0
  292. package/src/shared/game/eliteSystem.ts +8 -0
  293. package/src/shared/game/grid/allocator.ts +3 -0
  294. package/src/shared/game/grid/catalogueSlot.ts +7 -0
  295. package/src/shared/game/grid/placementRows.ts +36 -0
  296. package/src/shared/game/plugin/bossRecord.ts +3 -2
  297. package/src/shared/game/shardfile/ktx2Targets.ts +23 -3
  298. package/src/shared/game/shardfile/outputDescriptor.ts +9 -5
  299. package/src/shared/game/shardfile/packReads.ts +28 -3
  300. package/src/shared/game/shardfile/serverPack.ts +9 -0
  301. package/src/shared/game/systems/looks/trampleField.ts +2 -0
  302. package/src/shared/host/formatJsonSchema.ts +2 -2
  303. package/src/tsconfig.json +1 -27
  304. package/lint/host-reviews.json +0 -14
  305. package/lint/listed-view-code.json +0 -13
  306. package/lint/platform-pending-users.json +0 -2520
  307. package/lint/platform-pending-users.md +0 -18
  308. package/scripts/cell-mem/metrics-preload.mjs +0 -65
  309. package/scripts/check-closed-components.d.mts +0 -7
  310. package/scripts/check-closed-components.mjs +0 -87
  311. package/scripts/g571-hosts.d.mts +0 -3
  312. package/scripts/g571-hosts.mjs +0 -54
  313. package/scripts/host-ruler/register-migrate.mjs +0 -242
  314. package/scripts/listed-view-code.d.mts +0 -7
  315. package/scripts/listed-view-code.mjs +0 -93
  316. package/scripts/look-loop/fit-lut.py +0 -142
  317. package/scripts/look-loop/palette-delta.py +0 -183
  318. package/scripts/look-loop/regions/driftwood-isle.json +0 -25
  319. package/scripts/look-loop/regions/far-reach.json +0 -225
  320. package/scripts/look-loop/regions/open-plot-night-market.json +0 -855
  321. package/scripts/look-loop/regions/pine-hollow.json +0 -22
  322. package/scripts/look-loop/regions/sunscar-dunes.json +0 -281
  323. package/scripts/platform-chargebacks.d.mts +0 -9
  324. package/scripts/platform-chargebacks.mjs +0 -135
  325. package/scripts/shard-component-spans.d.mts +0 -15
  326. package/scripts/shard-component-spans.mjs +0 -18
  327. package/scripts/shard-components.d.mts +0 -16
  328. package/scripts/shard-components.mjs +0 -363
  329. package/scripts/shard-platform-hosts.d.mts +0 -2
  330. package/scripts/shard-platform-hosts.mjs +0 -93
  331. package/scripts/shard-platform-sections.d.mts +0 -2
  332. package/scripts/shard-platform-sections.mjs +0 -9
  333. package/scripts/shard-platform.d.mts +0 -76
  334. package/scripts/shard-platform.mjs +0 -1072
  335. package/src/.oxfmtrc.json +0 -48
  336. package/src/scripts/ask-new.sh +0 -29
  337. package/src/scripts/asks.mjs +0 -142
  338. package/src/scripts/herdr-prompt.sh +0 -34
  339. package/src/scripts/herdr-question-open.sh +0 -67
  340. package/src/scripts/md-wrap.mjs +0 -229
  341. /package/scripts/{shard-platform-format.d.mts → node-format.d.mts} +0 -0
  342. /package/scripts/{shard-platform-format.mjs → node-format.mjs} +0 -0
  343. /package/scripts/{shard-platform-output.d.mts → output-reader.d.mts} +0 -0
@@ -8,8 +8,12 @@
8
8
  // SHA-256 over every published path, its SHA-256 and its size. PZ3's `contentBase` is one base that mirrors page paths,
9
9
  // so the set keeps page paths under one content-named prefix: a build that changes no content names the same set and
10
10
  // uploads nothing, and every key is immutable. The set's index, `content/<set>/content.json`, goes last; its presence
11
- // with the same bytes means the set is complete. (A new set re-uploads files an older set already holds: cross-set
12
- // dedupe is a follow-up, with PZ10's content cache.)
11
+ // with the same bytes means the set is complete.
12
+ //
13
+ // Cross-set dedupe (PZ6): a new set copies, server side (S3 CopyObject, R2_ACCOUNT_ID and R2_PACKS_* from the
14
+ // environment, the same bucket-scoped keys server-release uses), every missing file whose bytes a recent complete set
15
+ // already holds, so a content change uploads only the files that changed, not the whole ~2.2 GB. A copy is checked by its
16
+ // size on R2, and a sample of copies is read back through the CDN like an upload; a failed copy is uploaded instead.
13
17
  //
14
18
  // A key that already answers (a cache-busted HEAD on packs.wildshard.io) is skipped only when it reads back with the
15
19
  // same decoded bytes and CORS; one with other bytes is uploaded again. The rest go up through
@@ -22,6 +26,7 @@ import {mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync} f
22
26
  import {join, resolve} from 'node:path';
23
27
  import {setTimeout as delay} from 'node:timers/promises';
24
28
  import {brotliDecompressSync} from 'node:zlib';
29
+ import {CopyObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client} from '@aws-sdk/client-s3';
25
30
  import {carriedPaths, classify, shardNames} from '../page-size.mjs';
26
31
  import {PACKS_ORIGIN} from './flow.mjs';
27
32
  import {runRedacted} from './redact.mjs';
@@ -46,9 +51,17 @@ const sha256 = (/** @type {Uint8Array} */ bytes) => createHash('sha256').update(
46
51
  * @property {(key: string) => Promise<boolean>} exists The key answers 200 now (never a cached 404).
47
52
  * @property {(puts: readonly Put[]) => Promise<void>} put
48
53
  * @property {(key: string, origin: string) => Promise<Read>} read A cache-busted CORS GET, decoded.
54
+ * @property {() => Promise<string[]>} [sets] PZ6: the complete sets (their `content.json` present), newest first.
55
+ * @property {(from: string, to: string, headers: Headers, bytes: number) => Promise<void>} [copy] PZ6: a server-side
56
+ * copy with the target's headers, refused unless the copy holds `bytes`.
49
57
  */
50
58
  /** @typedef {{set: string, base: string, files: number, bytes: number, uploaded: number, uploadedBytes: number, held: number,
51
- * kinds: Record<string, {files: number, bytes: number}>, cors: string}} ContentSummary */
59
+ * copied: number, copiedBytes: number, kinds: Record<string, {files: number, bytes: number}>, cors: string}} ContentSummary
60
+ * `uploaded` counts every key this run wrote (put or copied), `uploadedBytes` only the bytes it sent; `copied` and
61
+ * `copiedBytes` are the keys and bytes copied from an older set on R2 (PZ6). */
62
+ /** PZ6: how many recent complete sets a new set may copy from, and how many copies each publish reads back in full. */
63
+ export const DONOR_SETS = 4;
64
+ export const COPY_SAMPLE = 24;
52
65
 
53
66
  /** The cook's object headers (`objectHeaders()`, src/cook/game/cookWorker.ts), every key immutable: a set is content-named.
54
67
  * @param {string} key @returns {Headers} */
@@ -58,9 +71,16 @@ export function contentHeaders(key) {
58
71
  return {type, cache: IMMUTABLE};
59
72
  }
60
73
 
61
- /** The kind a page path publishes as, or undefined when it stays on the page (code, shell, runtime, maps).
74
+ /** PZ7: a shard file's I5 packs (`shardfiles/<slug>/blobs/pack/`, 475 MB). The wire meter read none of them on any deck
75
+ * card or the grid, phone and desktop (meta docs/plans/page-size/pz7-wire-meter-2026-11-13.json): spawn downloads read
76
+ * the loose blobs, and every pack reader falls back to them (`packOrLoose`, src/shared/game/shardfile/packReads.ts). */
77
+ export const UNREAD_PACK = /^shardfiles\/[^/]+\/blobs\/pack\//u;
78
+
79
+ /** The kind a page path publishes as, or undefined when it stays on the page (code, shell, runtime, maps) or nobody reads
80
+ * it (PZ7's S1 packs).
62
81
  * @param {string} path @param {ReadonlySet<string>} shards @returns {Kind | undefined} */
63
82
  export function contentKind(path, shards) {
83
+ if (UNREAD_PACK.test(path)) return undefined;
64
84
  const {kind, part} = classify(path, shards);
65
85
  if (kind === 'shard content') return /** @type {Kind} */ (part);
66
86
  return kind === 'engine art' ? 'E1' : undefined;
@@ -134,8 +154,8 @@ export async function publishContent({dir, store, scratch, origin = GAME_ORIGIN,
134
154
  const kinds = Object.fromEntries(KIND_NAMES.map(kind => [kind, {files: 0, bytes: 0}]));
135
155
  for (const file of files) { const row = kinds[file.kind]; if (row !== undefined) { row.files++; row.bytes += file.bytes; } }
136
156
  const bytes = files.reduce((sum, file) => sum + file.bytes, 0);
137
- const summary = (/** @type {number} */ uploaded, /** @type {number} */ uploadedBytes) =>
138
- ({set, base: contentBaseOf(set), files: files.length, bytes, uploaded, uploadedBytes, held: files.length - uploaded, kinds, cors: origin});
157
+ const summary = (/** @type {number} */ uploaded, /** @type {number} */ uploadedBytes, copied = 0, copiedBytes = 0) =>
158
+ ({set, base: contentBaseOf(set), files: files.length, bytes, uploaded, uploadedBytes, held: files.length - uploaded, copied, copiedBytes, kinds, cors: origin});
139
159
  const indexKey = contentKey(set, 'content.json');
140
160
  if (dryRun) {
141
161
  log(`DRY-RUN would publish ${files.length} files (${mb(bytes)} MB) as ${contentBaseOf(set)}: HEAD each on ${PACKS_ORIGIN}, wrangler r2 bulk put the missing ones to ${BUCKET}, read each back with CORS ${origin}, then ${indexKey}`);
@@ -160,8 +180,12 @@ export async function publishContent({dir, store, scratch, origin = GAME_ORIGIN,
160
180
  missing.push(file);
161
181
  });
162
182
  missing.sort((a, b) => (a.path < b.path ? -1 : 1));
183
+ const written = missing.length;
184
+ const copied = await copyFromSets({set, missing, store, origin, log, sleep});
185
+ const copiedBytes = copied.reduce((sum, file) => sum + file.bytes, 0);
186
+ if (copied.length > 0) { const done = new Set(copied); missing.splice(0, missing.length, ...missing.filter(file => !done.has(file))); }
163
187
  const uploadedBytes = missing.reduce((sum, file) => sum + file.bytes, 0);
164
- log(`content ${set}: ${files.length} files (${mb(bytes)} MB), ${files.length - missing.length} held (read back equal), ${missing.length} to upload (${mb(uploadedBytes)} MB, ${wrong} present with other bytes)`);
188
+ log(`content ${set}: ${files.length} files (${mb(bytes)} MB), ${files.length - written} held (read back equal), ${copied.length} copied on R2 (${mb(copiedBytes)} MB), ${missing.length} to upload (${mb(uploadedBytes)} MB, ${wrong} present with other bytes)`);
165
189
  if (missing.length > 0) {
166
190
  // `wrangler r2 bulk put` stops at the first failed object (a 502 and a 429 were seen); what landed answers a HEAD, so a retry
167
191
  // puts only the rest.
@@ -192,8 +216,95 @@ export async function publishContent({dir, store, scratch, origin = GAME_ORIGIN,
192
216
  await store.put([{key: indexKey, file: indexFile, headers: contentHeaders(indexKey)}]);
193
217
  await readBack(store, indexKey, sha256(index), origin, sleep);
194
218
  } finally { rmSync(indexFile, {force: true}); }
195
- log(`content ${set}: ${missing.length} uploaded (${mb(uploadedBytes)} MB), ${files.length - missing.length} held, all read back with CORS ${origin}; base ${contentBaseOf(set)}`);
196
- return summary(missing.length, uploadedBytes);
219
+ log(`content ${set}: ${missing.length} uploaded (${mb(uploadedBytes)} MB), ${copied.length} copied (${mb(copiedBytes)} MB), ${files.length - written} held, read back with CORS ${origin}; base ${contentBaseOf(set)}`);
220
+ return summary(written, uploadedBytes, copied.length, copiedBytes);
221
+ }
222
+
223
+ /**
224
+ * PZ6: copy each missing file whose bytes a recent complete set holds, server side, under this set's key with this
225
+ * path's headers. Returns the files copied; the rest are left to the upload. Each copy is size-checked by the store, and
226
+ * a sample of them is read back in full through the CDN with the game's CORS.
227
+ * @param {{set: string, missing: readonly ContentFile[], store: ContentStore, origin: string, log: (line: string) => void, sleep: (ms: number) => Promise<void>}} options
228
+ * @returns {Promise<ContentFile[]>}
229
+ */
230
+ export async function copyFromSets({set, missing, store, origin, log, sleep}) {
231
+ const {sets, copy} = store;
232
+ if (sets === undefined || copy === undefined || missing.length === 0) return [];
233
+ const donors = (await sets()).filter(name => name !== set).slice(0, DONOR_SETS);
234
+ /** @type {Map<string, string>} sha256 and size to a donor key, the newest set first */
235
+ const held = new Map();
236
+ for (const donor of donors) {
237
+ const read = await store.read(contentKey(donor, 'content.json'), origin);
238
+ if (read.status !== 200) { log(`content ${set}: donor ${donor}'s index answered ${read.status}; skipped`); continue; }
239
+ /** @type {{files?: {path: string, bytes: number, sha256: string}[]}} */ const index = JSON.parse(Buffer.from(read.bytes).toString('utf8'));
240
+ for (const file of index.files ?? []) {
241
+ const id = `${file.sha256}:${file.bytes}`;
242
+ if (!held.has(id)) held.set(id, contentKey(donor, file.path));
243
+ }
244
+ }
245
+ /** @type {ContentFile[]} */ const copied = [];
246
+ let failed = 0;
247
+ await pool(missing.filter(file => held.has(`${file.sha256}:${file.bytes}`)), 32, async file => {
248
+ const from = held.get(`${file.sha256}:${file.bytes}`) ?? '';
249
+ try { await copy(from, contentKey(set, file.path), contentHeaders(file.path), file.bytes); copied.push(file); }
250
+ catch (error) { failed++; log(`content ${set}: copy ${from} failed (${error instanceof Error ? error.message : String(error)}); uploading it`); }
251
+ });
252
+ copied.sort((a, b) => (a.path < b.path ? -1 : 1));
253
+ // a copy keeps R2's bytes, so a sample proves the headers and the CDN answer; every upload is still read back in full
254
+ const left = [...copied], sample = [];
255
+ while (sample.length < COPY_SAMPLE && left.length > 0) sample.push(...left.splice(randomInt(left.length), 1));
256
+ await pool(sample, 8, file => readBack(store, contentKey(set, file.path), file.decoded, origin, sleep).then(() => undefined));
257
+ if (donors.length > 0) log(`content ${set}: ${copied.length} copied from ${donors.join(', ')}, ${failed} copies failed, ${Math.min(COPY_SAMPLE, copied.length)} read back`);
258
+ return copied;
259
+ }
260
+
261
+ /**
262
+ * PZ6: R2's S3 API on the packs bucket, for the store's `sets` and `copy`. Credentials from the environment
263
+ * (R2_ACCOUNT_ID, R2_PACKS_ACCESS_KEY_ID, R2_PACKS_SECRET_ACCESS_KEY: server-release's bucket-scoped keys), else none.
264
+ * @param {NodeJS.ProcessEnv} env
265
+ * With them the store also puts through S3 (16 at a time), not `wrangler r2 bulk put`, so a CI token needs no R2 write.
266
+ * @returns {{sets: () => Promise<string[]>, copy: (from: string, to: string, headers: Headers, bytes: number) => Promise<void>, put: (puts: readonly Put[]) => Promise<void>, secrets: string[], close: () => void} | undefined}
267
+ */
268
+ export function r2Copies(env) {
269
+ const account = env.R2_ACCOUNT_ID?.trim(), accessKeyId = env.R2_PACKS_ACCESS_KEY_ID?.trim(), secretAccessKey = env.R2_PACKS_SECRET_ACCESS_KEY?.trim();
270
+ if (account === undefined || accessKeyId === undefined || secretAccessKey === undefined || account === '' || accessKeyId === '' || secretAccessKey === '') return undefined;
271
+ if (!/^[0-9a-f]{32}$/u.test(account)) throw new Error('Invalid R2_ACCOUNT_ID');
272
+ const client = new S3Client({region: 'auto', endpoint: `https://${account}.r2.cloudflarestorage.com`, credentials: {accessKeyId, secretAccessKey}});
273
+ return {
274
+ secrets: [accessKeyId, secretAccessKey],
275
+ close: () => client.destroy(),
276
+ async sets() {
277
+ /** @type {string[]} */ const prefixes = [];
278
+ for (let token; ;) {
279
+ const page = await client.send(new ListObjectsV2Command({Bucket: BUCKET, Prefix: `${CONTENT_ROOT}/`, Delimiter: '/', ContinuationToken: token}));
280
+ for (const row of page.CommonPrefixes ?? []) if (row.Prefix !== undefined) prefixes.push(row.Prefix);
281
+ if (page.IsTruncated !== true || page.NextContinuationToken === undefined) break;
282
+ token = page.NextContinuationToken;
283
+ }
284
+ /** @type {{set: string, at: number}[]} */ const complete = [];
285
+ await pool(prefixes, 8, async prefix => {
286
+ const set = prefix.slice(CONTENT_ROOT.length + 1, -1);
287
+ if (!/^[0-9a-f]{16}$/u.test(set)) return;
288
+ try {
289
+ const head = await client.send(new HeadObjectCommand({Bucket: BUCKET, Key: contentKey(set, 'content.json')}));
290
+ complete.push({set, at: head.LastModified?.getTime() ?? 0});
291
+ } catch (error) { if (!(error instanceof Error && (error.name === 'NotFound' || error.name === 'NoSuchKey'))) throw error; }
292
+ });
293
+ return complete.sort((a, b) => b.at - a.at).map(row => row.set);
294
+ },
295
+ async put(puts) {
296
+ await pool(puts, 16, async item => {
297
+ await client.send(new PutObjectCommand({Bucket: BUCKET, Key: item.key, Body: readFileSync(item.file),
298
+ ContentType: item.headers.type, CacheControl: item.headers.cache, ...(item.headers.encoding === undefined ? {} : {ContentEncoding: item.headers.encoding})}));
299
+ });
300
+ },
301
+ async copy(from, to, headers, bytes) {
302
+ await client.send(new CopyObjectCommand({Bucket: BUCKET, Key: to, CopySource: `${BUCKET}/${from.split('/').map(encodeURIComponent).join('/')}`,
303
+ MetadataDirective: 'REPLACE', ContentType: headers.type, CacheControl: headers.cache, ...(headers.encoding === undefined ? {} : {ContentEncoding: headers.encoding})}));
304
+ const head = await client.send(new HeadObjectCommand({Bucket: BUCKET, Key: to}));
305
+ if (head.ContentLength !== bytes) throw new Error(`${to} holds ${head.ContentLength} bytes after the copy, not ${bytes}`);
306
+ },
307
+ };
197
308
  }
198
309
 
199
310
  /**
@@ -219,10 +330,11 @@ export async function sampleContent({dir, store, count, origins, log, sleep = de
219
330
 
220
331
  /**
221
332
  * The real store: HEAD and GET through packs.wildshard.io, writes through `wrangler r2 bulk put`, one call per header set.
222
- * @param {{root: string, scratch: string, token?: string, fetch?: typeof fetch, exec?: typeof runRedacted}} options
333
+ * PZ6: with `copies` (r2Copies) the store also lists complete sets, copies keys server side, and puts through S3.
334
+ * @param {{root: string, scratch: string, token?: string, fetch?: typeof fetch, exec?: typeof runRedacted, copies?: Pick<ContentStore, 'sets' | 'copy'> & {put?: ContentStore['put']}}} options
223
335
  * @returns {ContentStore}
224
336
  */
225
- export function r2Content({root, scratch, token, fetch: get = fetch, exec = runRedacted}) {
337
+ export function r2Content({root, scratch, token, fetch: get = fetch, exec = runRedacted, copies}) {
226
338
  let reads = 0;
227
339
  const bust = (/** @type {string} */ key) => {
228
340
  const url = new URL(`/${key}`, PACKS_ORIGIN);
@@ -231,6 +343,7 @@ export function r2Content({root, scratch, token, fetch: get = fetch, exec = runR
231
343
  };
232
344
  const secrets = token === undefined || token === '' ? [] : [token];
233
345
  return {
346
+ ...(copies?.sets === undefined || copies.copy === undefined ? {} : {sets: copies.sets, copy: copies.copy}),
234
347
  async exists(key) {
235
348
  for (let attempt = 0; ; attempt++) {
236
349
  const response = await get(bust(key), {method: 'HEAD', cache: 'no-store', redirect: 'error', signal: AbortSignal.timeout(30_000)}).catch((/** @type {unknown} */ error) => { if (attempt < 3) return undefined; throw error; });
@@ -241,7 +354,7 @@ export function r2Content({root, scratch, token, fetch: get = fetch, exec = runR
241
354
  await delay(2_000);
242
355
  }
243
356
  },
244
- put(puts) {
357
+ put: copies?.put ?? (puts => {
245
358
  /** @type {Map<string, Put[]>} */
246
359
  const groups = new Map();
247
360
  for (const item of puts) {
@@ -260,7 +373,7 @@ export function r2Content({root, scratch, token, fetch: get = fetch, exec = runR
260
373
  } finally { rmSync(list, {force: true}); }
261
374
  }
262
375
  return Promise.resolve();
263
- },
376
+ }),
264
377
  async read(key, origin) {
265
378
  for (let attempt = 0; ; attempt++) {
266
379
  try {
@@ -277,10 +390,11 @@ async function main() {
277
390
  const value = (/** @type {string} */ name) => args.find(arg => arg.startsWith(`--${name}=`))?.split('=').slice(1).join('=');
278
391
  const dir = args.find(arg => !arg.startsWith('--'));
279
392
  if (dir === undefined) throw new Error('usage: node scripts/page-release/content.mjs <dist> [--origin=<page origin>] [--sample=<n>] [--dry-run]');
280
- const root = resolve(import.meta.dirname, '../..'), token = process.env.CLOUDFLARE_API_TOKEN;
281
- const hide = (/** @type {string} */ text) => token === undefined || token === '' ? text : text.split(token).join('<token>');
393
+ const root = resolve(import.meta.dirname, '../..'), token = process.env.CLOUDFLARE_API_TOKEN, copies = r2Copies(process.env);
394
+ const secrets = [...(token === undefined || token === '' ? [] : [token]), ...copies?.secrets ?? []];
395
+ const hide = (/** @type {string} */ text) => secrets.reduce((out, secret) => out.split(secret).join('<token>'), text);
282
396
  const log = (/** @type {string} */ line) => console.log(hide(line));
283
- const scratch = join(root, 'tmp/page-release/content'), store = r2Content({root, scratch, token});
397
+ const scratch = join(root, 'tmp/page-release/content'), store = r2Content({root, scratch, token, copies});
284
398
  const sample = value('sample');
285
399
  try {
286
400
  if (sample !== undefined) {
@@ -293,7 +407,7 @@ async function main() {
293
407
  } catch (error) {
294
408
  console.error(hide(`content failed: ${error instanceof Error ? error.message : String(error)}`));
295
409
  process.exitCode = 1;
296
- }
410
+ } finally { copies?.close(); }
297
411
  }
298
412
 
299
413
  if (process.argv[1] !== undefined && resolve(process.argv[1]) === import.meta.filename) await main();
@@ -78,7 +78,7 @@ export class RefusedError extends Error {
78
78
  * Parts first, the book last; an existing object must already have the same bytes.
79
79
  * @property {(context: {root: string, build: string, cors: string}) => Promise<PacksSummary>} verify Read-only.
80
80
  *
81
- * @typedef {{set: string, base: string, files: number, bytes: number, uploaded: number, uploadedBytes: number, held: number, cors: string}} ContentSummary
81
+ * @typedef {{set: string, base: string, files: number, bytes: number, uploaded: number, uploadedBytes: number, held: number, copied?: number, copiedBytes?: number, cors: string}} ContentSummary
82
82
  * @typedef {object} ContentPublisher
83
83
  * @property {(context: {dir: string, cors: string, dryRun: boolean}) => Promise<ContentSummary>} publish
84
84
  * PAGE-SIZE PZ2: the page's shard content and engine art to the packs bucket, read back with CORS (content.mjs).
@@ -90,7 +90,7 @@ export class RefusedError extends Error {
90
90
  * @property {PageHost} host
91
91
  * @property {PackStore} packs
92
92
  * @property {ContentPublisher} [content] PZ2: publish the page's content set before the page goes up (page-release.mjs
93
- * --content). Nothing stamps its base into the page yet: PZ4 and PZ6 do.
93
+ * --content). PZ6: the build then stamps the set's base into the page and keeps the split (dist-content/ is published).
94
94
  * @property {(file: string, args: string[], env: Record<string, string>) => void} run
95
95
  * @property {(url: string) => Promise<unknown>} fetchJSON 200 only, no cache, no redirects.
96
96
  * @property {(url: string) => Promise<string>} fetchText
@@ -146,17 +146,32 @@ export async function runPhase(phase, deps) {
146
146
  for (const line of deps.host.describe('prepare', context)) plan(line);
147
147
  if (!dry) await deps.host.prepare(context);
148
148
  plan(`run pnpm build:game with ${channel.solo ? 'no release file (solo)' : `WILDSHARD_RELEASE_URL=${env.WILDSHARD_RELEASE_URL}`} WILDSHARD_CATALOGUE=${env.WILDSHARD_CATALOGUE} WS_BUILD_ID=${deps.build} WS_CHANNEL=${channel.channel}`);
149
- if (!dry) deps.run('pnpm', ['build:game'], {...env, ...deps.host.buildEnv(context)});
150
- // PAGE-SIZE PZ4: build:game splits the page (dist/) from its content (dist-content/). No channel stamps a contentBase
151
- // yet, so each page still serves its own content: join the two before packaging, until PZ8 ships the split.
152
- plan('join dist-content/ back into dist/ (no channel has a contentBase yet)');
153
- if (!dry) deps.run('node', ['scripts/app-build/splitOutput.mjs', 'dist', '--join'], {});
149
+ // PZ5/PZ6: a build that publishes its content names its code only in deploy-chunks.json (`content: true`), so the
150
+ // next release carries its code only (app-build-pipeline.mjs reads WS_CONTENT_BASE)
151
+ const contentEnv = deps.content === undefined ? {} : {WS_CONTENT_BASE: `${PACKS_ORIGIN}/content/`};
152
+ if (!dry) deps.run('pnpm', ['build:game'], {...env, ...contentEnv, ...deps.host.buildEnv(context)});
153
+ // PAGE-SIZE PZ4: build:game splits the page (dist/) from its content (dist-content/). A release that publishes its
154
+ // content (--content, PZ6) stamps the set's base into the page and keeps the split; one that doesn't joins the two, so
155
+ // the page serves its own content.
156
+ if (deps.content === undefined) {
157
+ plan('join dist-content/ back into dist/ (no content base: the page serves its own content)');
158
+ if (!dry) deps.run('node', ['scripts/app-build/splitOutput.mjs', 'dist', '--join'], {});
159
+ } else {
160
+ plan(`stamp dist/index.html with the content set's base on ${PACKS_ORIGIN}/content/ and bake its service worker again (PZ6)`);
161
+ if (!dry) deps.run('node', ['--import', './scripts/bake-loader.mjs', 'scripts/page-release/stamp-content.mjs', 'dist'], {});
162
+ }
154
163
  for (const line of deps.host.describe('package', context)) plan(line);
155
164
  if (!dry) await deps.host.package(context);
156
165
  plan(`check ${deps.host.outputDir(context)} names ${channel.solo ? 'no release file' : channel.release} and build ${deps.build}`);
157
166
  if (dry) return {refusals};
158
167
  const files = checkOutput(channel, deps, context, refuse);
159
- deps.state.write('build', {build: deps.build, channel: channel.channel, host: deps.host.name, files});
168
+ /** @type {string|undefined} */
169
+ let contentBase;
170
+ if (deps.content !== undefined) {
171
+ contentBase = contentBaseIn(Buffer.from(deps.readFile(`${deps.host.outputDir(context)}/index.html`)).toString('utf8'));
172
+ if (contentBase === undefined || !contentBase.startsWith(`${PACKS_ORIGIN}/content/`)) refuse(`index.html content base is ${contentBase ?? 'missing'}, not a set on ${PACKS_ORIGIN}`);
173
+ }
174
+ deps.state.write('build', {build: deps.build, channel: channel.channel, host: deps.host.name, files, ...(contentBase === undefined ? {} : {contentBase})});
160
175
  deps.log(`PASS ${channel.channel} page built for ${deps.build}; nothing uploaded, alias not moved`);
161
176
  return {refusals};
162
177
  }
@@ -172,9 +187,11 @@ export async function runPhase(phase, deps) {
172
187
  /** @type {ContentSummary|undefined} */
173
188
  let content;
174
189
  if (deps.content !== undefined) {
175
- plan(`publish the page's content (S1 to S7, E1) from ${deps.host.outputDir(context)} to ${PACKS_ORIGIN}/content/<set>/ with CORS ${channel.origin} (PZ2, before the page)`);
176
- content = await deps.content.publish({dir: deps.host.outputDir(context), cors: channel.origin, dryRun: dry});
190
+ const contentDir = `${deps.host.outputDir(context)}-content`;
191
+ plan(`publish the page's content (S1 to S7, E1) from ${contentDir} to ${PACKS_ORIGIN}/content/<set>/ with CORS ${channel.origin} (PZ2, before the page)`);
192
+ content = await deps.content.publish({dir: contentDir, cors: channel.origin, dryRun: dry});
177
193
  if (!dry && content.held + content.uploaded !== content.files) refuse(`content: ${content.held} held + ${content.uploaded} uploaded is not ${content.files}`);
194
+ if (!dry && content.base !== built.contentBase) refuse(`content: published ${content.base}, but the page is stamped with ${built.contentBase ?? 'no base'}`);
178
195
  }
179
196
  /** @type {PacksSummary|undefined} */
180
197
  let packs;
@@ -261,6 +278,15 @@ export function buildEnv(channel, deps) {
261
278
  : {...common, WILDSHARD_RELEASE_URL: channel.release, WILDSHARD_RELEASE_KEY: deps.releaseKey ?? RELEASE_PUBLIC_KEY};
262
279
  }
263
280
 
281
+ /** PZ6: the boot bank's `contentBase` in a built document (undefined when it has none, or it is empty).
282
+ * @param {string} html @returns {string|undefined} */
283
+ export function contentBaseIn(html) {
284
+ const bank = /<script type="application\/json" id="boot-module-files">([^<]+)<\/script>/u.exec(html)?.[1];
285
+ if (bank === undefined) return undefined;
286
+ const base = JSON.parse(bank).contentBase;
287
+ return typeof base === 'string' && base !== '' ? base : undefined;
288
+ }
289
+
264
290
  /**
265
291
  * The packaged page names this build, this channel's release file and key, and never the other channel's file.
266
292
  * @param {ChannelConfig} channel @param {Deps} deps @param {HostContext} context @param {(message: string) => void} refuse
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env node
2
+ // PAGE-SIZE PZ6 (meta docs/plans/PAGE-SIZE-PLAN.md, J276): a split page (`dist/`, its content in `dist-content/`) reads its
3
+ // content from the CDN. The content set is named by the files content.mjs publishes from `dist-content/`, so the base is
4
+ // known before the upload: stamp it into `dist/index.html`, then bake the service worker again from the stamped document
5
+ // (content-sw-proof's order; the worker's precache lists move to CDN URLs). page-release.mjs runs this in the build phase
6
+ // when it publishes content, and its upload phase refuses a published set whose base differs from the stamped one.
7
+ //
8
+ // node --import ./scripts/bake-loader.mjs scripts/page-release/stamp-content.mjs <dist>
9
+ import {existsSync, readFileSync, writeFileSync} from 'node:fs';
10
+ import {join, resolve} from 'node:path';
11
+ import {stampContentBase} from '../app-build/contentBase.mjs';
12
+ import {contentBaseOf, contentSet, scanContent} from './content.mjs';
13
+
14
+ const dist = process.argv[2];
15
+ if (dist === undefined) throw new Error('usage: node --import ./scripts/bake-loader.mjs scripts/page-release/stamp-content.mjs <dist>');
16
+ const page = resolve(dist), content = `${page}-content`, root = resolve(import.meta.dirname, '../..');
17
+ if (!existsSync(content)) throw new Error(`${content} is missing: stamp a split page (pnpm build:game splits it)`);
18
+ const files = scanContent(content);
19
+ if (files.length === 0) throw new Error(`${content} holds no content`);
20
+ const base = contentBaseOf(contentSet(files));
21
+ writeFileSync(join(page, 'index.html'), stampContentBase(readFileSync(join(page, 'index.html'), 'utf8'), base));
22
+ const {writeServiceWorker} = await import('../app-build/serviceWorker.ts');
23
+ await writeServiceWorker(page, root);
24
+ console.log(`stamped ${dist}/index.html with ${base} (${files.length} content files) and baked its service worker again`);
@@ -10,21 +10,27 @@
10
10
  // --dry-run print every step and read the live gate; build, upload and alias nothing
11
11
  // --wait=<minutes> check: how long to wait for the signed serial and /health (default 45)
12
12
  // --after-staging-proofs production go-live: the strict staging proofs passed on this exact build
13
- // --content upload: publish the page's content set to packs.wildshard.io first (PAGE-SIZE PZ2,
14
- // page-release/content.mjs); off until PZ4 and PZ6 stamp contentBase into the page
13
+ // --content build: stamp the content set's base into dist/ and keep the split (PZ6); upload: publish
14
+ // dist-content/ to packs.wildshard.io first (PAGE-SIZE PZ2,
15
+ // page-release/content.mjs); preview.yml passes it (PZ6), staging and production with PZ8
15
16
  // --catalogue=<file> preview build: bake this catalogue, the grid borrowed from staging (SHARD-AUTHORING SA35,
16
17
  // scripts/preview-grid.ts); default test/pins/catalogue.json. Refused on staging and production
17
18
  //
19
+ // node scripts/page-release.mjs --print-env <preview|staging|production> [--root=<dir>] [--catalogue=<file>]
20
+ // print the channel's build environment for the root's HEAD (--catalogue on preview only),
21
+ // one sorted KEY=value a line, and run nothing (PAGE-SIZE PZ6: the three channels' builds
22
+ // differ only in named keys)
23
+ //
18
24
  // The flow and its gates are scripts/page-release/flow.mjs; the page hosts are scripts/page-release/hosts/*.mjs.
19
25
  // Phase results go to <root>/tmp/page-release/<channel>-<sha>.json, which the next phase reads.
20
26
  import {execFileSync} from 'node:child_process';
21
27
  import {setTimeout as delay} from 'node:timers/promises';
22
28
  import {existsSync, mkdirSync, readFileSync, writeFileSync} from 'node:fs';
23
29
  import {join, resolve} from 'node:path';
24
- import {RefusedError, runPhase} from './page-release/flow.mjs';
30
+ import {CHANNELS, RefusedError, buildEnv, runPhase} from './page-release/flow.mjs';
25
31
  import {workersHost} from './page-release/hosts/workers.mjs';
26
32
  import {r2Packs} from './page-release/packs.mjs';
27
- import {publishContent, r2Content} from './page-release/content.mjs';
33
+ import {publishContent, r2Content, r2Copies} from './page-release/content.mjs';
28
34
 
29
35
  const args = process.argv.slice(2);
30
36
  const flag = (/** @type {string} */ name) => args.find(arg => arg === `--${name}` || arg.startsWith(`--${name}=`));
@@ -32,6 +38,19 @@ const value = (/** @type {string} */ name) => flag(name)?.split('=').slice(1).jo
32
38
  /** Every channel is on Cloudflare Workers (OFF-VERCEL OV6 preview, OV7 staging, OV8 production), so `--host` has one value. */
33
39
  const [channel, phaseList] = args.filter(arg => !arg.startsWith('--'));
34
40
  const hostName = value('host') ?? 'workers';
41
+ if (flag('print-env') !== undefined) {
42
+ const config = channel !== undefined && Object.hasOwn(CHANNELS, channel) ? Reflect.get(CHANNELS, channel) : undefined;
43
+ const catalogue = value('catalogue');
44
+ if (config === undefined || phaseList !== undefined || (catalogue !== undefined && !config.solo)) {
45
+ console.error('usage: node scripts/page-release.mjs --print-env <preview|staging|production> [--root=<dir>] [--catalogue=<file>, preview only]');
46
+ process.exit(2);
47
+ }
48
+ const cwd = resolve(value('root') ?? '.');
49
+ const head = execFileSync('git', ['rev-parse', 'HEAD'], {cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe']}).trim();
50
+ const env = buildEnv(config, /** @type {any} */ ({build: head, ...(catalogue === undefined ? {} : {catalogue: resolve(catalogue)})}));
51
+ for (const name of Object.keys(env).sort()) console.log(`${name}=${env[name]}`);
52
+ process.exit(0);
53
+ }
35
54
  if (channel === undefined || phaseList === undefined || hostName !== 'workers') {
36
55
  console.error('usage: node scripts/page-release.mjs <preview|staging|production> <build|upload|check|go-live>[,...] [--host=workers] [--root=<dir>] [--dry-run] [--wait=<minutes>] [--after-staging-proofs] [--content] [--catalogue=<file>]');
37
56
  process.exit(2);
@@ -56,6 +75,9 @@ async function fetch200(url) {
56
75
  }
57
76
 
58
77
  const host = workersHost({token: () => cloudflareToken, log});
78
+ // PZ6: a content publish copies unchanged files from an older set on R2 when the bucket's S3 keys are in the environment
79
+ const contentCopies = flag('content') === undefined ? undefined : r2Copies(process.env);
80
+ if (contentCopies !== undefined) secrets.push(...contentCopies.secrets);
59
81
 
60
82
  let phase = '';
61
83
  try {
@@ -65,7 +87,7 @@ try {
65
87
  packs: r2Packs({log}),
66
88
  ...(flag('content') === undefined ? {} : {content: {
67
89
  publish: ({dir, cors, dryRun}) => publishContent({dir, origin: cors, dryRun, log,
68
- scratch: join(root, 'tmp/page-release/content'), store: r2Content({root, scratch: join(root, 'tmp/page-release/content'), token: cloudflareToken})}),
90
+ scratch: join(root, 'tmp/page-release/content'), store: r2Content({root, scratch: join(root, 'tmp/page-release/content'), token: cloudflareToken, copies: contentCopies})}),
69
91
  }}),
70
92
  run: (file, argv, env) => { execFileSync(file, argv, {cwd: root, env: {...process.env, ...env}, stdio: 'inherit'}); },
71
93
  fetchJSON: async url => (await fetch200(url)).json(),
@@ -102,4 +124,4 @@ try {
102
124
  console.error(redact(error instanceof RefusedError ? error.message : `${channel} ${phase} failed: ${error instanceof Error ? error.message : String(error)}`));
103
125
  console.error('the alias did not move unless an ALIAS MOVED line is above');
104
126
  process.exitCode = 1;
105
- }
127
+ } finally { contentCopies?.close(); }
@@ -152,7 +152,8 @@ export const PAGE_LIMITS = Object.freeze({files: 1000, bytes: 100e6});
152
152
  export function gatePage(report, limits = PAGE_LIMITS) {
153
153
  const errors = [], zero = tally(), maps = report.kinds.maps ?? zero;
154
154
  const content = [report.kinds['shard content'] ?? zero, report.kinds['engine art'] ?? zero];
155
- const contentFiles = content.reduce((sum, row) => sum + row.files, 0), contentBytes = content.reduce((sum, row) => sum + row.bytes, 0);
155
+ // a kept build's content stays on the page until it ages out (splitOutput): only this build's content is refused
156
+ const contentFiles = content.reduce((sum, row) => sum + row.files - row.carriedFiles, 0), contentBytes = content.reduce((sum, row) => sum + row.bytes - row.carriedBytes, 0);
156
157
  if (contentFiles > 0) errors.push(`page-size gate: dist/ holds ${n(contentFiles)} content files (${mb(contentBytes)} MB); shard content and engine art go to dist-content/ (node scripts/app-build/splitOutput.mjs)`);
157
158
  const pageFiles = report.total.files - maps.files, bytes = report.total.bytes - maps.bytes;
158
159
  if (pageFiles > limits.files) errors.push(`page-size gate: the page holds ${n(pageFiles)} files, past the ${n(limits.files)} a page may hold (PAGE-SIZE D1)`);
@@ -1,6 +1,6 @@
1
1
  // C0: a tarball leaves its packs out, because every member keeps its own path. Whoever serves a product rebuilds them, and a
2
2
  // pinned tree serves its products to the page and to the grid bake.
3
- import { packFromMembers } from '@wildshard/format/product-reads';
3
+ import { packFromMembers } from '@wildshard/format/shardfile-reads';
4
4
 
5
5
  type Members = Parameters<typeof packFromMembers>[1];
6
6
  type Pack = Parameters<typeof packFromMembers>[0];
package/scripts/pins.ts CHANGED
@@ -15,7 +15,7 @@
15
15
  //
16
16
  // `install` is what a build runs first. For each pin, in order:
17
17
  // 1. The tarball comes from the cache, the mirror or the API, and its sha256 has to be the pin.
18
- // 2. This tree's format validator admits it: files, hashes, schemas, budgets (`validateShardfileProduct`).
18
+ // 2. This tree's format validator admits it: files, hashes, schemas, budgets (`validateShardfileFolder`).
19
19
  // 3. Its product goes to `out/<folder>`, with any pack a tarball left out rebuilt from its members, so a page and the
20
20
  // grid bake read what they would from a build. Its receipt is issued under this tree's validator revision, by this file, only
21
21
  // after step 2 passed, as `productVerdict` earns one for a head a build never made. The cost numbers stay the
@@ -237,8 +237,8 @@ async function installLocked(only: readonly string[], source: 'pins' | 'format'
237
237
  // The validator hashes most members already; the manifest reuses those digests rather than hashing twice.
238
238
  const digests = new WeakMap<Uint8Array, string>();
239
239
  const digest = (bytes: Uint8Array): string => { let held = digests.get(bytes); if (held === undefined) { held = sha256(bytes); digests.set(bytes, held); } return held; };
240
- const { prepareAssetDecoders } = await import('@wildshard/format/product');
241
- const { validateShardfileProduct } = await import('@wildshard/format/productValidation');
240
+ const { prepareAssetDecoders } = await import('@wildshard/format/shardfileAssets');
241
+ const { validateShardfileFolder } = await import('@wildshard/format/shardfileValidation');
242
242
  const { readTar } = await import('@wildshard/format/tar');
243
243
  const { restorePacks } = await import('./pin-packs.ts');
244
244
  await prepareAssetDecoders();
@@ -286,7 +286,7 @@ async function installLocked(only: readonly string[], source: 'pins' | 'format'
286
286
  const started = performance.now(), files = readTar(bytes);
287
287
  const fileNamed = (path: string): Uint8Array => { const body = files.get(path); if (body === undefined) throw new Error(`${pin.slug}: no ${path}`); return body; };
288
288
  try {
289
- const { warnings } = validateShardfileProduct(
289
+ const { warnings } = validateShardfileFolder(
290
290
  { read: (path, maximum) => { const body = fileNamed(path); if (body.length > maximum) throw new Error(`${path} is over its cap`); return body; }, paths: () => files.keys() },
291
291
  { hash: digest },
292
292
  );
@@ -72,7 +72,7 @@ export async function stagedRatchet(files, executable, repository) {
72
72
  if (repository === undefined) throw new Error('Missing HEAD repository for staged ratchet comparison');
73
73
  const head = resolve('.commit-head');
74
74
  const prepared = spawnSync('python3', [resolve(import.meta.dirname, 'precommit-source.py'), repository, head,
75
- String(Date.now() / 1000 + 5), 'HEAD', JSON.stringify({ files: missing, guards: [], closed: false, deleted: false })],
75
+ String(Date.now() / 1000 + 5), 'HEAD', JSON.stringify({ files: missing, guards: [], deleted: false })],
76
76
  { encoding: 'utf8', maxBuffer: 1 << 30 });
77
77
  if (prepared.error !== undefined) throw prepared.error;
78
78
  if (prepared.status !== 0) throw new Error(`HEAD ratchet source failed: ${prepared.stderr}`);
@@ -104,12 +104,6 @@ if (import.meta.main) {
104
104
  const { deletedReferences } = await import('./push-checks.mjs');
105
105
  const errors = deletedReferences(process.cwd(), JSON.parse(readFileSync('.commit-deleted.json', 'utf8')));
106
106
  if (errors.length > 0) throw new Error(errors.join('\n'));
107
- } else if (mode === 'closed') {
108
- if (!existsSync('lint/ratchet.json')) throw new Error('Missing staged closed-component policy');
109
- const policy = JSON.parse(readFileSync('lint/ratchet.json', 'utf8'));
110
- const { closedComponentSources } = await import('./check-closed-components.mjs');
111
- const errors = closedComponentSources(process.cwd(), policy.closedComponents);
112
- if (errors.length > 0) throw new Error(errors.join('\n'));
113
107
  } else throw new Error('Unknown pre-commit guard');
114
108
  } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }
115
109
  }
@@ -69,7 +69,6 @@ export function commitGuards(changed) {
69
69
  ...(touches(/^src\/(?:format|sdk|sharddev)\//u) ? ['test/schema-reference.test.ts'] : []),
70
70
  ...(touches(/^(?:lint\/(?:host-callbacks\.mjs|wildshard-plugin\.js)|scripts\/gen-host-catalogue\.mjs|test\/host-callbacks\.test\.ts)$/u) ? ['test/host-callbacks.test.ts'] : []),
71
71
  ...(touches(/^(?:lint\/(?:shardfile-boundaries\.mjs|wildshard-plugin\.js)|test\/shardfile-boundaries\.test\.ts)$/u) ? ['test/shardfile-boundaries.test.ts'] : []),
72
- ...(touches(/^(?:src\/(?:engine|game|format|sdk|sharddev)\/|scripts\/(?:platform-chargebacks|inline-platform|host-approvals|g571-hosts|shard-components|shard-platform)[^/]*\.|docs\/HOST-BAR\.md$|test\/platform-chargebacks\.test\.ts$)/u) ? ['test/platform-chargebacks.test.ts'] : []),
73
72
  ...(touches(/^(?:src\/shards\/(?:nalati-grasslands|nine-dragon-stack)\/|src\/(?:client|cook|server|shared)\/game\/shard\/manifest\.ts$|scripts\/(?:authored-manifest|manifest-declaration)\.|test\/(?:manifest-declarations\.test\.ts|fixtures\/manifest-declarations\.json)$)/u) ? ['test/manifest-declarations.test.ts'] : []),
74
73
  ];
75
74
  }
@@ -160,14 +160,6 @@ def required_sources(blobs, specification, tree=None, executable=None, staged_pa
160
160
  if specification.get('deleted'):
161
161
  # Every active tracked reader is an input to the deleted-path grep.
162
162
  selected.update(name for name in blobs if name.split('/')[0] in ROOTS)
163
- if specification.get('closed'):
164
- for name in blobs:
165
- path = PurePosixPath(name)
166
- if len(path.parts) > 3 and path.parts[:2] == ('src', 'shards') and not path.parts[2].endswith('-legacy'):
167
- if path.suffix in {'.ts', '.tsx', '.glsl'} and '/generators/' not in name and not re.search(r'\.d\.ts$|\.(?:test|spec)\.tsx?$|/test/', name):
168
- selected.add(name)
169
- selected.update(name for name in blobs if re.search(r'/(?:generators/)?bake-[^/]+\.(?:mjs|ts)$', name))
170
- selected.add('scripts/source-generation-outputs.json')
171
163
  if 'test/schema-reference.test.ts' in specification.get('guards', []):
172
164
  for layer in ['sdk', 'sharddev']:
173
165
  manifest = json.loads(blobs.get(f'src/{layer}/package.json', b'{}'))
@@ -212,7 +204,7 @@ def required_sources(blobs, specification, tree=None, executable=None, staged_pa
212
204
  data = blobs[name] if name in blobs else (tree / name).read_bytes()
213
205
  if PurePosixPath(name).suffix not in {'.ts', '.tsx', '.mts', '.js', '.mjs', '.cjs', '.json'}:
214
206
  continue
215
- if name.endswith('.json') and name != 'lint/platform-pending-users.json':
207
+ if name.endswith('.json'):
216
208
  continue
217
209
  text = data.decode('utf8')
218
210
  imported = set() if name.endswith('.json') else imports(name, text)
@@ -3,7 +3,7 @@ import { resolve } from 'node:path';
3
3
  import { createHash } from 'node:crypto';
4
4
  import * as v from 'valibot';
5
5
  import { readBuiltShardfile } from './built-product.mjs';
6
- import { discoveryFormat, outputFormat } from './shard-platform-format.mjs';
6
+ import { discoveryFormat, outputFormat } from './node-format.mjs';
7
7
  import { encodeCanonicalJson } from '@wildshard/engine/host/immutable';
8
8
 
9
9
  /** @param {unknown} value @returns {value is Record<string, unknown>} */
@@ -11,6 +11,7 @@ const { createCookWorker, indexKey } = await import('../src/cook/game/cookWorker
11
11
  const { folderStore } = await import('../src/cook/game/packStore.ts');
12
12
  const { cookRevision } = await import('./cook-revision.ts');
13
13
  const { rendererRevision } = await import('./renderer-revision.mjs');
14
+ const { shardfileValidationRevision } = await import('./shardfile-validation-revision.mjs');
14
15
  const { values } = parseArgs({ options: { out: {type:'string',default:'tmp/ring-cook'}, build: {type:'string',default:'ring-local'},
15
16
  only: {type:'string'}, ring2: {type:'boolean',default:false}, 'timeout-ms': {type:'string',default:'120000'} } });
16
17
  const timeout = Number(values['timeout-ms']);
@@ -31,7 +32,7 @@ const heads = selected.map(slug => {
31
32
  });
32
33
  const store = folderStore(join(out,'store'));
33
34
  // A fresh run owns its empty cache. At least its first product always encodes cold.
34
- const worker = createCookWorker({ build: values.build, store, serverPacks: join(out,'server'), scratch: join(out,'jobs'),
35
+ const worker = createCookWorker({ build: values.build, store, serverPacks: join(out,'server'), scratch: join(out,'jobs'), validator: shardfileValidationRevision(root),
35
36
  source: { heads: () => Promise.resolve(heads.map(row => ({...row, revision:1}))), download: hash => Promise.resolve(readFileSync(join(cache,`${hash}.tar`))) },
36
37
  launcher: { root, execPath: process.execPath, loader:join(root,'scripts/bake-loader.mjs'), script:join(root,'scripts/cook-job.mjs'),
37
38
  wrap:[], limits:{seconds:timeout/1000,memoryMB:4096,heapMB:2048}, renderer: rendererRevision(root) } });
@@ -27,7 +27,7 @@ const sha256 = (text: string): string => createHash('sha256').update(text).diges
27
27
  const line = (tick: number): HeadlessCommand => ({ kind: 'player', moveX: 0.3 * Math.sin(tick / 90), moveZ: -1, yaw: 0, ...(tick % 47 === 0 ? { jump: true as const } : {}), ...(tick % 97 === 0 ? { dodge: true as const } : {}) });
28
28
  const only = option('only')?.split(',');
29
29
 
30
- const { prepareAssetDecoders } = await import('@wildshard/format/product');
30
+ const { prepareAssetDecoders } = await import('@wildshard/format/shardfileAssets');
31
31
  const { loadRapier } = await import('../src/shared/engine/physics/rapier.ts');
32
32
  const { loadHeadlessProduct } = await import('../src/server/host/headlessProduct.ts');
33
33
  const { createTrustedHeadlessResident } = await import('../src/server/host/headlessRuntime.ts');
@@ -4,7 +4,7 @@ import {existsSync,readdirSync,readFileSync,statSync} from 'node:fs';
4
4
  import {relative,resolve} from 'node:path';
5
5
  import {createHash} from 'node:crypto';
6
6
  import * as v from 'valibot';
7
- import {outputFormat} from './shard-platform-format.mjs';
7
+ import {outputFormat} from './node-format.mjs';
8
8
  import {formatSource} from './format-package.mjs';
9
9
 
10
10
  const hash=v.pipe(v.string(),v.regex(/^[a-f0-9]{64}$/u)),natural=v.pipe(v.number(),v.safeInteger(),v.minValue(0));
@@ -1,4 +1,4 @@
1
- import type {OutputReader} from './shard-platform-output.mjs';
1
+ import type {OutputReader} from './output-reader.mjs';
2
2
 
3
3
  export interface OutputContent {status:'built';product:string;privateBinaries:string[];privateBinaryBytes:number;tiles:{total:number;l0:number;l1:number};gate:{enforcement:'report-only';status:'candidate'|'incomplete';issues:string[]}}
4
4
  export function outputContent(root:string,slug:string,products:string,reader?:OutputReader):OutputContent;