@underpostnet/cyberia 3.3.73 → 3.3.76

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/.env.example +17 -5
  2. package/.github/workflows/coverall.cyberia.ci.yml +5 -0
  3. package/.github/workflows/docker-image.cyberia-client.ci.yml +1 -1
  4. package/.github/workflows/docker-image.cyberia-client.dev.ci.yml +1 -1
  5. package/.github/workflows/docker-image.cyberia-server.ci.yml +1 -1
  6. package/.github/workflows/docker-image.cyberia-server.dev.ci.yml +1 -1
  7. package/.github/workflows/docker-image.engine-cyberia.ci.yml +1 -1
  8. package/.github/workflows/docker-image.engine-cyberia.dev.ci.yml +1 -1
  9. package/.github/workflows/publish.cyberia.ci.yml +44 -0
  10. package/AGENTS.md +61 -12
  11. package/CHANGELOG.md +126 -41
  12. package/CLI-HELP.md +19 -12
  13. package/Dockerfile +1 -1
  14. package/Dockerfile.dev +1 -1
  15. package/Dockerfile.test +2 -2
  16. package/README.md +28 -28
  17. package/bin/build.js +37 -17
  18. package/bin/cyberia.js +1247 -1373
  19. package/bin/index.js +1247 -1373
  20. package/compose.env +17 -5
  21. package/conf.js +7 -3
  22. package/deploy/cyberia-client/deploy.sh +10 -11
  23. package/deploy/cyberia-client/package.sh +0 -1
  24. package/deploy/cyberia-client/state.sh +3 -2
  25. package/deploy/cyberia-server/deploy.sh +10 -11
  26. package/deploy/cyberia-server/package.sh +0 -1
  27. package/deploy/cyberia-server/state.sh +3 -2
  28. package/deploy/dd-cyberia/deploy.sh +5 -5
  29. package/deploy/dd-cyberia/init.sh +15 -14
  30. package/deploy/dd-cyberia/package.sh +0 -1
  31. package/deploy/dd-cyberia/state.sh +2 -2
  32. package/deploy/dd-cyberia/sync-deploy.sh +139 -75
  33. package/deploy/lib/config.sh +29 -0
  34. package/deploy/lib/github-actions-logging.sh +92 -76
  35. package/deploy/lib/host.sh +27 -18
  36. package/deploy/lib/state.sh +4 -1
  37. package/deploy/pwa-microservices-template/deploy.sh +10 -8
  38. package/deploy/release/deploy.sh +0 -2
  39. package/deployment.yaml +1 -1
  40. package/docker-compose.yml +23 -17
  41. package/hardhat/package-lock.json +2 -2
  42. package/hardhat/package.json +1 -1
  43. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
  44. package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +1 -1
  45. package/manifests/deployment/dd-cyberia-development/deployment.yaml +1 -1
  46. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  47. package/manifests/deployment/mongo-express/deployment.yaml +46 -27
  48. package/manifests/deployment/mongo-express/kustomization.yaml +9 -0
  49. package/manifests/deployment/mongo-express/mongo-express-nodeport.yaml +18 -0
  50. package/manifests/deployment/mongo-express/service.yaml +16 -0
  51. package/manifests/deployment/mongo-express-no-auth/disable-mongodb-auth.yaml +16 -0
  52. package/manifests/deployment/mongo-express-no-auth/kustomization.yaml +16 -0
  53. package/package.json +19 -17
  54. package/scripts/test-monitor.sh +3 -3
  55. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.controller.js +14 -9
  56. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.model.js +36 -5
  57. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.router.js +5 -0
  58. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.service.js +88 -184
  59. package/src/api/cyberia-action/cyberia-action.router.js +2 -2
  60. package/src/api/cyberia-action/cyberia-action.service.js +3 -4
  61. package/src/api/cyberia-audio/cyberia-audio.controller.js +6 -0
  62. package/src/api/cyberia-audio/cyberia-audio.model.js +61 -0
  63. package/src/api/cyberia-audio/cyberia-audio.router.js +21 -0
  64. package/src/api/cyberia-audio/cyberia-audio.service.js +220 -0
  65. package/src/api/cyberia-client-hints/cyberia-client-hints.controller.js +1 -3
  66. package/src/api/cyberia-client-hints/cyberia-client-hints.model.js +10 -19
  67. package/src/api/cyberia-client-hints/cyberia-client-hints.router.js +12 -34
  68. package/src/api/cyberia-client-hints/cyberia-client-hints.service.js +15 -32
  69. package/src/api/cyberia-dialogue/cyberia-dialogue.model.js +1 -1
  70. package/src/api/cyberia-dialogue/cyberia-dialogue.router.js +1 -1
  71. package/src/api/cyberia-entity/cyberia-entity.model.js +2 -0
  72. package/src/api/cyberia-entity/cyberia-entity.service.js +1 -1
  73. package/src/api/cyberia-entity-type-default/cyberia-entity-type-default.controller.js +8 -2
  74. package/src/api/cyberia-entity-type-default/cyberia-entity-type-default.model.js +31 -20
  75. package/src/api/cyberia-entity-type-default/cyberia-entity-type-default.router.js +16 -1
  76. package/src/api/cyberia-entity-type-default/cyberia-entity-type-default.service.js +466 -3
  77. package/src/api/cyberia-instance/cyberia-fallback-capture.js +87 -51
  78. package/src/api/cyberia-instance/cyberia-fallback-world.js +21 -40
  79. package/src/api/cyberia-instance/cyberia-instance-boot.service.js +5 -5
  80. package/src/api/cyberia-instance/cyberia-instance-items.js +98 -0
  81. package/src/api/cyberia-instance/cyberia-instance-map.service.js +34 -141
  82. package/src/api/cyberia-instance/cyberia-instance.controller.js +1 -4
  83. package/src/api/cyberia-instance/cyberia-instance.model.js +0 -14
  84. package/src/api/cyberia-instance/cyberia-instance.router.js +3 -16
  85. package/src/api/cyberia-instance/cyberia-instance.service.js +3 -43
  86. package/src/api/cyberia-instance/cyberia-world-generator.js +8 -13
  87. package/src/api/cyberia-instance-conf/cyberia-instance-conf.model.js +49 -75
  88. package/src/api/cyberia-instance-conf/cyberia-instance-conf.service.js +74 -1
  89. package/src/api/cyberia-map/cyberia-map.service.js +6 -2
  90. package/src/api/cyberia-map-audio-conf/cyberia-map-audio-conf.controller.js +31 -0
  91. package/src/api/cyberia-map-audio-conf/cyberia-map-audio-conf.model.js +56 -0
  92. package/src/api/cyberia-map-audio-conf/cyberia-map-audio-conf.router.js +33 -0
  93. package/src/api/cyberia-map-audio-conf/cyberia-map-audio-conf.service.js +189 -0
  94. package/src/api/cyberia-quest/cyberia-quest.router.js +3 -3
  95. package/src/api/cyberia-quest/cyberia-quest.service.js +3 -3
  96. package/src/api/cyberia-quest-progress/cyberia-quest-progress.model.js +1 -1
  97. package/src/api/cyberia-server-defaults/cyberia-server-defaults.js +403 -188
  98. package/src/api/cyberia-skill/cyberia-skill.model.js +6 -6
  99. package/src/api/file/file.ref.js +153 -0
  100. package/src/api/file/file.ref.json +11 -2
  101. package/src/api/object-layer/object-layer.model.js +47 -12
  102. package/src/api/object-layer/object-layer.router.js +10 -3
  103. package/src/api/object-layer/object-layer.service.js +113 -120
  104. package/src/cli/app.js +12 -1
  105. package/src/cli/client.js +2 -2
  106. package/src/cli/cluster.js +7 -0
  107. package/src/cli/deploy.js +4 -4
  108. package/src/cli/docker-compose.js +1 -1
  109. package/src/cli/fs-selection.js +115 -0
  110. package/src/cli/fs.js +158 -342
  111. package/src/cli/index.js +22 -10
  112. package/src/cli/repository.js +28 -0
  113. package/src/cli/run.js +71 -197
  114. package/src/cli/test.js +4 -0
  115. package/src/cli/wireguard.js +32 -31
  116. package/src/client/components/core/Docs.js +77 -100
  117. package/src/client/components/core/Modal.js +56 -1
  118. package/src/client/components/cyberia/ActionEngineCyberia.js +12 -36
  119. package/src/client/components/cyberia/EntityEngineCyberia.js +492 -121
  120. package/src/client/components/cyberia/InstanceEngineCyberia.js +129 -160
  121. package/src/client/components/cyberia/InstanceSelectionView.js +7 -1
  122. package/src/client/components/cyberia/MapEngineCyberia.js +175 -69
  123. package/src/client/components/cyberia/ObjectLayerEngine.js +1011 -97
  124. package/src/client/components/cyberia/ObjectLayerEngineModal.js +81 -155
  125. package/src/client/components/cyberia/ObjectLayerEngineViewer.js +113 -98
  126. package/src/client/components/cyberia/SharedDefaultsCyberia.js +215 -56
  127. package/src/client/components/cyberia-portal/AppShellCyberiaPortal.js +0 -34
  128. package/src/client/components/cyberia-portal/CssCyberiaPortal.js +1 -1
  129. package/src/client/components/cyberia-portal/MainBodyCyberiaPortal.js +712 -76
  130. package/src/client/components/cyberia-portal/TranslateCyberiaPortal.js +0 -4
  131. package/src/client/public/cyberia-docs/ARCHITECTURE.md +15 -15
  132. package/src/client/public/cyberia-docs/CYBERIA-CLI.md +280 -24
  133. package/src/client/public/cyberia-docs/CYBERIA-CLIENT.md +51 -29
  134. package/src/client/public/cyberia-docs/CYBERIA-SAGA.md +52 -53
  135. package/src/client/public/cyberia-docs/CYBERIA-SERVER.md +16 -11
  136. package/src/client/public/cyberia-docs/CYBERIA.md +28 -28
  137. package/src/client/public/cyberia-docs/ENTITY-PROFILE.md +11 -5
  138. package/src/client/public/cyberia-docs/ROADMAP.md +1 -1
  139. package/src/client/public/cyberia-docs/STATS-PROGRESSION.md +217 -0
  140. package/src/client/public/cyberia-docs/WHITE-PAPER.md +20 -12
  141. package/src/client/services/cyberia-audio/cyberia-audio.service.js +99 -0
  142. package/src/client/services/cyberia-entity-type-default/cyberia-entity-type-default.service.js +38 -0
  143. package/src/client/services/cyberia-instance/cyberia-instance.management.js +5 -4
  144. package/src/client/services/cyberia-instance/cyberia-instance.service.js +0 -40
  145. package/src/client/services/cyberia-map/cyberia-map.management.js +5 -4
  146. package/src/client/services/cyberia-map-audio-conf/cyberia-map-audio-conf.service.js +138 -0
  147. package/src/client/services/object-layer/object-layer.management.js +6 -10
  148. package/src/client/services/object-layer/object-layer.service.js +3 -1
  149. package/src/client-builder/client-build-docs.js +52 -52
  150. package/src/client-builder/client-build.js +67 -44
  151. package/src/client-builder/client-bundle.js +347 -0
  152. package/src/db/mongo/MongoExpress.js +187 -0
  153. package/src/grpc/cyberia/grpc-server.js +5 -7
  154. package/src/index.js +1 -1
  155. package/src/projects/cyberia/atlas-sprite-sheet-generator.js +143 -146
  156. package/src/projects/cyberia/atlas-sprite-sheet-store.js +391 -0
  157. package/src/projects/cyberia/besu-genesis-generator.js +5 -14
  158. package/src/projects/cyberia/boot-contract-fixtures.js +32 -0
  159. package/src/projects/cyberia/catalog-cyberia.js +3 -2
  160. package/src/projects/cyberia/gemini-client.js +3 -3
  161. package/src/projects/cyberia/generate-saga.js +4 -11
  162. package/src/projects/cyberia/hot-reload-trigger.js +17 -28
  163. package/src/projects/cyberia/instance-backup.js +210 -0
  164. package/src/projects/cyberia/instance-data.js +177 -257
  165. package/src/projects/cyberia/map-preview-generator.js +32 -68
  166. package/src/projects/cyberia/object-layer.js +152 -319
  167. package/src/projects/cyberia/seed-audio.js +139 -0
  168. package/src/projects/cyberia/semantic-layer-generator-skin.js +2 -8
  169. package/src/projects/cyberia/semantic-layer-generator.js +2 -18
  170. package/src/projects/cyberia/stat-balance.js +81 -0
  171. package/src/projects/cyberia/stat-commands.js +23 -0
  172. package/src/projects/cyberia/stat-contract-generator.js +101 -0
  173. package/src/runtime/cyberia-client/Dockerfile +8 -4
  174. package/src/runtime/cyberia-client/Dockerfile.dev +5 -2
  175. package/src/runtime/cyberia-server/Dockerfile +9 -2
  176. package/src/runtime/cyberia-server/Dockerfile.dev +8 -1
  177. package/src/runtime/engine-cyberia/Dockerfile +1 -1
  178. package/src/runtime/engine-cyberia/Dockerfile.dev +1 -1
  179. package/src/runtime/engine-cyberia/Dockerfile.test +2 -2
  180. package/src/runtime/engine-cyberia/compose.env +17 -5
  181. package/src/runtime/engine-cyberia/docker-compose.yml +23 -17
  182. package/src/server/build/coverage.js +113 -42
  183. package/src/server/build/package.js +55 -12
  184. package/src/server/build/testing.js +59 -9
  185. package/src/server/network/middlewares.js +8 -2
  186. package/src/server/ops/logger.js +70 -35
  187. package/src/server/runtime/conf.js +4 -2
  188. package/src/server/storage/data-query.js +16 -0
  189. package/src/server/storage/downloader.js +62 -14
  190. package/src/server/storage/zip.js +153 -0
  191. package/test/integration/app/cyberia/atlas-sprite-sheet-store.test.js +402 -0
  192. package/test/integration/app/cyberia/atlas-sprite-sheet.test.js +86 -0
  193. package/test/integration/app/cyberia/cyberia-cli-plain-reads.test.js +34 -0
  194. package/test/integration/app/cyberia/cyberia-entity-type-default.test.js +842 -0
  195. package/test/integration/app/cyberia/cyberia-instance-conf-coerce.test.js +53 -0
  196. package/test/integration/app/cyberia/cyberia-instance-conf-defaults.test.js +16 -22
  197. package/test/integration/app/cyberia/cyberia-instance-items.test.js +90 -0
  198. package/test/integration/app/cyberia/cyberia-load.test.js +6 -7
  199. package/test/integration/app/cyberia/cyberia-map-audio-conf.test.js +256 -0
  200. package/test/integration/app/cyberia/cyberia-stats.test.js +150 -0
  201. package/test/integration/app/cyberia/fallback-world-capture.test.js +87 -2
  202. package/test/integration/app/cyberia/object-layer-item-selection.test.js +56 -0
  203. package/test/integration/app/cyberia/object-layer-natural-key.test.js +32 -0
  204. package/test/integration/app/cyberia/seed-audio.test.js +275 -0
  205. package/test/integration/infra/2-network/wireguard-cli.test.js +33 -9
  206. package/test/integration/infra/2-network/wireguard-edge.test.js +58 -18
  207. package/test/integration/infra/3-cluster/mongo-express-deploy.test.js +156 -0
  208. package/test/integration/infra/4-ingress/api-cross-origin.test.js +79 -0
  209. package/test/unit/client-build-docs.test.js +97 -70
  210. package/test/unit/client-bundle.test.js +313 -0
  211. package/test/unit/conf-resolution.test.js +17 -0
  212. package/test/unit/coverage-artifact.test.js +120 -29
  213. package/test/unit/cyberia/instance-backup.test.js +183 -0
  214. package/test/unit/cyberia/instance-object-layer-items.test.js +46 -0
  215. package/test/unit/cyberia/publish-workflow.test.js +22 -0
  216. package/test/unit/cyberia/stat-balance.test.js +69 -0
  217. package/test/unit/deploy-log-table.test.js +76 -5
  218. package/test/unit/downloader-integrity.test.js +98 -0
  219. package/test/unit/file-reference-registry.test.js +141 -0
  220. package/test/unit/fs-storage-paths.test.js +596 -0
  221. package/test/unit/logger-redaction.test.js +29 -0
  222. package/test/unit/package.test.js +70 -16
  223. package/test/unit/prepare-host.test.js +84 -1
  224. package/test/unit/test-tiers.test.js +27 -0
  225. package/test/unit/zip-archive.test.js +142 -0
  226. package/vitest.config.js +11 -3
  227. package/src/api/cyberia-instance/cyberia-fallback-default-items.js +0 -63
  228. package/src/client/components/cyberia/FallbackWorldEngineCyberia.js +0 -368
@@ -17,7 +17,7 @@ import {
17
17
  } from '../client/components/core/CommonJs.js';
18
18
  import { readConfJson } from '../server/runtime/conf.js';
19
19
  import { minify } from 'html-minifier-terser';
20
- import AdmZip from 'adm-zip';
20
+ import { extractZipTo, findZipEntry, isZipBuffer, loadZip, zipFromLocalFiles } from '../server/storage/zip.js';
21
21
  import * as dir from 'path';
22
22
  import { shellExec } from '../server/runtime/process.js';
23
23
  import { SitemapStream, streamToPromise } from 'sitemap';
@@ -27,6 +27,7 @@ import { statusPageBuildSegment } from '../server/network/underpost-gateway.js';
27
27
  import { repositoryIdentityFactory } from '../server/storage/repository.js';
28
28
  import Underpost from '../index.js';
29
29
  import { buildDocs } from './client-build-docs.js';
30
+ import { coverageReportsFactory } from '../server/build/coverage.js';
30
31
  import { ssrFactory } from './ssr.js';
31
32
 
32
33
  // Static Site Generation (SSG)
@@ -180,6 +181,17 @@ const splitFileByMb = ({ filePath, partSizeMb, logger }) => {
180
181
  return partPaths;
181
182
  };
182
183
 
184
+ /**
185
+ * Names the zip artifact a served route builds to. Single source of truth for the bundle
186
+ * identity: the build writes `<id>.zip`, and push/pull address the same file by this name.
187
+ * @function clientBundleIdFactory
188
+ * @param {string} host - The served host.
189
+ * @param {string} routePath - The route's proxy sub-path (`/`, `/peer`, ...).
190
+ * @returns {string} The bundle id, e.g. `underpost.net-` for `/` or `underpost.net-peer`.
191
+ * @memberof clientBuild
192
+ */
193
+ const clientBundleIdFactory = (host, routePath) => `${host}-${`${routePath ?? ''}`.replaceAll('/', '')}`;
194
+
183
195
  const getZipPartPaths = (zipPath) => {
184
196
  const zipDir = dir.dirname(zipPath);
185
197
  const zipBase = dir.basename(zipPath);
@@ -269,9 +281,9 @@ const resolveClientBuildZip = (buildPrefix) => {
269
281
  * @param {object} options
270
282
  * @param {string} options.buildPrefix - The build prefix path (e.g. build/underpost.net/underpost.net-).
271
283
  * @param {object} options.logger - Logger instance.
272
- * @returns {{ zipPath: string, partPaths: string[], mergedBytes: number }}
284
+ * @returns {Promise<{ zipPath: string, partPaths: string[], mergedBytes: number }>}
273
285
  */
274
- const mergeClientBuildZip = ({ buildPrefix, logger }) => {
286
+ const mergeClientBuildZip = async ({ buildPrefix, logger }) => {
275
287
  // Normalize to get the zip path, then look for parts directly (bypassing resolveClientBuildZip
276
288
  // which prefers an existing monolithic zip over parts).
277
289
  const normalizedPrefix = buildPrefix.replace(/\.zip(?:[.-]part\d+)?$/, '').replace(/[-.]$/, '') + '-';
@@ -301,31 +313,41 @@ const mergeClientBuildZip = ({ buildPrefix, logger }) => {
301
313
  // For each part, extract raw bytes: if the part file is a Cloudinary wrapper zip
302
314
  // (downloaded via pull without --omit-unzip or with --omit-unzip keeping the .zip),
303
315
  // extract the inner entry rather than using the wrapper bytes.
304
- const readPartBytes = (partPath) => {
316
+ const readPartBytes = async (partPath) => {
305
317
  const rawBytes = fs.readFileSync(partPath);
306
- // Check for ZIP magic bytes (PK\x03\x04)
307
- if (rawBytes[0] === 0x50 && rawBytes[1] === 0x4b && rawBytes[2] === 0x03 && rawBytes[3] === 0x04) {
308
- try {
309
- const wrapperZip = new AdmZip(rawBytes);
310
- const entries = wrapperZip.getEntries();
311
- // The inner entry is the original part file (without the outer .zip wrapper)
312
- const partBase = dir.basename(partPath).replace(/\.zip$/i, '');
313
- const entry = entries.find((e) => e.entryName === partBase || e.entryName.endsWith('/' + partBase));
314
- if (entry) {
315
- return entry.getData();
316
- }
317
- // Fallback: single-entry archive
318
- if (entries.length === 1) {
319
- return entries[0].getData();
320
- }
321
- } catch (_) {
322
- // Not a valid zip or extraction failed — use raw bytes
323
- }
318
+ if (!isZipBuffer(rawBytes)) return rawBytes;
319
+
320
+ let wrapper;
321
+ try {
322
+ wrapper = await loadZip(rawBytes);
323
+ } catch (_) {
324
+ // A raw split part carries the zip signature but no central directory, so failing to load
325
+ // is how a part that was never wrapped identifies itself.
326
+ return rawBytes;
324
327
  }
325
- return rawBytes;
328
+
329
+ // A part pulled from storage arrives inside the archive the download produced. Falling back
330
+ // to that wrapper's own bytes merges compressed data into the bundle and surfaces much later
331
+ // as a corrupt archive, so a wrapper whose part cannot be found fails here and names itself.
332
+ const entry = findZipEntry(wrapper, dir.basename(partPath).replace(/\.zip$/i, ''));
333
+ if (!entry) throw new Error(`Bundle part wrapper holds no matching entry: ${partPath}`);
334
+ return await entry.async('nodebuffer');
326
335
  };
327
336
 
328
- const mergedBuffer = Buffer.concat(partPaths.map(readPartBytes));
337
+ const partBuffers = [];
338
+ for (const partPath of partPaths) partBuffers.push(await readPartBytes(partPath));
339
+ const mergedBuffer = Buffer.concat(partBuffers);
340
+
341
+ // A short part anywhere in the sequence only shows up as a byte count against the assembled
342
+ // archive, which names nothing. Reporting each part's contribution is what makes the one that
343
+ // came up short identifiable.
344
+ logger.info('merge-zip: part sizes', {
345
+ zipPath,
346
+ parts: partPaths.map((partPath, index) => ({
347
+ part: dir.basename(partPath),
348
+ bytes: partBuffers[index].length,
349
+ })),
350
+ });
329
351
  fs.writeFileSync(zipPath, mergedBuffer);
330
352
 
331
353
  logger.warn('merge-zip: merged split parts into zip', {
@@ -337,19 +359,16 @@ const mergeClientBuildZip = ({ buildPrefix, logger }) => {
337
359
  return { zipPath, partPaths, mergedBytes: mergedBuffer.length };
338
360
  };
339
361
 
340
- const unzipClientBuild = ({ buildPrefix, logger }) => {
362
+ const unzipClientBuild = async ({ buildPrefix, logger }) => {
341
363
  const { zipPath, partPaths, buildPrefix: resolvedBuildPrefix } = resolveClientBuildZip(buildPrefix);
342
364
  const outputPath = resolvedBuildPrefix.replace(/-$/, '');
343
365
 
344
366
  fs.removeSync(outputPath);
345
- fs.mkdirSync(outputPath, { recursive: true });
346
-
347
- const zip =
348
- partPaths.length > 0
349
- ? new AdmZip(Buffer.concat(partPaths.map((partPath) => fs.readFileSync(partPath))))
350
- : new AdmZip(zipPath);
351
367
 
352
- zip.extractAllTo(outputPath, true);
368
+ await extractZipTo(
369
+ partPaths.length > 0 ? Buffer.concat(partPaths.map((partPath) => fs.readFileSync(partPath))) : zipPath,
370
+ outputPath,
371
+ );
353
372
 
354
373
  logger.warn('unzip build', {
355
374
  source: partPaths.length > 0 ? partPaths : [zipPath],
@@ -704,6 +723,8 @@ const buildClient = async (
704
723
  apiBasePath: process.env.BASE_API,
705
724
  version: Underpost.version,
706
725
  repository: repositoryIdentityFactory(),
726
+ // The reports the docs menu offers, each published at /docs/coverage/<id>.
727
+ coverage: coverageReportsFactory(docs).map(({ id, label }) => ({ id, label })),
707
728
  ...(isDevelopment ? { dev: true } : undefined),
708
729
  };
709
730
 
@@ -833,14 +854,12 @@ const buildClient = async (
833
854
  }
834
855
 
835
856
  if (!ssrOnly && views) {
836
- if (
837
- !(
838
- enableLiveRebuild &&
839
- !options.liveClientBuildPaths.find(
840
- (p) => p.srcBuildPath.startsWith(`./src/client/ssr`) || p.srcBuildPath.slice(-9) === '.index.js',
841
- )
857
+ if (!(
858
+ enableLiveRebuild &&
859
+ !options.liveClientBuildPaths.find(
860
+ (p) => p.srcBuildPath.startsWith(`./src/client/ssr`) || p.srcBuildPath.slice(-9) === '.index.js',
842
861
  )
843
- )
862
+ ))
844
863
  for (const view of views) {
845
864
  const buildPath = `${
846
865
  rootClientPath[rootClientPath.length - 1] === '/' ? rootClientPath.slice(0, -1) : rootClientPath
@@ -1147,23 +1166,25 @@ ${swTransformedJs}`,
1147
1166
 
1148
1167
  if (!fs.existsSync('./build')) fs.mkdirSync('./build');
1149
1168
 
1150
- const zip = new AdmZip();
1151
1169
  const files = await fs.readdir(rootClientPath, { recursive: true });
1170
+ const zipEntries = [];
1152
1171
 
1153
1172
  for (const relativePath of files) {
1154
1173
  const filePath = dir.resolve(`${rootClientPath}/${relativePath}`);
1155
1174
  if (!fs.lstatSync(filePath).isDirectory()) {
1156
1175
  const folder = dir.relative(`public/${host}${path}`, dir.dirname(filePath));
1157
- zip.addLocalFile(filePath, folder);
1176
+ zipEntries.push({
1177
+ localPath: filePath,
1178
+ entryName: folder ? `${folder}/${dir.basename(filePath)}` : dir.basename(filePath),
1179
+ });
1158
1180
  }
1159
1181
  }
1160
1182
 
1161
- const buildId = `${host}-${path.replaceAll('/', '')}`;
1162
- const zipPath = `./build/${buildId}.zip`;
1183
+ const zipPath = `./build/${clientBundleIdFactory(host, path)}.zip`;
1163
1184
 
1164
1185
  logger.warn('write zip', zipPath);
1165
1186
 
1166
- zip.writeZip(zipPath);
1187
+ fs.writeFileSync(zipPath, await zipFromLocalFiles(zipEntries));
1167
1188
 
1168
1189
  if (options.split) {
1169
1190
  splitFileByMb({
@@ -1181,7 +1202,9 @@ ${swTransformedJs}`,
1181
1202
 
1182
1203
  export {
1183
1204
  buildClient,
1205
+ clientBundleIdFactory,
1184
1206
  copyNonExistingFiles,
1207
+ getZipPartPaths,
1185
1208
  unzipClientBuild,
1186
1209
  mergeClientBuildZip,
1187
1210
  staticContextRoutesFactory,
@@ -0,0 +1,347 @@
1
+ /**
2
+ * Client bundle transport: enumerates the zip artifacts a deployment's routes build, uploads
3
+ * them to file storage, and restores them into the served public directory.
4
+ *
5
+ * Push and pull are two directions of one contract, so both resolve their routes, bundle ids,
6
+ * storage keys and part files through the helpers here rather than re-deriving them. Route
7
+ * selection mirrors {@link buildClient}: a route that the build skips produces no artifact for
8
+ * either direction to carry.
9
+ * @module src/client-builder/client-bundle.js
10
+ * @namespace clientBundle
11
+ */
12
+
13
+ import fs from 'fs-extra';
14
+ import UnderpostFileStorage from '../cli/fs.js';
15
+ import * as dir from 'path';
16
+ import { cli } from '../server/build/execution.js';
17
+ import { loadConfServerJson, loadReplicas } from '../server/runtime/conf.js';
18
+ import { loggerFactory } from '../server/ops/logger.js';
19
+ import { shellArgumentFactory, shellExec } from '../server/runtime/process.js';
20
+ import { clientBundleIdFactory, getZipPartPaths, mergeClientBuildZip, unzipClientBuild } from './client-build.js';
21
+
22
+ const logger = loggerFactory(import.meta);
23
+
24
+ /** @type {string} Directory `buildClient --build-zip` writes bundles to, and the only place push and pull look. */
25
+ const CLIENT_BUNDLE_DIRECTORY = './build';
26
+
27
+ /** @type {number} Part size in MB applied when the caller names none. */
28
+ const DEFAULT_SPLIT_MB = 8;
29
+
30
+ /** @type {string} `--split` value that turns splitting off. */
31
+ const SPLIT_DISABLED = 'none';
32
+
33
+ const CLIENT_BUNDLE_STORAGE_ID = 'bundle';
34
+
35
+ /**
36
+ * Resolves the conf.server path a deployment's routes are read from.
37
+ * @function clientBundleConfServerPath
38
+ * @param {string} deployId - The deployment identifier.
39
+ * @returns {string} Path to the deployment's `conf.server.json`.
40
+ * @memberof clientBundle
41
+ */
42
+ const clientBundleConfServerPath = (deployId) => `./engine-private/conf/${deployId}/conf.server.json`;
43
+
44
+ /**
45
+ * Resolves the storage manifest that tracks a deployment's uploaded bundle parts. Push writes
46
+ * it and pull reads it, so both must name the same file.
47
+ * @function clientBundleStorageFilePath
48
+ * @param {string} deployId - The deployment identifier.
49
+ * @returns {string} Path to the deployment's bundle storage manifest.
50
+ * @memberof clientBundle
51
+ */
52
+ const clientBundleStorageFilePath = (deployId) =>
53
+ UnderpostFileStorage.API.resolveManifest({ deployId, storageId: CLIENT_BUNDLE_STORAGE_ID });
54
+
55
+ /**
56
+ * Parses the comma-separated host filter both runners accept as their positional argument.
57
+ * @function clientBundleHostFilter
58
+ * @param {string} [hosts] - Comma-separated host names, or empty for every host.
59
+ * @returns {string[]} The requested hosts, empty when no filter was given.
60
+ * @memberof clientBundle
61
+ */
62
+ const clientBundleHostFilter = (hosts = '') =>
63
+ `${hosts ?? ''}`
64
+ .split(',')
65
+ .map((host) => host.trim())
66
+ .filter(Boolean);
67
+
68
+ /**
69
+ * Resolves the `--split` option into the flag `underpost client` expects.
70
+ * @function clientBundleSplitFlag
71
+ * @param {string|number} [split] - Part size in MB, or `none` to disable splitting.
72
+ * @returns {string} The flag to append, empty when splitting is disabled.
73
+ * @memberof clientBundle
74
+ */
75
+ const clientBundleSplitFlag = (split) => {
76
+ const requested = `${split ?? ''}`.trim();
77
+ if (!requested) return `--split ${DEFAULT_SPLIT_MB}`;
78
+ if (requested === SPLIT_DISABLED) return '';
79
+
80
+ const splitMb = Number(requested);
81
+ if (Number.isFinite(splitMb) && splitMb > 0) return `--split ${splitMb}`;
82
+
83
+ logger.warn('invalid split option, falling back to the default', { split: requested, default: DEFAULT_SPLIT_MB });
84
+ return `--split ${DEFAULT_SPLIT_MB}`;
85
+ };
86
+
87
+ /**
88
+ * Enumerates the bundles a deployment's routes build to.
89
+ *
90
+ * A route is skipped exactly where {@link buildClient} skips it — redirects and
91
+ * rebuild-disabled routes emit nothing, and a `singleReplica` route's replicas are built under
92
+ * their own deploy ids, so they carry their own manifests rather than the parent's.
93
+ * @function clientBundleEntriesFactory
94
+ * @param {object} options
95
+ * @param {string} options.deployId - The deployment identifier.
96
+ * @param {string[]} [options.hosts] - Optional host filter; empty selects every configured host.
97
+ * @returns {Array<{host: string, routePath: string, buildId: string, zipPath: string, publicPath: string}>}
98
+ * One entry per bundle, in configuration order.
99
+ * @memberof clientBundle
100
+ */
101
+ const clientBundleEntriesFactory = ({ deployId, hosts = [] }) => {
102
+ const confServerPath = clientBundleConfServerPath(deployId);
103
+ if (!fs.existsSync(confServerPath)) {
104
+ logger.warn('conf.server not found', { deployId, confServerPath });
105
+ return [];
106
+ }
107
+
108
+ const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath));
109
+ const selectedHosts = hosts.length > 0 ? hosts : Object.keys(confServer);
110
+
111
+ return selectedHosts.flatMap((host) => {
112
+ if (!confServer[host]) {
113
+ logger.warn('host not present in conf.server', { deployId, host });
114
+ return [];
115
+ }
116
+ return Object.keys(confServer[host])
117
+ .filter((routePath) => {
118
+ const { redirect, disabledRebuild, singleReplica } = confServer[host][routePath] ?? {};
119
+ return !redirect && !disabledRebuild && !singleReplica;
120
+ })
121
+ .map((routePath) => {
122
+ const buildId = clientBundleIdFactory(host, routePath);
123
+ return {
124
+ host,
125
+ routePath,
126
+ buildId,
127
+ zipPath: `${CLIENT_BUNDLE_DIRECTORY}/${buildId}.zip`,
128
+ publicPath: routePath === '/' ? `public/${host}` : `public/${host}${routePath}`,
129
+ };
130
+ });
131
+ });
132
+ };
133
+
134
+ /**
135
+ * Reads the storage manifest as a plain lookup, for the recorded size of each uploaded artifact.
136
+ * Read-only and tolerant: a manifest that is absent or unreadable disables verification rather
137
+ * than failing the pull, since the artifacts themselves are still what the bundle is made of.
138
+ * @function clientBundleManifestFactory
139
+ * @param {string} deployId - The deployment identifier.
140
+ * @returns {Object<string, {bytes?: number}>} The tracked entries, empty when unavailable.
141
+ * @memberof clientBundle
142
+ */
143
+ const clientBundleManifestFactory = (deployId) => {
144
+ const manifestPath = clientBundleStorageFilePath(deployId);
145
+ try {
146
+ return UnderpostFileStorage.API.readManifest({ deployId, storageId: CLIENT_BUNDLE_STORAGE_ID }).storage;
147
+ } catch (error) {
148
+ logger.warn('bundle manifest unreadable, size verification skipped', { manifestPath, error: error?.message });
149
+ return {};
150
+ }
151
+ };
152
+
153
+ /**
154
+ * The tracked keys that belong to one bundle.
155
+ *
156
+ * A bundle owns its zip and every split part of it, and nothing else: the id already carries the
157
+ * route, so `underpost.net-` and `underpost.net-peer` cannot claim each other's parts.
158
+ * @function clientBundleTrackedKeys
159
+ * @param {object} manifest - The bundle storage manifest.
160
+ * @param {string} buildId - The bundle's build id.
161
+ * @returns {string[]} The manifest keys for that bundle.
162
+ * @memberof clientBundle
163
+ */
164
+ const clientBundleTrackedKeys = (manifest, buildId) =>
165
+ Object.keys(manifest).filter((key) =>
166
+ key.replace(/^\.\//, '').startsWith(`${CLIENT_BUNDLE_DIRECTORY.replace(/^\.\//, '')}/${buildId}.zip`),
167
+ );
168
+
169
+ /**
170
+ * Lists the local artifacts that make up one bundle: its split parts when the build split it,
171
+ * otherwise the single zip.
172
+ * @function clientBundleArtifactPaths
173
+ * @param {string} zipPath - The bundle's zip path.
174
+ * @returns {{zipPath: string, partPaths: string[], artifactPaths: string[]}} The resolved artifacts.
175
+ * @memberof clientBundle
176
+ */
177
+ const clientBundleArtifactPaths = (zipPath) => {
178
+ const partPaths = fs.existsSync(dir.dirname(zipPath)) ? getZipPartPaths(zipPath) : [];
179
+ const artifactPaths = partPaths.length > 0 ? partPaths : fs.existsSync(zipPath) ? [zipPath] : [];
180
+ return { zipPath, partPaths, artifactPaths };
181
+ };
182
+
183
+ /**
184
+ * Builds a deployment's client bundles and uploads them to file storage.
185
+ *
186
+ * The build and the uploads run as child processes because `underpost app load` writes the
187
+ * `.env` this deployment's credentials live in, and only a freshly started process reads it.
188
+ * @function pushClientBundle
189
+ * @param {object} options
190
+ * @param {string} options.deployId - The deployment identifier.
191
+ * @param {string[]} [options.hosts] - Optional host filter; empty selects every configured host.
192
+ * @param {string|number} [options.split] - Part size in MB, or `none` to upload one zip.
193
+ * @param {boolean} [options.dev] - Build for development; defaults to production.
194
+ * @returns {{pushed: number, skipped: number}} How many bundles were uploaded and skipped.
195
+ * @memberof clientBundle
196
+ */
197
+ const pushClientBundle = ({ deployId, hosts = [], split, dev = false }) => {
198
+ const baseCommand = cli('underpost', { local: true });
199
+ const env = dev ? 'development' : 'production';
200
+ const splitFlag = clientBundleSplitFlag(split);
201
+
202
+ shellExec(`${baseCommand} app load --env ${env} --args deploy-id=${deployId}`);
203
+ shellExec(`${baseCommand} client ${deployId} --env ${env} --build-zip${splitFlag ? ` ${splitFlag}` : ''}`);
204
+
205
+ const entries = clientBundleEntriesFactory({ deployId, hosts });
206
+ if (entries.length === 0) logger.warn('no bundle routes resolved', { deployId, hosts });
207
+
208
+ let pushed = 0;
209
+ let skipped = 0;
210
+ for (const { host, routePath, buildId, zipPath } of entries) {
211
+ const { artifactPaths } = clientBundleArtifactPaths(zipPath);
212
+ if (artifactPaths.length === 0) {
213
+ logger.warn('no bundle artifacts found', { host, routePath, buildId });
214
+ skipped++;
215
+ continue;
216
+ }
217
+ for (const artifactPath of artifactPaths) {
218
+ shellExec(
219
+ `${baseCommand} fs ${artifactPath} --deploy-id ${deployId} --storage-id ${CLIENT_BUNDLE_STORAGE_ID} --force`,
220
+ );
221
+ }
222
+
223
+ // Remove obsolete manifest entries only after all current artifacts upload.
224
+ const current = new Set(artifactPaths.map((artifactPath) => artifactPath.replace(/^\.\//, '')));
225
+ const stale = clientBundleTrackedKeys(clientBundleManifestFactory(deployId), buildId).filter(
226
+ (key) => !current.has(key.replace(/^\.\//, '')),
227
+ );
228
+ for (const key of stale) {
229
+ shellExec(
230
+ `${baseCommand} fs --tracked --key ${shellArgumentFactory(key)} ` +
231
+ `--deploy-id ${deployId} --storage-id ${CLIENT_BUNDLE_STORAGE_ID} --rm`,
232
+ );
233
+ }
234
+ if (stale.length > 0) logger.warn('removed bundle parts a smaller build no longer produces', { buildId, stale });
235
+
236
+ logger.info('pushed bundle', { host, routePath, artifacts: artifactPaths.length });
237
+ pushed++;
238
+ }
239
+ return { pushed, skipped };
240
+ };
241
+
242
+ /**
243
+ * Downloads a deployment's client bundles from file storage and installs them into the served
244
+ * public directory.
245
+ *
246
+ * Merging and extraction run in-process through the build module; only the download needs a
247
+ * child process, for the same `.env` reason {@link pushClientBundle} documents.
248
+ * @function pullClientBundle
249
+ * @param {object} options
250
+ * @param {string} options.deployId - The deployment identifier.
251
+ * @param {string[]} [options.hosts] - Optional host filter; empty selects every configured host.
252
+ * @param {boolean} [options.dev] - Pull the development environment; defaults to production.
253
+ * @returns {Promise<{pulled: number, skipped: number}>} How many bundles were installed and skipped.
254
+ * @memberof clientBundle
255
+ */
256
+ const expectedBundleBytes = ({ manifest, partPaths }) => {
257
+ const sizes = partPaths.map((partPath) => manifest[clientBundleManifestKey(partPath)]?.bytes);
258
+ return sizes.every((bytes) => Number.isFinite(bytes)) ? sizes.reduce((total, bytes) => total + bytes, 0) : undefined;
259
+ };
260
+
261
+ /**
262
+ * The manifest key a locally pulled part corresponds to. A pull writes the downloaded archive as
263
+ * `<key>.zip`, so the wrapper suffix is what has to come off to address the entry that produced it.
264
+ * @function clientBundleManifestKey
265
+ * @param {string} partPath - Local path of a pulled part.
266
+ * @returns {string} The tracked storage key.
267
+ * @memberof clientBundle
268
+ */
269
+ const clientBundleManifestKey = (partPath) => partPath.replace(/^\.\//, '').replace(/\.zip$/i, '');
270
+
271
+ const pullClientBundle = async ({ deployId, hosts = [], dev = false }) => {
272
+ const baseCommand = cli('underpost', { local: true });
273
+ const env = dev ? 'development' : 'production';
274
+
275
+ const entries = clientBundleEntriesFactory({ deployId, hosts });
276
+ if (entries.length === 0) {
277
+ logger.error('no bundle routes resolved', { deployId, hosts });
278
+ return { pulled: 0, skipped: 0 };
279
+ }
280
+
281
+ shellExec(`${baseCommand} app load --env ${env} --args deploy-id=${deployId}`);
282
+ fs.mkdirSync(CLIENT_BUNDLE_DIRECTORY, { recursive: true });
283
+ shellExec(
284
+ `${baseCommand} fs ${CLIENT_BUNDLE_DIRECTORY.replace(/^\.\//, '')} --tracked --deploy-id ${deployId} ` +
285
+ `--storage-id ${CLIENT_BUNDLE_STORAGE_ID} --pull --omit-unzip`,
286
+ );
287
+
288
+ const manifest = clientBundleManifestFactory(deployId);
289
+
290
+ let pulled = 0;
291
+ let skipped = 0;
292
+ for (const { host, routePath, buildId, zipPath, publicPath } of entries) {
293
+ const { partPaths, artifactPaths } = clientBundleArtifactPaths(zipPath);
294
+ if (artifactPaths.length === 0) {
295
+ logger.warn('bundle not found, skipping', { host, routePath, buildId, zipPath });
296
+ skipped++;
297
+ continue;
298
+ }
299
+
300
+ if (partPaths.length > 0) {
301
+ const { mergedBytes } = await mergeClientBuildZip({ buildPrefix: zipPath, logger });
302
+ // The parts are reassembled blind, so a single stale or short one is only visible as a
303
+ // corrupt archive far downstream. The pushed sizes are what identify it here instead.
304
+ const expectedBytes = expectedBundleBytes({ manifest, partPaths });
305
+ if (expectedBytes !== undefined && expectedBytes !== mergedBytes)
306
+ throw new Error(
307
+ `Bundle ${buildId} reassembled to ${mergedBytes} bytes, expected ${expectedBytes} from ${partPaths.length} pushed parts`,
308
+ );
309
+ }
310
+ const { outputPath } = await unzipClientBuild({ buildPrefix: zipPath, logger });
311
+
312
+ fs.removeSync(zipPath);
313
+ for (const partPath of partPaths) fs.removeSync(partPath);
314
+
315
+ if (!fs.existsSync(outputPath)) {
316
+ logger.warn('extracted bundle directory not found, skipping', { host, routePath, outputPath });
317
+ skipped++;
318
+ continue;
319
+ }
320
+
321
+ // The previous deployment's document root can be owned by the web server user, so the
322
+ // replace goes through sudo; the extracted tree this process just wrote does not.
323
+ shellExec(`sudo rm -rf ${publicPath}`);
324
+ if (routePath !== '/') shellExec(`sudo mkdir -p public/${host}`);
325
+ fs.copySync(outputPath, publicPath);
326
+
327
+ logger.info('pulled bundle', { host, routePath, publicPath });
328
+ pulled++;
329
+ }
330
+ return { pulled, skipped };
331
+ };
332
+
333
+ export {
334
+ CLIENT_BUNDLE_DIRECTORY,
335
+ clientBundleTrackedKeys,
336
+ clientBundleManifestFactory,
337
+ clientBundleManifestKey,
338
+ DEFAULT_SPLIT_MB,
339
+ clientBundleArtifactPaths,
340
+ clientBundleConfServerPath,
341
+ clientBundleEntriesFactory,
342
+ clientBundleHostFilter,
343
+ clientBundleSplitFlag,
344
+ clientBundleStorageFilePath,
345
+ pullClientBundle,
346
+ pushClientBundle,
347
+ };