cyberia 3.0.2 → 3.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/{.env.production → .env.example} +20 -2
  2. package/.github/workflows/engine-cyberia.cd.yml +41 -10
  3. package/.github/workflows/engine-cyberia.ci.yml +53 -14
  4. package/.github/workflows/ghpkg.ci.yml +1 -1
  5. package/.github/workflows/gitlab.ci.yml +1 -1
  6. package/.github/workflows/hardhat.ci.yml +82 -0
  7. package/.github/workflows/npmpkg.ci.yml +37 -8
  8. package/.github/workflows/publish.ci.yml +5 -5
  9. package/.github/workflows/publish.cyberia.ci.yml +5 -5
  10. package/.github/workflows/pwa-microservices-template-page.cd.yml +3 -3
  11. package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
  12. package/.github/workflows/release.cd.yml +3 -2
  13. package/.vscode/extensions.json +9 -8
  14. package/.vscode/settings.json +3 -2
  15. package/CHANGELOG.md +533 -290
  16. package/CLI-HELP.md +79 -53
  17. package/WHITE-PAPER.md +1540 -0
  18. package/bin/build.js +16 -11
  19. package/bin/cyberia.js +959 -8
  20. package/bin/deploy.js +103 -270
  21. package/bin/file.js +2 -1
  22. package/bin/index.js +959 -8
  23. package/bin/vs.js +3 -3
  24. package/conf.js +277 -77
  25. package/deployment.yaml +218 -4
  26. package/hardhat/.env.example +31 -0
  27. package/hardhat/README.md +531 -0
  28. package/hardhat/WHITE-PAPER.md +1540 -0
  29. package/hardhat/contracts/ObjectLayerToken.sol +391 -0
  30. package/hardhat/deployments/.gitkeep +0 -0
  31. package/hardhat/deployments/hardhat-ObjectLayerToken.json +11 -0
  32. package/hardhat/hardhat.config.js +136 -0
  33. package/hardhat/ignition/modules/ObjectLayerToken.js +21 -0
  34. package/hardhat/networks/besu-object-layer.network.json +138 -0
  35. package/hardhat/package-lock.json +7628 -0
  36. package/hardhat/package.json +45 -0
  37. package/hardhat/scripts/deployObjectLayerToken.js +98 -0
  38. package/hardhat/test/ObjectLayerToken.js +590 -0
  39. package/jsdoc.dd-cyberia.json +59 -0
  40. package/jsdoc.json +20 -13
  41. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
  42. package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
  43. package/manifests/deployment/dd-cyberia-development/deployment.yaml +490 -0
  44. package/manifests/deployment/dd-cyberia-development/proxy.yaml +261 -0
  45. package/manifests/deployment/dd-cyberia-development/pv-pvc.yaml +132 -0
  46. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  47. package/manifests/deployment/dd-test-development/deployment.yaml +52 -52
  48. package/manifests/deployment/dd-test-development/proxy.yaml +4 -4
  49. package/manifests/pv-pvc-dd.yaml +1 -1
  50. package/package.json +60 -50
  51. package/proxy.yaml +128 -9
  52. package/pv-pvc.yaml +132 -0
  53. package/scripts/k3s-node-setup.sh +1 -1
  54. package/scripts/ports-ls.sh +2 -0
  55. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.controller.js +3 -1
  56. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.model.js +1 -2
  57. package/src/api/atlas-sprite-sheet/atlas-sprite-sheet.service.js +40 -7
  58. package/src/api/document/document.service.js +1 -1
  59. package/src/api/file/file.controller.js +3 -1
  60. package/src/api/file/file.service.js +28 -5
  61. package/src/api/ipfs/ipfs.service.js +2 -2
  62. package/src/api/object-layer/object-layer.controller.js +6 -2
  63. package/src/api/object-layer/object-layer.model.js +67 -21
  64. package/src/api/object-layer/object-layer.router.js +668 -42
  65. package/src/api/object-layer/object-layer.service.js +10 -16
  66. package/src/api/object-layer-render-frames/object-layer-render-frames.model.js +1 -2
  67. package/src/api/user/user.router.js +10 -5
  68. package/src/api/user/user.service.js +7 -7
  69. package/src/cli/baremetal.js +6 -10
  70. package/src/cli/cloud-init.js +0 -3
  71. package/src/cli/db.js +54 -71
  72. package/src/cli/deploy.js +64 -12
  73. package/src/cli/env.js +5 -5
  74. package/src/cli/fs.js +0 -2
  75. package/src/cli/image.js +0 -3
  76. package/src/cli/index.js +41 -13
  77. package/src/cli/monitor.js +5 -6
  78. package/src/cli/repository.js +329 -46
  79. package/src/cli/run.js +210 -122
  80. package/src/cli/secrets.js +1 -3
  81. package/src/cli/ssh.js +1 -1
  82. package/src/client/Itemledger.index.js +1 -959
  83. package/src/client/Underpost.index.js +36 -0
  84. package/src/client/components/core/AgGrid.js +20 -5
  85. package/src/client/components/core/Alert.js +2 -2
  86. package/src/client/components/core/Content.js +22 -3
  87. package/src/client/components/core/Docs.js +30 -6
  88. package/src/client/components/core/FileExplorer.js +71 -4
  89. package/src/client/components/core/Input.js +1 -1
  90. package/src/client/components/core/Modal.js +22 -6
  91. package/src/client/components/core/PublicProfile.js +3 -3
  92. package/src/client/components/core/RichText.js +1 -2
  93. package/src/client/components/core/Router.js +34 -1
  94. package/src/client/components/core/Worker.js +1 -1
  95. package/src/client/components/cryptokoyn/CssCryptokoyn.js +63 -1
  96. package/src/client/components/cyberia/ObjectLayerEngineModal.js +145 -119
  97. package/src/client/components/cyberia/ObjectLayerEngineViewer.js +64 -6
  98. package/src/client/components/cyberia-portal/CommonCyberiaPortal.js +1 -0
  99. package/src/client/components/cyberia-portal/CssCyberiaPortal.js +44 -2
  100. package/src/client/components/cyberia-portal/LogInCyberiaPortal.js +0 -1
  101. package/src/client/components/cyberia-portal/MenuCyberiaPortal.js +64 -2
  102. package/src/client/components/cyberia-portal/RoutesCyberiaPortal.js +1 -0
  103. package/src/client/components/itemledger/CssItemledger.js +62 -0
  104. package/src/client/components/underpost/CommonUnderpost.js +29 -0
  105. package/src/client/components/underpost/CssUnderpost.js +281 -0
  106. package/src/client/components/underpost/CyberpunkBloggerUnderpost.js +879 -0
  107. package/src/client/components/underpost/DocumentSearchProvider.js +448 -0
  108. package/src/client/components/underpost/ElementsUnderpost.js +38 -0
  109. package/src/client/components/underpost/LabGalleryUnderpost.js +82 -0
  110. package/src/client/components/underpost/LogInUnderpost.js +23 -0
  111. package/src/client/components/underpost/LogOutUnderpost.js +15 -0
  112. package/src/client/components/underpost/MenuUnderpost.js +691 -0
  113. package/src/client/components/underpost/RoutesUnderpost.js +47 -0
  114. package/src/client/components/underpost/SettingsUnderpost.js +16 -0
  115. package/src/client/components/underpost/SignUpUnderpost.js +9 -0
  116. package/src/client/components/underpost/SocketIoUnderpost.js +54 -0
  117. package/src/client/components/underpost/TranslateUnderpost.js +10 -0
  118. package/src/client/public/cryptokoyn/assets/logo/base-icon.png +0 -0
  119. package/src/client/public/cryptokoyn/browserconfig.xml +12 -0
  120. package/src/client/public/cryptokoyn/microdata.json +85 -0
  121. package/src/client/public/cryptokoyn/site.webmanifest +57 -0
  122. package/src/client/public/cryptokoyn/sitemap +3 -3
  123. package/src/client/public/default/sitemap +3 -3
  124. package/src/client/public/itemledger/browserconfig.xml +2 -2
  125. package/src/client/public/itemledger/manifest.webmanifest +4 -4
  126. package/src/client/public/itemledger/microdata.json +71 -0
  127. package/src/client/public/itemledger/sitemap +3 -3
  128. package/src/client/public/itemledger/yandex-browser-manifest.json +2 -2
  129. package/src/client/public/test/sitemap +3 -3
  130. package/src/client/services/object-layer/object-layer.management.js +23 -4
  131. package/src/client/ssr/body/404.js +15 -11
  132. package/src/client/ssr/body/500.js +15 -11
  133. package/src/client/ssr/body/SwaggerDarkMode.js +285 -0
  134. package/src/client/ssr/body/UnderpostDefaultSplashScreen.js +83 -0
  135. package/src/client/ssr/head/PwaItemledger.js +60 -0
  136. package/src/client/ssr/head/UnderpostScripts.js +6 -0
  137. package/src/client/ssr/offline/NoNetworkConnection.js +11 -10
  138. package/src/client/ssr/pages/Test.js +11 -10
  139. package/src/client.build.js +0 -3
  140. package/src/client.dev.js +0 -3
  141. package/src/db/DataBaseProvider.js +17 -2
  142. package/src/db/mariadb/MariaDB.js +14 -9
  143. package/src/db/mongo/MongooseDB.js +17 -1
  144. package/src/index.js +1 -1
  145. package/src/proxy.js +0 -3
  146. package/src/runtime/express/Express.js +15 -9
  147. package/src/runtime/lampp/Lampp.js +6 -13
  148. package/src/server/auth.js +12 -14
  149. package/src/server/backup.js +2 -3
  150. package/src/server/besu-genesis-generator.js +1630 -0
  151. package/src/server/client-build-docs.js +126 -17
  152. package/src/server/client-build-live.js +9 -18
  153. package/src/server/client-build.js +203 -75
  154. package/src/server/client-dev-server.js +14 -13
  155. package/src/server/conf.js +376 -164
  156. package/src/server/cron.js +2 -1
  157. package/src/server/dns.js +28 -12
  158. package/src/server/downloader.js +0 -2
  159. package/src/server/logger.js +27 -9
  160. package/src/server/object-layer.js +92 -16
  161. package/src/server/peer.js +0 -2
  162. package/src/server/process.js +1 -50
  163. package/src/server/proxy.js +4 -8
  164. package/src/server/runtime.js +5 -8
  165. package/src/server/semantic-layer-generator.js +1 -0
  166. package/src/server/ssr.js +0 -3
  167. package/src/server/start.js +19 -12
  168. package/src/server/tls.js +0 -2
  169. package/src/server.js +0 -4
  170. package/.env.development +0 -43
  171. package/.env.test +0 -43
  172. package/hardhat/contracts/CryptoKoyn.sol +0 -59
  173. package/hardhat/contracts/ItemLedger.sol +0 -73
  174. package/hardhat/contracts/Lock.sol +0 -34
  175. package/hardhat/hardhat.config.cjs +0 -45
  176. package/hardhat/ignition/modules/Lock.js +0 -18
  177. package/hardhat/networks/cryptokoyn-itemledger.network.json +0 -29
  178. package/hardhat/scripts/deployCryptokoyn.cjs +0 -25
  179. package/hardhat/scripts/deployItemledger.cjs +0 -25
  180. package/hardhat/test/Lock.js +0 -126
  181. package/hardhat/white-paper.md +0 -581
  182. package/white-paper.md +0 -581
@@ -46,7 +46,7 @@ const ObjectLayerService = {
46
46
  * - Default — Create an object layer directly from the request body.
47
47
  *
48
48
  * The `/metadata` and default routes delegate to {@link ObjectLayerEngine.createObjectLayerDocuments}
49
- * for centralized document creation, atlas generation, SHA-256 computation, and IPFS pinning.
49
+ * for document creation, atlas generation, SHA-256 computation, and IPFS pinning.
50
50
  *
51
51
  * @async
52
52
  * @function post
@@ -148,7 +148,7 @@ const ObjectLayerService = {
148
148
  fs.writeFileSync(`${folder}/metadata.json`, metadataContent);
149
149
  fs.writeFileSync(`${publicFolder}/metadata.json`, metadataContent);
150
150
 
151
- // Build object layer data from the asset directory using centralized logic
151
+ // Build object layer data from the asset directory
152
152
  const ObjectLayer = DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.ObjectLayer;
153
153
  const ObjectLayerRenderFrames =
154
154
  DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.ObjectLayerRenderFrames;
@@ -161,7 +161,6 @@ const ObjectLayerService = {
161
161
  metadataOverride: req.body,
162
162
  });
163
163
 
164
- // Create documents using centralized engine method (with atlas generation)
165
164
  const { objectLayer } = await ObjectLayerEngine.createObjectLayerDocuments({
166
165
  ObjectLayer,
167
166
  ObjectLayerRenderFrames,
@@ -189,14 +188,14 @@ const ObjectLayerService = {
189
188
  DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.ObjectLayerRenderFrames;
190
189
  let newObjectLayer = await new ObjectLayer(req.body).save();
191
190
 
192
- // Generate atlas sprite sheet – this sets data.atlasSpriteSheetCid and saves
191
+ // Generate atlas sprite sheet – this sets data.render.cid and saves
193
192
  try {
194
193
  await AtlasSpriteSheetService.generate({ params: { id: newObjectLayer._id }, auth: req.auth }, res, options);
195
194
  } catch (atlasError) {
196
195
  logger.error('Failed to auto-generate atlas for new ObjectLayer:', atlasError);
197
196
  }
198
197
 
199
- // Re-read so data.atlasSpriteSheetCid is up-to-date, then recompute SHA-256 & IPFS CID
198
+ // Re-read so data.render.cid is up-to-date, then recompute SHA-256 & IPFS CID
200
199
  newObjectLayer = await ObjectLayer.findById(newObjectLayer._id).populate('objectLayerRenderFramesId');
201
200
  if (newObjectLayer) {
202
201
  newObjectLayer = await ObjectLayerEngine.computeAndSaveFinalSha256({
@@ -519,7 +518,7 @@ const ObjectLayerService = {
519
518
  * - `/:id` — Standard update from request body.
520
519
  *
521
520
  * The `/metadata` route delegates to {@link ObjectLayerEngine.updateObjectLayerDocuments}
522
- * for centralized document update, atlas regeneration, SHA-256 computation, and IPFS pinning.
521
+ * for document update, atlas regeneration, SHA-256 computation, and IPFS pinning.
523
522
  *
524
523
  * @async
525
524
  * @function put
@@ -636,7 +635,7 @@ const ObjectLayerService = {
636
635
  const ObjectLayerRenderFrames =
637
636
  DataBaseProvider.instance[`${options.host}${options.path}`].mongoose.models.ObjectLayerRenderFrames;
638
637
 
639
- // Build object layer data from the asset directory using centralized logic
638
+ // Build object layer data from the asset directory
640
639
  const { objectLayerRenderFramesData, objectLayerData } =
641
640
  await ObjectLayerEngine.buildObjectLayerDataFromDirectory({
642
641
  folder,
@@ -645,12 +644,7 @@ const ObjectLayerService = {
645
644
  metadataOverride: req.body,
646
645
  });
647
646
 
648
- // Preserve the existing seed if provided in the request body
649
- if (req.body.data && req.body.data.seed) {
650
- objectLayerData.data.seed = req.body.data.seed;
651
- }
652
-
653
- // Update documents using centralized engine method (with atlas generation)
647
+ // Update documents using engine method (with atlas generation)
654
648
  const { objectLayer } = await ObjectLayerEngine.updateObjectLayerDocuments({
655
649
  objectLayerId,
656
650
  ObjectLayer,
@@ -674,17 +668,17 @@ const ObjectLayerService = {
674
668
  }
675
669
 
676
670
  // PUT /:id - Standard update
677
- let updatedObjectLayer = await ObjectLayer.findByIdAndUpdate(req.params.id, req.body, { new: true });
671
+ let updatedObjectLayer = await ObjectLayer.findByIdAndUpdate(req.params.id, req.body, { returnDocument: 'after' });
678
672
 
679
673
  if (updatedObjectLayer) {
680
- // Generate atlas sprite sheet – this sets data.atlasSpriteSheetCid and saves
674
+ // Generate atlas sprite sheet – this sets data.render.cid and saves
681
675
  try {
682
676
  await AtlasSpriteSheetService.generate({ params: { id: req.params.id }, auth: req.auth }, res, options);
683
677
  } catch (atlasError) {
684
678
  logger.error('Failed to auto-update atlas for ObjectLayer:', atlasError);
685
679
  }
686
680
 
687
- // Re-read so data.atlasSpriteSheetCid is up-to-date, then recompute SHA-256 & IPFS CID
681
+ // Re-read so data.render.cid is up-to-date, then recompute SHA-256 & IPFS CID
688
682
  updatedObjectLayer = await ObjectLayer.findById(req.params.id).populate('objectLayerRenderFramesId');
689
683
  if (updatedObjectLayer) {
690
684
  updatedObjectLayer = await ObjectLayerEngine.computeAndSaveFinalSha256({
@@ -77,12 +77,11 @@ const ObjectLayerRenderFramesSchema = new Schema(
77
77
  );
78
78
 
79
79
  // Pre-save hook to ensure data consistency
80
- ObjectLayerRenderFramesSchema.pre('save', function (next) {
80
+ ObjectLayerRenderFramesSchema.pre('save', function () {
81
81
  // Ensure all required fields are present
82
82
  if (!this.frames || !this.colors || this.frame_duration === undefined) {
83
83
  throw new Error('Missing required fields: frames, colors, or frame_duration');
84
84
  }
85
- next();
86
85
  });
87
86
 
88
87
  // Create and export the model
@@ -30,12 +30,15 @@ const UserRouter = (options) => {
30
30
  emailConfirmed: true,
31
31
  publicKey: [],
32
32
  });
33
- logger.warn('Default admin user created. Please change the default password immediately!', result._doc);
33
+ logger.warn('Default admin user created. Please change the default password immediately!', {
34
+ username: result._doc.username,
35
+ email: result._doc.email,
36
+ role: result._doc.role,
37
+ });
34
38
  }
35
39
  }
36
40
  } catch (error) {
37
- logger.error('Error checking/creating admin user');
38
- console.log(error);
41
+ logger.error('Error checking/creating admin user', { error: error.message });
39
42
  }
40
43
 
41
44
  // Cache mailer images
@@ -46,10 +49,12 @@ const UserRouter = (options) => {
46
49
  check: fs.readFileSync(`./src/client/public/default/assets/mailer/api-user-check.png`),
47
50
  avatar: fs.readFileSync(`./src/client/public/default/assets/mailer/api-user-default-avatar.png`),
48
51
  },
49
- header: (res) => {
52
+ header: (res, req) => {
50
53
  res.set('Cross-Origin-Resource-Policy', 'cross-origin');
51
- res.set('Access-Control-Allow-Origin', '*');
52
54
  res.set('Access-Control-Allow-Headers', '*');
55
+ if (req && req.headers && req.headers.origin) {
56
+ res.set('Access-Control-Allow-Origin', req.headers.origin);
57
+ } else res.setHeader('Access-Control-Allow-Origin', '*');
53
58
  res.set('Content-Type', 'image/png');
54
59
  },
55
60
  };
@@ -265,7 +265,7 @@ const UserService = {
265
265
  }
266
266
 
267
267
  if (req.path.startsWith('/assets')) {
268
- options.png.header(res);
268
+ options.png.header(res, req);
269
269
  return options.png.buffer[req.params.id];
270
270
  }
271
271
 
@@ -281,7 +281,7 @@ const UserService = {
281
281
  payload = verifyJWT(req.params.id, options);
282
282
  } catch (error) {
283
283
  logger.error(error, { 'req.params.id': req.params.id });
284
- options.png.header(res);
284
+ options.png.header(res, req);
285
285
  return options.png.buffer['invalid-token'];
286
286
  }
287
287
  const user = await User.findOne({
@@ -294,10 +294,10 @@ const UserService = {
294
294
  { recoverTimeOut: new Date(+new Date() + 1000 * 60 * 15) },
295
295
  { runValidators: true },
296
296
  );
297
- options.png.header(res);
297
+ options.png.header(res, req);
298
298
  return options.png.buffer['recover'];
299
299
  } else {
300
- options.png.header(res);
300
+ options.png.header(res, req);
301
301
  return options.png.buffer['invalid-token'];
302
302
  }
303
303
  }
@@ -308,7 +308,7 @@ const UserService = {
308
308
  payload = verifyJWT(req.params.id, options);
309
309
  } catch (error) {
310
310
  logger.error(error, { 'req.params.id': req.params.id });
311
- options.png.header(res);
311
+ options.png.header(res, req);
312
312
  return options.png.buffer['invalid-token'];
313
313
  }
314
314
  const user = await User.findOne({
@@ -324,10 +324,10 @@ const UserService = {
324
324
  status: 'email-confirmed',
325
325
  id: userWsId,
326
326
  });
327
- options.png.header(res);
327
+ options.png.header(res, req);
328
328
  return options.png.buffer['check'];
329
329
  } else {
330
- options.png.header(res);
330
+ options.png.header(res, req);
331
331
  return options.png.buffer['invalid-token'];
332
332
  }
333
333
  }
@@ -135,9 +135,6 @@ class UnderpostBaremetal {
135
135
  ) {
136
136
  let { ipAddress, hostname, ipFileServer, ipConfig, netmask, dnsServer } = options;
137
137
 
138
- // Load environment variables from .env file, overriding existing ones if present.
139
- dotenv.config({ path: `${getUnderpostRootPath()}/.env`, override: true });
140
-
141
138
  // Determine the root path for npm and underpost.
142
139
  const npmRoot = getNpmRootPath();
143
140
  const underpostRoot = options?.dev === true ? '.' : `${npmRoot}/underpost`;
@@ -1147,9 +1144,8 @@ rm -rf ${artifacts.join(' ')}`);
1147
1144
  machine: machine ? machine.system_id : null,
1148
1145
  });
1149
1146
 
1150
- const { discovery, machine: discoveredMachine } = await Underpost.baremetal.commissionMonitor(
1151
- commissionMonitorPayload,
1152
- );
1147
+ const { discovery, machine: discoveredMachine } =
1148
+ await Underpost.baremetal.commissionMonitor(commissionMonitorPayload);
1153
1149
  if (discoveredMachine) machine = discoveredMachine;
1154
1150
  }
1155
1151
  },
@@ -2494,10 +2490,10 @@ fi
2494
2490
  const discoverHostname = discovery.hostname
2495
2491
  ? discovery.hostname
2496
2492
  : discovery.mac_organization
2497
- ? discovery.mac_organization
2498
- : discovery.domain
2499
- ? discovery.domain
2500
- : `generic-host-${s4()}${s4()}`;
2493
+ ? discovery.mac_organization
2494
+ : discovery.domain
2495
+ ? discovery.domain
2496
+ : `generic-host-${s4()}${s4()}`;
2501
2497
 
2502
2498
  console.log(discoverHostname.bgBlue.bold.white);
2503
2499
  console.log('ip target:'.green + ipAddress, 'ip discovered:'.green + discovery.ip);
@@ -5,15 +5,12 @@
5
5
  * @namespace UnderpostCloudInit
6
6
  */
7
7
 
8
- import dotenv from 'dotenv';
9
8
  import { shellExec } from '../server/process.js';
10
9
  import fs from 'fs-extra';
11
10
  import { loggerFactory } from '../server/logger.js';
12
11
  import { getNpmRootPath } from '../server/conf.js';
13
12
  import Underpost from '../index.js';
14
13
 
15
- dotenv.config();
16
-
17
14
  const logger = loggerFactory(import.meta);
18
15
 
19
16
  /**
package/src/cli/db.js CHANGED
@@ -6,15 +6,30 @@
6
6
  * Supports MariaDB and MongoDB with import/export capabilities, Git integration, and multi-pod operations.
7
7
  */
8
8
 
9
- import { mergeFile, splitFileFactory } from '../server/conf.js';
9
+ import { mergeFile, splitFileFactory, loadConfServerJson, resolveConfSecrets } from '../server/conf.js';
10
10
  import { loggerFactory } from '../server/logger.js';
11
11
  import { shellExec } from '../server/process.js';
12
12
  import fs from 'fs-extra';
13
13
  import { DataBaseProvider } from '../db/DataBaseProvider.js';
14
- import { loadReplicas, pathPortAssignmentFactory } from '../server/conf.js';
14
+ import { loadReplicas, pathPortAssignmentFactory, loadCronDeployEnv } from '../server/conf.js';
15
15
  import Underpost from '../index.js';
16
16
  const logger = loggerFactory(import.meta);
17
17
 
18
+ /**
19
+ * Redacts credentials from shell command strings before logging.
20
+ * Masks passwords in `-p<password>`, `--password=<password>`, and `-P <password>` patterns.
21
+ * @param {string} cmd - The raw command string.
22
+ * @memberof UnderpostDB
23
+ * @returns {string} The command with credentials replaced by `***`.
24
+ */
25
+ const sanitizeCommand = (cmd) => {
26
+ if (typeof cmd !== 'string') return cmd;
27
+ return cmd
28
+ .replace(/-p['"]?[^\s'"]+/g, '-p***')
29
+ .replace(/--password=['"]?[^\s'"]+/g, '--password=***')
30
+ .replace(/-P\s+['"]?[^\s'"]+/g, '-P ***');
31
+ };
32
+
18
33
  /**
19
34
  * Constants for database operations
20
35
  * @constant {number} MAX_BACKUP_RETENTION - Maximum number of backups to retain
@@ -133,10 +148,10 @@ class UnderpostDB {
133
148
  const { context = '' } = options;
134
149
 
135
150
  try {
136
- logger.info(`Executing kubectl command`, { command, context });
137
- return shellExec(command, { stdout: true });
151
+ logger.info(`Executing kubectl command`, { command: sanitizeCommand(command), context });
152
+ return shellExec(command, { stdout: true, disableLog: true });
138
153
  } catch (error) {
139
- logger.error(`kubectl command failed`, { command, error: error.message, context });
154
+ logger.error(`kubectl command failed`, { command: sanitizeCommand(command), error: error.message, context });
140
155
  throw error;
141
156
  }
142
157
  },
@@ -200,11 +215,30 @@ class UnderpostDB {
200
215
  const kubectlCmd = `sudo kubectl exec -n ${namespace} -i ${podName} -- sh -c "${command}"`;
201
216
  return Underpost.db._executeKubectl(kubectlCmd, { context: `exec in pod ${podName}` });
202
217
  } catch (error) {
203
- logger.error('Failed to execute command in pod', { podName, command, error: error.message });
218
+ logger.error('Failed to execute command in pod', {
219
+ podName,
220
+ command: sanitizeCommand(command),
221
+ error: error.message,
222
+ });
204
223
  throw error;
205
224
  }
206
225
  },
207
226
 
227
+ /**
228
+ * Helper: Resolves the latest backup timestamp from an existing backup directory.
229
+ * Scans the directory for numeric (epoch) sub-folders and returns the most recent one.
230
+ * @method _getLatestBackupTimestamp
231
+ * @memberof UnderpostDB
232
+ * @param {string} backupDir - Path to the host-folder backup directory.
233
+ * @return {string|null} The latest timestamp string, or null if none found.
234
+ */
235
+ _getLatestBackupTimestamp(backupDir) {
236
+ if (!fs.existsSync(backupDir)) return null;
237
+ const entries = fs.readdirSync(backupDir).filter((e) => /^\d+$/.test(e));
238
+ if (entries.length === 0) return null;
239
+ return entries.sort((a, b) => parseInt(b) - parseInt(a))[0];
240
+ },
241
+
208
242
  /**
209
243
  * Helper: Manages Git repository for backups.
210
244
  * @method _manageGitRepo
@@ -275,55 +309,6 @@ class UnderpostDB {
275
309
  }
276
310
  },
277
311
 
278
- /**
279
- * Helper: Manages backup timestamps and cleanup.
280
- * @method _manageBackupTimestamps
281
- * @memberof UnderpostDB
282
- * @param {string} backupPath - Backup directory path.
283
- * @param {number} newTimestamp - New backup timestamp.
284
- * @param {boolean} shouldCleanup - Whether to cleanup old backups.
285
- * @return {Object} Backup info with current and removed timestamps.
286
- */
287
- _manageBackupTimestamps(backupPath, newTimestamp, shouldCleanup) {
288
- try {
289
- if (!fs.existsSync(backupPath)) {
290
- fs.mkdirSync(backupPath, { recursive: true });
291
- }
292
-
293
- // Delete empty folders
294
- shellExec(`cd ${backupPath} && find . -type d -empty -delete`);
295
-
296
- const times = fs.readdirSync(backupPath);
297
- const validTimes = times.map((t) => parseInt(t)).filter((t) => !isNaN(t));
298
-
299
- const currentBackupTimestamp = validTimes.length > 0 ? Math.max(...validTimes) : null;
300
- const removeBackupTimestamp = validTimes.length > 0 ? Math.min(...validTimes) : null;
301
-
302
- // Cleanup old backups if we have too many
303
- if (shouldCleanup && validTimes.length >= MAX_BACKUP_RETENTION && removeBackupTimestamp) {
304
- const removeDir = `${backupPath}/${removeBackupTimestamp}`;
305
- logger.info('Removing old backup', { path: removeDir });
306
- fs.removeSync(removeDir);
307
- }
308
-
309
- // Create new backup directory
310
- if (shouldCleanup) {
311
- const newBackupDir = `${backupPath}/${newTimestamp}`;
312
- logger.info('Creating new backup directory', { path: newBackupDir });
313
- fs.mkdirSync(newBackupDir, { recursive: true });
314
- }
315
-
316
- return {
317
- current: currentBackupTimestamp,
318
- removed: removeBackupTimestamp,
319
- count: validTimes.length,
320
- };
321
- } catch (error) {
322
- logger.error('Error managing backup timestamps', { backupPath, error: error.message });
323
- return { current: null, removed: null, count: 0 };
324
- }
325
- },
326
-
327
312
  /**
328
313
  * Helper: Performs MariaDB import operation.
329
314
  * @method _importMariaDB
@@ -624,7 +609,7 @@ class UnderpostDB {
624
609
  logger.info('Getting MariaDB table statistics', { podName, dbName });
625
610
 
626
611
  const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mariadb -u ${user} -p${password} ${dbName} -e "SELECT TABLE_NAME as 'table', TABLE_ROWS as 'count' FROM information_schema.TABLES WHERE TABLE_SCHEMA = '${dbName}' ORDER BY TABLE_NAME;" --skip-column-names --batch`;
627
- const output = shellExec(command, { stdout: true, silent: true });
612
+ const output = shellExec(command, { stdout: true, silent: true, disableLog: true });
628
613
 
629
614
  if (!output || output.trim() === '') {
630
615
  logger.warn('No tables found or empty output');
@@ -788,6 +773,7 @@ class UnderpostDB {
788
773
  kind: false,
789
774
  },
790
775
  ) {
776
+ loadCronDeployEnv();
791
777
  const newBackupTimestamp = new Date().getTime();
792
778
  const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
793
779
 
@@ -844,7 +830,7 @@ class UnderpostDB {
844
830
  continue;
845
831
  }
846
832
 
847
- const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
833
+ const confServer = loadConfServerJson(confServerPath, { resolve: true });
848
834
 
849
835
  // Build database configuration map
850
836
  for (const host of Object.keys(confServer)) {
@@ -943,16 +929,11 @@ class UnderpostDB {
943
929
 
944
930
  logger.info('Processing database', { hostFolder, provider, dbName, deployId });
945
931
 
946
- const backUpPath = `../${repoName}/${hostFolder}`;
947
- const backupInfo = Underpost.db._manageBackupTimestamps(
948
- backUpPath,
949
- newBackupTimestamp,
950
- options.export === true,
951
- );
932
+ const latestBackupTimestamp = Underpost.db._getLatestBackupTimestamp(`../${repoName}/${hostFolder}`);
952
933
 
953
- dbs[provider][dbName].currentBackupTimestamp = backupInfo.current;
934
+ dbs[provider][dbName].currentBackupTimestamp = latestBackupTimestamp;
954
935
 
955
- const currentTimestamp = backupInfo.current || newBackupTimestamp;
936
+ const currentTimestamp = latestBackupTimestamp || newBackupTimestamp;
956
937
  const sqlContainerPath = `/home/${dbName}.sql`;
957
938
  const fromPartsPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}-parths.json`;
958
939
  const toSqlPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}.sql`;
@@ -1149,6 +1130,7 @@ class UnderpostDB {
1149
1130
  host = process.env.DEFAULT_DEPLOY_HOST,
1150
1131
  path = process.env.DEFAULT_DEPLOY_PATH,
1151
1132
  ) {
1133
+ loadCronDeployEnv();
1152
1134
  deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
1153
1135
  host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
1154
1136
  path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
@@ -1171,7 +1153,7 @@ class UnderpostDB {
1171
1153
  throw new Error(`Server configuration not found: ${confServerPath}`);
1172
1154
  }
1173
1155
 
1174
- const { db } = JSON.parse(fs.readFileSync(confServerPath, 'utf8'))[host][path];
1156
+ const { db } = loadConfServerJson(confServerPath, { resolve: true })[host][path];
1175
1157
 
1176
1158
  try {
1177
1159
  await DataBaseProvider.load({ apis: ['instance', 'cron'], host, path, db });
@@ -1194,7 +1176,7 @@ class UnderpostDB {
1194
1176
  continue;
1195
1177
  }
1196
1178
 
1197
- const confServer = loadReplicas(deployId, JSON.parse(fs.readFileSync(confServerPath, 'utf8')));
1179
+ const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath, { resolve: true }));
1198
1180
  const router = await Underpost.deploy.routerFactory(deployId, env);
1199
1181
  const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);
1200
1182
 
@@ -1257,7 +1239,7 @@ class UnderpostDB {
1257
1239
  }
1258
1240
  }
1259
1241
  } catch (error) {
1260
- logger.error('Failed to create instance metadata', { error: error.message, stack: error.stack });
1242
+ logger.error('Failed to create instance metadata', { error: error.message });
1261
1243
  throw error;
1262
1244
  }
1263
1245
 
@@ -1297,7 +1279,7 @@ class UnderpostDB {
1297
1279
  await new Cron(body).save();
1298
1280
  }
1299
1281
  } catch (error) {
1300
- logger.error('Failed to create cron metadata', { error: error.message, stack: error.stack });
1282
+ logger.error('Failed to create cron metadata', { error: error.message });
1301
1283
  }
1302
1284
 
1303
1285
  await DataBaseProvider.instance[`${host}${path}`].mongoose.close();
@@ -1325,6 +1307,7 @@ class UnderpostDB {
1325
1307
  dryRun: false,
1326
1308
  },
1327
1309
  ) {
1310
+ loadCronDeployEnv();
1328
1311
  if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
1329
1312
 
1330
1313
  logger.info('Starting File collection cleanup', { deployList, options });
@@ -1359,7 +1342,7 @@ class UnderpostDB {
1359
1342
  continue;
1360
1343
  }
1361
1344
 
1362
- const confServer = JSON.parse(fs.readFileSync(confServerPath, 'utf8'));
1345
+ const confServer = loadConfServerJson(confServerPath, { resolve: true });
1363
1346
 
1364
1347
  // Process each host+path combination
1365
1348
  for (const host of Object.keys(confServer)) {
@@ -1499,7 +1482,6 @@ class UnderpostDB {
1499
1482
  host,
1500
1483
  path,
1501
1484
  error: error.message,
1502
- stack: error.stack,
1503
1485
  });
1504
1486
  }
1505
1487
  }
@@ -1553,6 +1535,7 @@ class UnderpostDB {
1553
1535
  crons: false,
1554
1536
  },
1555
1537
  ) {
1538
+ loadCronDeployEnv();
1556
1539
  deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
1557
1540
  host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
1558
1541
  path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
package/src/cli/deploy.js CHANGED
@@ -11,6 +11,7 @@ import {
11
11
  Config,
12
12
  deployRangePortFactory,
13
13
  getDataDeploy,
14
+ loadConfServerJson,
14
15
  loadReplicas,
15
16
  pathPortAssignmentFactory,
16
17
  } from '../server/conf.js';
@@ -132,7 +133,7 @@ class UnderpostDeploy {
132
133
  `npm install -g npm@11.2.0`,
133
134
  `npm install -g underpost`,
134
135
  `underpost secret underpost --create-from-file /etc/config/.env.${env}`,
135
- `underpost start --build --run --underpost-quickly-install ${deployId} ${env}`,
136
+ `underpost start --build --run ${deployId} ${env}`,
136
137
  ];
137
138
  const packageJson = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
138
139
  if (!volumes)
@@ -230,7 +231,7 @@ spec:
230
231
  if (!deployId) continue;
231
232
  const confServer = loadReplicas(
232
233
  deployId,
233
- JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')),
234
+ loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`),
234
235
  );
235
236
  const router = await Underpost.deploy.routerFactory(deployId, env);
236
237
  const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);
@@ -259,6 +260,27 @@ ${Underpost.deploy
259
260
  }
260
261
  fs.writeFileSync(`./engine-private/conf/${deployId}/build/${env}/deployment.yaml`, deploymentYamlParts, 'utf8');
261
262
 
263
+ const confVolume = fs.existsSync(`./engine-private/conf/${deployId}/conf.volume.json`)
264
+ ? JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.volume.json`, 'utf8'))
265
+ : [];
266
+ if (confVolume.length > 0) {
267
+ let volumeYaml = '';
268
+ for (const deploymentVersion of deploymentVersions) {
269
+ for (const volume of confVolume) {
270
+ if (!volume.claimName) continue;
271
+ const pvcId = `${volume.claimName}-${deployId}-${env}-${deploymentVersion}`;
272
+ const pvId = pvcId.replace(/^pvc-/, 'pv-');
273
+ const hostPath = `/home/dd/engine/volume/${pvId}`;
274
+ volumeYaml += `---\n${Underpost.deploy.persistentVolumeFactory({
275
+ pvcId,
276
+ namespace: options.namespace,
277
+ hostPath,
278
+ })}\n`;
279
+ }
280
+ }
281
+ fs.writeFileSync(`./engine-private/conf/${deployId}/build/${env}/pv-pvc.yaml`, volumeYaml, 'utf8');
282
+ }
283
+
262
284
  let proxyYaml = '';
263
285
  let secretYaml = '';
264
286
  const customServices = fs.existsSync(`./engine-private/conf/${deployId}/conf.services.json`)
@@ -335,7 +357,7 @@ ${Underpost.deploy
335
357
  const yamlPath = `./engine-private/conf/${deployId}/build/${env}/secret.yaml`;
336
358
  fs.writeFileSync(yamlPath, secretYaml, 'utf8');
337
359
  } else {
338
- const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml'];
360
+ const deploymentsFiles = ['Dockerfile', 'proxy.yaml', 'deployment.yaml', 'pv-pvc.yaml'];
339
361
  for (const file of deploymentsFiles) {
340
362
  if (fs.existsSync(`./engine-private/conf/${deployId}/build/${env}/${file}`)) {
341
363
  fs.copyFileSync(
@@ -385,7 +407,7 @@ spec:
385
407
  // kubectl get deploy,sts,svc,configmap,secret -n default -o yaml --export > default.yaml
386
408
  const hostTest = options?.hostTest
387
409
  ? options.hostTest
388
- : Object.keys(JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8')))[0];
410
+ : Object.keys(loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`))[0];
389
411
  const info = shellExec(`sudo kubectl get HTTPProxy/${hostTest} -n ${options.namespace} -o yaml`, {
390
412
  silent: true,
391
413
  stdout: true,
@@ -550,7 +572,7 @@ EOF`);
550
572
  if (!(options.versions && typeof options.versions === 'string')) options.versions = 'blue,green';
551
573
  if (!options.replicas) options.replicas = 1;
552
574
  if (options.sync)
553
- getDataDeploy({
575
+ await getDataDeploy({
554
576
  buildSingleReplica: true,
555
577
  });
556
578
  if (options.buildManifest === true) await Underpost.deploy.buildManifest(deployList, env, options);
@@ -591,7 +613,7 @@ EOF`);
591
613
  continue;
592
614
  }
593
615
 
594
- const confServer = JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.server.json`, 'utf8'));
616
+ const confServer = loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`);
595
617
  const confVolume = fs.existsSync(`./engine-private/conf/${deployId}/conf.volume.json`)
596
618
  ? JSON.parse(fs.readFileSync(`./engine-private/conf/${deployId}/conf.volume.json`, 'utf8'))
597
619
  : [];
@@ -844,6 +866,7 @@ EOF`);
844
866
  ${Underpost.deploy.persistentVolumeFactory({
845
867
  hostPath: rootVolumeHostPath,
846
868
  pvcId,
869
+ namespace,
847
870
  })}
848
871
  EOF
849
872
  `);
@@ -914,15 +937,44 @@ EOF
914
937
  * @param {object} options - Options for the persistent volume and claim creation.
915
938
  * @param {string} options.hostPath - Host path for the persistent volume.
916
939
  * @param {string} options.pvcId - Persistent volume claim ID.
940
+ * @param {string} [options.namespace='default'] - Kubernetes namespace for the PVC claimRef.
917
941
  * @returns {string} - YAML configuration for the persistent volume and claim.
918
942
  * @memberof UnderpostDeploy
919
943
  */
920
- persistentVolumeFactory({ hostPath, pvcId }) {
921
- return fs
922
- .readFileSync(`./manifests/pv-pvc-dd.yaml`, 'utf8')
923
- .replace('/home/dd', hostPath)
924
- .replace('pv-dd', pvcId.replace('pvc-', 'pv-'))
925
- .replace('pvc-dd', pvcId);
944
+ persistentVolumeFactory({ hostPath, pvcId, namespace = 'default' }) {
945
+ const pvId = pvcId.replace(/^pvc-/, 'pv-');
946
+ return `apiVersion: v1
947
+ kind: PersistentVolume
948
+ metadata:
949
+ name: ${pvId}
950
+ spec:
951
+ capacity:
952
+ storage: 5Gi
953
+ accessModes:
954
+ - ReadWriteOnce
955
+ persistentVolumeReclaimPolicy: Retain
956
+ storageClassName: manual
957
+ claimRef:
958
+ apiVersion: v1
959
+ kind: PersistentVolumeClaim
960
+ name: ${pvcId}
961
+ namespace: ${namespace}
962
+ hostPath:
963
+ path: ${hostPath}
964
+ type: DirectoryOrCreate
965
+ ---
966
+ apiVersion: v1
967
+ kind: PersistentVolumeClaim
968
+ metadata:
969
+ name: ${pvcId}
970
+ spec:
971
+ accessModes:
972
+ - ReadWriteOnce
973
+ storageClassName: manual
974
+ volumeName: ${pvId}
975
+ resources:
976
+ requests:
977
+ storage: 5Gi`;
926
978
  },
927
979
 
928
980
  /**