@underpostnet/cyberia 3.2.90 → 3.3.73

package/src/api/object-layer/object-layer.model.js

@@ -3,6 +3,8 @@
  * @module src/api/object-layer/object-layer.model.js
  * @namespace CyberiaObjectLayerModel
  */
+import crypto from 'crypto';
+import stringify from 'fast-json-stable-stringify';
 import { Schema, model } from 'mongoose';
 /**
  * @typedef {Object} Stats
@@ -132,8 +134,19 @@ const ObjectLayerSchema = new Schema(
     toObject: { virtuals: true },
   },
 );
-// Index for faster querying
-ObjectLayerSchema.index({ 'data.item.id': 1 });
+/**
+ * Name of the `data.item.id` index. Kept stable across the non-unique → unique
+ * upgrade so {@link ObjectLayerModel.ensureUniqueItemIdIndex} can detect and
+ * replace the legacy definition instead of leaving two overlapping indexes.
+ * @memberof CyberiaObjectLayerModel
+ */
+const ITEM_ID_INDEX_NAME = 'data.item.id_1';
+// `data.item.id` is the natural key: exactly one document per item id.
+// autoIndex runs at model-compile time, so on a collection that still holds
+// legacy duplicates (or a legacy non-unique index) this build fails and is
+// logged by the model's `error` listener. ensureUniqueItemIdIndex() then
+// dedupes and rebuilds it, after which autoIndex agrees and stays quiet.
+ObjectLayerSchema.index({ 'data.item.id': 1 }, { name: ITEM_ID_INDEX_NAME, unique: true });
 ObjectLayerSchema.index({ 'data.item.type': 1 });
 // Add text index for searchable fields
 ObjectLayerSchema.index(
@@ -158,6 +171,186 @@ ObjectLayerSchema.pre('save', function () {
   }
   // cid (object layer data JSON) and data.render.cid (atlas PNG) are optional – default to ''
 });
+
+/**
+ * Computes the canonical SHA-256 of an object layer `data` sub-document using
+ * deterministic JSON serialisation. Single source of truth for the hash;
+ * `ObjectLayerEngine.computeSha256` delegates here.
+ *
+ * @param {Object} data - The `data` sub-document (item, stats, ledger, render).
+ * @returns {string} Hex-encoded SHA-256 hash.
+ * @memberof CyberiaObjectLayerModel
+ */
+const computeObjectLayerSha256 = (data) => crypto.createHash('sha256').update(stringify(data)).digest('hex');
+
+const isMergeableObject = (value) => {
+  if (value === null || typeof value !== 'object') return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+};
+
+/**
+ * An incoming attribute only wins when it actually carries a value. `null`,
+ * `undefined` and `''` mean "not provided by this writer" — importers emit them
+ * whenever a stage is skipped or degraded (IPFS unreachable, atlas not
+ * generated, saga metadata with no render yet), and they must never erase a
+ * value another writer already persisted. `false` and `0` are real values.
+ *
+ * @param {*} value - Candidate attribute value.
+ * @returns {boolean} Whether the value should override the stored one.
+ * @memberof CyberiaObjectLayerModel
+ */
+const hasValue = (value) => value !== null && value !== undefined && value !== '';
+
+/**
+ * Deep-merges `incoming` over `existing`, keeping the last attribute that
+ * actually carries a value. Plain objects merge key by key; every other value
+ * (scalars, arrays, ObjectIds, Dates, Buffers) is replaced atomically.
+ *
+ * @param {*} existing - Currently persisted value.
+ * @param {*} incoming - Value received from the writer.
+ * @returns {*} The merged value.
+ * @memberof CyberiaObjectLayerModel
+ */
+function mergeObjectLayerData(existing, incoming) {
+  if (isMergeableObject(incoming)) {
+    if (!isMergeableObject(existing)) return incoming;
+    const merged = { ...existing };
+    for (const key of Object.keys(incoming)) merged[key] = mergeObjectLayerData(existing[key], incoming[key]);
+    return merged;
+  }
+  return hasValue(incoming) ? incoming : existing;
+}
+
+/**
+ * Document fields an upsert may carry. `sha256` is excluded on purpose: it is
+ * always recomputed from the merged `data`, so it can never drift from the
+ * payload the writer hashed before merging.
+ * @memberof CyberiaObjectLayerModel
+ */
+const UPSERTABLE_FIELDS = ['data', 'cid', 'objectLayerRenderFramesId', 'atlasSpriteSheetId'];
+
+/**
+ * Collapses any pre-existing duplicates of a given `data.item.id` down to a
+ * single document, keeping the oldest one so `_id` references held elsewhere
+ * (instances, inventories, atlas metadata) stay valid.
+ *
+ * @param {import('mongoose').Model} Model - The bound ObjectLayer model.
+ * @param {string} itemId - The item id to collapse.
+ * @returns {Promise<{ survivor: Object|null, removedIds: Array }>} Survivor and removed document ids.
+ * @memberof CyberiaObjectLayerModel
+ */
+async function collapseItemIdDuplicates(Model, itemId) {
+  const [survivor, ...duplicates] = await Model.find({ 'data.item.id': itemId }).sort({ createdAt: 1, _id: 1 });
+  if (!survivor || duplicates.length === 0) return { survivor: survivor || null, removedIds: [] };
+
+  const removedIds = duplicates.map((duplicate) => duplicate._id);
+  await Model.deleteMany({ _id: { $in: removedIds } });
+
+  // Orphaned render frames have no other cleanup path once their owner is gone.
+  const renderFramesIds = duplicates.map((duplicate) => duplicate.objectLayerRenderFramesId).filter(Boolean);
+  if (renderFramesIds.length > 0 && Model.db.models.ObjectLayerRenderFrames) {
+    await Model.db.models.ObjectLayerRenderFrames.deleteMany({ _id: { $in: renderFramesIds } });
+  }
+
+  return { survivor, removedIds };
+}
+
+/**
+ * Resolves the canonical document for an item id. While a legacy collection can
+ * still hold duplicates, an unsorted `findOne` may hand back a document that the
+ * next upsert collapses away, so readers and writers must agree on the same
+ * survivor: the oldest one.
+ *
+ * @param {string} itemId - The `data.item.id` to look up.
+ * @returns {import('mongoose').Query} Query resolving to the canonical document or `null`.
+ * @memberof CyberiaObjectLayerModel
+ */
+ObjectLayerSchema.statics.findByItemId = function (itemId) {
+  return this.findOne({ 'data.item.id': itemId }).sort({ createdAt: 1, _id: 1 });
+};
+
+/**
+ * Dedupes the whole collection by `data.item.id`.
+ *
+ * @returns {Promise<Array>} Ids of the removed duplicate documents.
+ * @memberof CyberiaObjectLayerModel
+ */
+ObjectLayerSchema.statics.dedupeByItemId = async function () {
+  const duplicated = await this.aggregate([
+    { $group: { _id: '$data.item.id', count: { $sum: 1 } } },
+    { $match: { count: { $gt: 1 } } },
+  ]);
+  const removedIds = [];
+  for (const { _id: itemId } of duplicated) {
+    if (itemId === null || itemId === undefined) continue;
+    const { removedIds: removed } = await collapseItemIdDuplicates(this, itemId);
+    removedIds.push(...removed);
+  }
+  return removedIds;
+};
+
+/**
+ * Idempotent migration that enforces the one-document-per-`data.item.id`
+ * invariant at the storage layer: dedupes first, then upgrades the legacy
+ * non-unique index to a unique one. Safe to rerun; a no-op once applied.
+ *
+ * @returns {Promise<{ removedIds: Array, indexUpgraded: boolean }>} Migration outcome.
+ * @memberof CyberiaObjectLayerModel
+ */
+ObjectLayerSchema.statics.ensureUniqueItemIdIndex = async function () {
+  const removedIds = await this.dedupeByItemId();
+
+  const indexes = await this.collection.indexes().catch(() => []);
+  const current = indexes.find((index) => index.name === ITEM_ID_INDEX_NAME);
+  if (current?.unique) return { removedIds, indexUpgraded: false };
+
+  if (current) await this.collection.dropIndex(ITEM_ID_INDEX_NAME);
+  await this.collection.createIndex({ 'data.item.id': 1 }, { name: ITEM_ID_INDEX_NAME, unique: true });
+  return { removedIds, indexUpgraded: true };
+};
+
+/**
+ * The single write path for object layers keyed by item id.
+ *
+ * Guarantees that exactly one document exists per `data.item.id`: pre-existing
+ * duplicates are collapsed onto the oldest document, which is then updated in
+ * place. Attributes are merged with {@link mergeObjectLayerData}, so a writer
+ * that omits a value (or sends `null` / `''` because a stage was skipped) keeps
+ * whatever a previous writer stored instead of erasing it. `sha256` is always
+ * recomputed from the merged result.
+ *
+ * @param {Object} payload - Document payload; `data.item.id` is required.
+ * @param {Object} payload.data - Object layer data (item, stats, ledger, render).
+ * @param {Object} [options] - Upsert options.
+ * @param {Object} [options.setOnInsert=null] - Partial document applied only when the item is new,
+ *   for fields a writer wants to seed but never refresh (mirrors Mongo's `$setOnInsert`).
+ * @returns {Promise<Object>} The single surviving ObjectLayer document.
+ * @memberof CyberiaObjectLayerModel
+ */
+ObjectLayerSchema.statics.upsertByItemId = async function (payload, { setOnInsert = null } = {}) {
+  const itemId = payload?.data?.item?.id;
+  if (!itemId) throw new Error('ObjectLayer.upsertByItemId requires data.item.id');
+
+  const { survivor } = await collapseItemIdDuplicates(this, itemId);
+
+  if (!survivor) {
+    const inserted = setOnInsert ? mergeObjectLayerData(setOnInsert, payload) : payload;
+    return await this.create({ ...inserted, sha256: computeObjectLayerSha256(inserted.data) });
+  }
+
+  const existing = survivor.toObject({ virtuals: false, depopulate: true });
+  const update = {};
+  for (const field of UPSERTABLE_FIELDS) {
+    if (!(field in payload)) continue;
+    const merged = mergeObjectLayerData(existing[field], payload[field]);
+    if (merged !== undefined) update[field] = merged;
+  }
+  update.sha256 = computeObjectLayerSha256(update.data ?? existing.data);
+
+  return await this.findByIdAndUpdate(survivor._id, { $set: update }, { returnDocument: 'after' });
+};
+
 // Create and export the model
 const ObjectLayerModel = model('ObjectLayer', ObjectLayerSchema);
 const ProviderSchema = ObjectLayerSchema;
@@ -197,4 +390,12 @@ class ObjectLayerDto {
     },
   };
 }
-export { ObjectLayerSchema, ObjectLayerModel, ProviderSchema, ObjectLayerDto };
+export {
+  ObjectLayerSchema,
+  ObjectLayerModel,
+  ProviderSchema,
+  ObjectLayerDto,
+  computeObjectLayerSha256,
+  mergeObjectLayerData,
+  ITEM_ID_INDEX_NAME,
+};

package/src/api/object-layer/object-layer.router.js

@@ -1,7 +1,7 @@
-import { loggerFactory } from '../../server/logger.js';
+import { loggerFactory } from '../../server/ops/logger.js';
 import { ObjectLayerController } from './object-layer.controller.js';
 import express from 'express';
-import { moderatorGuard, adminGuard } from '../../server/auth.js';
+import { moderatorGuard, adminGuard } from '../../server/security/auth.js';
 
 const logger = loggerFactory(import.meta);
 

package/src/api/object-layer/object-layer.service.js

@@ -11,14 +11,14 @@
  */
 
 import { DataBaseProviderService } from '../../db/DataBaseProvider.js';
-import { loggerFactory } from '../../server/logger.js';
+import { loggerFactory } from '../../server/ops/logger.js';
 import { ObjectLayerRenderFramesDto } from '../object-layer-render-frames/object-layer-render-frames.model.js';
 import { FileFactory } from '../file/file.service.js';
 import fs from 'fs-extra';
 import { ObjectLayerDto } from './object-layer.model.js';
 import { ObjectLayerEngine } from '../../projects/cyberia/object-layer.js';
-import { shellExec } from '../../server/process.js';
-import { DataQuery } from '../../server/data-query.js';
+import { shellExec } from '../../server/runtime/process.js';
+import { DataQuery } from '../../server/storage/data-query.js';
 import { AtlasSpriteSheetService } from '../atlas-sprite-sheet/atlas-sprite-sheet.service.js';
 import { IpfsClient } from '../../projects/cyberia/ipfs-client.js';
 import { createPinRecord, removePinRecordsAndUnpin } from '../ipfs/ipfs.service.js';
@@ -215,9 +215,6 @@ class ObjectLayerService {
       }
     }
 
-    // Compute final SHA-256 with all CIDs
-    bodyData.sha256 = ObjectLayerEngine.computeSha256(bodyData.data);
-
     // Pin data JSON to IPFS
     try {
       const itemId = bodyData.data.item.id;
@@ -232,17 +229,7 @@ class ObjectLayerService {
     }
 
     // Atomic create/replace – ObjectLayer is fully populated with all CIDs
-    let newObjectLayer;
-    const existingByItemId = await ObjectLayer.findOne({ 'data.item.id': bodyData.data.item.id });
-    if (existingByItemId) {
-      newObjectLayer = await ObjectLayer.findByIdAndUpdate(existingByItemId._id, bodyData, {
-        returnDocument: 'after',
-      }).populate('objectLayerRenderFramesId');
-    } else {
-      newObjectLayer = await (await new ObjectLayer(bodyData).save()).populate('objectLayerRenderFramesId');
-    }
-
-    return newObjectLayer;
+    return await (await ObjectLayer.upsertByItemId(bodyData)).populate('objectLayerRenderFramesId');
   };
 
   /**

package/src/api/object-layer-render-frames/object-layer-render-frames.controller.js

@@ -1,4 +1,4 @@
-import { buildCrudController } from '../../server/middlewares.js';
+import { buildCrudController } from '../../server/network/middlewares.js';
 import { ObjectLayerRenderFramesService } from './object-layer-render-frames.service.js';
 
 const ObjectLayerRenderFramesController = buildCrudController(ObjectLayerRenderFramesService);

package/src/api/object-layer-render-frames/object-layer-render-frames.router.js

@@ -1,5 +1,5 @@
 import express from 'express';
-import { registerCrudRoutes } from '../../server/middlewares.js';
+import { registerCrudRoutes } from '../../server/network/middlewares.js';
 import { ObjectLayerRenderFramesController } from './object-layer-render-frames.controller.js';
 
 class ObjectLayerRenderFramesRouter {

package/src/api/object-layer-render-frames/object-layer-render-frames.service.js

@@ -1,6 +1,6 @@
 import { DataBaseProviderService } from '../../db/DataBaseProvider.js';
-import { loggerFactory } from '../../server/logger.js';
-import { DataQuery } from '../../server/data-query.js';
+import { loggerFactory } from '../../server/ops/logger.js';
+import { DataQuery } from '../../server/storage/data-query.js';
 
 const logger = loggerFactory(import.meta);
 

package/src/api/test/test.controller.js

@@ -1,4 +1,4 @@
-import { controllerHandler, sendSuccess, serviceHandler } from '../../server/middlewares.js';
+import { controllerHandler, sendSuccess, serviceHandler } from '../../server/network/middlewares.js';
 import { TestService } from './test.service.js';
 
 class TestController {

package/src/api/test/test.service.js

@@ -1,5 +1,5 @@
 import validator from 'validator';
-import { loggerFactory } from '../../server/logger.js';
+import { loggerFactory } from '../../server/ops/logger.js';
 import { TestModel } from './test.model.js';
 import { getYouTubeID, validatePassword } from '../../client/components/core/CommonJs.js';
 

package/src/api/user/guest.service.js

@@ -1,7 +1,7 @@
 import mongoose from 'mongoose';
 import { UserDto } from './user.model.js';
-import { ValkeyAPI } from '../../server/valkey.js';
-import { hashPassword, getBearerToken, jwtSign } from '../../server/auth.js';
+import { ValkeyAPI } from '../../db/valkey/Valkey.js';
+import { hashPassword, getBearerToken, jwtSign } from '../../server/security/auth.js';
 
 // ─── TTL ──────────────────────────────────────────────────────────────────────
 
@@ -15,7 +15,7 @@ const _guestTtlMs = () => {
 /**
  * Constructs a new ephemeral guest user object.
  * This is domain logic specific to the guest lifecycle; it does not belong
- * in the generic Valkey storage module.
+ * in the Valkey database adapter.
  *
  * @param {{ host?: string }} options
  * @returns {object}

package/src/api/user/user.controller.js

@@ -1,4 +1,4 @@
-import { loggerFactory } from '../../server/logger.js';
+import { loggerFactory } from '../../server/ops/logger.js';
 import { UserService } from './user.service.js';
 
 const logger = loggerFactory(import.meta);

package/src/api/user/user.router.js

@@ -1,6 +1,6 @@
-import { hashPassword } from '../../server/auth.js';
+import { hashPassword } from '../../server/security/auth.js';
 import fs from 'fs-extra';
-import { loggerFactory } from '../../server/logger.js';
+import { loggerFactory } from '../../server/ops/logger.js';
 import { UserController } from './user.controller.js';
 import express from 'express';
 import { DataBaseProviderService } from '../../db/DataBaseProvider.js';

package/src/api/user/user.service.js

@@ -7,8 +7,8 @@
  * @namespace UserService
  */
 
-import { loggerFactory } from '../../server/logger.js';
-import { DataQuery } from '../../server/data-query.js';
+import { loggerFactory } from '../../server/ops/logger.js';
+import { DataQuery } from '../../server/storage/data-query.js';
 import {
   hashPassword,
   verifyPassword,
@@ -19,7 +19,7 @@ import {
   logoutSession,
   jwtSign,
   validatePasswordMiddleware,
-} from '../../server/auth.js';
+} from '../../server/security/auth.js';
 import { MailerProvider } from '../../mailer/MailerProvider.js';
 import { CoreWsEmitter } from '../../ws/core/core.ws.emit.js';
 import { CoreWsMailerChannel } from '../../ws/core/channels/core.ws.mailer.js';
@@ -29,7 +29,7 @@ import { FileFactory, FileCleanup } from '../file/file.service.js';
 import { UserDto } from './user.model.js';
 import { timer } from '../../client/components/core/CommonJs.js';
 import { GuestService } from './guest.service.js';
-import { resolveHostKeyContext } from '../../server/conf.js';
+import { resolveHostKeyContext } from '../../server/runtime/conf.js';
 
 const logger = loggerFactory(import.meta);
 

package/src/api.js

@@ -1,18 +1,14 @@
 'use strict';
 
-// https://nodejs.org/api
-// https://expressjs.com/en/4x/api.html
-
 import dotenv from 'dotenv';
-import { loggerFactory } from './server/logger.js';
-import { buildClient } from './server/client-build.js';
-import { buildRuntime } from './server/runtime.js';
-import { ProcessController } from './server/process.js';
-import { Config } from './server/conf.js';
+import { loggerFactory } from './server/ops/logger.js';
+import { buildRuntime } from './server/runtime/runtime.js';
+import { ProcessController } from './server/runtime/process.js';
+import { Config, buildApiConf } from './server/runtime/conf.js';
 
 dotenv.config();
 
-await Config.build();
+await Config.build(undefined, undefined, await buildApiConf());
 
 const logger = loggerFactory(import.meta);