@naturalcycles/db-lib 10.42.0 → 10.42.1

package/dist/adapter/file/file.db.d.ts

@@ -1,4 +1,4 @@
-import { type ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { ObjectWithId } from '@naturalcycles/js-lib/types';
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv';
 import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { BaseCommonDB } from '../../commondb/base.common.db.js';

package/dist/adapter/file/file.db.js

@@ -3,7 +3,7 @@ import { _by, _sortBy } from '@naturalcycles/js-lib/array';
 import { _since, localTime } from '@naturalcycles/js-lib/datetime';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object';
-import { _stringMapValues, } from '@naturalcycles/js-lib/types';
+import { _stringMapValues } from '@naturalcycles/js-lib/types';
 import { generateJsonSchemaFromData } from '@naturalcycles/nodejs-lib/ajv';
 import { dimGrey } from '@naturalcycles/nodejs-lib/colors';
 import { Pipeline } from '@naturalcycles/nodejs-lib/stream';

package/dist/commondao/common.dao.d.ts

@@ -1,4 +1,4 @@
-import { type BaseDBEntity, type NonNegativeInteger, type StringMap, type Unsaved } from '@naturalcycles/js-lib/types';
+import type { BaseDBEntity, NonNegativeInteger, ObjectWithId, StringMap, Unsaved } from '@naturalcycles/js-lib/types';
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv';
 import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDBTransactionOptions, RunQueryResult } from '../db.model.js';
@@ -9,9 +9,12 @@ import { CommonDaoTransaction } from './commonDaoTransaction.js';
 /**
  * Lowest common denominator API between supported Databases.
  *
- * DBM = Database model (how it's stored in DB)
  * BM = Backend model (optimized for API access)
+ * DBM = Database model (logical representation, before compression)
  * TM = Transport model (optimized to be sent over the wire)
+ *
+ * Note: When auto-compression is enabled, the physical storage format differs from DBM.
+ * Compression/decompression is handled transparently at the storage boundary.
  */
 export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM, ID extends string = BM['id']> {
     cfg: CommonDaoCfg<BM, DBM, ID>;
@@ -46,7 +49,6 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     runQueryCount(q: DBQuery<DBM>, opt?: CommonDaoReadOptions): Promise<number>;
     streamQueryAsDBM(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<DBM>): Pipeline<DBM>;
     streamQuery(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<BM>): Pipeline<BM>;
-    private streamQueryRaw;
     queryIds(q: DBQuery<DBM>, opt?: CommonDaoReadOptions): Promise<ID[]>;
     streamQueryIds(q: DBQuery<DBM>, opt?: CommonDaoStreamOptions<ID>): Pipeline<ID>;
     /**
@@ -131,14 +133,43 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     bmToDBM(bm: undefined, opt?: CommonDaoOptions): Promise<null>;
     bmToDBM(bm?: BM, opt?: CommonDaoOptions): Promise<DBM>;
     bmsToDBM(bms: BM[], opt?: CommonDaoOptions): Promise<DBM[]>;
+    /**
+     * Converts a DBM to storage format, applying compression if configured.
+     *
+     * Use this when you need to write directly to the database, bypassing the DAO save methods.
+     * The returned value is opaque and should only be passed to db.saveBatch() or similar.
+     *
+     * @example
+     * const storageRow = await dao.dbmToStorageRow(dbm)
+     * await db.saveBatch(table, [storageRow])
+     */
+    dbmToStorageRow(dbm: DBM): Promise<ObjectWithId>;
+    /**
+     * Converts multiple DBMs to storage rows.
+     */
+    dbmsToStorageRows(dbms: DBM[]): Promise<ObjectWithId[]>;
+    /**
+     * Converts a storage row back to a DBM, applying decompression if needed.
+     *
+     * Use this when you need to read directly from the database, bypassing the DAO load methods.
+     *
+     * @example
+     * const rows = await db.getByIds(table, ids)
+     * const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
+     */
+    storageRowToDBM(row: ObjectWithId): Promise<DBM>;
+    /**
+     * Converts multiple storage rows to DBMs.
+     */
+    storageRowsToDBMs(rows: ObjectWithId[]): Promise<DBM[]>;
     /**
      * Mutates `dbm`.
      */
-    compress(dbm: DBM): Promise<void>;
+    private compress;
     /**
      * Mutates `dbm`.
      */
-    decompress(dbm: DBM): Promise<void>;
+    private decompress;
     anyToDBM(dbm: undefined, opt?: CommonDaoOptions): Promise<null>;
     anyToDBM(dbm?: any, opt?: CommonDaoOptions): Promise<DBM>;
     anyToDBMs(rows: DBM[], opt?: CommonDaoOptions): Promise<DBM[]>;
@@ -158,6 +189,43 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
     withIds(ids: ID[]): DaoWithIds<CommonDao<BM, DBM, ID>>;
     withRowsToSave(rows: Unsaved<BM>[]): DaoWithRows<CommonDao<BM, DBM, ID>>;
     withRowToSave(row: Unsaved<BM>, opt?: DaoWithRowOptions<BM>): DaoWithRow<CommonDao<BM, DBM, ID>>;
+    /**
+     * Helper to decompress legacy compressed data when migrating away from auto-compression.
+     * Use as your `beforeDBMToBM` hook to decompress legacy rows on read.
+     *
+     * @example
+     * const dao = new CommonDao({
+     *   hooks: {
+     *     beforeDBMToBM: CommonDao.decompressLegacyRow,
+     *   }
+     * })
+     *
+     * // Or within an existing hook:
+     * beforeDBMToBM: async (dbm) => {
+     *   await CommonDao.decompressLegacyRow(dbm)
+     *   // ... other transformations
+     *   return dbm
+     * }
+     */
+    static decompressLegacyRow<T extends ObjectWithId>(row: T): Promise<T>;
+    /**
+     * Temporary helper to migrate from the old `data` compressed property to the new `__compressed` property.
+     * Use as your `beforeDBMToBM` hook during the migration period.
+     *
+     * Migration steps:
+     * 1. Add `beforeDBMToBM: CommonDao.migrateCompressedDataProperty` to your hooks
+     * 2. Deploy - old data (with `data` property) will be decompressed on read and recompressed to `__compressed` on write
+     * 3. Once all data has been naturally rewritten, remove the hook
+     *
+     * @example
+     * const dao = new CommonDao({
+     *   compress: { keys: ['field1', 'field2'] },
+     *   hooks: {
+     *     beforeDBMToBM: CommonDao.migrateCompressedDataProperty,
+     *   }
+     * })
+     */
+    static migrateCompressedDataProperty<T extends ObjectWithId>(row: T): Promise<T>;
     /**
      * Load rows (by their ids) from Multiple tables at once.
      * An optimized way to load data, minimizing DB round-trips.

package/dist/commondao/common.dao.js

@@ -14,9 +14,12 @@ import { CommonDaoTransaction } from './commonDaoTransaction.js';
 /**
  * Lowest common denominator API between supported Databases.
  *
- * DBM = Database model (how it's stored in DB)
  * BM = Backend model (optimized for API access)
+ * DBM = Database model (logical representation, before compression)
  * TM = Transport model (optimized to be sent over the wire)
+ *
+ * Note: When auto-compression is enabled, the physical storage format differs from DBM.
+ * Compression/decompression is handled transparently at the storage boundary.
  */
 export class CommonDao {
     cfg;
@@ -44,6 +47,15 @@ export class CommonDao {
         else {
             delete this.cfg.hooks.createRandomId;
         }
+        // If the auto-compression is enabled,
+        // then we need to ensure that the '__compressed' property is part of the index exclusion list.
+        if (this.cfg.compress?.keys) {
+            const current = this.cfg.excludeFromIndexes;
+            this.cfg.excludeFromIndexes = current ? [...current] : [];
+            if (!this.cfg.excludeFromIndexes.includes('__compressed')) {
+                this.cfg.excludeFromIndexes.push('__compressed');
+            }
+        }
     }
     // CREATE
     create(part = {}, opt = {}) {
@@ -92,7 +104,8 @@ export class CommonDao {
         if (!ids.length)
             return [];
         const table = opt.table || this.cfg.table;
-        return await (opt.tx || this.cfg.db).getByIds(table, ids, opt);
+        const rows = await (opt.tx || this.cfg.db).getByIds(table, ids, opt);
+        return await this.storageRowsToDBMs(rows);
     }
     async getBy(by, value, limit = 0, opt) {
         return await this.query().filterEq(by, value).limit(limit).runQuery(opt);
@@ -133,8 +146,9 @@ export class CommonDao {
     async runQueryExtended(q, opt = {}) {
         this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
         q.table = opt.table || q.table;
-        const { rows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
+        const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
         const isPartialQuery = !!q._selectedFieldNames;
+        const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows);
         const bms = isPartialQuery ? rows : await this.dbmsToBM(rows, opt);
         return {
             rows: bms,
@@ -148,8 +162,9 @@ export class CommonDao {
     async runQueryExtendedAsDBM(q, opt = {}) {
         this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
         q.table = opt.table || q.table;
-        const { rows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
+        const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
         const isPartialQuery = !!q._selectedFieldNames;
+        const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows);
         const dbms = isPartialQuery ? rows : await this.anyToDBMs(rows, opt);
         return { rows: dbms, ...queryResult };
     }
@@ -159,7 +174,12 @@ export class CommonDao {
         return await this.cfg.db.runQueryCount(q, opt);
     }
     streamQueryAsDBM(q, opt = {}) {
-        const pipeline = this.streamQueryRaw(q, opt);
+        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
+        q.table = opt.table || q.table;
+        let pipeline = this.cfg.db.streamQuery(q, opt);
+        if (this.cfg.compress?.keys.length) {
+            pipeline = pipeline.map(async (row) => await this.storageRowToDBM(row));
+        }
         const isPartialQuery = !!q._selectedFieldNames;
         if (isPartialQuery)
             return pipeline;
@@ -168,7 +188,12 @@ export class CommonDao {
         return pipeline.map(async (dbm) => await this.anyToDBM(dbm, opt), { errorMode: opt.errorMode });
     }
     streamQuery(q, opt = {}) {
-        const pipeline = this.streamQueryRaw(q, opt);
+        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
+        q.table = opt.table || q.table;
+        let pipeline = this.cfg.db.streamQuery(q, opt);
+        if (this.cfg.compress?.keys.length) {
+            pipeline = pipeline.map(async (row) => await this.storageRowToDBM(row));
+        }
         const isPartialQuery = !!q._selectedFieldNames;
         if (isPartialQuery)
             return pipeline;
@@ -176,11 +201,6 @@ export class CommonDao {
         opt.errorMode ||= ErrorMode.SUPPRESS;
         return pipeline.map(async (dbm) => await this.dbmToBM(dbm, opt), { errorMode: opt.errorMode });
     }
-    streamQueryRaw(q, opt = {}) {
-        this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
-        q.table = opt.table || q.table;
-        return this.cfg.db.streamQuery(q, opt);
-    }
     async queryIds(q, opt = {}) {
         this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
         q.table = opt.table || q.table;
@@ -331,7 +351,8 @@ export class CommonDao {
         this.cfg.hooks.beforeSave?.(dbm);
         const table = opt.table || this.cfg.table;
         const saveOptions = this.prepareSaveOptions(opt);
-        await (opt.tx || this.cfg.db).saveBatch(table, [dbm], saveOptions);
+        const row = await this.dbmToStorageRow(dbm);
+        await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions);
         if (saveOptions.assignGeneratedIds) {
             bm.id = dbm.id;
         }
@@ -340,15 +361,16 @@ export class CommonDao {
     async saveAsDBM(dbm, opt = {}) {
         this.requireWriteAccess();
         this.assignIdCreatedUpdated(dbm, opt); // mutates
-        const row = await this.anyToDBM(dbm, opt);
-        this.cfg.hooks.beforeSave?.(row);
+        const validDbm = await this.anyToDBM(dbm, opt);
+        this.cfg.hooks.beforeSave?.(validDbm);
         const table = opt.table || this.cfg.table;
         const saveOptions = this.prepareSaveOptions(opt);
+        const row = await this.dbmToStorageRow(validDbm);
         await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions);
         if (saveOptions.assignGeneratedIds) {
-            dbm.id = row.id;
+            dbm.id = validDbm.id;
         }
-        return row;
+        return validDbm;
     }
     async saveBatch(bms, opt = {}) {
         if (!bms.length)
@@ -361,7 +383,8 @@ export class CommonDao {
         }
         const table = opt.table || this.cfg.table;
         const saveOptions = this.prepareSaveOptions(opt);
-        await (opt.tx || this.cfg.db).saveBatch(table, dbms, saveOptions);
+        const rows = await this.dbmsToStorageRows(dbms);
+        await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions);
         if (saveOptions.assignGeneratedIds) {
             dbms.forEach((dbm, i) => (bms[i].id = dbm.id));
         }
@@ -372,24 +395,28 @@ export class CommonDao {
             return [];
         this.requireWriteAccess();
         dbms.forEach(dbm => this.assignIdCreatedUpdated(dbm, opt));
-        const rows = await this.anyToDBMs(dbms, opt);
+        const validDbms = await this.anyToDBMs(dbms, opt);
         if (this.cfg.hooks.beforeSave) {
-            rows.forEach(row => this.cfg.hooks.beforeSave(row));
+            validDbms.forEach(dbm => this.cfg.hooks.beforeSave(dbm));
         }
         const table = opt.table || this.cfg.table;
         const saveOptions = this.prepareSaveOptions(opt);
+        const rows = await this.dbmsToStorageRows(validDbms);
         await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions);
         if (saveOptions.assignGeneratedIds) {
-            rows.forEach((row, i) => (dbms[i].id = row.id));
+            validDbms.forEach((dbm, i) => (dbms[i].id = dbm.id));
         }
-        return rows;
+        return validDbms;
     }
     prepareSaveOptions(opt) {
         let { saveMethod, assignGeneratedIds = this.cfg.assignGeneratedIds, excludeFromIndexes = this.cfg.excludeFromIndexes, } = opt;
+        // If the user passed in custom `excludeFromIndexes` with the save() call,
+        // and the auto-compression is enabled,
+        // then we need to ensure that the '__compressed' property is part of the list.
         if (this.cfg.compress?.keys) {
             excludeFromIndexes ??= [];
-            if (!excludeFromIndexes.includes('data')) {
-                excludeFromIndexes.push('data');
+            if (!excludeFromIndexes.includes('__compressed')) {
+                excludeFromIndexes.push('__compressed');
             }
         }
         if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
@@ -397,7 +424,7 @@ export class CommonDao {
         }
         return {
             ...opt,
-            excludeFromIndexes,
+            excludeFromIndexes: excludeFromIndexes,
             saveMethod,
             assignGeneratedIds,
         };
@@ -416,10 +443,7 @@ export class CommonDao {
         const table = opt.table || this.cfg.table;
         opt.skipValidation ??= true;
         opt.errorMode ||= ErrorMode.SUPPRESS;
-        if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
-            opt = { ...opt, saveMethod: 'insert' };
-        }
-        const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
+        const saveOptions = this.prepareSaveOptions(opt);
         const { beforeSave } = this.cfg.hooks;
         const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt;
         await p
@@ -427,14 +451,11 @@ export class CommonDao {
             this.assignIdCreatedUpdated(bm, opt);
             const dbm = await this.bmToDBM(bm, opt);
             beforeSave?.(dbm);
-            return dbm;
+            return await this.dbmToStorageRow(dbm);
         }, { errorMode })
             .chunk(chunkSize)
             .map(async (batch) => {
-            await this.cfg.db.saveBatch(table, batch, {
-                ...opt,
-                excludeFromIndexes,
-            });
+            await this.cfg.db.saveBatch(table, batch, saveOptions);
             return batch;
         }, {
             concurrency: chunkConcurrency,
@@ -545,8 +566,6 @@ export class CommonDao {
         // optimization: no need to run full joi DBM validation, cause BM validation will be run
         // const dbm = this.anyToDBM(_dbm, opt)
         const dbm = { ..._dbm, ...this.cfg.hooks.parseNaturalId(_dbm.id) };
-        // Decompress
-        await this.decompress(dbm);
         // DBM > BM
         const bm = ((await this.cfg.hooks.beforeDBMToBM?.(dbm)) || dbm);
         // Validate/convert BM
@@ -562,15 +581,65 @@ export class CommonDao {
         bm = this.validateAndConvert(bm, 'save', opt);
         // BM > DBM
         const dbm = ((await this.cfg.hooks.beforeBMToDBM?.(bm)) || bm);
-        // Compress
-        if (this.cfg.compress)
-            await this.compress(dbm);
         return dbm;
     }
     async bmsToDBM(bms, opt = {}) {
         // try/catch?
         return await pMap(bms, async (bm) => await this.bmToDBM(bm, opt));
     }
+    // STORAGE LAYER (compression/decompression at DB boundary)
+    // These methods convert between DBM (logical model) and storage format (physical, possibly compressed).
+    // Public methods allow external code to bypass the DAO layer for direct DB access
+    // (e.g., cross-environment data copy).
+    /**
+     * Converts a DBM to storage format, applying compression if configured.
+     *
+     * Use this when you need to write directly to the database, bypassing the DAO save methods.
+     * The returned value is opaque and should only be passed to db.saveBatch() or similar.
+     *
+     * @example
+     * const storageRow = await dao.dbmToStorageRow(dbm)
+     * await db.saveBatch(table, [storageRow])
+     */
+    async dbmToStorageRow(dbm) {
+        if (!this.cfg.compress?.keys.length)
+            return dbm;
+        const row = { ...dbm };
+        await this.compress(row);
+        return row;
+    }
+    /**
+     * Converts multiple DBMs to storage rows.
+     */
+    async dbmsToStorageRows(dbms) {
+        if (!this.cfg.compress?.keys.length)
+            return dbms;
+        return await pMap(dbms, async (dbm) => await this.dbmToStorageRow(dbm));
+    }
+    /**
+     * Converts a storage row back to a DBM, applying decompression if needed.
+     *
+     * Use this when you need to read directly from the database, bypassing the DAO load methods.
+     *
+     * @example
+     * const rows = await db.getByIds(table, ids)
+     * const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
+     */
+    async storageRowToDBM(row) {
+        if (!this.cfg.compress?.keys.length)
+            return row;
+        const dbm = { ...row };
+        await this.decompress(dbm);
+        return dbm;
+    }
+    /**
+     * Converts multiple storage rows to DBMs.
+     */
+    async storageRowsToDBMs(rows) {
+        if (!this.cfg.compress?.keys.length)
+            return rows;
+        return await pMap(rows, async (row) => await this.storageRowToDBM(row));
+    }
     /**
      * Mutates `dbm`.
      */
@@ -579,26 +648,22 @@ export class CommonDao {
             return; // No compression requested
         const { keys } = this.cfg.compress;
         const properties = _pick(dbm, keys);
-        _assert(!('data' in dbm) || 'data' in properties, `Data (${dbm.id}) already has a "data" property. When using compression, this property must be included in the compression keys list.`);
         const bufferString = JSON.stringify(properties);
-        const data = await zstdCompress(bufferString);
+        const __compressed = await zstdCompress(bufferString);
         _omitWithUndefined(dbm, _objectKeys(properties), { mutate: true });
-        Object.assign(dbm, { data });
+        Object.assign(dbm, { __compressed });
     }
     /**
      * Mutates `dbm`.
      */
     async decompress(dbm) {
         _typeCast(dbm);
-        if (!this.cfg.compress)
-            return; // Auto-compression not turned on
-        if (!Buffer.isBuffer(dbm.data))
+        if (!Buffer.isBuffer(dbm.__compressed))
             return; // No compressed data
-        // try-catch to avoid a `data` with Buffer which is not compressed, but legit data
         try {
-            const bufferString = await decompressZstdOrInflateToString(dbm.data);
+            const bufferString = await decompressZstdOrInflateToString(dbm.__compressed);
             const properties = JSON.parse(bufferString);
-            dbm.data = undefined;
+            dbm.__compressed = undefined;
             Object.assign(dbm, properties);
         }
         catch { }
@@ -609,8 +674,6 @@ export class CommonDao {
         // this shouldn't be happening on load! but should on save!
         // this.assignIdCreatedUpdated(dbm, opt)
         dbm = { ...dbm, ...this.cfg.hooks.parseNaturalId(dbm.id) };
-        // Decompress
-        await this.decompress(dbm);
         // Validate/convert DBM
         // return this.validateAndConvert(dbm, this.cfg.dbmSchema, DBModelType.DBM, opt)
         return dbm;
@@ -688,6 +751,73 @@ export class CommonDao {
             opt: opt,
         };
     }
+    /**
+     * Helper to decompress legacy compressed data when migrating away from auto-compression.
+     * Use as your `beforeDBMToBM` hook to decompress legacy rows on read.
+     *
+     * @example
+     * const dao = new CommonDao({
+     *   hooks: {
+     *     beforeDBMToBM: CommonDao.decompressLegacyRow,
+     *   }
+     * })
+     *
+     * // Or within an existing hook:
+     * beforeDBMToBM: async (dbm) => {
+     *   await CommonDao.decompressLegacyRow(dbm)
+     *   // ... other transformations
+     *   return dbm
+     * }
+     */
+    static async decompressLegacyRow(row) {
+        // Check both __compressed (current) and data (legacy) for backward compatibility
+        const compressed = row.__compressed ?? row.data;
+        if (!Buffer.isBuffer(compressed))
+            return row;
+        try {
+            const bufferString = await decompressZstdOrInflateToString(compressed);
+            const properties = JSON.parse(bufferString);
+            row.__compressed = undefined;
+            row.data = undefined;
+            Object.assign(row, properties);
+        }
+        catch {
+            // Decompression failed - field is not compressed, leave as-is
+        }
+        return row;
+    }
+    /**
+     * Temporary helper to migrate from the old `data` compressed property to the new `__compressed` property.
+     * Use as your `beforeDBMToBM` hook during the migration period.
+     *
+     * Migration steps:
+     * 1. Add `beforeDBMToBM: CommonDao.migrateCompressedDataProperty` to your hooks
+     * 2. Deploy - old data (with `data` property) will be decompressed on read and recompressed to `__compressed` on write
+     * 3. Once all data has been naturally rewritten, remove the hook
+     *
+     * @example
+     * const dao = new CommonDao({
+     *   compress: { keys: ['field1', 'field2'] },
+     *   hooks: {
+     *     beforeDBMToBM: CommonDao.migrateCompressedDataProperty,
+     *   }
+     * })
+     */
+    static async migrateCompressedDataProperty(row) {
+        const data = row.data;
+        if (!Buffer.isBuffer(data))
+            return row;
+        try {
+            const bufferString = await decompressZstdOrInflateToString(data);
+            const properties = JSON.parse(bufferString);
+            row.data = undefined;
+            Object.assign(row, properties);
+        }
+        catch {
+            // Decompression failed - data field is not compressed, leave as-is
+        }
+        return row;
+    }
     /**
      * Load rows (by their ids) from Multiple tables at once.
      * An optimized way to load data, minimizing DB round-trips.
@@ -748,14 +878,18 @@ export class CommonDao {
             const { table } = dao.cfg;
             if ('id' in input) {
                 // Singular
-                const dbm = dbmByTableById[table][input.id];
+                const row = dbmByTableById[table][input.id];
+                // Decompress before converting to BM
+                const dbm = row ? await dao.storageRowToDBM(row) : undefined;
                 bmsByProp[prop] = (await dao.dbmToBM(dbm, opt)) || null;
             }
             else {
                 // Plural
                 // We apply filtering, to be able to support multiple input props fetching from the same table.
                 // Without filtering - every prop will get ALL rows from that table.
-                const dbms = input.ids.map(id => dbmByTableById[table][id]).filter(_isTruthy);
+                const rows = input.ids.map(id => dbmByTableById[table][id]).filter(_isTruthy);
+                // Decompress before converting to BM
+                const dbms = await dao.storageRowsToDBMs(rows);
                 bmsByProp[prop] = await dao.dbmsToBM(dbms, opt);
             }
         });
@@ -809,7 +943,8 @@ export class CommonDao {
                 dao.assignIdCreatedUpdated(row, opt);
                 const dbm = await dao.bmToDBM(row, opt);
                 dao.cfg.hooks.beforeSave?.(dbm);
-                dbmsByTable[table].push(dbm);
+                const storageRow = await dao.dbmToStorageRow(dbm);
+                dbmsByTable[table].push(storageRow);
             }
             else {
                 // Plural
@@ -818,7 +953,8 @@ export class CommonDao {
                 if (dao.cfg.hooks.beforeSave) {
                     dbms.forEach(dbm => dao.cfg.hooks.beforeSave(dbm));
                 }
-                dbmsByTable[table].push(...dbms);
+                const storageRows = await dao.dbmsToStorageRows(dbms);
+                dbmsByTable[table].push(...storageRows);
             }
         });
         await db.multiSave(dbmsByTable);

package/dist/commondao/common.dao.model.d.ts

@@ -168,14 +168,15 @@ export interface CommonDaoCfg<BM extends BaseDBEntity, DBM extends BaseDBEntity
      */
     patchInTransaction?: boolean;
     /**
-     * When specified, the listed properties will be compressed under a `data` property in the DBM.
-     * If DBM already has a `data` property and you don't add it to the list, an error will be thrown.
-     *
-     * When specified with an empty `keys` list, then compression will be skipped, but all previously compressed data
-     * will be decompressed, so the Dao can still work.
+     * When specified, the listed properties will be compressed into the `__compressed` property.
      *
      * Compression happens after the `beforeBMToDBM` hook and before the DBM is saved to the database.
      * Decompression happens after the DBM is loaded from the database and before the `beforeDBMToBM` hook.
+     *
+     * To migrate away from compression:
+     * 1. Remove this config (or set `keys` to empty array)
+     * 2. Add `beforeDBMToBM: CommonDao.decompressLegacyRow` to your hooks to decompress legacy data on read
+     * 3. Once all data has been naturally rewritten without compression, remove the hook
      */
     compress?: {
         keys: (keyof DBM)[];

package/dist/inmemory/inMemory.db.d.ts

@@ -1,5 +1,5 @@
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { type AnyObjectWithId, type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
+import type { AnyObjectWithId, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types';
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv';
 import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB, CommonDBSupport } from '../commondb/common.db.js';

package/dist/inmemory/inMemory.db.js

@@ -1,7 +1,7 @@
 import { _isEmptyObject } from '@naturalcycles/js-lib';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object';
-import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
+import { _stringMapEntries, _stringMapValues } from '@naturalcycles/js-lib/types';
 import { generateJsonSchemaFromData } from '@naturalcycles/nodejs-lib/ajv';
 import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { bufferReviver } from '@naturalcycles/nodejs-lib/stream/ndjson/transformJsonParse.js';

package/dist/kv/commonKeyValueDao.d.ts

@@ -1,5 +1,5 @@
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { type Integer, type KeyValueTuple } from '@naturalcycles/js-lib/types';
+import type { Integer, KeyValueTuple } from '@naturalcycles/js-lib/types';
 import type { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js';
 import type { CommonDBCreateOptions } from '../db.model.js';

package/dist/pipeline/dbPipelineRestore.d.ts

@@ -1,6 +1,7 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
 import type { AsyncMapper, UnixTimestamp } from '@naturalcycles/js-lib/types';
-import { NDJsonStats, type TransformLogProgressOptions, type TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
+import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
+import type { TransformLogProgressOptions, TransformMapOptions } from '@naturalcycles/nodejs-lib/stream';
 import type { CommonDB } from '../commondb/common.db.js';
 import type { CommonDBSaveOptions } from '../db.model.js';
 export interface DBPipelineRestoreOptions extends TransformLogProgressOptions {

package/dist/pipeline/dbPipelineRestore.js

@@ -6,7 +6,7 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { _passthroughMapper } from '@naturalcycles/js-lib/types';
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
-import { NDJsonStats, Pipeline, } from '@naturalcycles/nodejs-lib/stream';
+import { NDJsonStats, Pipeline } from '@naturalcycles/nodejs-lib/stream';
 /**
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
  * Allows to define a mapper and a predicate to map/filter objects between input and output.

package/dist/testing/test.model.d.ts

@@ -1,5 +1,5 @@
 import type { BaseDBEntity } from '@naturalcycles/js-lib/types';
-import { type JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv';
+import type { JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv';
 export declare const TEST_TABLE = "TEST_TABLE";
 export declare const TEST_TABLE_2 = "TEST_TABLE_2";
 export interface TestItemBM extends BaseDBEntity {

package/dist/validation/index.d.ts

@@ -1,5 +1,5 @@
 import type { ObjectWithId } from '@naturalcycles/js-lib/types';
-import { type JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv';
+import type { JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv';
 import type { CommonDBOptions } from '../db.model.js';
 export declare const commonDBOptionsSchema: () => JsonSchemaObjectBuilder<CommonDBOptions, CommonDBOptions>;
 export declare const commonDBSaveOptionsSchema: <ROW extends ObjectWithId>() => any;

package/dist/validation/index.js

@@ -1,5 +1,5 @@
-import { j, JsonSchemaAnyBuilder, } from '@naturalcycles/nodejs-lib/ajv';
-import { dbQueryFilterOperatorValues, } from '../query/dbQuery.js';
+import { j, JsonSchemaAnyBuilder } from '@naturalcycles/nodejs-lib/ajv';
+import { dbQueryFilterOperatorValues } from '../query/dbQuery.js';
 // oxlint-disable typescript/explicit-function-return-type
 // DBTransaction schema - validates presence without deep validation
 const dbTransactionSchema = j.object.any().castAs();

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@naturalcycles/db-lib",
   "type": "module",
-  "version": "10.42.0",
+  "version": "10.42.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@naturalcycles/nodejs-lib": "^15"
   },
   "devDependencies": {
+    "@typescript/native-preview": "^7.0.0-0",
     "@naturalcycles/dev-lib": "18.4.2"
   },
   "files": [

package/src/adapter/file/file.db.ts

@@ -3,11 +3,8 @@ import { _by, _sortBy } from '@naturalcycles/js-lib/array'
 import { _since, localTime } from '@naturalcycles/js-lib/datetime'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import { _deepEquals, _filterUndefinedValues, _sortObjectDeep } from '@naturalcycles/js-lib/object'
-import {
-  _stringMapValues,
-  type ObjectWithId,
-  type UnixTimestampMillis,
-} from '@naturalcycles/js-lib/types'
+import { _stringMapValues } from '@naturalcycles/js-lib/types'
+import type { ObjectWithId, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
 import { generateJsonSchemaFromData } from '@naturalcycles/nodejs-lib/ajv'
 import { dimGrey } from '@naturalcycles/nodejs-lib/colors'

package/src/commondao/common.dao.model.ts

@@ -207,14 +207,15 @@ export interface CommonDaoCfg<
   patchInTransaction?: boolean
 
   /**
-   * When specified, the listed properties will be compressed under a `data` property in the DBM.
-   * If DBM already has a `data` property and you don't add it to the list, an error will be thrown.
-   *
-   * When specified with an empty `keys` list, then compression will be skipped, but all previously compressed data
-   * will be decompressed, so the Dao can still work.
+   * When specified, the listed properties will be compressed into the `__compressed` property.
    *
    * Compression happens after the `beforeBMToDBM` hook and before the DBM is saved to the database.
    * Decompression happens after the DBM is loaded from the database and before the `beforeDBMToBM` hook.
+   *
+   * To migrate away from compression:
+   * 1. Remove this config (or set `keys` to empty array)
+   * 2. Add `beforeDBMToBM: CommonDao.decompressLegacyRow` to your hooks to decompress legacy data on read
+   * 3. Once all data has been naturally rewritten without compression, remove the hook
    */
   compress?: {
     keys: (keyof DBM)[]

package/src/commondao/common.dao.ts

@@ -16,11 +16,13 @@ import {
   _stringMapEntries,
   _stringMapValues,
   _typeCast,
-  type BaseDBEntity,
-  type NonNegativeInteger,
-  type ObjectWithId,
-  type StringMap,
-  type Unsaved,
+} from '@naturalcycles/js-lib/types'
+import type {
+  BaseDBEntity,
+  NonNegativeInteger,
+  ObjectWithId,
+  StringMap,
+  Unsaved,
 } from '@naturalcycles/js-lib/types'
 import { stringId } from '@naturalcycles/nodejs-lib'
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
@@ -53,9 +55,12 @@ import { CommonDaoTransaction } from './commonDaoTransaction.js'
 /**
  * Lowest common denominator API between supported Databases.
  *
- * DBM = Database model (how it's stored in DB)
  * BM = Backend model (optimized for API access)
+ * DBM = Database model (logical representation, before compression)
  * TM = Transport model (optimized to be sent over the wire)
+ *
+ * Note: When auto-compression is enabled, the physical storage format differs from DBM.
+ * Compression/decompression is handled transparently at the storage boundary.
  */
 export class CommonDao<
   BM extends BaseDBEntity,
@@ -85,6 +90,16 @@ export class CommonDao<
     } else {
       delete this.cfg.hooks!.createRandomId
     }
+
+    // If the auto-compression is enabled,
+    // then we need to ensure that the '__compressed' property is part of the index exclusion list.
+    if (this.cfg.compress?.keys) {
+      const current = this.cfg.excludeFromIndexes
+      this.cfg.excludeFromIndexes = current ? [...current] : []
+      if (!this.cfg.excludeFromIndexes.includes('__compressed' as any)) {
+        this.cfg.excludeFromIndexes.push('__compressed' as any)
+      }
+    }
   }
 
   // CREATE
@@ -139,7 +154,8 @@ export class CommonDao<
   private async loadByIds(ids: ID[], opt: CommonDaoReadOptions = {}): Promise<DBM[]> {
     if (!ids.length) return []
     const table = opt.table || this.cfg.table
-    return await (opt.tx || this.cfg.db).getByIds<DBM>(table, ids, opt)
+    const rows = await (opt.tx || this.cfg.db).getByIds<DBM>(table, ids, opt)
+    return await this.storageRowsToDBMs(rows)
   }
 
   async getBy(by: keyof DBM, value: any, limit = 0, opt?: CommonDaoReadOptions): Promise<BM[]> {
@@ -198,8 +214,9 @@ export class CommonDao<
   ): Promise<RunQueryResult<BM>> {
     this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
     q.table = opt.table || q.table
-    const { rows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
+    const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
     const isPartialQuery = !!q._selectedFieldNames
+    const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows)
     const bms = isPartialQuery ? (rows as any[]) : await this.dbmsToBM(rows, opt)
     return {
       rows: bms,
@@ -218,8 +235,9 @@ export class CommonDao<
   ): Promise<RunQueryResult<DBM>> {
     this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
     q.table = opt.table || q.table
-    const { rows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
+    const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
     const isPartialQuery = !!q._selectedFieldNames
+    const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows)
     const dbms = isPartialQuery ? rows : await this.anyToDBMs(rows, opt)
     return { rows: dbms, ...queryResult }
   }
@@ -231,7 +249,13 @@ export class CommonDao<
   }
 
   streamQueryAsDBM(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<DBM> = {}): Pipeline<DBM> {
-    const pipeline = this.streamQueryRaw(q, opt)
+    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
+    q.table = opt.table || q.table
+    let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
+
+    if (this.cfg.compress?.keys.length) {
+      pipeline = pipeline.map(async row => await this.storageRowToDBM(row))
+    }
 
     const isPartialQuery = !!q._selectedFieldNames
     if (isPartialQuery) return pipeline
@@ -243,7 +267,13 @@ export class CommonDao<
   }
 
   streamQuery(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<BM> = {}): Pipeline<BM> {
-    const pipeline = this.streamQueryRaw(q, opt)
+    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
+    q.table = opt.table || q.table
+    let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
+
+    if (this.cfg.compress?.keys.length) {
+      pipeline = pipeline.map(async row => await this.storageRowToDBM(row))
+    }
 
     const isPartialQuery = !!q._selectedFieldNames
     if (isPartialQuery) return pipeline as any as Pipeline<BM>
@@ -254,12 +284,6 @@ export class CommonDao<
     return pipeline.map(async dbm => await this.dbmToBM(dbm, opt), { errorMode: opt.errorMode })
   }
 
-  private streamQueryRaw(q: DBQuery<DBM>, opt: CommonDaoStreamOptions<any> = {}): Pipeline<DBM> {
-    this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
-    q.table = opt.table || q.table
-    return this.cfg.db.streamQuery<DBM>(q, opt)
-  }
-
   async queryIds(q: DBQuery<DBM>, opt: CommonDaoReadOptions = {}): Promise<ID[]> {
     this.validateQueryIndexes(q) // throws if query uses `excludeFromIndexes` property
     q.table = opt.table || q.table
@@ -447,7 +471,8 @@ export class CommonDao<
     const table = opt.table || this.cfg.table
     const saveOptions = this.prepareSaveOptions(opt)
 
-    await (opt.tx || this.cfg.db).saveBatch(table, [dbm], saveOptions)
+    const row = await this.dbmToStorageRow(dbm)
+    await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions)
 
     if (saveOptions.assignGeneratedIds) {
       bm.id = dbm.id
@@ -459,18 +484,19 @@ export class CommonDao<
   async saveAsDBM(dbm: Unsaved<DBM>, opt: CommonDaoSaveOptions<BM, DBM> = {}): Promise<DBM> {
     this.requireWriteAccess()
     this.assignIdCreatedUpdated(dbm, opt) // mutates
-    const row = await this.anyToDBM(dbm, opt)
-    this.cfg.hooks!.beforeSave?.(row)
+    const validDbm = await this.anyToDBM(dbm, opt)
+    this.cfg.hooks!.beforeSave?.(validDbm)
     const table = opt.table || this.cfg.table
     const saveOptions = this.prepareSaveOptions(opt)
 
+    const row = await this.dbmToStorageRow(validDbm)
     await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions)
 
     if (saveOptions.assignGeneratedIds) {
-      dbm.id = row.id
+      dbm.id = validDbm.id
     }
 
-    return row
+    return validDbm
   }
 
   async saveBatch(bms: Unsaved<BM>[], opt: CommonDaoSaveBatchOptions<DBM> = {}): Promise<BM[]> {
@@ -484,7 +510,8 @@ export class CommonDao<
     const table = opt.table || this.cfg.table
     const saveOptions = this.prepareSaveOptions(opt)
 
-    await (opt.tx || this.cfg.db).saveBatch(table, dbms, saveOptions)
+    const rows = await this.dbmsToStorageRows(dbms)
+    await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions)
 
     if (saveOptions.assignGeneratedIds) {
       dbms.forEach((dbm, i) => (bms[i]!.id = dbm.id))
@@ -500,33 +527,39 @@ export class CommonDao<
     if (!dbms.length) return []
     this.requireWriteAccess()
     dbms.forEach(dbm => this.assignIdCreatedUpdated(dbm, opt))
-    const rows = await this.anyToDBMs(dbms as DBM[], opt)
+    const validDbms = await this.anyToDBMs(dbms as DBM[], opt)
     if (this.cfg.hooks!.beforeSave) {
-      rows.forEach(row => this.cfg.hooks!.beforeSave!(row))
+      validDbms.forEach(dbm => this.cfg.hooks!.beforeSave!(dbm))
     }
     const table = opt.table || this.cfg.table
     const saveOptions = this.prepareSaveOptions(opt)
 
+    const rows = await this.dbmsToStorageRows(validDbms)
     await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions)
 
     if (saveOptions.assignGeneratedIds) {
-      rows.forEach((row, i) => (dbms[i]!.id = row.id))
+      validDbms.forEach((dbm, i) => (dbms[i]!.id = dbm.id))
     }
 
-    return rows
+    return validDbms
   }
 
-  private prepareSaveOptions(opt: CommonDaoSaveOptions<BM, DBM>): CommonDBSaveOptions<DBM> {
+  private prepareSaveOptions(
+    opt: CommonDaoSaveOptions<BM, DBM>,
+  ): CommonDBSaveOptions<ObjectWithId> {
     let {
       saveMethod,
       assignGeneratedIds = this.cfg.assignGeneratedIds,
       excludeFromIndexes = this.cfg.excludeFromIndexes,
     } = opt
 
+    // If the user passed in custom `excludeFromIndexes` with the save() call,
+    // and the auto-compression is enabled,
+    // then we need to ensure that the '__compressed' property is part of the list.
     if (this.cfg.compress?.keys) {
       excludeFromIndexes ??= []
-      if (!excludeFromIndexes.includes('data' as any)) {
-        excludeFromIndexes.push('data' as any)
+      if (!excludeFromIndexes.includes('__compressed' as any)) {
+        excludeFromIndexes.push('__compressed' as any)
       }
     }
 
@@ -536,7 +569,7 @@ export class CommonDao<
 
     return {
       ...opt,
-      excludeFromIndexes,
+      excludeFromIndexes: excludeFromIndexes as (keyof ObjectWithId)[],
       saveMethod,
       assignGeneratedIds,
     }
@@ -558,11 +591,7 @@ export class CommonDao<
     opt.skipValidation ??= true
     opt.errorMode ||= ErrorMode.SUPPRESS
 
-    if (this.cfg.immutable && !opt.allowMutability && !opt.saveMethod) {
-      opt = { ...opt, saveMethod: 'insert' }
-    }
-
-    const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
+    const saveOptions = this.prepareSaveOptions(opt)
     const { beforeSave } = this.cfg.hooks!
 
     const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt
@@ -573,17 +602,14 @@ export class CommonDao<
           this.assignIdCreatedUpdated(bm, opt)
           const dbm = await this.bmToDBM(bm, opt)
           beforeSave?.(dbm)
-          return dbm
+          return await this.dbmToStorageRow(dbm)
         },
         { errorMode },
       )
       .chunk(chunkSize)
       .map(
         async batch => {
-          await this.cfg.db.saveBatch(table, batch, {
-            ...opt,
-            excludeFromIndexes,
-          })
+          await this.cfg.db.saveBatch(table, batch, saveOptions)
           return batch
         },
         {
@@ -722,9 +748,6 @@ export class CommonDao<
     // const dbm = this.anyToDBM(_dbm, opt)
     const dbm: DBM = { ..._dbm, ...this.cfg.hooks!.parseNaturalId!(_dbm.id as ID) }
 
-    // Decompress
-    await this.decompress(dbm)
-
     // DBM > BM
     const bm = ((await this.cfg.hooks!.beforeDBMToBM?.(dbm)) || dbm) as Partial<BM>
 
@@ -751,9 +774,6 @@ export class CommonDao<
     // BM > DBM
     const dbm = ((await this.cfg.hooks!.beforeBMToDBM?.(bm)) || bm) as DBM
 
-    // Compress
-    if (this.cfg.compress) await this.compress(dbm)
-
     return dbm
   }
 
@@ -762,37 +782,85 @@ export class CommonDao<
     return await pMap(bms, async bm => await this.bmToDBM(bm, opt))
   }
 
+  // STORAGE LAYER (compression/decompression at DB boundary)
+  // These methods convert between DBM (logical model) and storage format (physical, possibly compressed).
+  // Public methods allow external code to bypass the DAO layer for direct DB access
+  // (e.g., cross-environment data copy).
+
+  /**
+   * Converts a DBM to storage format, applying compression if configured.
+   *
+   * Use this when you need to write directly to the database, bypassing the DAO save methods.
+   * The returned value is opaque and should only be passed to db.saveBatch() or similar.
+   *
+   * @example
+   * const storageRow = await dao.dbmToStorageRow(dbm)
+   * await db.saveBatch(table, [storageRow])
+   */
+  async dbmToStorageRow(dbm: DBM): Promise<ObjectWithId> {
+    if (!this.cfg.compress?.keys.length) return dbm
+    const row = { ...dbm }
+    await this.compress(row)
+    return row
+  }
+
+  /**
+   * Converts multiple DBMs to storage rows.
+   */
+  async dbmsToStorageRows(dbms: DBM[]): Promise<ObjectWithId[]> {
+    if (!this.cfg.compress?.keys.length) return dbms
+    return await pMap(dbms, async dbm => await this.dbmToStorageRow(dbm))
+  }
+
+  /**
+   * Converts a storage row back to a DBM, applying decompression if needed.
+   *
+   * Use this when you need to read directly from the database, bypassing the DAO load methods.
+   *
+   * @example
+   * const rows = await db.getByIds(table, ids)
+   * const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
+   */
+  async storageRowToDBM(row: ObjectWithId): Promise<DBM> {
+    if (!this.cfg.compress?.keys.length) return row as DBM
+    const dbm = { ...(row as DBM) }
+    await this.decompress(dbm)
+    return dbm
+  }
+
+  /**
+   * Converts multiple storage rows to DBMs.
+   */
+  async storageRowsToDBMs(rows: ObjectWithId[]): Promise<DBM[]> {
+    if (!this.cfg.compress?.keys.length) return rows as DBM[]
+    return await pMap(rows, async row => await this.storageRowToDBM(row))
+  }
+
   /**
    * Mutates `dbm`.
    */
-  async compress(dbm: DBM): Promise<void> {
+  private async compress(dbm: DBM): Promise<void> {
     if (!this.cfg.compress?.keys.length) return // No compression requested
 
     const { keys } = this.cfg.compress
     const properties = _pick(dbm, keys)
-    _assert(
-      !('data' in dbm) || 'data' in properties,
-      `Data (${dbm.id}) already has a "data" property. When using compression, this property must be included in the compression keys list.`,
-    )
     const bufferString = JSON.stringify(properties)
-    const data = await zstdCompress(bufferString)
+    const __compressed = await zstdCompress(bufferString)
     _omitWithUndefined(dbm as any, _objectKeys(properties), { mutate: true })
-    Object.assign(dbm, { data })
+    Object.assign(dbm, { __compressed })
   }
 
   /**
    * Mutates `dbm`.
    */
-  async decompress(dbm: DBM): Promise<void> {
+  private async decompress(dbm: DBM): Promise<void> {
     _typeCast<Compressed<DBM>>(dbm)
-    if (!this.cfg.compress) return // Auto-compression not turned on
-    if (!Buffer.isBuffer(dbm.data)) return // No compressed data
+    if (!Buffer.isBuffer(dbm.__compressed)) return // No compressed data
 
-    // try-catch to avoid a `data` with Buffer which is not compressed, but legit data
     try {
-      const bufferString = await decompressZstdOrInflateToString(dbm.data)
+      const bufferString = await decompressZstdOrInflateToString(dbm.__compressed)
       const properties = JSON.parse(bufferString)
-      dbm.data = undefined
+      dbm.__compressed = undefined
       Object.assign(dbm, properties)
     } catch {}
   }
@@ -807,9 +875,6 @@ export class CommonDao<
 
     dbm = { ...dbm, ...this.cfg.hooks!.parseNaturalId!(dbm.id as ID) }
 
-    // Decompress
-    await this.decompress(dbm)
-
     // Validate/convert DBM
     // return this.validateAndConvert(dbm, this.cfg.dbmSchema, DBModelType.DBM, opt)
     return dbm
@@ -906,6 +971,75 @@ export class CommonDao<
     }
   }
 
+  /**
+   * Helper to decompress legacy compressed data when migrating away from auto-compression.
+   * Use as your `beforeDBMToBM` hook to decompress legacy rows on read.
+   *
+   * @example
+   * const dao = new CommonDao({
+   *   hooks: {
+   *     beforeDBMToBM: CommonDao.decompressLegacyRow,
+   *   }
+   * })
+   *
+   * // Or within an existing hook:
+   * beforeDBMToBM: async (dbm) => {
+   *   await CommonDao.decompressLegacyRow(dbm)
+   *   // ... other transformations
+   *   return dbm
+   * }
+   */
+  static async decompressLegacyRow<T extends ObjectWithId>(row: T): Promise<T> {
+    // Check both __compressed (current) and data (legacy) for backward compatibility
+    const compressed = (row as any).__compressed ?? (row as any).data
+    if (!Buffer.isBuffer(compressed)) return row
+
+    try {
+      const bufferString = await decompressZstdOrInflateToString(compressed)
+      const properties = JSON.parse(bufferString)
+      ;(row as any).__compressed = undefined
+      ;(row as any).data = undefined
+      Object.assign(row, properties)
+    } catch {
+      // Decompression failed - field is not compressed, leave as-is
+    }
+
+    return row
+  }
+
+  /**
+   * Temporary helper to migrate from the old `data` compressed property to the new `__compressed` property.
+   * Use as your `beforeDBMToBM` hook during the migration period.
+   *
+   * Migration steps:
+   * 1. Add `beforeDBMToBM: CommonDao.migrateCompressedDataProperty` to your hooks
+   * 2. Deploy - old data (with `data` property) will be decompressed on read and recompressed to `__compressed` on write
+   * 3. Once all data has been naturally rewritten, remove the hook
+   *
+   * @example
+   * const dao = new CommonDao({
+   *   compress: { keys: ['field1', 'field2'] },
+   *   hooks: {
+   *     beforeDBMToBM: CommonDao.migrateCompressedDataProperty,
+   *   }
+   * })
+   */
+  static async migrateCompressedDataProperty<T extends ObjectWithId>(row: T): Promise<T> {
+    const data = (row as any).data
+    if (!Buffer.isBuffer(data)) return row
+
+    try {
+      const bufferString = await decompressZstdOrInflateToString(data)
+      const properties = JSON.parse(bufferString)
+      ;(row as any).data = undefined
+      Object.assign(row, properties)
+    } catch {
+      // Decompression failed - data field is not compressed, leave as-is
+    }
+
+    return row
+  }
+
   /**
    * Load rows (by their ids) from Multiple tables at once.
    * An optimized way to load data, minimizing DB round-trips.
@@ -991,13 +1125,17 @@ export class CommonDao<
       const { table } = dao.cfg
       if ('id' in input) {
         // Singular
-        const dbm = dbmByTableById[table]![input.id]
+        const row = dbmByTableById[table]![input.id]
+        // Decompress before converting to BM
+        const dbm = row ? await dao.storageRowToDBM(row) : undefined
         bmsByProp[prop] = (await dao.dbmToBM(dbm, opt)) || null
       } else {
         // Plural
         // We apply filtering, to be able to support multiple input props fetching from the same table.
         // Without filtering - every prop will get ALL rows from that table.
-        const dbms = input.ids.map(id => dbmByTableById[table]![id]).filter(_isTruthy)
+        const rows = input.ids.map(id => dbmByTableById[table]![id]).filter(_isTruthy)
+        // Decompress before converting to BM
+        const dbms = await dao.storageRowsToDBMs(rows)
         bmsByProp[prop] = await dao.dbmsToBM(dbms, opt)
       }
     })
@@ -1062,7 +1200,8 @@ export class CommonDao<
         dao.assignIdCreatedUpdated(row, opt)
         const dbm = await dao.bmToDBM(row, opt)
         dao.cfg.hooks!.beforeSave?.(dbm)
-        dbmsByTable[table].push(dbm)
+        const storageRow = await dao.dbmToStorageRow(dbm)
+        dbmsByTable[table].push(storageRow)
       } else {
         // Plural
         input.rows.forEach(bm => dao.assignIdCreatedUpdated(bm, opt))
@@ -1070,7 +1209,8 @@ export class CommonDao<
         if (dao.cfg.hooks!.beforeSave) {
           dbms.forEach(dbm => dao.cfg.hooks!.beforeSave!(dbm))
         }
-        dbmsByTable[table].push(...dbms)
+        const storageRows = await dao.dbmsToStorageRows(dbms)
+        dbmsByTable[table].push(...storageRows)
       }
     })
 
@@ -1205,4 +1345,4 @@ export type AnyDao = CommonDao<any>
  * Used internally during compression/decompression so that DBM instances can
  * carry their compressed payload alongside the original type shape.
  */
-type Compressed<DBM> = DBM & { data?: Buffer }
+type Compressed<DBM> = DBM & { __compressed?: Buffer }

package/src/inmemory/inMemory.db.ts

@@ -2,13 +2,8 @@ import { _isEmptyObject } from '@naturalcycles/js-lib'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { _deepCopy, _sortObjectDeep } from '@naturalcycles/js-lib/object'
-import {
-  _stringMapEntries,
-  _stringMapValues,
-  type AnyObjectWithId,
-  type ObjectWithId,
-  type StringMap,
-} from '@naturalcycles/js-lib/types'
+import { _stringMapEntries, _stringMapValues } from '@naturalcycles/js-lib/types'
+import type { AnyObjectWithId, ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
 import { generateJsonSchemaFromData } from '@naturalcycles/nodejs-lib/ajv'
 import { Pipeline } from '@naturalcycles/nodejs-lib/stream'

package/src/kv/commonKeyValueDao.ts

@@ -1,7 +1,8 @@
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
-import { type Integer, type KeyValueTuple, SKIP } from '@naturalcycles/js-lib/types'
+import { SKIP } from '@naturalcycles/js-lib/types'
+import type { Integer, KeyValueTuple } from '@naturalcycles/js-lib/types'
 import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
 import {
   decompressZstdOrInflateToString,

package/src/pipeline/dbPipelineRestore.ts

@@ -8,11 +8,10 @@ import { _passthroughMapper } from '@naturalcycles/js-lib/types'
 import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
 import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
 import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
-import {
-  NDJsonStats,
-  Pipeline,
-  type TransformLogProgressOptions,
-  type TransformMapOptions,
+import { NDJsonStats, Pipeline } from '@naturalcycles/nodejs-lib/stream'
+import type {
+  TransformLogProgressOptions,
+  TransformMapOptions,
 } from '@naturalcycles/nodejs-lib/stream'
 import type { CommonDB } from '../commondb/common.db.js'
 import type { CommonDBSaveOptions } from '../db.model.js'

package/src/testing/test.model.ts

@@ -1,6 +1,7 @@
 import { _range } from '@naturalcycles/js-lib/array/range.js'
 import type { BaseDBEntity, UnixTimestamp } from '@naturalcycles/js-lib/types'
-import { j, type JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv'
+import { j } from '@naturalcycles/nodejs-lib/ajv'
+import type { JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv'
 
 const MOCK_TS_2018_06_21 = 1529539200 as UnixTimestamp
 

package/src/validation/index.ts

@@ -1,15 +1,9 @@
 import type { ObjectWithId } from '@naturalcycles/js-lib/types'
-import {
-  j,
-  JsonSchemaAnyBuilder,
-  type JsonSchemaObjectBuilder,
-} from '@naturalcycles/nodejs-lib/ajv'
+import { j, JsonSchemaAnyBuilder } from '@naturalcycles/nodejs-lib/ajv'
+import type { JsonSchemaObjectBuilder } from '@naturalcycles/nodejs-lib/ajv'
 import type { CommonDBOptions, CommonDBSaveOptions, DBTransaction } from '../db.model.js'
-import {
-  type DBQueryFilter,
-  dbQueryFilterOperatorValues,
-  type DBQueryOrder,
-} from '../query/dbQuery.js'
+import { dbQueryFilterOperatorValues } from '../query/dbQuery.js'
+import type { DBQueryFilter, DBQueryOrder } from '../query/dbQuery.js'
 
 // oxlint-disable typescript/explicit-function-return-type