@cap-js-community/common 0.2.4 → 0.2.6

package/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## Version 0.2.6 - 2025-08-04
+
+### Fixed
+
+- Migration Check `ReleasedElementCompatibleTypeChangeIsNotWhitelisted` to allow compatible type changes
+- Admin tracking writes an admin changes file to keep track of incompatible changes as well
+
+## Version 0.2.5 - 2025-07-07
+
+### Fixed
+
+- Normalize newline character in hash calculation
+
 ## Version 0.2.4 - 2025-07-03
 
 ### Fixed

package/README.md

@@ -65,6 +65,7 @@ Options can be passed to replication cache via CDS environment via `cds.replicat
 - `check: Number`: Interval to check size and prune. Default is `60000` (1 minute)
 - `stats: Number`: Interval to log statistics. Default is `300000` (5 minutes)
 - `size: Number`: Maximal cache size in bytes. Default is `10485760` (10 MB) and in production: `104857600` (100 MB)
+- `pipe: Boolean`: Replication is streamed through pipeline. `chunks` is not used. Default is `true`
 - `chunks: Number`: Replication chunk size. Default is `1000`
 - `retries: Number`: Replication retries for failed replications. Default is `3`
 - `auto: Boolean`: Replication is managed automatically. Default is `true`
@@ -96,31 +97,91 @@ Replication cache is inactive per default for tests (`test` profile). It can be
 
 ## Migration Check
 
+The migration check allows to check for incompatible changes in the CDS model and
+to maintain a whitelist for compatible changes via `cdsmc` command line tool.
+
 ### Options
 
 Options can be passed to migration check via CDS environment via `cds.migrationCheck` section:
 
 - `baseDir: String`: Specifies the base directory for migration check. Default is `"migration-check"`
-- `whitelist: Boolean`: Requires to maintain a whitelist for compatible changes. Default is `true`
-- `checkMtx: Boolean`: Includes CDS MTXS persistence into check. Default is `true`
+- `whitelist: Boolean`: Requires maintaining a whitelist for compatible changes. Default is `true`
+- `checkMtx: Boolean`: Includes CDS MTXS persistence in check. Default is `true`
 - `keep: Boolean`: Keeps whitelist after update, otherwise whitelist is cleared. Default is `false`
 - `freeze: Boolean`: Freeze the persistence. Event compatible changes are not allowed, Default is `false`
 - `label: String`: Label to describe the updated hash files in addition to the timestamp. Default is `""`
-- `buildPath: String`: Path to the build CSN. If not specified it derived from CAP project type. Default is `null`
+- `buildPath: String`: Path to the build CSN. If not specified, it is derived from the CAP project type. Default is `null`
 - `adminHash: String`: Specify admin hash to acknowledge incompatible changes. Default is `null`
+- `adminTracking: Boolean`: Track changes acknowledged by admin in an admin changes file. Default is `true`
 
 ### Usage
 
-#### Basic Flow
+#### Build Production CSN
+
+Production CSN is built for first time when not existing (otherwise it is updated):
+
+- Build CSN: `cds build --production`
+- Update Production CSN: `cdsmc -u`
+
+> Production CSN MUST be added to version control.
+
+#### Migration Check
+
+Migration check is used to check for incompatible changes in a repetitive way along development:
+
+- Build CSN: `cds build --production`
+- Check Changes: `cdsmc`
+
+Incompatible changes are detected and reported as error.
+Compatible changes need to be whitelisted (can be disabled via options).
+
+##### Checks
+
+**Incompatible Changes:**
+
+- A released entity cannot be removed (`ReleasedEntityCannotBeRemoved`)
+- The draft enablement state of a released entity cannot be changed (`ReleasedEntityDraftEnablementCannotBeChanged`)
+- A released element cannot be removed (`ReleasedElementCannotBeRemoved`)
+- The key of a released element cannot be changed (`ReleasedElementKeyCannotBeChanged`)
+- The managed/unmanaged state of a released element cannot be changed (`ReleasedElementManagedUnmanagedCannotBeChanged`)
+- The virtual state of a released element cannot be changed (`ReleasedElementVirtualCannotBeChanged`)
+- The localization state of a released element cannot be changed (`ReleasedElementLocalizationCannotBeChanged`)
+- A released element cannot be changed to not-nullable (`ReleasedElementNullableCannotBeChanged`)
+- The data type of a released element cannot be changed (`ReleasedElementTypeCannotBeChanged`)
+- The data type of a released element cannot be shortened (`ReleasedElementTypeCannotBeShortened`)
+- The scale or precision of a released element cannot be reduced (`ReleasedElementScalePrecisionCannotBeLower`)
+- The target of a released element cannot be changed (`ReleasedElementTargetCannotBeChanged`)
+- The cardinality of a released element cannot be changed (`ReleasedElementCardinalityCannotBeChanged`)
+- The ON condition of a released element cannot be changed (`ReleasedElementOnConditionCannotBeChanged`)
+- The keys condition of a released element cannot be changed (`ReleasedElementKeysConditionCannotBeChanged`)
+- Enabling journal mode and changing entity in same cycle is not allowed (`ReleasedEntityJournalModeAndEntityChangeIsNotAllowed`)
+- Changes to the index of a released entity are not allowed (`ReleasedEntityIndexChangeIsNotAllowed`)
+
+**Compatible Changes:**
+
+- Changes to the index of a released entity must be whitelisted (`ReleasedEntityIndexChangeIsNotWhitelisted`)
+- Extending the type of a released element requires whitelisting (`ReleasedElementTypeExtensionIsNotWhitelisted`)
+- Extending the scale or precision of a released element requires whitelisting (`ReleasedElementScalePrecisionExtensionIsNotWhitelisted`)
+- Changing the type of a released element to a compatible type requires whitelisting (`ReleasedElementTypeChangeIsNotWhitelisted`)
+- The new entity is not whitelisted (`NewEntityIsNotWhitelisted`)
+- The new entity element is not whitelisted (`NewEntityElementIsNotWhitelisted`)
+- A new entity element must have a default value if it is not nullable (`NewEntityElementNotNullableDefault`)
+- The new entity index is not whitelisted (`NewEntityIndexIsNotWhitelisted`)
+
+#### Update Production CSN
+
+The Production CSN can be updated when no migration check errors occur:
 
 - Build CSN: `cds build --production`
-- Check migrations: `cdsmc`
 - Update Production CSN: `cdsmc -u`
 
+> Production CSN MUST be added to version control.
+
 ### Whitelisting
 
-- Maintain the whitelist extension file `migration-extension-whitelist.json` for compatible changes:
-  - **Whitelist Entity**:
+Maintain the whitelist extension file `migration-extension-whitelist.json` for compatible changes:
+
+- **Whitelist Entity**:
 
   ```json
   {
@@ -130,7 +191,7 @@ Options can be passed to migration check via CDS environment via `cds.migrationC
   }
   ```
 
-  - **Whitelist Entity Element**:
+- **Whitelist Entity Element**:
 
   ```json
   {
@@ -146,18 +207,35 @@ Options can be passed to migration check via CDS environment via `cds.migrationC
 
 ### Admin Mode
 
-- Get Admin Hash: `cdsmc -a`
-- (Un-)Freeze Persistence (based on options): `cdsmc -u -a`
+#### Incompatible Changes
+
+Accepted incompatible changes can be acknowledged and will not be reported as error anymore:
+
+- Get current admin hash for incompatible changes: `cdsmc -a`
+- Set admin hash in env: `cds.migrationCheck.adminHash`
+
+#### Freeze Persistence
+
+CDS persistence can be (temporarily) frozen to prevent any changes (also compatible) to the persistence model:
+
+- Activate/Deactivate persistence freeze in env `cds.migrationCheck.freeze`
+- Freeze/Unfreeze Persistence: `cdsmc -u -a`
+  - File `./csn-prod.freeze` is created to indicate that persistence is frozen
 
 ### Pipeline
 
+Migration check can be used in a pipeline (e.g. part of Pull Request voter)
+to ensure that incompatible changes are not introduced:
+
 - Build & Check: `cds build --production && cdsmc`
-- Update Production CSN: `cdsmc -u`
+- Update Production CSN: `cds build --production && cdsmc -u`
 
-> Production CSN MUST be added to version control
+> Production CSN MUST be added to version control.
 
 ## Rate Limiting
 
+The rate limiting allows to limit the number of requests per service and tenant.
+
 ### Usage
 
 ```cds

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js-community/common",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "CAP Node.js Community Common",
   "homepage": "https://cap.cloud.sap/",
   "engines": {
@@ -53,17 +53,17 @@
   "devDependencies": {
     "@cap-js-community/common": "./",
     "@cap-js/cds-test": "^0.4.0",
-    "@sap/cds": "^9.1.0",
+    "@sap/cds": "^9.2.0",
     "@sap/cds-common-content": "^3.0.1",
-    "@sap/cds-dk": "^9.1.0",
-    "eslint": "9.30.1",
-    "eslint-config-prettier": "10.1.5",
-    "eslint-plugin-jest": "29.0.1",
-    "eslint-plugin-n": "^17.20.0",
-    "jest": "30.0.4",
-    "jest-html-reporters": "3.1.7",
-    "jest-junit": "16.0.0",
-    "prettier": "3.6.2",
+    "@sap/cds-dk": "^9.2.0",
+    "eslint": "^9.32.0",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-jest": "^29.0.1",
+    "eslint-plugin-n": "^17.21.3",
+    "jest": "^30.0.5",
+    "jest-html-reporters": "^3.1.7",
+    "jest-junit": "^16.0.0",
+    "prettier": "^3.6.2",
     "shelljs": "^0.10.0"
   },
   "cds": {
@@ -82,7 +82,8 @@
       "freeze": false,
       "label": null,
       "buildPath": null,
-      "adminHash": null
+      "adminHash": null,
+      "adminTracking": true
     },
     "rateLimiting": {
       "plugin": true,
@@ -115,6 +116,7 @@
       "[production]": {
         "size": 104857600
       },
+      "pipe": true,
       "chunks": 1000,
       "retries": 3,
       "auto": true,

package/src/migration-check/MigrationCheck.js

@@ -32,6 +32,8 @@ const Messages = {
   ReleasedElementTypeExtensionIsNotWhitelisted: "Extending the type of a released element requires whitelisting",
   ReleasedElementScalePrecisionExtensionIsNotWhitelisted:
     "Extending the scale or precision of a released element requires whitelisting",
+  ReleasedElementCompatibleTypeChangeIsNotWhitelisted:
+    "Changing the type of a released element to a compatible type requires whitelisting",
 
   NewEntityIsNotWhitelisted: "The new entity is not whitelisted",
   NewEntityElementIsNotWhitelisted: "The new entity element is not whitelisted",
@@ -60,6 +62,7 @@ class MigrationCheck {
       prodHashPath: path.join(basePath, "./csn-prod-hash.json"),
       prodWhitelistPath: path.join(basePath, "./migration-extension-whitelist.json"),
       prodWhitelistHashPath: path.join(basePath, "./migration-extension-whitelist-hash.json"),
+      prodAdminChangesPath: path.join(basePath, "./migration-admin-changes.json"),
       prodFreeze: path.join(basePath, "./csn-prod.freeze"),
     };
     this.setup();
@@ -207,6 +210,9 @@ class MigrationCheck {
           for (const message of result.messages) {
             message.severity = message.severity === "error" ? "warning" : message.severity;
           }
+          if (this.options.adminTracking) {
+            fs.writeFileSync(this.paths.prodAdminChangesPath, JSON.stringify(messages, null, 2) + "\n");
+          }
           messages.push({
             code: "AcceptedByAdmin",
             text: "Migration check errors accepted by admin",
@@ -214,6 +220,9 @@ class MigrationCheck {
           });
           result.success = true;
         } else {
+          if (this.options.adminTracking) {
+            fs.writeFileSync(this.paths.prodAdminChangesPath, JSON.stringify(messages, null, 2) + "\n");
+          }
           messages.push({
             code: "AdminHashInvalid",
             text: "Admin hash is not valid for current migration check state",
@@ -225,6 +234,9 @@ class MigrationCheck {
         result.success = false;
       }
     }
+    if (!this.options.adminHash && fs.existsSync(this.paths.prodAdminChangesPath)) {
+      fs.rmSync(this.paths.prodAdminChangesPath);
+    }
     return result;
   }
 
@@ -366,7 +378,20 @@ function releasedEntityCheck(csnBuild, csnProd, whitelist, options) {
         } else if (!elementProd.notNull && elementBuild.notNull) {
           report(messages, MessagesCodes.ReleasedElementNullableCannotBeChanged, definitionProd.name, elementProdName);
         } else if (normalizeType(csnProd, elementProd.type) !== normalizeType(csnBuild, elementBuild.type)) {
-          report(messages, MessagesCodes.ReleasedElementTypeCannotBeChanged, definitionProd.name, elementProdName);
+          const prodType = normalizeType(csnProd, elementProd.type);
+          const buildType = normalizeType(csnBuild, elementBuild.type);
+          if (prodType === "cds.String" && buildType === "cds.LargeString") {
+            if (!elementWhitelist && options.whitelist) {
+              report(
+                messages,
+                MessagesCodes.ReleasedElementCompatibleTypeChangeIsNotWhitelisted,
+                definitionProd.name,
+                elementProdName,
+              );
+            }
+          } else {
+            report(messages, MessagesCodes.ReleasedElementTypeCannotBeChanged, definitionProd.name, elementProdName);
+          }
         } else if ((elementProd.length || STRING_DEFAULT_LENGTH) > (elementBuild.length || STRING_DEFAULT_LENGTH)) {
           report(messages, MessagesCodes.ReleasedElementTypeCannotBeShortened, definitionProd.name, elementProdName);
         } else if ((elementProd.length || STRING_DEFAULT_LENGTH) < (elementBuild.length || STRING_DEFAULT_LENGTH)) {
@@ -652,6 +677,7 @@ function normalizeType(csn, type) {
   return typeof type === "object" ? JSON.stringify(type) : type;
 }
 
-const hash = (buffer, algorithm = "sha256") => crypto.createHash(algorithm).update(buffer).digest("hex");
+const hash = (buffer, algorithm = "sha256") =>
+  crypto.createHash(algorithm).update(buffer.toString().replace(/\r\n/g, "\n")).digest("hex");
 
 module.exports = MigrationCheck;

package/src/replication-cache/ReplicationCache.js

@@ -492,9 +492,13 @@ class ReplicationCache {
         return result;
       })(),
     ]);
-    const percent = ((timeService - timeCache) / timeService) * 100;
-    this.log.info("Replication cache measurement", Math.round(percent), timeCache, timeService);
-    this.stats.measureTotal += percent;
+    const savedPercent = ((timeService - timeCache) / timeService) * 100;
+    this.log.info("Replication cache measurement", {
+      timeCache,
+      timeService,
+      savedPercent: Math.round(savedPercent),
+    });
+    this.stats.measureTotal += savedPercent;
     this.stats.measureCount += 1;
     this.stats.measureRatio = Math.round(this.stats.measureTotal / this.stats.measureCount);
     return cacheResult;
@@ -696,14 +700,14 @@ class ReplicationCacheEntry {
     if (thread && cds.context && this.service instanceof SQLiteService) {
       const srcTx = this.service.tx(cds.context);
       await this.db.tx({ tenant: this.tenant.id }, async (destTx) => {
-        await this.loadChunked(srcTx, destTx);
+        await this.loadRecords(srcTx, destTx);
         await this.checkRecords(srcTx, destTx);
         await this.calcSize(destTx);
       });
     } else {
       await this.service.tx({ tenant: this.tenant.id }, async (srcTx) => {
         await this.db.tx({ tenant: this.tenant.id }, async (destTx) => {
-          await this.loadChunked(srcTx, destTx);
+          await this.loadRecords(srcTx, destTx);
           await this.checkRecords(srcTx, destTx);
           await this.calcSize(destTx);
         });
@@ -712,6 +716,24 @@ class ReplicationCacheEntry {
     this.timestamp = Date.now();
   }
 
+  async loadRecords(srcTx, destTx) {
+    if (this.cache.options.pipe) {
+      await this.loadPiped(srcTx, destTx);
+    } else {
+      await this.loadChunked(srcTx, destTx);
+    }
+  }
+
+  async loadPiped(srcTx, destTx) {
+    const keys = Object.keys(this.definition.keys);
+    const selectQuery = SELECT.from(this.definition).orderBy(keys);
+    selectQuery.replication = true;
+    selectQuery.bind(srcTx);
+    const insertQuery = INSERT.into(this.definition);
+    insertQuery.bind(destTx);
+    await selectQuery.pipeline(insertQuery);
+  }
+
   async loadChunked(srcTx, destTx) {
     const keys = Object.keys(this.definition.keys);
     const selectQuery = SELECT.from(this.definition).orderBy(keys);