ic-mops 1.4.0 → 1.5.1

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Mops CLI Changelog
 
+## 1.5.1
+- Collapsible output of `mops bench` in a CI environment
+- Fix regression in `mops bench` without `dfx.json` file (by @rvanasa)
+
+## 1.5.0
+- Compile benchmarks with `--release` flag by default
+- Respect `profile` field in `dfx.json` for benchmarks
+
 ## 1.4.0
 - Update `mops bench` command output:
   - Print only final results if benchmarks run in a CI environment or there is no vertical space to progressively print the results

package/commands/bench.ts

@@ -12,7 +12,7 @@ import {filesize} from 'filesize';
 import terminalSize from 'terminal-size';
 import {SemVer} from 'semver';
 
-import {getRootDir, readConfig} from '../mops.js';
+import {getRootDir, readConfig, readDfxJson} from '../mops.js';
 import {parallel} from '../parallel.js';
 import {absToRel} from './test/utils.js';
 import {getMocVersion} from '../helpers/get-moc-version.js';
@@ -49,11 +49,12 @@ type BenchOptions = {
 	compare : boolean,
 	verbose : boolean,
 	silent : boolean,
-
+	profile : 'Debug' | 'Release',
 };
 
 export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}) : Promise<Benchmarks> {
 	let config = readConfig();
+	let dfxJson = readDfxJson();
 
 	let defaultOptions : BenchOptions = {
 		replica: config.toolchain?.['pocket-ic'] ? 'pocket-ic' : 'dfx',
@@ -66,6 +67,7 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 		compare: false,
 		verbose: false,
 		silent: false,
+		profile: dfxJson?.profile || 'Release',
 	};
 
 	let options : BenchOptions = {...defaultOptions, ...optionsArg};
@@ -123,7 +125,7 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 	fs.rmSync(benchDir, {recursive: true, force: true});
 	fs.mkdirSync(benchDir, {recursive: true});
 
-	if (!options.silent) {
+	if (!options.silent && !process.env.CI) {
 		console.log('Benchmark files:');
 		for (let file of files) {
 			console.log(chalk.gray(`• ${absToRel(file)}`));
@@ -181,14 +183,75 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 	return benchResults;
 }
 
+function computeDiff(
+	results : Map<string, BenchResult>,
+	prevResults : Map<string, BenchResult> | undefined,
+	rows : string[],
+	cols : string[],
+	metric : keyof BenchResult,
+) : number {
+	let diff = 0;
+	let count = 0;
+
+	for (let [_rowIndex, row] of rows.entries()) {
+		for (let [_colIndex, col] of cols.entries()) {
+			let res = results.get(`${row}:${col}`);
+			if (res && prevResults) {
+				let prevRes = prevResults.get(`${row}:${col}`);
+				if (prevRes) {
+					let denom = Number(prevRes[metric]);
+					let numerator = Number(res[metric]) - denom;
+					let percent = 0;
+					if (denom !== 0) {
+						percent = (numerator / denom) * 100;
+					}
+					if (!Number.isFinite(percent)) {
+						percent = 0;
+					}
+					diff += percent;
+					count++;
+				}
+			}
+		}
+	}
+
+	return count > 0 ? diff / count : 0;
+}
+
+function computeDiffAll(
+	currentResults : Map<string, BenchResult>,
+	prevResults : Map<string, BenchResult> | undefined,
+	rows : string[],
+	cols : string[],
+) : number {
+	let metrics : (keyof BenchResult)[] = ['instructions', 'rts_heap_size', 'rts_logical_stable_memory_size', 'rts_reclaimed'];
+	let diff = 0;
+
+	for (let metric of metrics) {
+		diff += computeDiff(currentResults, prevResults, rows, cols, metric);
+	}
+
+	return diff;
+}
+
 function getMocArgs(options : BenchOptions) : string {
 	let args = '';
+
 	if (options.forceGc) {
 		args += ' --force-gc';
 	}
+
 	if (options.gc) {
 		args += ` --${options.gc}-gc`;
 	}
+
+	if (options.profile === 'Debug') {
+		args += ' --debug';
+	}
+	else if (options.profile === 'Release') {
+		args += ' --release';
+	}
+
 	return args;
 }
 
@@ -261,6 +324,19 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 		return filesize(n, {standard: 'iec', round: 2});
 	};
 
+	let colorizePercent = (percent : number, wrapInParens = false) : string => {
+		let sign = percent > 0 ? '+' : '';
+		let percentText = percent == 0 ? '0%' : sign + percent.toFixed(2) + '%';
+		let color : keyof typeof chalk = percent == 0 ? 'gray' : (percent > 0 ? 'red' : 'green');
+		let parens = wrapInParens ? ['(', ')'] : ['', ''];
+		if (process.env.CI) {
+			return `$${parens[0]}{\\color{${color}}${percentText.replace('%', '\\\\%')}}${parens[1]}$`;
+		}
+		else {
+			return `${parens[0]}${chalk[color](percentText)}${parens[1]}`;
+		}
+	};
+
 	let getTable = (prop : keyof BenchResult) : string => {
 		let resArr = [['', ...schema.cols]];
 		let allZero = true;
@@ -284,15 +360,7 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 							if (Object.is(percent, NaN)) {
 								percent = 0;
 							}
-							let sign = percent > 0 ? '+' : '';
-							let percentText = percent == 0 ? '0%' : sign + percent.toFixed(2) + '%';
-							let color : keyof typeof chalk = percent == 0 ? 'gray' : (percent > 0 ? 'red' : 'green');
-							if (process.env.CI) {
-								diff = ` $({\\color{${color}}${percentText.replace('%', '\\\\%')}})$`;
-							}
-							else {
-								diff = ` (${chalk[color](percentText)})`;
-							}
+							diff = ' ' + colorizePercent(percent, true);
 						}
 						else {
 							diff = chalk.yellow(' (no previous results)');
@@ -333,14 +401,33 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 	let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
 
 	let getOutput = () => {
-		return `
-			\n${process.env.CI ? `## ${schema.name}` : chalk.bold(schema.name)}
-			${schema.description ? '\n' + (process.env.CI ? `_${schema.description}_` : chalk.gray(schema.description)) : ''}
-			\n\n${chalk.blue('Instructions')}\n\n${getTable('instructions')}
-			\n\n${chalk.blue('Heap')}\n\n${getTable('rts_heap_size')}
-			\n\n${chalk.blue('Garbage Collection')}\n\n${getTable('rts_reclaimed')}
-			${getTable('rts_logical_stable_memory_size') ? `\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}` : ''}
-		`;
+		if (process.env.CI) {
+			return [
+				'\n<details>',
+				`\n<summary>${absToRel(file)} ${colorizePercent(computeDiffAll(results, prevResults, schema.rows, schema.cols), true).replace('\\\\%', '\\%')}</summary>`,
+				`\n${process.env.CI ? `### ${schema.name}` : chalk.bold(schema.name)}`,
+				`${schema.description ? '\n' + (process.env.CI ? `_${schema.description}_` : chalk.gray(schema.description)) : ''}`,
+				`\n\nInstructions: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'instructions'), false)}`,
+				`Heap: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'rts_heap_size'), false)}`,
+				`Stable Memory: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'rts_logical_stable_memory_size'), false)}`,
+				`Garbage Collection: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'rts_reclaimed'), false)}`,
+				`\n\n**Instructions**\n\n${getTable('instructions')}`,
+				`\n\n**Heap**\n\n${getTable('rts_heap_size')}`,
+				`\n\n**Garbage Collection**\n\n${getTable('rts_reclaimed')}`,
+				`${getTable('rts_logical_stable_memory_size') ? `\n\n**Stable Memory**\n\n${getTable('rts_logical_stable_memory_size')}` : ''}`,
+				'\n</details>',
+			].join('\n');
+		}
+		else {
+			return `
+				\n${chalk.bold(schema.name)}
+				${schema.description ? '\n' + chalk.gray(schema.description) : ''}
+				\n\n${chalk.blue('Instructions')}\n\n${getTable('instructions')}
+				\n\n${chalk.blue('Heap')}\n\n${getTable('rts_heap_size')}
+				\n\n${chalk.blue('Garbage Collection')}\n\n${getTable('rts_reclaimed')}
+				${getTable('rts_logical_stable_memory_size') ? `\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}` : ''}
+			`;
+		}
 	};
 
 	let canUpdateLog = !process.env.CI && !options.silent && terminalSize().rows > getOutput().split('\n').length;
@@ -433,4 +520,4 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 			['rts_reclaimed', reclaimedCells],
 		],
 	};
-}
+}

package/commands/watch/parseDfxJson.ts

@@ -5,6 +5,7 @@ import {getRootDir} from '../../mops.js';
 type DfxConfig = {
 	$schema : string;
 	version : number;
+	profile : 'Debug' | 'Release';
 	canisters : {
 		[key : string] : {
 			type : 'motoko' | 'assets';
@@ -41,7 +42,7 @@ type DfxConfig = {
 	};
 };
 
-function readDfxJson() : DfxConfig | Record<string, never> {
+export function readDfxJson() : DfxConfig | Record<string, never> {
 	let dfxJsonPath = path.resolve(getRootDir(), 'dfx.json');
 	if (!existsSync(dfxJsonPath)) {
 		return {};

package/dist/commands/bench.d.ts

@@ -11,6 +11,7 @@ type BenchOptions = {
     compare: boolean;
     verbose: boolean;
     silent: boolean;
+    profile: 'Debug' | 'Release';
 };
 export declare function bench(filter?: string, optionsArg?: Partial<BenchOptions>): Promise<Benchmarks>;
 export {};

package/dist/commands/bench.js

@@ -11,7 +11,7 @@ import stringWidth from 'string-width';
 import { filesize } from 'filesize';
 import terminalSize from 'terminal-size';
 import { SemVer } from 'semver';
-import { getRootDir, readConfig } from '../mops.js';
+import { getRootDir, readConfig, readDfxJson } from '../mops.js';
 import { parallel } from '../parallel.js';
 import { absToRel } from './test/utils.js';
 import { getMocVersion } from '../helpers/get-moc-version.js';
@@ -31,6 +31,7 @@ let globConfig = {
 };
 export async function bench(filter = '', optionsArg = {}) {
     let config = readConfig();
+    let dfxJson = readDfxJson();
     let defaultOptions = {
         replica: config.toolchain?.['pocket-ic'] ? 'pocket-ic' : 'dfx',
         replicaVersion: '',
@@ -42,6 +43,7 @@ export async function bench(filter = '', optionsArg = {}) {
         compare: false,
         verbose: false,
         silent: false,
+        profile: dfxJson?.profile || 'Release',
     };
     let options = { ...defaultOptions, ...optionsArg };
     let replicaType = options.replica ?? (config.toolchain?.['pocket-ic'] ? 'pocket-ic' : 'dfx');
@@ -88,7 +90,7 @@ export async function bench(filter = '', optionsArg = {}) {
     let benchDir = `${getRootDir()}/.mops/.bench/`;
     fs.rmSync(benchDir, { recursive: true, force: true });
     fs.mkdirSync(benchDir, { recursive: true });
-    if (!options.silent) {
+    if (!options.silent && !process.env.CI) {
         console.log('Benchmark files:');
         for (let file of files) {
             console.log(chalk.gray(`• ${absToRel(file)}`));
@@ -137,6 +139,40 @@ export async function bench(filter = '', optionsArg = {}) {
     fs.rmSync(benchDir, { recursive: true, force: true });
     return benchResults;
 }
+function computeDiff(results, prevResults, rows, cols, metric) {
+    let diff = 0;
+    let count = 0;
+    for (let [_rowIndex, row] of rows.entries()) {
+        for (let [_colIndex, col] of cols.entries()) {
+            let res = results.get(`${row}:${col}`);
+            if (res && prevResults) {
+                let prevRes = prevResults.get(`${row}:${col}`);
+                if (prevRes) {
+                    let denom = Number(prevRes[metric]);
+                    let numerator = Number(res[metric]) - denom;
+                    let percent = 0;
+                    if (denom !== 0) {
+                        percent = (numerator / denom) * 100;
+                    }
+                    if (!Number.isFinite(percent)) {
+                        percent = 0;
+                    }
+                    diff += percent;
+                    count++;
+                }
+            }
+        }
+    }
+    return count > 0 ? diff / count : 0;
+}
+function computeDiffAll(currentResults, prevResults, rows, cols) {
+    let metrics = ['instructions', 'rts_heap_size', 'rts_logical_stable_memory_size', 'rts_reclaimed'];
+    let diff = 0;
+    for (let metric of metrics) {
+        diff += computeDiff(currentResults, prevResults, rows, cols, metric);
+    }
+    return diff;
+}
 function getMocArgs(options) {
     let args = '';
     if (options.forceGc) {
@@ -145,6 +181,12 @@ function getMocArgs(options) {
     if (options.gc) {
         args += ` --${options.gc}-gc`;
     }
+    if (options.profile === 'Debug') {
+        args += ' --debug';
+    }
+    else if (options.profile === 'Release') {
+        args += ' --release';
+    }
     return args;
 }
 async function deployBenchFile(file, options, replica) {
@@ -203,6 +245,18 @@ async function runBenchFile(file, options, replica) {
     let formatSize = (n) => {
         return filesize(n, { standard: 'iec', round: 2 });
     };
+    let colorizePercent = (percent, wrapInParens = false) => {
+        let sign = percent > 0 ? '+' : '';
+        let percentText = percent == 0 ? '0%' : sign + percent.toFixed(2) + '%';
+        let color = percent == 0 ? 'gray' : (percent > 0 ? 'red' : 'green');
+        let parens = wrapInParens ? ['(', ')'] : ['', ''];
+        if (process.env.CI) {
+            return `$${parens[0]}{\\color{${color}}${percentText.replace('%', '\\\\%')}}${parens[1]}$`;
+        }
+        else {
+            return `${parens[0]}${chalk[color](percentText)}${parens[1]}`;
+        }
+    };
     let getTable = (prop) => {
         let resArr = [['', ...schema.cols]];
         let allZero = true;
@@ -223,15 +277,7 @@ async function runBenchFile(file, options, replica) {
                             if (Object.is(percent, NaN)) {
                                 percent = 0;
                             }
-                            let sign = percent > 0 ? '+' : '';
-                            let percentText = percent == 0 ? '0%' : sign + percent.toFixed(2) + '%';
-                            let color = percent == 0 ? 'gray' : (percent > 0 ? 'red' : 'green');
-                            if (process.env.CI) {
-                                diff = ` $({\\color{${color}}${percentText.replace('%', '\\\\%')}})$`;
-                            }
-                            else {
-                                diff = ` (${chalk[color](percentText)})`;
-                            }
+                            diff = ' ' + colorizePercent(percent, true);
                         }
                         else {
                             diff = chalk.yellow(' (no previous results)');
@@ -267,14 +313,33 @@ async function runBenchFile(file, options, replica) {
     };
     let logUpdate = createLogUpdate(process.stdout, { showCursor: true });
     let getOutput = () => {
-        return `
-			\n${process.env.CI ? `## ${schema.name}` : chalk.bold(schema.name)}
-			${schema.description ? '\n' + (process.env.CI ? `_${schema.description}_` : chalk.gray(schema.description)) : ''}
-			\n\n${chalk.blue('Instructions')}\n\n${getTable('instructions')}
-			\n\n${chalk.blue('Heap')}\n\n${getTable('rts_heap_size')}
-			\n\n${chalk.blue('Garbage Collection')}\n\n${getTable('rts_reclaimed')}
-			${getTable('rts_logical_stable_memory_size') ? `\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}` : ''}
-		`;
+        if (process.env.CI) {
+            return [
+                '\n<details>',
+                `\n<summary>${absToRel(file)} ${colorizePercent(computeDiffAll(results, prevResults, schema.rows, schema.cols), true).replace('\\\\%', '\\%')}</summary>`,
+                `\n${process.env.CI ? `### ${schema.name}` : chalk.bold(schema.name)}`,
+                `${schema.description ? '\n' + (process.env.CI ? `_${schema.description}_` : chalk.gray(schema.description)) : ''}`,
+                `\n\nInstructions: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'instructions'), false)}`,
+                `Heap: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'rts_heap_size'), false)}`,
+                `Stable Memory: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'rts_logical_stable_memory_size'), false)}`,
+                `Garbage Collection: ${colorizePercent(computeDiff(results, prevResults, schema.rows, schema.cols, 'rts_reclaimed'), false)}`,
+                `\n\n**Instructions**\n\n${getTable('instructions')}`,
+                `\n\n**Heap**\n\n${getTable('rts_heap_size')}`,
+                `\n\n**Garbage Collection**\n\n${getTable('rts_reclaimed')}`,
+                `${getTable('rts_logical_stable_memory_size') ? `\n\n**Stable Memory**\n\n${getTable('rts_logical_stable_memory_size')}` : ''}`,
+                '\n</details>',
+            ].join('\n');
+        }
+        else {
+            return `
+				\n${chalk.bold(schema.name)}
+				${schema.description ? '\n' + chalk.gray(schema.description) : ''}
+				\n\n${chalk.blue('Instructions')}\n\n${getTable('instructions')}
+				\n\n${chalk.blue('Heap')}\n\n${getTable('rts_heap_size')}
+				\n\n${chalk.blue('Garbage Collection')}\n\n${getTable('rts_reclaimed')}
+				${getTable('rts_logical_stable_memory_size') ? `\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}` : ''}
+			`;
+        }
     };
     let canUpdateLog = !process.env.CI && !options.silent && terminalSize().rows > getOutput().split('\n').length;
     let log = () => {

package/dist/commands/watch/parseDfxJson.d.ts

@@ -1,2 +1,43 @@
+type DfxConfig = {
+    $schema: string;
+    version: number;
+    profile: 'Debug' | 'Release';
+    canisters: {
+        [key: string]: {
+            type: 'motoko' | 'assets';
+            main?: string;
+            specified_id?: string;
+            declarations?: {
+                output: string;
+                node_compatibility: boolean;
+            };
+            build?: string[];
+            frontend?: {
+                entrypoint: string;
+            };
+            source?: string[];
+            remote?: {
+                id: {
+                    ic: string;
+                    staging: string;
+                };
+            };
+        };
+    };
+    defaults: {
+        build: {
+            packtool: string;
+        };
+    };
+    dfx: string;
+    networks: {
+        [key: string]: {
+            type: string;
+            providers: string[];
+        };
+    };
+};
+export declare function readDfxJson(): DfxConfig | Record<string, never>;
 export declare function getMotokoCanisters(): Record<string, string>;
 export declare function getMotokoCanistersWithDeclarations(): Record<string, string>;
+export {};

package/dist/commands/watch/parseDfxJson.js

@@ -1,7 +1,7 @@
 import { existsSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import { getRootDir } from '../../mops.js';
-function readDfxJson() {
+export function readDfxJson() {
     let dfxJsonPath = path.resolve(getRootDir(), 'dfx.json');
     if (!existsSync(dfxJsonPath)) {
         return {};

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "1.4.0",
+  "version": "1.5.1",
   "type": "module",
   "bin": {
     "mops": "bin/mops.js",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "1.4.0",
+  "version": "1.5.1",
   "type": "module",
   "bin": {
     "mops": "dist/bin/mops.js",