ic-mops 1.3.0 → 1.4.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Mops CLI Changelog
 
+## 1.4.0
+- Update `mops bench` command output:
+  - Print only final results if benchmarks run in a CI environment or there is no vertical space to progressively print the results
+  - Hide "Stable Memory" table if it has no data
+  - Hide verbose output when running in a CI environment ("Starting replica...", "Running simple.bench.mo...", etc.)
+  - Add LaTeX colors to the diffs when running in a CI environment with `--compare` flag
+- CLI now fails if excess arguments are passed to it
+
 ## 1.3.0
 - Show error on `mops install <pkg>` command. Use `mops add <pkg>` instead.
 - Added support for pocket-ic replica that comes with dfx in `mops bench` command. To activate it, remove `pocket-ic` from `mops.toml` and run `mops bench --replica pocket-ic`. Requires dfx 0.24.1 or higher.

package/commands/bench-replica.ts

@@ -22,7 +22,9 @@ export class BenchReplica {
 	}
 
 	async start({silent = false} = {}) {
-		silent || console.log(`Starting ${this.type} replica...`);
+		if (!process.env.CI && !silent) {
+			console.log(`Starting ${this.type} replica...`);
+		}
 
 		if (this.type == 'dfx' || this.type === 'dfx-pocket-ic') {
 			await this.stop();

package/commands/bench.ts

@@ -5,9 +5,12 @@ import os from 'node:os';
 import chalk from 'chalk';
 import {globSync} from 'glob';
 import {markdownTable} from 'markdown-table';
-import logUpdate from 'log-update';
+import {createLogUpdate} from 'log-update';
 import {execaCommand} from 'execa';
 import stringWidth from 'string-width';
+import {filesize} from 'filesize';
+import terminalSize from 'terminal-size';
+import {SemVer} from 'semver';
 
 import {getRootDir, readConfig} from '../mops.js';
 import {parallel} from '../parallel.js';
@@ -20,8 +23,6 @@ import {sources} from './sources.js';
 import {Benchmark, Benchmarks} from '../declarations/main/main.did.js';
 import {BenchResult, _SERVICE} from '../declarations/bench/bench.did.js';
 import {BenchReplica} from './bench-replica.js';
-import {filesize} from 'filesize';
-import {SemVer} from 'semver';
 
 type ReplicaName = 'dfx' | 'pocket-ic' | 'dfx-pocket-ic';
 
@@ -83,6 +84,10 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 
 	options.replica = replicaType;
 
+	if (process.env.CI) {
+		console.log('# Benchmark Results\n\n');
+	}
+
 	if (replicaType == 'dfx') {
 		options.replicaVersion = getDfxVersion();
 	}
@@ -123,14 +128,19 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 		for (let file of files) {
 			console.log(chalk.gray(`• ${absToRel(file)}`));
 		}
-		console.log('');
-		console.log('='.repeat(50));
-		console.log('');
+		if (!process.env.CI) {
+			console.log('');
+			console.log('='.repeat(50));
+			console.log('');
+		}
 	}
 
 	await replica.start({silent: options.silent});
 
-	options.silent || console.log('Deploying canisters...');
+	if (!process.env.CI && !options.silent) {
+		console.log('Deploying canisters...');
+	}
+
 	await parallel(os.cpus().length, files, async (file : string) => {
 		try {
 			await deployBenchFile(file, options, replica);
@@ -145,8 +155,8 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 	let benchResults : Benchmarks = [];
 
 	await parallel(1, files, async (file : string) => {
-		if (!options.silent) {
-			console.log('\n' + '—'.repeat(50));
+		if (!options.silent && !process.env.CI) {
+			console.log('\n' + '-'.repeat(50));
 			console.log(`\nRunning ${chalk.gray(absToRel(file))}...`);
 			console.log('');
 		}
@@ -161,7 +171,9 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 		}
 	});
 
-	options.silent || console.log('Stopping replica...');
+	if (!process.env.CI && !options.silent) {
+		console.log('Stopping replica...');
+	}
 	await replica.stop();
 
 	fs.rmSync(benchDir, {recursive: true, force: true});
@@ -251,6 +263,7 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 
 	let getTable = (prop : keyof BenchResult) : string => {
 		let resArr = [['', ...schema.cols]];
+		let allZero = true;
 
 		for (let [_rowIndex, row] of schema.rows.entries()) {
 			let curRow = [row];
@@ -258,6 +271,9 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 			for (let [_colIndex, col] of schema.cols.entries()) {
 				let res = results.get(`${row}:${col}`);
 				if (res) {
+					if (res[prop] != 0n) {
+						allZero = false;
+					}
 
 					// compare with previous results
 					let diff = '';
@@ -265,10 +281,18 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 						let prevRes = prevResults.get(`${row}:${col}`);
 						if (prevRes) {
 							let percent = (Number(res[prop]) - Number(prevRes[prop])) / Number(prevRes[prop]) * 100;
+							if (Object.is(percent, NaN)) {
+								percent = 0;
+							}
 							let sign = percent > 0 ? '+' : '';
 							let percentText = percent == 0 ? '0%' : sign + percent.toFixed(2) + '%';
 							let color : keyof typeof chalk = percent == 0 ? 'gray' : (percent > 0 ? 'red' : 'green');
-							diff = ` (${chalk[color](percentText)})`;
+							if (process.env.CI) {
+								diff = ` $({\\color{${color}}${percentText.replace('%', '\\\\%')}})$`;
+							}
+							else {
+								diff = ` (${chalk[color](percentText)})`;
+							}
 						}
 						else {
 							diff = chalk.yellow(' (no previous results)');
@@ -295,25 +319,47 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 			resArr.push(curRow);
 		}
 
+		// don't show Stable Memory table if all values are 0
+		if (allZero && prop == 'rts_logical_stable_memory_size') {
+			return '';
+		}
+
 		return markdownTable(resArr, {
 			align: ['l', ...'r'.repeat(schema.cols.length)],
 			stringLength: stringWidth,
 		});
 	};
 
-	let printResults = () => {
-		logUpdate(`
-			\n${chalk.bold(schema.name)}
-			${schema.description ? '\n' + chalk.gray(schema.description) : ''}
+	let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
+
+	let getOutput = () => {
+		return `
+			\n${process.env.CI ? `## ${schema.name}` : chalk.bold(schema.name)}
+			${schema.description ? '\n' + (process.env.CI ? `_${schema.description}_` : chalk.gray(schema.description)) : ''}
 			\n\n${chalk.blue('Instructions')}\n\n${getTable('instructions')}
 			\n\n${chalk.blue('Heap')}\n\n${getTable('rts_heap_size')}
-			\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}
 			\n\n${chalk.blue('Garbage Collection')}\n\n${getTable('rts_reclaimed')}
-		`);
+			${getTable('rts_logical_stable_memory_size') ? `\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}` : ''}
+		`;
 	};
 
-	if (!process.env.CI && !options.silent) {
-		printResults();
+	let canUpdateLog = !process.env.CI && !options.silent && terminalSize().rows > getOutput().split('\n').length;
+
+	let log = () => {
+		if (options.silent) {
+			return;
+		}
+		let output = getOutput();
+		if (process.env.CI || terminalSize().rows <= output.split('\n').length) {
+			console.log(output);
+		}
+		else {
+			logUpdate(output);
+		}
+	};
+
+	if (canUpdateLog) {
+		log();
 	}
 
 	// run all cells
@@ -331,16 +377,18 @@ async function runBenchFile(file : string, options : BenchOptions, replica : Ben
 			// @ts-ignore
 			reclaimedCells[rowIndex][colIndex] = res.rts_reclaimed;
 
-			if (!process.env.CI && !options.silent) {
-				printResults();
+			if (canUpdateLog) {
+				log();
 			}
 		}
 	}
 
-	if (process.env.CI) {
-		printResults();
+	if (canUpdateLog) {
+		logUpdate.done();
+	}
+	else {
+		log();
 	}
-	logUpdate.done();
 
 	// save results
 	if (options.save) {

package/dist/commands/bench-replica.js

@@ -16,7 +16,9 @@ export class BenchReplica {
         this.verbose = verbose;
     }
     async start({ silent = false } = {}) {
-        silent || console.log(`Starting ${this.type} replica...`);
+        if (!process.env.CI && !silent) {
+            console.log(`Starting ${this.type} replica...`);
+        }
         if (this.type == 'dfx' || this.type === 'dfx-pocket-ic') {
             await this.stop();
             let dir = path.join(getRootDir(), '.mops/.bench');

package/dist/commands/bench.js

@@ -5,9 +5,12 @@ import os from 'node:os';
 import chalk from 'chalk';
 import { globSync } from 'glob';
 import { markdownTable } from 'markdown-table';
-import logUpdate from 'log-update';
+import { createLogUpdate } from 'log-update';
 import { execaCommand } from 'execa';
 import stringWidth from 'string-width';
+import { filesize } from 'filesize';
+import terminalSize from 'terminal-size';
+import { SemVer } from 'semver';
 import { getRootDir, readConfig } from '../mops.js';
 import { parallel } from '../parallel.js';
 import { absToRel } from './test/utils.js';
@@ -16,8 +19,6 @@ import { getDfxVersion } from '../helpers/get-dfx-version.js';
 import { getMocPath } from '../helpers/get-moc-path.js';
 import { sources } from './sources.js';
 import { BenchReplica } from './bench-replica.js';
-import { filesize } from 'filesize';
-import { SemVer } from 'semver';
 let ignore = [
     '**/node_modules/**',
     '**/.mops/**',
@@ -55,6 +56,9 @@ export async function bench(filter = '', optionsArg = {}) {
         }
     }
     options.replica = replicaType;
+    if (process.env.CI) {
+        console.log('# Benchmark Results\n\n');
+    }
     if (replicaType == 'dfx') {
         options.replicaVersion = getDfxVersion();
     }
@@ -89,12 +93,16 @@ export async function bench(filter = '', optionsArg = {}) {
         for (let file of files) {
             console.log(chalk.gray(`• ${absToRel(file)}`));
         }
-        console.log('');
-        console.log('='.repeat(50));
-        console.log('');
+        if (!process.env.CI) {
+            console.log('');
+            console.log('='.repeat(50));
+            console.log('');
+        }
     }
     await replica.start({ silent: options.silent });
-    options.silent || console.log('Deploying canisters...');
+    if (!process.env.CI && !options.silent) {
+        console.log('Deploying canisters...');
+    }
     await parallel(os.cpus().length, files, async (file) => {
         try {
             await deployBenchFile(file, options, replica);
@@ -107,8 +115,8 @@ export async function bench(filter = '', optionsArg = {}) {
     });
     let benchResults = [];
     await parallel(1, files, async (file) => {
-        if (!options.silent) {
-            console.log('\n' + '—'.repeat(50));
+        if (!options.silent && !process.env.CI) {
+            console.log('\n' + '-'.repeat(50));
             console.log(`\nRunning ${chalk.gray(absToRel(file))}...`);
             console.log('');
         }
@@ -122,7 +130,9 @@ export async function bench(filter = '', optionsArg = {}) {
             throw err;
         }
     });
-    options.silent || console.log('Stopping replica...');
+    if (!process.env.CI && !options.silent) {
+        console.log('Stopping replica...');
+    }
     await replica.stop();
     fs.rmSync(benchDir, { recursive: true, force: true });
     return benchResults;
@@ -195,21 +205,33 @@ async function runBenchFile(file, options, replica) {
     };
     let getTable = (prop) => {
         let resArr = [['', ...schema.cols]];
+        let allZero = true;
         for (let [_rowIndex, row] of schema.rows.entries()) {
             let curRow = [row];
             for (let [_colIndex, col] of schema.cols.entries()) {
                 let res = results.get(`${row}:${col}`);
                 if (res) {
+                    if (res[prop] != 0n) {
+                        allZero = false;
+                    }
                     // compare with previous results
                     let diff = '';
                     if (options.compare && prevResults) {
                         let prevRes = prevResults.get(`${row}:${col}`);
                         if (prevRes) {
                             let percent = (Number(res[prop]) - Number(prevRes[prop])) / Number(prevRes[prop]) * 100;
+                            if (Object.is(percent, NaN)) {
+                                percent = 0;
+                            }
                             let sign = percent > 0 ? '+' : '';
                             let percentText = percent == 0 ? '0%' : sign + percent.toFixed(2) + '%';
                             let color = percent == 0 ? 'gray' : (percent > 0 ? 'red' : 'green');
-                            diff = ` (${chalk[color](percentText)})`;
+                            if (process.env.CI) {
+                                diff = ` $({\\color{${color}}${percentText.replace('%', '\\\\%')}})$`;
+                            }
+                            else {
+                                diff = ` (${chalk[color](percentText)})`;
+                            }
                         }
                         else {
                             diff = chalk.yellow(' (no previous results)');
@@ -234,23 +256,41 @@ async function runBenchFile(file, options, replica) {
             }
             resArr.push(curRow);
         }
+        // don't show Stable Memory table if all values are 0
+        if (allZero && prop == 'rts_logical_stable_memory_size') {
+            return '';
+        }
         return markdownTable(resArr, {
             align: ['l', ...'r'.repeat(schema.cols.length)],
             stringLength: stringWidth,
         });
     };
-    let printResults = () => {
-        logUpdate(`
-			\n${chalk.bold(schema.name)}
-			${schema.description ? '\n' + chalk.gray(schema.description) : ''}
+    let logUpdate = createLogUpdate(process.stdout, { showCursor: true });
+    let getOutput = () => {
+        return `
+			\n${process.env.CI ? `## ${schema.name}` : chalk.bold(schema.name)}
+			${schema.description ? '\n' + (process.env.CI ? `_${schema.description}_` : chalk.gray(schema.description)) : ''}
 			\n\n${chalk.blue('Instructions')}\n\n${getTable('instructions')}
 			\n\n${chalk.blue('Heap')}\n\n${getTable('rts_heap_size')}
-			\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}
 			\n\n${chalk.blue('Garbage Collection')}\n\n${getTable('rts_reclaimed')}
-		`);
+			${getTable('rts_logical_stable_memory_size') ? `\n\n${chalk.blue('Stable Memory')}\n\n${getTable('rts_logical_stable_memory_size')}` : ''}
+		`;
     };
-    if (!process.env.CI && !options.silent) {
-        printResults();
+    let canUpdateLog = !process.env.CI && !options.silent && terminalSize().rows > getOutput().split('\n').length;
+    let log = () => {
+        if (options.silent) {
+            return;
+        }
+        let output = getOutput();
+        if (process.env.CI || terminalSize().rows <= output.split('\n').length) {
+            console.log(output);
+        }
+        else {
+            logUpdate(output);
+        }
+    };
+    if (canUpdateLog) {
+        log();
     }
     // run all cells
     for (let [rowIndex, row] of schema.rows.entries()) {
@@ -265,15 +305,17 @@ async function runBenchFile(file, options, replica) {
             logicalStableMemoryCells[rowIndex][colIndex] = res.rts_logical_stable_memory_size;
             // @ts-ignore
             reclaimedCells[rowIndex][colIndex] = res.rts_reclaimed;
-            if (!process.env.CI && !options.silent) {
-                printResults();
+            if (canUpdateLog) {
+                log();
             }
         }
     }
-    if (process.env.CI) {
-        printResults();
+    if (canUpdateLog) {
+        logUpdate.done();
+    }
+    else {
+        log();
     }
-    logUpdate.done();
     // save results
     if (options.save) {
         console.log(`Saving results to ${chalk.gray(absToRel(resultsJsonFile))}`);

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "type": "module",
   "bin": {
     "mops": "bin/mops.js",
@@ -41,7 +41,7 @@
     "chalk": "5.4.1",
     "change-case": "5.4.4",
     "chokidar": "3.6.0",
-    "commander": "12.1.0",
+    "commander": "13.1.0",
     "debounce": "2.1.1",
     "decomp-tarxz": "0.1.1",
     "decompress": "4.2.1",
@@ -69,7 +69,8 @@
     "semver": "7.7.1",
     "stream-to-promise": "3.0.0",
     "string-width": "7.2.0",
-    "tar": "7.4.3"
+    "tar": "7.4.3",
+    "terminal-size": "4.0.0"
   },
   "devDependencies": {
     "@tsconfig/strictest": "2.0.5",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "type": "module",
   "bin": {
     "mops": "dist/bin/mops.js",
@@ -57,7 +57,7 @@
     "chalk": "5.4.1",
     "change-case": "5.4.4",
     "chokidar": "3.6.0",
-    "commander": "12.1.0",
+    "commander": "13.1.0",
     "debounce": "2.1.1",
     "decomp-tarxz": "0.1.1",
     "decompress": "4.2.1",
@@ -85,7 +85,8 @@
     "semver": "7.7.1",
     "stream-to-promise": "3.0.0",
     "string-width": "7.2.0",
-    "tar": "7.4.3"
+    "tar": "7.4.3",
+    "terminal-size": "4.0.0"
   },
   "devDependencies": {
     "@tsconfig/strictest": "2.0.5",