ic-mops 1.1.0-pre.0 → 1.1.1

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Mops CLI Changelog
 
+## 1.1.1
+- `moc-wrapper` now adds hostname to the moc path cache(`.mops/moc-*` filename) to avoid errors when running in Dev Containers
+- `mops watch` now deploys canisters with the `--yes` flag to skip data loss confirmation
+
+## 1.1.0
+- New `mops watch` command to check for syntax errors, show warnings, run tests, generate declarations and deploy canisters ([docs](https://docs.mops.one/cli/mops-watch))
+- New flag `--no-toolchain` in `mops install` command to skip toolchain installation
+- New lock file format v3 ([docs](https://docs.mops.one/mops.lock))
+- Faster `mops install` from lock file when lock file is up-to-date and there are no cached packages
+- Fixed replica test hanging in watch mode bug
+- Fixed mops failing when dfx is not installed
+- Fixed `mops test` Github Action template
+
 ## 1.0.1
 - Fixed `mops user *` commands
 

package/bin/moc-wrapper.sh

@@ -22,7 +22,7 @@ else
     mopsTomlHash=$(shasum $mopsToml -a 256 | awk -F' ' '{print $1}')
   fi;
 
-  cached="$rootDir/.mops/moc-$mopsTomlHash"
+  cached="$rootDir/.mops/moc-$(uname -n)-$mopsTomlHash"
 
   if [ -f $cached ]; then
     mocPath=$(cat $cached)

package/check-requirements.ts

@@ -11,7 +11,7 @@ export async function checkRequirements({verbose = false} = {}) {
 	let config = readConfig();
 	let mocVersion = config.toolchain?.moc;
 	if (!mocVersion) {
-		mocVersion = getMocVersion();
+		mocVersion = getMocVersion(false);
 	}
 	if (!mocVersion) {
 		return;

package/cli.ts

@@ -1,5 +1,6 @@
 import process from 'node:process';
 import fs from 'node:fs';
+import events from 'node:events';
 import {Command, Argument, Option} from 'commander';
 
 import {init} from './commands/init.js';
@@ -25,6 +26,7 @@ import {toolchain} from './commands/toolchain/index.js';
 import {Tool} from './types.js';
 import * as self from './commands/self.js';
 import {resolvePackages} from './resolve-packages.js';
+import {watch} from './commands/watch/watch.js';
 
 declare global {
 	// eslint-disable-next-line no-var
@@ -33,6 +35,8 @@ declare global {
 	var mopsReplicaTestRunning : boolean;
 }
 
+events.setMaxListeners(20);
+
 let networkFile = getNetworkFile();
 if (fs.existsSync(networkFile)) {
 	globalThis.MOPS_NETWORK = fs.readFileSync(networkFile).toString() || 'ic';
@@ -398,4 +402,18 @@ selfCommand
 
 program.addCommand(selfCommand);
 
+// watch
+program
+	.command('watch')
+	.description('Watch *.mo files and check for syntax errors, warnings, run tests, generate declarations and deploy canisters')
+	.option('-e, --error', 'Check Motoko canisters or *.mo files for syntax errors')
+	.option('-w, --warning', 'Check Motoko canisters or *.mo files for warnings')
+	.option('-t, --test', 'Run tests')
+	.option('-g, --generate', 'Generate declarations for Motoko canisters')
+	.option('-d, --deploy', 'Deploy Motoko canisters')
+	.action(async (options) => {
+		checkConfigFile(true);
+		await watch(options);
+	});
+
 program.parse();

package/commands/bench.ts

@@ -55,7 +55,7 @@ export async function bench(filter = '', optionsArg : Partial<BenchOptions> = {}
 		replica: config.toolchain?.['pocket-ic'] ? 'pocket-ic' : 'dfx',
 		replicaVersion: '',
 		compiler: 'moc',
-		compilerVersion: getMocVersion(),
+		compilerVersion: getMocVersion(true),
 		gc: 'copying',
 		forceGc: true,
 		save: false,

package/commands/install/install-all.ts

@@ -34,10 +34,10 @@ export async function installAll({verbose = false, silent = false, threads, lock
 		if (lockFileJson && lockFileJson.version === 3) {
 			verbose && console.log('Installing from lock file...');
 			installedFromLockFile = true;
-			let deps = Object.entries(lockFileJson.deps).map(([name, version]) => {
+			let lockedDeps = Object.entries(lockFileJson.deps).map(([name, version]) => {
 				return parseDepValue(name, version);
 			});
-			let ok = await installDeps(deps, {silent, verbose, threads, ignoreTransitive: true});
+			let ok = await installDeps(lockedDeps, {silent, verbose, threads, ignoreTransitive: true});
 			if (!ok) {
 				return false;
 			}

package/commands/replica.ts

@@ -3,6 +3,7 @@ import {ChildProcessWithoutNullStreams, execSync, spawn} from 'node:child_proces
 import path from 'node:path';
 import fs from 'node:fs';
 import {PassThrough} from 'node:stream';
+import {spawn as spawnAsync} from 'promisify-child-process';
 
 import {IDL} from '@dfinity/candid';
 import {Actor, HttpAgent} from '@dfinity/agent';
@@ -126,7 +127,7 @@ export class Replica {
 		}
 	}
 
-	async deploy(name : string, wasm : string, idlFactory : IDL.InterfaceFactory, cwd : string = process.cwd()) {
+	async deploy(name : string, wasm : string, idlFactory : IDL.InterfaceFactory, cwd : string = process.cwd(), signal ?: AbortSignal) {
 		if (this.type === 'dfx') {
 			// prepare dfx.json for current canister
 			let dfxJson = path.join(this.dir, 'dfx.json');
@@ -144,8 +145,27 @@ export class Replica {
 			fs.mkdirSync(this.dir, {recursive: true});
 			fs.writeFileSync(dfxJson, JSON.stringify(newDfxJsonData, null, 2));
 
-			execSync(`dfx deploy ${name} --mode reinstall --yes --identity anonymous`, {cwd: this.dir, stdio: this.verbose ? 'pipe' : ['pipe', 'ignore', 'pipe']});
-			execSync(`dfx ledger fabricate-cycles --canister ${name} --t 100`, {cwd: this.dir, stdio: this.verbose ? 'pipe' : ['pipe', 'ignore', 'pipe']});
+			await spawnAsync('dfx', ['deploy', name, '--mode', 'reinstall', '--yes', '--identity', 'anonymous'], {cwd: this.dir, signal, stdio: this.verbose ? 'pipe' : ['pipe', 'ignore', 'pipe']}).catch((error) => {
+				if (error.code === 'ABORT_ERR') {
+					return {stderr: ''};
+				}
+				throw error;
+			});
+
+			if (signal?.aborted) {
+				return;
+			}
+
+			await spawnAsync('dfx', ['ledger', 'fabricate-cycles', '--canister', name, '--t', '100'], {cwd: this.dir, signal, stdio: this.verbose ? 'pipe' : ['pipe', 'ignore', 'pipe']}).catch((error) => {
+				if (error.code === 'ABORT_ERR') {
+					return {stderr: ''};
+				}
+				throw error;
+			});
+
+			if (signal?.aborted) {
+				return;
+			}
 
 			let canisterId = execSync(`dfx canister id ${name}`, {cwd: this.dir}).toString().trim();
 
@@ -170,7 +190,17 @@ export class Replica {
 				idlFactory,
 				wasm,
 			});
+
+			if (signal?.aborted) {
+				return;
+			}
+
 			await this.pocketIc.addCycles(canisterId, 1_000_000_000_000);
+
+			if (signal?.aborted) {
+				return;
+			}
+
 			this.canisters[name] = {
 				cwd,
 				canisterId: canisterId.toText(),

package/commands/sources.ts

@@ -45,5 +45,5 @@ export async function sources({conflicts = 'ignore' as 'warning' | 'error' | 'ig
 		}
 
 		return `--package ${name} ${pkgBaseDir}`;
-	});
+	}).filter(x => x != null);
 }

package/commands/sync.ts

@@ -1,4 +1,3 @@
-import process from 'node:process';
 import path from 'node:path';
 import {execSync} from 'node:child_process';
 import {globSync} from 'glob';
@@ -7,6 +6,7 @@ import {checkConfigFile, getRootDir, readConfig} from '../mops.js';
 import {add} from './add.js';
 import {remove} from './remove.js';
 import {checkIntegrity} from '../integrity.js';
+import {getMocPath} from '../helpers/get-moc-path.js';
 
 type SyncOptions = {
 	lock ?: 'update' | 'ignore';
@@ -48,25 +48,10 @@ let ignore = [
 	'**/.mops/**',
 ];
 
-let mocPath = '';
-function getMocPath() : string {
-	if (!mocPath) {
-		mocPath = process.env.DFX_MOC_PATH || '';
-	}
-	if (!mocPath) {
-		try {
-			mocPath = execSync('dfx cache show').toString().trim() + '/moc';
-		}
-		catch {}
-	}
-	if (!mocPath) {
-		mocPath = 'moc';
-	}
-	return mocPath;
-}
-
 async function getUsedPackages() : Promise<string[]> {
 	let rootDir = getRootDir();
+	let mocPath = getMocPath();
+
 	let files = globSync('**/*.mo', {
 		cwd: rootDir,
 		nocase: true,
@@ -76,7 +61,7 @@ async function getUsedPackages() : Promise<string[]> {
 	let packages : Set<string> = new Set;
 
 	for (let file of files) {
-		let deps : string[] = execSync(`${getMocPath()} --print-deps ${path.join(rootDir, file)}`).toString().trim().split('\n');
+		let deps : string[] = execSync(`${mocPath} --print-deps ${path.join(rootDir, file)}`).toString().trim().split('\n');
 
 		for (let dep of deps) {
 			if (dep.startsWith('mo:') && !dep.startsWith('mo:prim') && !dep.startsWith('mo:⛔')) {

package/commands/test/mmf1.ts

@@ -53,6 +53,10 @@ export class MMF1 {
 		this.output = [];
 	}
 
+	getErrorMessages() {
+		return this.output.filter(out => out.type === 'fail').map(out => out.message);
+	}
+
 	parseLine(line : string) {
 		if (line.startsWith('mops:1:start ')) {
 			this._testStart(line.split('mops:1:start ')[1] || '');

package/commands/test/reporters/silent-reporter.ts

@@ -5,14 +5,25 @@ import {Reporter} from './reporter.js';
 import {TestMode} from '../../../types.js';
 
 export class SilentReporter implements Reporter {
+	total = 0;
 	passed = 0;
 	failed = 0;
 	skipped = 0;
 	passedFiles = 0;
 	failedFiles = 0;
 	passedNamesFlat : string[] = [];
+	flushOnError = true;
+	errorOutput = '';
+	onProgress = () => {};
 
-	addFiles(_files : string[]) {}
+	constructor(flushOnError = true, onProgress = () => {}) {
+		this.flushOnError = flushOnError;
+		this.onProgress = onProgress;
+	}
+
+	addFiles(files : string[]) {
+		this.total = files.length;
+	}
 
 	addRun(file : string, mmf : MMF1, state : Promise<void>, _mode : TestMode) {
 		state.then(() => {
@@ -30,10 +41,17 @@ export class SilentReporter implements Reporter {
 			this.failedFiles += Number(mmf.failed !== 0);
 
 			if (mmf.failed) {
-				console.log(chalk.red('✖'), absToRel(file));
-				mmf.flush('fail');
-				console.log('-'.repeat(50));
+				let output = `${chalk.red('✖')} ${absToRel(file)}\n${mmf.getErrorMessages().join('\n')}\n${'-'.repeat(50)}`;
+
+				if (this.flushOnError) {
+					console.log(output);
+				}
+				else {
+					this.errorOutput = `${this.errorOutput}\n${output}`.trim();
+				}
 			}
+
+			this.onProgress();
 		});
 	}
 

package/commands/test/test.ts

@@ -85,14 +85,28 @@ export async function test(filter = '', options : Partial<TestOptions> = {}) {
 					process.exit(0);
 				});
 			}
+			else {
+				process.exit(0);
+			}
 		});
 
 		// todo: run only changed for *.test.mo?
 		// todo: run all for *.mo?
+
+		let curRun = Promise.resolve(true);
+		let controller = new AbortController();
+
 		let run = debounce(async () => {
+			controller.abort();
+			await curRun;
+
 			console.clear();
 			process.stdout.write('\x1Bc');
-			await runAll(options.reporter, filter, options.mode, replicaType, true);
+
+			controller = new AbortController();
+			curRun = runAll(options.reporter, filter, options.mode, replicaType, true, controller.signal);
+			await curRun;
+
 			console.log('-'.repeat(50));
 			console.log('Waiting for file changes...');
 			console.log(chalk.gray((`Press ${chalk.gray('Ctrl+C')} to exit.`)));
@@ -122,12 +136,12 @@ export async function test(filter = '', options : Partial<TestOptions> = {}) {
 let mocPath = '';
 let wasmtimePath = '';
 
-async function runAll(reporterName : ReporterName | undefined, filter = '', mode : TestMode = 'interpreter', replicaType : ReplicaName, watch = false) : Promise<boolean> {
-	let done = await testWithReporter(reporterName, filter, mode, replicaType, watch);
+async function runAll(reporterName : ReporterName | undefined, filter = '', mode : TestMode = 'interpreter', replicaType : ReplicaName, watch = false, signal ?: AbortSignal) : Promise<boolean> {
+	let done = await testWithReporter(reporterName, filter, mode, replicaType, watch, signal);
 	return done;
 }
 
-export async function testWithReporter(reporterName : ReporterName | Reporter | undefined, filter = '', defaultMode : TestMode = 'interpreter', replicaType : ReplicaName, watch = false) : Promise<boolean> {
+export async function testWithReporter(reporterName : ReporterName | Reporter | undefined, filter = '', defaultMode : TestMode = 'interpreter', replicaType : ReplicaName, watch = false, signal ?: AbortSignal) : Promise<boolean> {
 	let rootDir = getRootDir();
 	let files : string[] = [];
 	let libFiles = globSync('**/test?(s)/lib.mo', globConfig);
@@ -191,6 +205,10 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 	fs.mkdirSync(testTempDir, {recursive: true});
 
 	await parallel(os.cpus().length, files, async (file : string) => {
+		if (signal?.aborted) {
+			return;
+		}
+
 		let mmf = new MMF1('store', absToRel(file));
 
 		// mode overrides
@@ -221,7 +239,13 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 
 			// interpret
 			if (mode === 'interpreter') {
-				let proc = spawn(mocPath, ['-r', '-ref-system-api', ...mocArgs]);
+				let proc = spawn(mocPath, ['-r', '-ref-system-api', ...mocArgs], {signal});
+				proc.addListener('error', (error : any) => {
+					if (error?.code === 'ABORT_ERR') {
+						return;
+					}
+					throw error;
+				});
 				pipeMMF(proc, mmf).then(resolve);
 			}
 			// build and run wasm
@@ -229,7 +253,13 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 				let wasmFile = `${path.join(testTempDir, path.parse(file).name)}.wasm`;
 
 				// build
-				let buildProc = spawn(mocPath, [`-o=${wasmFile}`, '-wasi-system-api', ...mocArgs]);
+				let buildProc = spawn(mocPath, [`-o=${wasmFile}`, '-wasi-system-api', ...mocArgs], {signal});
+				buildProc.addListener('error', (error : any) => {
+					if (error?.code === 'ABORT_ERR') {
+						return;
+					}
+					throw error;
+				});
 				pipeMMF(buildProc, mmf).then(async () => {
 					if (mmf.failed > 0) {
 						return;
@@ -252,7 +282,14 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 						process.exit(1);
 					}
 
-					let proc = spawn(wasmtimePath, wasmtimeArgs);
+					let proc = spawn(wasmtimePath, wasmtimeArgs, {signal});
+					proc.addListener('error', (error : any) => {
+						if (error?.code === 'ABORT_ERR') {
+							return;
+						}
+						throw error;
+					});
+
 					await pipeMMF(proc, mmf);
 				}).finally(() => {
 					fs.rmSync(wasmFile, {force: true});
@@ -265,7 +302,13 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 				let wasmFile = `${path.join(testTempDir, path.parse(file).name)}.wasm`;
 
 				// build
-				let buildProc = spawn(mocPath, [`-o=${wasmFile}`, ...mocArgs]);
+				let buildProc = spawn(mocPath, [`-o=${wasmFile}`, ...mocArgs], {signal});
+				buildProc.addListener('error', (error : any) => {
+					if (error?.code === 'ABORT_ERR') {
+						return;
+					}
+					throw error;
+				});
 
 				pipeMMF(buildProc, mmf).then(async () => {
 					if (mmf.failed > 0) {
@@ -274,15 +317,23 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 
 					await startReplicaOnce(replica, replicaType);
 
+					if (signal?.aborted) {
+						return;
+					}
+
 					let canisterName = path.parse(file).name;
 					let idlFactory = ({IDL} : any) => {
 						return IDL.Service({'runTests': IDL.Func([], [], [])});
 					};
 					interface _SERVICE {'runTests' : ActorMethod<[], undefined>;}
 
-					let {stream} = await replica.deploy(canisterName, wasmFile, idlFactory);
+					let canister = await replica.deploy(canisterName, wasmFile, idlFactory, undefined, signal);
 
-					pipeStdoutToMMF(stream, mmf);
+					if (signal?.aborted || !canister) {
+						return;
+					}
+
+					pipeStdoutToMMF(canister.stream, mmf);
 
 					let actor = await replica.getActor(canisterName) as _SERVICE;
 
@@ -298,6 +349,10 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 							});
 						}
 
+						if (signal?.aborted) {
+							return;
+						}
+
 						globalThis.mopsReplicaTestRunning = true;
 						await actor.runTests();
 						globalThis.mopsReplicaTestRunning = false;
@@ -310,11 +365,16 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 						stderrStream.write(e.message);
 					}
 				}).finally(async () => {
+					globalThis.mopsReplicaTestRunning = false;
 					fs.rmSync(wasmFile, {force: true});
 				}).then(resolve);
 			}
 		});
 
+		if (signal?.aborted) {
+			return;
+		}
+
 		reporter.addRun(file, mmf, promise, mode);
 
 		await promise;
@@ -325,6 +385,10 @@ export async function testWithReporter(reporterName : ReporterName | Reporter |
 		fs.rmSync(testTempDir, {recursive: true, force: true});
 	}
 
+	if (signal?.aborted) {
+		return false;
+	}
+
 	return reporter.done();
 }
 

package/commands/watch/deployer.ts

@@ -0,0 +1,155 @@
+import chalk from 'chalk';
+import os from 'node:os';
+import {promisify} from 'node:util';
+import {execFile, execSync} from 'node:child_process';
+
+import {ErrorChecker} from './error-checker.js';
+import {Generator} from './generator.js';
+import {parallel} from '../../parallel.js';
+import {getRootDir} from '../../mops.js';
+
+export class Deployer {
+	verbose = false;
+	canisters : Record<string, string> = {};
+	status : 'pending' | 'running' | 'syntax-error' | 'dfx-error' | 'error' | 'success' = 'pending';
+	errorChecker : ErrorChecker;
+	generator : Generator;
+	success = 0;
+	errors : string[] = [];
+	aborted = false;
+	controllers = new Map<string, AbortController>();
+	currentRun : Promise<any> | undefined;
+
+	constructor({verbose, canisters, errorChecker, generator} : {verbose : boolean, canisters : Record<string, string>, errorChecker : ErrorChecker, generator : Generator}) {
+		this.verbose = verbose;
+		this.canisters = canisters;
+		this.errorChecker = errorChecker;
+		this.generator = generator;
+	}
+
+	reset() {
+		this.status = 'pending';
+		this.success = 0;
+		this.errors = [];
+	}
+
+	async abortCurrent() {
+		this.aborted = true;
+		for (let controller of this.controllers.values()) {
+			controller.abort();
+		}
+		this.controllers.clear();
+		await this.currentRun;
+		this.reset();
+		this.aborted = false;
+	}
+
+	async run(onProgress : () => void) {
+		await this.abortCurrent();
+
+		if (this.errorChecker.status === 'error') {
+			this.status = 'syntax-error';
+			onProgress();
+			return;
+		}
+
+		if (Object.keys(this.canisters).length === 0) {
+			this.status = 'success';
+			onProgress();
+			return;
+		}
+
+		let rootDir = getRootDir();
+
+		try {
+			execSync('dfx ping', {cwd: rootDir});
+		}
+		catch (error) {
+			this.status = 'dfx-error';
+			onProgress();
+			return;
+		}
+
+		this.status = 'running';
+		onProgress();
+
+		// create canisters (sequentially to avoid DFX errors)
+		let resolve : (() => void) | undefined;
+		this.currentRun = new Promise<void>((res) => resolve = res);
+		for (let canister of Object.keys(this.canisters)) {
+			let controller = new AbortController();
+			let {signal} = controller;
+			this.controllers.set(canister, controller);
+
+			await promisify(execFile)('dfx', ['canister', 'create', canister], {cwd: rootDir, signal}).catch((error) => {
+				if (error.code === 'ABORT_ERR') {
+					return {stderr: ''};
+				}
+				throw error;
+			});
+
+			this.controllers.delete(canister);
+		}
+
+		resolve?.();
+
+		if (this.aborted) {
+			return;
+		}
+
+		this.currentRun = parallel(os.cpus().length, [...Object.keys(this.canisters)], async (canister) => {
+			let controller = new AbortController();
+			let {signal} = controller;
+			this.controllers.set(canister, controller);
+
+			// build
+			if (this.generator.status !== 'success' || !this.generator.canisters[canister]) {
+				await promisify(execFile)('dfx', ['build', canister], {cwd: rootDir, signal}).catch((error) => {
+					if (error.code === 'ABORT_ERR') {
+						return {stderr: ''};
+					}
+					throw error;
+				});
+			}
+
+			// install
+			await promisify(execFile)('dfx', ['canister', 'install', '--mode=auto', '--yes', canister], {cwd: rootDir, signal}).catch((error) => {
+				if (error.code === 'ABORT_ERR') {
+					return {stderr: ''};
+				}
+				throw error;
+			});
+
+			this.success += 1;
+			this.controllers.delete(canister);
+			onProgress();
+		});
+
+		await this.currentRun;
+
+		if (!this.aborted) {
+			this.status = 'success';
+		}
+		onProgress();
+	}
+
+	getOutput() : string {
+		let get = (v : number) => v.toString();
+		let count = (this.status === 'running' ? get : chalk.bold[this.errors.length > 0 ? 'redBright' : 'green'])(this.errors.length || this.success);
+
+		if (this.status === 'pending') {
+			return `Deploy: ${chalk.gray('(pending)')}`;
+		}
+		if (this.status === 'running') {
+			return `Deploy: ${count}/${Object.keys(this.canisters).length} ${chalk.gray('(running)')}`;
+		}
+		if (this.status === 'syntax-error') {
+			return `Deploy: ${chalk.gray('(errors)')}`;
+		}
+		if (this.status === 'dfx-error') {
+			return `Deploy: ${chalk.gray('(dfx not running)')}`;
+		}
+
+		return `Deploy: ${count}`;
+	}
+}