ic-mops 0.41.1 → 0.42.1

package/bundle/declarations/bench/index.js

@@ -4,14 +4,6 @@ import { Actor, HttpAgent } from "@dfinity/agent";
 import { idlFactory } from "./bench.did.js";
 export { idlFactory } from "./bench.did.js";
 
-/* CANISTER_ID is replaced by webpack based on node environment
- * Note: canister environment variable will be standardized as
- * process.env.CANISTER_ID_<CANISTER_NAME_UPPERCASE>
- * beginning in dfx 0.15.0
- */
-export const canisterId =
-  process.env.CANISTER_ID_BENCH;
-
 export const createActor = (canisterId, options = {}) => {
   const agent = options.agent || new HttpAgent({ ...options.agentOptions });
 

package/bundle/declarations/storage/storage.did.d.ts

@@ -1,5 +1,6 @@
 import type { Principal } from '@dfinity/principal';
 import type { ActorMethod } from '@dfinity/agent';
+import type { IDL } from '@dfinity/candid';
 
 export type Chunk = Array<number>;
 export type Err = string;
@@ -35,3 +36,5 @@ export interface StorageStats {
   'memorySize' : bigint,
 }
 export interface _SERVICE extends Storage {}
+export declare const idlFactory: IDL.InterfaceFactory;
+export declare const init: (args: { IDL: typeof IDL }) => IDL.Type[];

package/bundle/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "0.41.0",
+  "version": "0.42.0",
   "type": "module",
   "bin": {
     "mops": "bin/mops.js",
@@ -24,7 +24,7 @@
     "node": ">=18.0.0"
   },
   "dependencies": {
-    "dhall-to-json-cli": "^1.7.6",
+    "dhall-to-json-cli": "1.7.6",
     "decomp-tarxz": "0.1.1",
     "buffer": "6.0.3"
   }

package/cache.ts

@@ -3,15 +3,33 @@ import path from 'node:path';
 import ncp from 'ncp';
 import getFolderSize from 'get-folder-size';
 
-import {globalCacheDir} from './mops.js';
+import {getDependencyType, globalCacheDir, parseGithubURL} from './mops.js';
 
-export let isCached = (pkgId : string) => {
-	let dir = path.join(globalCacheDir, 'packages', pkgId);
+export let getDepCacheDir = (cacheName : string) => {
+	return path.join(globalCacheDir, 'packages', cacheName);
+};
+
+export let isDepCached = (cacheName : string) => {
+	let dir = getDepCacheDir(cacheName);
 	return fs.existsSync(dir);
 };
 
-export let addCache = (pkgId : string, source : string) => {
-	let dest = path.join(globalCacheDir, 'packages', pkgId);
+export function getDepCacheName(name : string, version : string) {
+	let depType = getDependencyType(version);
+	return depType === 'mops' ? getMopsDepCacheName(name, version) : getGithubDepCacheName(name, version);
+}
+
+export function getMopsDepCacheName(name : string, version : string) {
+	return `${name}@${version}`;
+}
+
+export function getGithubDepCacheName(name : string, repo : string) {
+	const {branch, commitHash} = parseGithubURL(repo);
+	return `_github/${name}#${branch}` + (commitHash ? `@${commitHash}` : '');
+}
+
+export let addCache = (cacheName : string, source : string) => {
+	let dest = path.join(globalCacheDir, 'packages', cacheName);
 	fs.mkdirSync(dest, {recursive: true});
 
 	return new Promise<void>((resolve, reject) => {
@@ -24,8 +42,8 @@ export let addCache = (pkgId : string, source : string) => {
 	});
 };
 
-export let copyCache = (pkgId : string, dest : string) => {
-	let source = path.join(globalCacheDir, 'packages', pkgId);
+export let copyCache = (cacheName : string, dest : string) => {
+	let source = path.join(globalCacheDir, 'packages', cacheName);
 	fs.mkdirSync(dest, {recursive: true});
 
 	return new Promise<void>((resolve, reject) => {

package/check-requirements.ts

@@ -2,7 +2,7 @@ import path from 'node:path';
 import {SemVer} from 'semver';
 import chalk from 'chalk';
 
-import {getDependencyType, readConfig} from './mops.js';
+import {getDependencyType, getRootDir, readConfig} from './mops.js';
 import {resolvePackages} from './resolve-packages.js';
 import {getMocVersion} from './helpers/get-moc-version.js';
 
@@ -18,12 +18,13 @@ export async function checkRequirements({verbose = false} = {}) {
 	let installedMoc = new SemVer(mocVersion);
 	let highestRequiredMoc = new SemVer('0.0.0');
 	let highestRequiredMocPkgId = '';
+	let rootDir = getRootDir();
 
 	let resolvedPackages = await resolvePackages();
 	for (let [name, version] of Object.entries(resolvedPackages)) {
 		if (getDependencyType(version) === 'mops') {
 			let pkgId = `${name}@${version}`;
-			let depConfig = readConfig(path.join('.mops', pkgId, 'mops.toml'));
+			let depConfig = readConfig(path.join(rootDir, '.mops', pkgId, 'mops.toml'));
 			let moc = depConfig.requirements?.moc;
 
 			if (moc) {

package/cli.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import fs from 'node:fs';
 import {Command, Argument, Option} from 'commander';
 import chalk from 'chalk';
@@ -9,7 +10,7 @@ import {sources} from './commands/sources.js';
 import {checkApiCompatibility, setNetwork, apiVersion, checkConfigFile, getNetworkFile, version} from './mops.js';
 import {getNetwork} from './api/network.js';
 import {whoami} from './commands/whoami.js';
-import {installAll} from './commands/install-all.js';
+import {installAll} from './commands/install/install-all.js';
 import {search} from './commands/search.js';
 import {add} from './commands/add.js';
 import {cacheSize, cleanCache} from './cache.js';
@@ -109,8 +110,11 @@ program
 			await add(pkg, options);
 		}
 		else {
-			await installAll(options);
+			let ok = await installAll(options);
 			await toolchain.installAll(options);
+			if (!ok) {
+				process.exit(1);
+			}
 		}
 	});
 
@@ -169,7 +173,7 @@ program
 		if (!checkConfigFile()) {
 			process.exit(1);
 		}
-		await installAll({silent: true, lock: 'ignore'});
+		await installAll({silent: true, lock: 'ignore', threads: 6});
 		await toolchain.ensureToolchainInited({strict: false});
 		let sourcesArr = await sources(options);
 		console.log(sourcesArr.join('\n'));

package/commands/add.ts

@@ -1,13 +1,15 @@
+import process from 'node:process';
 import path from 'node:path';
 import chalk from 'chalk';
 import {createLogUpdate} from 'log-update';
 import {checkConfigFile, getGithubCommit, parseGithubURL, readConfig, writeConfig} from '../mops.js';
 import {getHighestVersion} from '../api/getHighestVersion.js';
+import {installMopsDep} from './install/install-mops-dep.js';
 import {installFromGithub} from '../vessel.js';
-import {install} from './install.js';
-import {notifyInstalls} from '../notify-installs.js';
 import {checkIntegrity} from '../integrity.js';
 import {checkRequirements} from '../check-requirements.js';
+import {syncLocalCache} from './install/sync-local-cache.js';
+import {notifyInstalls} from '../notify-installs.js';
 
 type AddOptions = {
 	verbose ?: boolean;
@@ -86,17 +88,14 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
 		};
 	}
 
-	let installedPackages = {};
-
 	if (pkgDetails.repo) {
 		await installFromGithub(pkgDetails.name, pkgDetails.repo, {verbose: verbose});
 	}
 	else if (!pkgDetails.path) {
-		let res = await install(pkgDetails.name, pkgDetails.version, {verbose: verbose});
+		let res = await installMopsDep(pkgDetails.name, pkgDetails.version, {verbose: verbose});
 		if (res === false) {
 			return;
 		}
-		installedPackages = {...installedPackages, ...res};
 	}
 
 	const depsProp = dev ? 'dev-dependencies' : 'dependencies';
@@ -115,8 +114,11 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
 	if (lock !== 'ignore') {
 		logUpdate('Checking integrity...');
 	}
+
+	let installedPackages = await syncLocalCache();
+
 	await Promise.all([
-		notifyInstalls(Object.keys(installedPackages)),
+		notifyInstalls(installedPackages),
 		checkIntegrity(lock),
 	]);
 

package/commands/available-updates.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import chalk from 'chalk';
 import {mainActor} from '../api/actors.js';
 import {Config} from '../types.js';

package/commands/bench-replica.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import {execSync} from 'node:child_process';
 import path from 'node:path';
 import fs from 'node:fs';

package/commands/bench.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import path from 'node:path';
 import fs from 'node:fs';
 import os from 'node:os';

package/commands/bump.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import prompts from 'prompts';
 import chalk from 'chalk';
 import {checkConfigFile, readConfig, writeConfig} from '../mops.js';

package/commands/docs.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import {spawn} from 'node:child_process';
 import fs from 'node:fs';
 import path from 'node:path';

package/commands/import-identity.ts

@@ -1,5 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
+import {Buffer} from 'node:buffer';
 import chalk from 'chalk';
 import prompts from 'prompts';
 import {deleteSync} from 'del';

package/commands/init.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import {execSync} from 'node:child_process';
 import path from 'node:path';
 import {existsSync, readFileSync, writeFileSync} from 'node:fs';
@@ -6,7 +7,7 @@ import prompts from 'prompts';
 
 import {checkApiCompatibility, writeConfig} from '../mops.js';
 import {mainActor} from '../api/actors.js';
-import {installAll} from './install-all.js';
+import {installAll} from './install/install-all.js';
 import {VesselConfig, readVesselConfig} from '../vessel.js';
 import {Config, Dependencies} from '../types.js';
 import {template} from './template.js';

package/commands/install/install-all.ts

@@ -0,0 +1,53 @@
+import process from 'node:process';
+import chalk from 'chalk';
+import {createLogUpdate} from 'log-update';
+import {checkConfigFile, readConfig} from '../../mops.js';
+import {checkIntegrity} from '../../integrity.js';
+import {installDeps} from './install-deps.js';
+import {checkRequirements} from '../../check-requirements.js';
+import {syncLocalCache} from './sync-local-cache.js';
+import {notifyInstalls} from '../../notify-installs.js';
+
+type InstallAllOptions = {
+	verbose ?: boolean;
+	silent ?: boolean;
+	lock ?: 'check' | 'update' | 'ignore';
+	threads ?: number;
+}
+
+export async function installAll({verbose = false, silent = false, threads, lock} : InstallAllOptions = {}) : Promise<boolean> {
+	if (!checkConfigFile()) {
+		return false;
+	}
+
+	let config = readConfig();
+	let deps = Object.values(config.dependencies || {});
+	let devDeps = Object.values(config['dev-dependencies'] || {});
+	let allDeps = [...deps, ...devDeps];
+
+	let ok = await installDeps(allDeps, {silent, verbose, threads});
+	if (!ok) {
+		return false;
+	}
+
+	let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
+
+	if (!silent && lock !== 'ignore') {
+		logUpdate('Checking integrity...');
+	}
+
+	let installedPackages = await syncLocalCache();
+
+	await Promise.all([
+		notifyInstalls(installedPackages),
+		checkIntegrity(lock),
+	]);
+
+	if (!silent) {
+		logUpdate.clear();
+		await checkRequirements();
+		console.log(chalk.green('Packages installed'));
+	}
+
+	return true;
+}

package/commands/install/install-dep.ts

@@ -0,0 +1,34 @@
+import path from 'node:path';
+import {installFromGithub} from '../../vessel.js';
+import {installMopsDep} from './install-mops-dep.js';
+import {Dependency} from '../../types.js';
+import {installLocalDep} from './install-local-dep.js';
+import {getRootDir} from '../../mops.js';
+
+type InstallDepOptions = {
+	verbose ?: boolean;
+	silent ?: boolean;
+	threads ?: number;
+};
+
+// install dependency
+// returns false if failed
+export async function installDep(dep : Dependency, {verbose, silent, threads} : InstallDepOptions = {}, parentPkgPath ?: string) : Promise<boolean> {
+	if (dep.repo) {
+		await installFromGithub(dep.name, dep.repo, {silent, verbose});
+		return true;
+	}
+	else if (dep.path) {
+		let depPath = dep.path;
+		parentPkgPath = parentPkgPath || getRootDir();
+		if (parentPkgPath) {
+			depPath = path.resolve(parentPkgPath, dep.path);
+		}
+		return installLocalDep(dep.name, depPath, {silent, verbose});
+	}
+	else if (dep.version) {
+		return installMopsDep(dep.name, dep.version, {silent, verbose, threads});
+	}
+
+	return true;
+}

package/commands/install/install-deps.ts

@@ -0,0 +1,24 @@
+import {Dependency} from '../../types.js';
+import {installDep} from './install-dep.js';
+
+type InstallDepsOptions = {
+	verbose ?: boolean;
+	silent ?: boolean;
+	threads ?: number;
+};
+
+// install all dependencies
+// returns actual installed dependencies
+// returns false if failed
+export async function installDeps(deps : Dependency[], {verbose, silent, threads} : InstallDepsOptions = {}, parentPkgPath ?: string) : Promise<boolean> {
+	let ok = true;
+
+	for (const dep of deps) {
+		let res = await installDep(dep, {verbose, silent, threads}, parentPkgPath);
+		if (!res) {
+			ok = false;
+		}
+	}
+
+	return ok;
+}

package/commands/install/install-local-dep.ts

@@ -0,0 +1,31 @@
+import process from 'node:process';
+import path from 'node:path';
+import {createLogUpdate} from 'log-update';
+import {getRootDir, readConfig} from '../../mops.js';
+import {installDeps} from './install-deps.js';
+
+type InstallLocalDepOptions = {
+	verbose ?: boolean;
+	silent ?: boolean;
+};
+
+// skip install and just find non-local dependencies to install
+// pkgPath should be relative to the current root dir or absolute
+export async function installLocalDep(pkg : string, pkgPath = '', {verbose, silent} : InstallLocalDepOptions = {}) : Promise<boolean> {
+	if (!silent) {
+		let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
+		logUpdate(`Local dependency ${pkg} = "${pkgPath}"`);
+
+		if (verbose) {
+			silent || logUpdate.done();
+		}
+		else {
+			logUpdate.clear();
+		}
+	}
+
+	// install dependencies
+	let dir = path.resolve(getRootDir(), pkgPath);
+	let config = readConfig(path.join(dir, 'mops.toml'));
+	return installDeps(Object.values(config.dependencies || {}), {silent, verbose}, pkgPath);
+}

package/commands/install/install-mops-dep.ts

@@ -0,0 +1,122 @@
+import process from 'node:process';
+import fs from 'node:fs';
+import path from 'node:path';
+import {Buffer} from 'node:buffer';
+import {createLogUpdate} from 'log-update';
+import chalk from 'chalk';
+import {deleteSync} from 'del';
+import {checkConfigFile, progressBar, readConfig} from '../../mops.js';
+import {getHighestVersion} from '../../api/getHighestVersion.js';
+import {storageActor} from '../../api/actors.js';
+import {parallel} from '../../parallel.js';
+import {getDepCacheDir, getMopsDepCacheName, isDepCached} from '../../cache.js';
+import {downloadFile, getPackageFilesInfo} from '../../api/downloadPackageFiles.js';
+import {installDeps} from './install-deps.js';
+
+type InstallMopsDepOptions = {
+	verbose ?: boolean;
+	silent ?: boolean;
+	dep ?: boolean;
+	threads ?: number;
+};
+
+export async function installMopsDep(pkg : string, version = '', {verbose, silent, dep, threads} : InstallMopsDepOptions = {}) : Promise<boolean> {
+	threads = threads || 12;
+
+	if (!checkConfigFile()) {
+		return false;
+	}
+	let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
+
+	// progress
+	let total = Infinity;
+	let step = 0;
+	let progress = () => {
+		step++;
+		silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${pkg}@${version} ${progressBar(step, total)}`);
+	};
+	progress();
+
+	if (!version) {
+		let versionRes = await getHighestVersion(pkg);
+		if ('err' in versionRes) {
+			console.log(chalk.red('Error: ') + versionRes.err);
+			return false;
+		}
+		version = versionRes.ok;
+	}
+
+	let cacheName = getMopsDepCacheName(pkg, version);
+	let cacheDir = getDepCacheDir(cacheName);
+
+	// global cache hit
+	if (isDepCached(cacheName)) {
+		silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${pkg}@${version} (cache)`);
+	}
+	// download
+	else {
+		// GitHub Actions fails with "fetch failed" if there are multiple concurrent actions
+		if (process.env.GITHUB_ENV) {
+			threads = 4;
+		}
+
+		try {
+			let {storageId, fileIds} = await getPackageFilesInfo(pkg, version);
+
+			total = fileIds.length + 2;
+
+			let filesData = new Map;
+			let storage = await storageActor(storageId);
+
+			await parallel(threads, fileIds, async (fileId : string) => {
+				let {path, data} = await downloadFile(storage, fileId);
+				filesData.set(path, data);
+				progress();
+			});
+
+			let onSigInt = () => {
+				deleteSync([cacheDir], {force: true});
+				process.exit();
+			};
+			process.on('SIGINT', onSigInt);
+
+			// write files to global cache
+			try {
+				await Promise.all(Array.from(filesData.entries()).map(async ([filePath, data]) => {
+					await fs.promises.mkdir(path.join(cacheDir, path.dirname(filePath)), {recursive: true});
+					await fs.promises.writeFile(path.join(cacheDir, filePath), Buffer.from(data));
+				}));
+			}
+			catch (err) {
+				console.error(chalk.red('Error: ') + err);
+				deleteSync([cacheDir], {force: true});
+				return false;
+			}
+
+			process.off('SIGINT', onSigInt);
+		}
+		catch (err) {
+			console.error(chalk.red('Error: ') + err);
+			return false;
+		}
+
+		progress();
+	}
+
+	if (verbose) {
+		silent || logUpdate.done();
+	}
+	else {
+		logUpdate.clear();
+	}
+
+	// install dependencies
+	let config = readConfig(path.join(cacheDir, 'mops.toml'));
+	let res = await installDeps(Object.values(config.dependencies || {}), {silent, verbose});
+
+	if (!res) {
+		return false;
+	}
+
+	return true;
+}

package/commands/install/sync-local-cache.ts

@@ -0,0 +1,34 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {copyCache, getDepCacheName} from '../../cache.js';
+import {getDependencyType, getRootDir} from '../../mops.js';
+import {resolvePackages} from '../../resolve-packages.js';
+
+export async function syncLocalCache({verbose = false} = {}) : Promise<Record<string, string>> {
+	let resolvedPackages = await resolvePackages();
+	let rootDir = getRootDir();
+
+	verbose && console.log('Syncing local cache...');
+
+	let installedDeps : Record<string, string> = {};
+
+	await Promise.all(Object.entries(resolvedPackages).map(([name, value]) => {
+		let depType = getDependencyType(value);
+
+		if (depType === 'mops' || depType === 'github') {
+			let cacheName = getDepCacheName(name, value);
+			let dest = path.join(rootDir, '.mops', cacheName);
+
+			if (!fs.existsSync(dest)) {
+				if (depType === 'mops') {
+					installedDeps[name] = value;
+				}
+				return copyCache(cacheName, path.join(rootDir, '.mops', cacheName));
+			}
+		}
+
+		return Promise.resolve();
+	}));
+
+	return installedDeps;
+}

package/commands/publish.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import fs from 'node:fs';
 import path from 'node:path';
 import chalk from 'chalk';

package/commands/remove.ts

@@ -1,9 +1,12 @@
 import fs from 'node:fs';
 import {deleteSync} from 'del';
 import chalk from 'chalk';
-import {formatDir, formatGithubDir, checkConfigFile, readConfig, writeConfig} from '../mops.js';
+import {checkConfigFile, getRootDir, readConfig, writeConfig} from '../mops.js';
 import {Config, Dependency} from '../types.js';
 import {checkIntegrity} from '../integrity.js';
+import {getDepCacheDir, getDepCacheName} from '../cache.js';
+import path from 'node:path';
+import {syncLocalCache} from './install/sync-local-cache.js';
 
 type RemoveOptions = {
 	verbose ?: boolean;
@@ -12,7 +15,6 @@ type RemoveOptions = {
 	lock ?: 'update' | 'ignore';
 };
 
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 export async function remove(name : string, {dev = false, verbose = false, dryRun = false, lock} : RemoveOptions = {}) {
 	if (!checkConfigFile()) {
 		return;
@@ -31,13 +33,12 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
 	}
 
 	function getTransitiveDependenciesOf(name : string, version : string | undefined, repo ?: string) {
-		let pkgDir = '';
-		if (repo) {
-			pkgDir = formatGithubDir(name, repo);
-		}
-		else if (version) {
-			pkgDir = formatDir(name, version);
+		let value = version || repo;
+		if (!value) {
+			return [];
 		}
+		let cacheName = getDepCacheName(name, value);
+		let pkgDir = getDepCacheDir(cacheName);
 		let configFile = pkgDir + '/mops.toml';
 		if (!fs.existsSync(configFile)) {
 			verbose && console.log('no config', configFile);
@@ -78,16 +79,11 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
 			verbose && console.log(`Ignored transitive dependency ${depId} (other deps depend on it)`);
 			continue;
 		}
-		let pkgDir;
-		if (dep.repo) {
-			pkgDir = formatGithubDir(dep.name, dep.repo);
-		}
-		else if (dep.version) {
-			pkgDir = formatDir(dep.name, dep.version);
-		}
-		if (pkgDir && fs.existsSync(pkgDir)) {
-			dryRun || deleteSync([`${pkgDir}`], {force: true});
-			verbose && console.log(`Removed local cache ${pkgDir}`);
+		let cacheName = getDepCacheName(dep.name, dep.version || dep.repo || '');
+		let localCacheDir = path.join(getRootDir(), '.mops', cacheName);
+		if (localCacheDir && fs.existsSync(localCacheDir)) {
+			dryRun || deleteSync([localCacheDir], {force: true});
+			verbose && console.log(`Removed local cache ${localCacheDir}`);
 		}
 	}
 
@@ -100,6 +96,7 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
 	}
 	dryRun || writeConfig(config);
 
+	await syncLocalCache();
 	await checkIntegrity(lock);
 
 	console.log(chalk.green('Package removed ') + `${name} = "${version}"`);

package/commands/search.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import asTable from 'as-table';
 import chalk from 'chalk';
 import {mainActor} from '../api/actors.js';

package/commands/self.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import child_process, {execSync} from 'node:child_process';
 import chalk from 'chalk';
 import {version, globalConfigDir} from '../mops.js';

package/commands/sources.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import path from 'node:path';
 import fs from 'node:fs';
 import {checkConfigFile, formatDir, formatGithubDir, getDependencyType, readConfig} from '../mops.js';

package/commands/sync.ts

@@ -1,3 +1,4 @@
+import process from 'node:process';
 import path from 'node:path';
 import {execSync} from 'node:child_process';
 import {globSync} from 'glob';