ic-mops 0.1.12 → 0.2.0

package/cli.js

@@ -4,25 +4,21 @@ import fs from 'fs';
 import path from 'path';
 import {program} from 'commander';
 import chalk from 'chalk';
-import TOML from '@iarna/toml';
 
 import {init} from './commands/init.js';
 import {install} from './commands/install.js';
 import {publish} from './commands/publish.js';
 import {importPem} from './commands/import-identity.js';
 import {sources} from './commands/sources.js';
-import {checkApiCompatibility, getHighestVersion, getNetwork, setNetwork} from './mops.js';
+import {checkApiCompatibility, getHighestVersion, getNetwork, parseGithubURL, readConfig, setNetwork, writeConfig} from './mops.js';
 import {whoami} from './commands/whoami.js';
 import {installAll} from './commands/install-all.js';
 import logUpdate from 'log-update';
+import { installFromGithub } from './vessel.js';
 
 let cwd = process.cwd();
 let configFile = path.join(cwd, 'mops.toml');
 
-function wirteConfig(config) {
-	fs.writeFileSync(configFile, TOML.stringify(config).trim());
-}
-
 program.name('mops');
 
 // init
@@ -44,8 +40,7 @@ program
 		let config = {};
 		let exists = fs.existsSync(configFile);
 		if (exists) {
-			let text = fs.readFileSync(configFile).toString();
-			config = TOML.parse(text);
+			config = readConfig(configFile);
 		}
 		else {
 			console.log(chalk.red('Error: ') + `mops.toml not found. Please run ${chalk.green('mops init')} first`);
@@ -62,22 +57,63 @@ program
 
 		if (!pkg) {
 			installAll(options);
+			return;
 		}
-		else {
+
+		let pkgDetails;
+		let existingPkg = config.dependencies[pkg];
+
+		if (pkg.startsWith('https://github.com') || pkg.split('/') > 1){
+			const {org, gitName, branch} = parseGithubURL(pkg);
+
+			pkgDetails = {
+				name: parseGithubURL(pkg).gitName,
+				repo: `https://github.com/${org}/${gitName}#${branch}`,
+				version: ''
+			};
+
+			existingPkg = config.dependencies[pkgDetails.name];
+
+		}else if (!existingPkg || !existingPkg.repo){
 			let versionRes = await getHighestVersion(pkg);
 			if (versionRes.err) {
 				console.log(chalk.red('Error: ') + versionRes.err);
 				return;
 			}
-			let version = versionRes.ok;
 
-			await install(pkg, version, {verbose: options.verbose});
+			pkgDetails = {
+				name: pkg,
+				repo: '',
+				version:  versionRes.ok
+			};
 
-			config.dependencies[pkg] = version;
-			wirteConfig(config);
-			logUpdate.clear();
-			console.log(chalk.green('Package installed ') + `${pkg} = "${version}"`);
+		}else{
+			options.silent || logUpdate(`Installing ${existingPkg.name}@${existingPkg.version} (cache) from Github`);
+			return;
 		}
+
+		const {name, repo, version} = pkgDetails;
+
+		if (repo){
+			// pkg name conflict with an installed mops pkg
+			if (existingPkg && !existingPkg.repo){
+				console.log(chalk.red('Error: ') + `Conflicting Package Name '${name}`);
+				console.log('Consider entering the repo url and assigning a new name in the \'mops.toml\' file');
+				return;
+			}
+
+			await installFromGithub(name, repo, {verbose: options.verbose});
+		}else{
+			await install(name, version, {verbose: options.verbose});
+		}
+
+		config.dependencies[name] = pkgDetails;
+		writeConfig(config);
+
+		logUpdate.clear();
+		console.log(
+			chalk.green('Package installed ') + `${name} = "${repo || version}"`
+		);
 	});
 
 // publish

package/commands/init.js

@@ -1,7 +1,9 @@
-import TOML from '@iarna/toml';
 import chalk from 'chalk';
 import path from 'path';
 import fs from 'fs';
+import {checkApiCompatibility, mainActor, readDfxJson, writeConfig} from '../mops.js';
+import {installAll} from './install-all.js';
+import { readVesselConfig } from '../vessel.js';
 
 export async function init(name = '') {
 	let configFile = path.join(process.cwd(), 'mops.toml');
@@ -10,23 +12,67 @@ export async function init(name = '') {
 		console.log(chalk.yellow('mops.toml already exists'));
 		return;
 	}
+
+	console.log('Initializing...');
+
 	let config = {};
+	let vesselConfig = {};
+
+	const vesselFile = path.join(process.cwd(), 'vessel.dhall');
+
+	if (fs.existsSync(vesselFile)){
+		console.log('Reading vessel.dhall file');
+		const res = await readVesselConfig(process.cwd(), { cache: false });
+		vesselConfig = {...res};
+	}
+
+	if (vesselConfig.dependencies){
+		config.dependencies = {};
+
+		for (const dep of (vesselConfig.dependencies || [])){
+			config.dependencies[dep.name] = dep;
+		}
+	}
+
+	// lib mode
 	if (name) {
-		config = {
-			package: {
-				name: name,
-				version: '0.1.0',
-				description: '',
-				repository: '',
-			}
+		config.package = {
+			name,
+			version: '0.1.0',
+			description: '',
+			repository: '',
 		};
+
+		writeConfig(config);
+
+		if (Object.keys(config.dependencies || {}).length)
+			await installAll({verbose: true});
 	}
-	// TODO: add last version of 'base' package?
-	// else {
-	// 	config.dependencies = {};
-	// }
 
-	fs.writeFileSync(configFile, TOML.stringify(config).trim());
+	// project mode
+	if (!name) {
+		let compatible = await checkApiCompatibility();
+		if (!compatible) {
+			return;
+		}
+
+		let dfxJson = readDfxJson();
+		let dfxVersion = dfxJson?.dfx || '';
+
+		let actor = await mainActor();
+		let defaultPackages = await actor.getDefaultPackages(dfxVersion);
+
+		if (!config.dependencies)
+			config.dependencies = {};
+
+		defaultPackages.forEach(([name, version]) => {
+			config.dependencies[name] =  version;
+		});
+
+		writeConfig(config);
+
+		await installAll({verbose: true});
+	}
 
 	console.log(chalk.green('mops.toml has been created'));
 }

package/commands/install-all.js

@@ -2,6 +2,7 @@ import chalk from 'chalk';
 import logUpdate from 'log-update';
 import {checkConfigFile, readConfig} from '../mops.js';
 import {install} from './install.js';
+import {installFromGithub} from '../vessel.js';
 
 export async function installAll({verbose} = {}) {
 	if (!checkConfigFile()) {
@@ -9,10 +10,14 @@ export async function installAll({verbose} = {}) {
 	}
 
 	let config = readConfig();
-	let deps = Object.entries(config.dependencies || {});
+	const deps = Object.values(config.dependencies || {});
 
-	for (let [pkg, ver] of deps) {
-		await install(pkg, ver, {verbose});
+	for (let {name, repo, version} of deps) {
+		if (repo){
+			await installFromGithub(name, repo, {verbose});
+		}else{
+			await install(name, version, {verbose});
+		}
 	}
 
 	logUpdate.clear();

package/commands/install.js

@@ -1,9 +1,10 @@
 import path from 'path';
 import fs from 'fs';
 import logUpdate from 'log-update';
-import {checkConfigFile, getHighestVersion, mainActor, progressBar, readConfig, storageActor} from '../mops.js';
+import {checkConfigFile, formatDir, getHighestVersion, mainActor, progressBar, readConfig, storageActor} from '../mops.js';
 import {parallel} from '../parallel.js';
 import chalk from 'chalk';
+import { installFromGithub } from '../vessel.js';
 
 export async function install(pkg, version = '', {verbose, silent, dep} = {}) {
 	if (!checkConfigFile()) {
@@ -19,7 +20,7 @@ export async function install(pkg, version = '', {verbose, silent, dep} = {}) {
 		version = versionRes.ok;
 	}
 
-	let dir = path.join(process.cwd(), '.mops', `${pkg}@${version}`);
+	let dir = formatDir(pkg, version);
 	let actor = await mainActor();
 
 	// cache
@@ -30,9 +31,7 @@ export async function install(pkg, version = '', {verbose, silent, dep} = {}) {
 	else {
 		fs.mkdirSync(dir, {recursive: true});
 
-		if (!dep) {
-			actor.notifyInstall(pkg, version);
-		}
+		actor.notifyInstall(pkg, version);
 
 		let packageDetailsRes = await actor.getPackageDetails(pkg, version);
 		if (packageDetailsRes.err) {
@@ -91,7 +90,11 @@ export async function install(pkg, version = '', {verbose, silent, dep} = {}) {
 
 	// install dependencies
 	let config = readConfig(path.join(dir, 'mops.toml'));
-	for (let [name, version] of Object.entries(config.dependencies || {})) {
-		await install(name, version, {verbose, silent, dep: true});
+	for (const {name, repo, version} of Object.values(config.dependencies || {})) {
+		if (repo){
+			await installFromGithub(name, repo, {verbose});
+		}else{
+			await install(name, version, {verbose});
+		}
 	}
 }

package/commands/publish.js

@@ -136,9 +136,7 @@ export async function publish() {
 		dfx: config.package.dfx || '',
 		moc: config.package.moc || '',
 		donation: config.package.donation || '',
-		dependencies: (Object.entries(config.dependencies || {})).map(([name, version]) => {
-			return {name, version};
-		}),
+		dependencies: Object.values(config.dependencies || {}),
 		devDependencies: [],
 		scripts: [],
 	};

package/commands/sources.js

@@ -2,7 +2,8 @@ import path from 'path';
 import fs from 'fs';
 import chalk from 'chalk';
 import {install} from './install.js';
-import {getHighestVersion, readConfig} from '../mops.js';
+import {formatDir, formatGithubDir, getHighestVersion, parseGithubURL, readConfig} from '../mops.js';
+import { readVesselConfig } from '../vessel.js';
 
 function rootDir(cwd = process.cwd()) {
 	let configFile = path.join(cwd, 'mops.toml');
@@ -16,8 +17,7 @@ function rootDir(cwd = process.cwd()) {
 	return rootDir(path.join(cwd, '..'));
 }
 
-// TODO: resolve deps for a specific file to avoid conflicts
-// TODO: remove base-unofficial
+// TODO: resolve conflicts
 export async function sources({verbose} = {}) {
 	let root = rootDir();
 	if (!root) {
@@ -42,48 +42,63 @@ export async function sources({verbose} = {}) {
 		return 0;
 	};
 
-	let collectDeps = (config, isRoot = false) => {
-		for (let [dep, ver] of Object.entries(config.dependencies || {})) {
-			// take root dep ver or bigger one
-			if (isRoot || !packages[dep] || compareVersions(packages[dep], ver) === -1) {
-				packages[dep] = ver;
-			}
+	const gitVerRegex = new RegExp(/v(\d{1,2}\.\d{1,2}\.\d{1,2})(-.*)?$/);
 
-			let config = readConfig(path.join(root, `.mops/${dep}@${ver}/mops.toml`));
-			collectDeps(config);
+	const compareGitVersions = (repoA, repoB) => {
+		const {branch: a} = parseGithubURL(repoA);
+		const {branch: b} = parseGithubURL(repoB);
 
-			if (!versions[dep]) {
-				versions[dep] = [];
-			}
-			versions[dep].push(ver);
+		if (gitVerRegex.test(a) && gitVerRegex.test(b)){
+			return compareVersions(a.substring(1) , b.substring(1));
+		}else if (!gitVerRegex.test(a)){
+			return -1;
+		}else{
+			return 1;
 		}
 	};
 
-	let config = readConfig(path.join(root, 'mops.toml'));
-	collectDeps(config, true);
+	let collectDeps = async (config, isRoot = false) => {
+		for (const pkgDetails of Object.values(config.dependencies || {})) {
+			const {name, repo, version} = pkgDetails;
 
-	// install base-unofficial
-	if (!packages['base-unofficial']) {
-		let dirs = fs.readdirSync(path.join(root, '.mops'));
-		let downloadedPackages = Object.fromEntries(dirs.map((dir) => {
-			return dir.split('@');
-		}));
+			// take root dep version or bigger one
+			if (
+				isRoot ||
+				!packages[name] ||
+				repo && compareGitVersions(packages[name].repo, repo) === -1 ||
+				compareVersions(packages[name].version, version) === -1
+			) {
+				packages[name] = pkgDetails;
+			}
 
-		if (downloadedPackages['base-unofficial']) {
-			packages['base-unofficial'] = downloadedPackages['base-unofficial'];
-		}
-		else {
-			let versionRes = await getHighestVersion('base-unofficial');
-			if (versionRes.err) {
-				console.log(chalk.red('Error: ') + versionRes.err);
-				return;
+			let nestedConfig;
+
+			if (repo){
+				const dir = formatGithubDir(name, repo);
+				nestedConfig = await readVesselConfig(dir) || {};
+			}
+			else{
+				const dir = formatDir(name, version) + '/mops.toml';
+				nestedConfig = readConfig(dir);
+			}
+
+			await collectDeps(nestedConfig);
+
+			if (!versions[name]) {
+				versions[name] = [];
 			}
-			let version = versionRes.ok;
 
-			await install('base-unofficial', version, {silent: true, dep: true});
-			packages['base-unofficial'] = version;
+			if (repo){
+				const {branch} = parseGithubURL(repo);
+				versions[name].push(branch);
+			}else{
+				versions[name].push(version);
+			}
 		}
-	}
+	};
+
+	let config = readConfig(path.join(root, 'mops.toml'));
+	await collectDeps(config, true);
 
 	// show conflicts
 	if (verbose) {
@@ -95,13 +110,14 @@ export async function sources({verbose} = {}) {
 	}
 
 	// sources
-	for (let [name, ver] of Object.entries(packages)) {
-		let pkgDir = path.relative(process.cwd(), path.join(root, `.mops/${name}@${ver}/src`));
-		console.log(`--package ${name} ${pkgDir}`);
-
-		// fallback base to base-unofficial
-		if (name == 'base-unofficial' && !packages.base) {
-			console.log(`--package base ${pkgDir}`);
+	for (let [name, {repo, version}] of Object.entries(packages)) {
+		let pkgDir;
+		if (repo){
+			pkgDir =  path.relative(process.cwd(), formatGithubDir(name, repo)) + '/src';
+		}else{
+			pkgDir = path.relative(process.cwd(), formatDir(name, version)) + '/src';
 		}
+
+		console.log(`--package ${name} ${pkgDir}`);
 	}
 }

package/commands/uninstall.js

@@ -1,8 +1,8 @@
-import {checkConfigFile} from '../mops.js';
-import path from 'path';
+import {checkConfigFile, readConfig} from '../mops.js';
 import fs from 'fs';
 import del from 'del';
 import chalk from 'chalk';
+import { formatDir, formatGithubDir } from '../mops.js';
 
 export async function uninstall(pkg, version) {
 	if (!checkConfigFile()) {
@@ -10,7 +10,23 @@ export async function uninstall(pkg, version) {
 	}
 
 	// TODO: check if deps relate on this package
-	let pkgDir = path.join(process.cwd(), '.mops', `${pkg}@${version}`);
+	const config = readConfig();
+
+	const pkgDetails = config.dependencies[pkg];
+
+	if (!pkgDetails){
+		console.log(`No dependency to remove ${pkg} = "${version}"`);
+		return;
+	}
+
+	const {repo} = pkgDetails;
+	let pkgDir;
+
+	if (repo){
+		pkgDir = formatGithubDir(pkg, repo);
+	}else{
+		pkgDir = formatDir(pkg, version);
+	}
 
 	if (!fs.existsSync(pkgDir)) {
 		console.log(`No cache to remove ${pkg} = "${version}"`);

package/declarations/main/index.d.ts

@@ -0,0 +1,16 @@
+import { ActorSubclass, HttpAgentOptions, ActorConfig } from '@dfinity/agent';
+import { Principal } from '@dfinity/principal';
+
+import { _SERVICE } from './main.did';
+
+export declare interface CreateActorOptions {
+  agentOptions?: HttpAgentOptions;
+  actorOptions?: ActorConfig;
+}
+
+export declare const createActor: (
+  canisterId: string | Principal,
+  options: CreateActorOptions
+) => ActorSubclass<_SERVICE>;
+
+export declare const main: ActorSubclass<_SERVICE>;

package/declarations/main/index.js

@@ -7,17 +7,21 @@ export { idlFactory } from './main.did.js';
 export const canisterId = process.env.MAIN_CANISTER_ID;
 
 /**
- * 
+ * @deprecated since dfx 0.11.1
+ * Do not import from `.dfx`, instead switch to using `dfx generate` to generate your JS interface.
  * @param {string | import("@dfinity/principal").Principal} canisterId Canister ID of Agent
- * @param {{agentOptions?: import("@dfinity/agent").HttpAgentOptions; actorOptions?: import("@dfinity/agent").ActorConfig}} [options]
+ * @param {{agentOptions?: import("@dfinity/agent").HttpAgentOptions; actorOptions?: import("@dfinity/agent").ActorConfig} | { agent?: import("@dfinity/agent").Agent; actorOptions?: import("@dfinity/agent").ActorConfig }} [options]
  * @return {import("@dfinity/agent").ActorSubclass<import("./main.did.js")._SERVICE>}
  */
- export const createActor = (canisterId, options) => {
-  const agent = new HttpAgent({ ...options?.agentOptions });
+export const createActor = (canisterId, options = {}) => {
+  console.warn(`Deprecation warning: you are currently importing code from .dfx. Going forward, refactor to use the dfx generate command for JavaScript bindings.
+
+See https://internetcomputer.org/docs/current/developer-docs/updates/release-notes/ for migration instructions`);
+  const agent = options.agent || new HttpAgent({ ...options.agentOptions });
   
   // Fetch root key for certificate validation during development
-  if(process.env.NODE_ENV !== "production") {
-    agent.fetchRootKey().catch(err=>{
+  if (process.env.DFX_NETWORK !== "ic") {
+    agent.fetchRootKey().catch(err => {
       console.warn("Unable to fetch root key. Check to ensure that your local replica is running");
       console.error(err);
     });
@@ -27,7 +31,7 @@ export const canisterId = process.env.MAIN_CANISTER_ID;
   return Actor.createActor(idlFactory, {
     agent,
     canisterId,
-    ...options?.actorOptions,
+    ...(options ? options.actorOptions : {}),
   });
 };
   
@@ -35,4 +39,4 @@ export const canisterId = process.env.MAIN_CANISTER_ID;
  * A ready-to-use agent for the main canister
  * @type {import("@dfinity/agent").ActorSubclass<import("./main.did.js")._SERVICE>}
  */
- export const main = createActor(canisterId);
+export const main = createActor(canisterId);

package/declarations/main/main.did

@@ -1,3 +1,4 @@
+type Version = text;
 type Ver = text;
 type Time = int;
 type Text = text;
@@ -55,18 +56,18 @@ type PackageName__1 = text;
 type PackageName = text;
 type PackageDetails = 
  record {
-   config: PackageConfig__1;
+   config: PackageConfigV2__1;
    downloadsInLast30Days: nat;
    downloadsTotal: nat;
    owner: principal;
    publication: PackagePublication;
  };
-type PackageConfig__1 = 
+type PackageConfigV2__1 = 
  record {
    baseDir: text;
-   dependencies: vec Dependency;
+   dependencies: vec DependencyV2;
    description: text;
-   devDependencies: vec Dependency;
+   devDependencies: vec DependencyV2;
    dfx: text;
    documentation: text;
    donation: text;
@@ -80,12 +81,12 @@ type PackageConfig__1 =
    scripts: vec Script;
    version: text;
  };
-type PackageConfig = 
+type PackageConfigV2 = 
  record {
    baseDir: text;
-   dependencies: vec Dependency;
+   dependencies: vec DependencyV2;
    description: text;
-   devDependencies: vec Dependency;
+   devDependencies: vec DependencyV2;
    dfx: text;
    documentation: text;
    donation: text;
@@ -101,25 +102,33 @@ type PackageConfig =
  };
 type FileId = text;
 type Err = text;
-type Dependency = 
+type DependencyV2 = 
  record {
    name: PackageName;
+   repo: text;
    version: text;
  };
 service : {
   finishPublish: (PublishingId) -> (Result);
   getApiVersion: () -> (Text) query;
+  getDefaultPackages: (text) -> (vec record {
+                                       PackageName__1;
+                                       Version;
+                                     }) query;
   getFileIds: (PackageName__1, Ver) -> (Result_5) query;
   getHighestVersion: (PackageName__1) -> (Result_4) query;
+  getMostDownloadedPackages: () -> (vec PackageDetails) query;
   getPackageDetails: (PackageName__1, Ver) -> (Result_3) query;
   getRecentlyUpdatedPackages: () -> (vec PackageDetails) query;
   getStoragesStats: () -> (vec record {
                                  StorageId;
                                  StorageStats;
                                }) query;
+  getTotalDownloads: () -> (nat);
+  getTotalPackages: () -> (nat);
   notifyInstall: (PackageName__1, Ver) -> () oneway;
   search: (Text) -> (vec PackageDetails) query;
   startFileUpload: (PublishingId, Text, nat, blob) -> (Result_2);
-  startPublish: (PackageConfig) -> (Result_1);
+  startPublish: (PackageConfigV2) -> (Result_1);
   uploadFileChunk: (PublishingId, FileId, nat, blob) -> (Result);
 }

package/declarations/main/main.did.d.ts

@@ -1,10 +1,14 @@
 import type { Principal } from '@dfinity/principal';
 import type { ActorMethod } from '@dfinity/agent';
 
-export interface Dependency { 'name' : PackageName, 'version' : string }
+export interface DependencyV2 {
+  'name' : PackageName,
+  'repo' : string,
+  'version' : string,
+}
 export type Err = string;
 export type FileId = string;
-export interface PackageConfig {
+export interface PackageConfigV2 {
   'dfx' : string,
   'moc' : string,
   'scripts' : Array<Script>,
@@ -16,13 +20,13 @@ export interface PackageConfig {
   'version' : string,
   'keywords' : Array<string>,
   'donation' : string,
-  'devDependencies' : Array<Dependency>,
+  'devDependencies' : Array<DependencyV2>,
   'repository' : string,
-  'dependencies' : Array<Dependency>,
+  'dependencies' : Array<DependencyV2>,
   'license' : string,
   'readme' : string,
 }
-export interface PackageConfig__1 {
+export interface PackageConfigV2__1 {
   'dfx' : string,
   'moc' : string,
   'scripts' : Array<Script>,
@@ -34,9 +38,9 @@ export interface PackageConfig__1 {
   'version' : string,
   'keywords' : Array<string>,
   'donation' : string,
-  'devDependencies' : Array<Dependency>,
+  'devDependencies' : Array<DependencyV2>,
   'repository' : string,
-  'dependencies' : Array<Dependency>,
+  'dependencies' : Array<DependencyV2>,
   'license' : string,
   'readme' : string,
 }
@@ -44,7 +48,7 @@ export interface PackageDetails {
   'owner' : Principal,
   'downloadsTotal' : bigint,
   'downloadsInLast30Days' : bigint,
-  'config' : PackageConfig__1,
+  'config' : PackageConfigV2__1,
   'publication' : PackagePublication,
 }
 export type PackageName = string;
@@ -78,23 +82,31 @@ export interface StorageStats {
 export type Text = string;
 export type Time = bigint;
 export type Ver = string;
+export type Version = string;
 export interface _SERVICE {
   'finishPublish' : ActorMethod<[PublishingId], Result>,
   'getApiVersion' : ActorMethod<[], Text>,
+  'getDefaultPackages' : ActorMethod<
+    [string],
+    Array<[PackageName__1, Version]>,
+  >,
   'getFileIds' : ActorMethod<[PackageName__1, Ver], Result_5>,
   'getHighestVersion' : ActorMethod<[PackageName__1], Result_4>,
+  'getMostDownloadedPackages' : ActorMethod<[], Array<PackageDetails>>,
   'getPackageDetails' : ActorMethod<[PackageName__1, Ver], Result_3>,
   'getRecentlyUpdatedPackages' : ActorMethod<[], Array<PackageDetails>>,
   'getStoragesStats' : ActorMethod<[], Array<[StorageId, StorageStats]>>,
+  'getTotalDownloads' : ActorMethod<[], bigint>,
+  'getTotalPackages' : ActorMethod<[], bigint>,
   'notifyInstall' : ActorMethod<[PackageName__1, Ver], undefined>,
   'search' : ActorMethod<[Text], Array<PackageDetails>>,
   'startFileUpload' : ActorMethod<
-    [PublishingId, Text, bigint, Array<number>],
+    [PublishingId, Text, bigint, Uint8Array],
     Result_2,
   >,
-  'startPublish' : ActorMethod<[PackageConfig], Result_1>,
+  'startPublish' : ActorMethod<[PackageConfigV2], Result_1>,
   'uploadFileChunk' : ActorMethod<
-    [PublishingId, FileId, bigint, Array<number>],
+    [PublishingId, FileId, bigint, Uint8Array],
     Result,
   >,
 }

package/declarations/main/main.did.js

@@ -4,14 +4,19 @@ export const idlFactory = ({ IDL }) => {
   const Result = IDL.Variant({ 'ok' : IDL.Null, 'err' : Err });
   const Text = IDL.Text;
   const PackageName__1 = IDL.Text;
+  const Version = IDL.Text;
   const Ver = IDL.Text;
   const FileId = IDL.Text;
   const Result_5 = IDL.Variant({ 'ok' : IDL.Vec(FileId), 'err' : Err });
   const Result_4 = IDL.Variant({ 'ok' : Ver, 'err' : Err });
   const Script = IDL.Record({ 'value' : IDL.Text, 'name' : IDL.Text });
   const PackageName = IDL.Text;
-  const Dependency = IDL.Record({ 'name' : PackageName, 'version' : IDL.Text });
-  const PackageConfig__1 = IDL.Record({
+  const DependencyV2 = IDL.Record({
+    'name' : PackageName,
+    'repo' : IDL.Text,
+    'version' : IDL.Text,
+  });
+  const PackageConfigV2__1 = IDL.Record({
     'dfx' : IDL.Text,
     'moc' : IDL.Text,
     'scripts' : IDL.Vec(Script),
@@ -23,9 +28,9 @@ export const idlFactory = ({ IDL }) => {
     'version' : IDL.Text,
     'keywords' : IDL.Vec(IDL.Text),
     'donation' : IDL.Text,
-    'devDependencies' : IDL.Vec(Dependency),
+    'devDependencies' : IDL.Vec(DependencyV2),
     'repository' : IDL.Text,
-    'dependencies' : IDL.Vec(Dependency),
+    'dependencies' : IDL.Vec(DependencyV2),
     'license' : IDL.Text,
     'readme' : IDL.Text,
   });
@@ -39,7 +44,7 @@ export const idlFactory = ({ IDL }) => {
     'owner' : IDL.Principal,
     'downloadsTotal' : IDL.Nat,
     'downloadsInLast30Days' : IDL.Nat,
-    'config' : PackageConfig__1,
+    'config' : PackageConfigV2__1,
     'publication' : PackagePublication,
   });
   const Result_3 = IDL.Variant({ 'ok' : PackageDetails, 'err' : Err });
@@ -50,7 +55,7 @@ export const idlFactory = ({ IDL }) => {
     'memorySize' : IDL.Nat,
   });
   const Result_2 = IDL.Variant({ 'ok' : FileId, 'err' : Err });
-  const PackageConfig = IDL.Record({
+  const PackageConfigV2 = IDL.Record({
     'dfx' : IDL.Text,
     'moc' : IDL.Text,
     'scripts' : IDL.Vec(Script),
@@ -62,9 +67,9 @@ export const idlFactory = ({ IDL }) => {
     'version' : IDL.Text,
     'keywords' : IDL.Vec(IDL.Text),
     'donation' : IDL.Text,
-    'devDependencies' : IDL.Vec(Dependency),
+    'devDependencies' : IDL.Vec(DependencyV2),
     'repository' : IDL.Text,
-    'dependencies' : IDL.Vec(Dependency),
+    'dependencies' : IDL.Vec(DependencyV2),
     'license' : IDL.Text,
     'readme' : IDL.Text,
   });
@@ -73,8 +78,18 @@ export const idlFactory = ({ IDL }) => {
   return IDL.Service({
     'finishPublish' : IDL.Func([PublishingId], [Result], []),
     'getApiVersion' : IDL.Func([], [Text], ['query']),
+    'getDefaultPackages' : IDL.Func(
+        [IDL.Text],
+        [IDL.Vec(IDL.Tuple(PackageName__1, Version))],
+        ['query'],
+      ),
     'getFileIds' : IDL.Func([PackageName__1, Ver], [Result_5], ['query']),
     'getHighestVersion' : IDL.Func([PackageName__1], [Result_4], ['query']),
+    'getMostDownloadedPackages' : IDL.Func(
+        [],
+        [IDL.Vec(PackageDetails)],
+        ['query'],
+      ),
     'getPackageDetails' : IDL.Func(
         [PackageName__1, Ver],
         [Result_3],
@@ -90,6 +105,8 @@ export const idlFactory = ({ IDL }) => {
         [IDL.Vec(IDL.Tuple(StorageId, StorageStats))],
         ['query'],
       ),
+    'getTotalDownloads' : IDL.Func([], [IDL.Nat], []),
+    'getTotalPackages' : IDL.Func([], [IDL.Nat], []),
     'notifyInstall' : IDL.Func([PackageName__1, Ver], [], ['oneway']),
     'search' : IDL.Func([Text], [IDL.Vec(PackageDetails)], ['query']),
     'startFileUpload' : IDL.Func(
@@ -97,7 +114,7 @@ export const idlFactory = ({ IDL }) => {
         [Result_2],
         [],
       ),
-    'startPublish' : IDL.Func([PackageConfig], [Result_1], []),
+    'startPublish' : IDL.Func([PackageConfigV2], [Result_1], []),
     'uploadFileChunk' : IDL.Func(
         [PublishingId, FileId, IDL.Nat, IDL.Vec(IDL.Nat8)],
         [Result],

package/mops.js

@@ -13,7 +13,7 @@ import {decodeFile} from './pem.js';
 global.fetch = fetch;
 
 // (!) make changes in pair with backend
-let apiVersion = '0.1';
+let apiVersion = '1.2';
 
 let networkFile = new URL('./network.txt', import.meta.url);
 
@@ -107,9 +107,71 @@ export async function getHighestVersion(pkgName) {
 	return actor.getHighestVersion(pkgName);
 }
 
+export function parseGithubURL(href){
+	const url = new URL(href);
+	const branch =  url.hash?.substring(1) || 'master';
+
+	let [org, gitName] = url.pathname.split('/').filter(path => !!path);
+
+	if (gitName.endsWith('.git')){
+		gitName = gitName.substring(0, gitName.length - 4);
+	}
+
+	return { org, gitName, branch };
+}
+
 export function readConfig(configFile = path.join(process.cwd(), 'mops.toml')) {
 	let text = fs.readFileSync(configFile).toString();
-	return TOML.parse(text);
+	let toml = TOML.parse(text);
+
+	const deps = toml.dependencies || {};
+
+	Object.entries(deps).forEach(([name, data])=>{
+		if (data.startsWith('https://github.com/')){
+			deps[name] = {name, repo: data, version: ''};
+		}else{
+			deps[name] = {name, repo: '', version: data};
+		}
+	});
+
+	return toml;
+}
+
+export function writeConfig(config, configFile = path.join(process.cwd(), 'mops.toml')) {
+	const deps = config.dependencies || {};
+
+	Object.entries(deps).forEach(([name, {repo, version}])=>{
+		if (repo){
+			deps[name] = repo;
+		}else{
+			deps[name] = version;
+		}
+	});
+
+	fs.writeFileSync(configFile, TOML.stringify(config).trim());
+}
+
+export function formatDir(name, version){
+	return path.join(process.cwd(), '.mops', `${name}@${version}`);
+}
+
+export function formatGithubDir(name, repo){
+	const { branch } = parseGithubURL(repo);
+	return path.join(process.cwd(), '.mops/_github', `${name}@${branch}`);
+}
+
+export function readDfxJson() {
+	let dir = process.cwd();
+	let dfxJson = null;
+	for (let i = 0; i < 5; i++) {
+		let file = path.resolve(dir, 'dfx.json');
+		if (fs.existsSync(file)) {
+			dfxJson = JSON.parse(fs.readFileSync(file).toString());
+			break;
+		}
+		dir = path.resolve(dir, '..');
+	}
+	return dfxJson;
 }
 
 // warn on minor mismatch

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ic-mops",
-	"version": "0.1.12",
+	"version": "0.2.0",
 	"type": "module",
 	"bin": {
 		"mops": "cli.js"
@@ -13,9 +13,13 @@
 		"@iarna/toml": "^2.2.5",
 		"chalk": "^4.1.2",
 		"commander": "^9.2.0",
+		"decompress": "^4.2.1",
 		"del": "^6.0.0",
+		"dhall-to-json-cli": "^1.7.6",
 		"eslint": "^8.15.0",
+		"execa": "^6.1.0",
 		"globby": "^13.1.1",
+		"got": "^12.5.3",
 		"log-update": "^5.0.1",
 		"minimatch": "^5.0.1",
 		"node-fetch": "^2.6.7",

package/pem.js

@@ -1,5 +1,5 @@
 import fs from 'fs';
-import {Ed25519KeyIdentity} from '@dfinity/identity';
+import {Ed25519KeyIdentity, Secp256k1KeyIdentity} from '@dfinity/identity';
 import pemfile from 'pem-file';
 
 export function decodeFile(file) {
@@ -9,6 +9,12 @@ export function decodeFile(file) {
 
 export function decode(rawKey) {
 	var buf = pemfile.decode(rawKey);
+	if (rawKey.includes('EC PRIVATE KEY')) {
+		if (buf.length != 118) {
+			throw 'expecting byte length 118 but got ' + buf.length;
+		}
+		return Secp256k1KeyIdentity.fromSecretKey(buf.slice(7, 39));
+	}
 	if (buf.length != 85) {
 		throw 'expecting byte length 85 but got ' + buf.length;
 	}

package/vessel.js

@@ -0,0 +1,162 @@
+import {
+	existsSync, mkdirSync, createWriteStream, readFileSync, writeFileSync
+} from 'fs';
+import del from 'del';
+import { execaCommand} from 'execa';
+import chalk from 'chalk';
+import logUpdate from 'log-update';
+import { formatGithubDir, parseGithubURL, progressBar } from './mops.js';
+import path from 'path';
+import got from 'got';
+import decompress from 'decompress';
+import {pipeline} from 'stream/promises';
+
+const dhallFileToJson = async (filePath) => {
+	if (existsSync(filePath)) {
+		let cwd = new URL(path.dirname(import.meta.url)).pathname;
+		const res = await execaCommand(`dhall-to-json --file ${filePath}`, {preferLocal:true, cwd});
+
+		if (res.exitCode === 0){
+			return JSON.parse(res.stdout);
+		}
+		else {
+			return res;
+		}
+	}
+
+	return null;
+};
+
+export const readVesselConfig = async (
+	configFile,
+	{ cache = true } = { cache: true }
+) => {
+	const cachedFile = (configFile || process.cwd()) + '/vessel.json';
+
+	if (existsSync(cachedFile)) {
+		let cachedConfig = readFileSync(cachedFile);
+		return JSON.parse(cachedConfig);
+	}
+
+	const [vessel, packageSetArray] = await Promise.all([
+		dhallFileToJson((configFile || process.cwd()) + '/vessel.dhall'),
+		dhallFileToJson((configFile || process.cwd()) + '/package-set.dhall')
+	]);
+
+	if (!vessel || !packageSetArray) return null;
+
+	let repos = {};
+	for (const { name, repo, version } of packageSetArray) {
+		const { org, gitName } = parseGithubURL(repo);
+		repos[name] = `https://github.com/${org}/${gitName}#${version}`;
+	}
+
+	let config = {
+		compiler: vessel.compiler,
+		dependencies: vessel.dependencies.map((name) => {
+			return { name, repo: repos[name], version: '' };
+		}),
+	};
+
+	if (cache === true) {
+		writeFileSync(cachedFile, JSON.stringify(config), 'utf-8');
+	}
+
+	return config;
+};
+
+export const downloadFromGithub = async (repo, dest, onProgress = null) => {
+	const {branch, org, gitName} = parseGithubURL(repo);
+
+	const zipFile = `https://github.com/${org}/${gitName}/archive/${branch}.zip`;
+	const readStream = got.stream(zipFile);
+
+	const promise = new Promise((resolve, reject) => {
+
+		readStream.on('downloadProgress', ({ transferred, total}) => {
+			onProgress?.(transferred, total || 2 * (1024 ** 2) );
+		});
+
+		readStream.on('response', (response) => {
+			if (response.headers.age > 3600) {
+				console.log(chalk.red('Error: ') +  'Failure - response too old');
+				readStream.destroy(); // Destroy the stream to prevent hanging resources.
+				reject();
+				return;
+			}
+
+			// Prevent `onError` being called twice.
+			readStream.off('error', reject);
+			const tmpDir = process.cwd() + '/.mops/_tmp/';
+			const tmpFile = tmpDir + `/${gitName}@${branch}.zip`;
+
+			try {
+				mkdirSync(tmpDir, {recursive: true});
+
+				pipeline(readStream, createWriteStream(tmpFile))
+					.then(() => {
+						let options = {
+							extract: true,
+							strip: 1,
+							headers: {
+								accept: 'application/zip'
+							}
+						};
+
+						return decompress(tmpFile, dest, options);
+
+					}).then((unzippedFiles) => {
+						del.sync([tmpDir]);
+						resolve(unzippedFiles);
+
+					}).catch(err => {
+						del.sync([tmpDir]);
+						reject(err);
+					});
+
+			} catch (err) {
+				del.sync([tmpDir]);
+				reject(err);
+			}
+		});
+	});
+
+	return promise;
+};
+
+export const installFromGithub = async (name, repo, options = {})=>{
+
+	const {verbose, dep, silent} = options;
+
+	const {branch} = parseGithubURL(repo);
+	const dir = formatGithubDir(name, repo);
+
+	if (existsSync(dir)){
+		silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${name}@${branch} (cache) from Github`);
+	}
+	else {
+		mkdirSync(dir, {recursive: true});
+
+		let progress = (step, total) => {
+			silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${name}@${branch} ${progressBar(step, total)}`);
+		};
+
+		progress(0, 2 * (1024 ** 2));
+		await downloadFromGithub(repo, dir, progress).catch((err)=> {
+			del.sync([dir]);
+			console.log(chalk.red('Error: ') + err);
+		});
+	}
+
+	if (verbose) {
+		silent || logUpdate.done();
+	}
+
+	const config = await readVesselConfig(dir);
+
+	if (config){
+		for (const {name, repo} of config.dependencies){
+			await installFromGithub(name, repo, {verbose, silent, dep: true });
+		}
+	}
+};