ic-mops 0.33.0 → 0.34.0-0

package/commands/install.ts

@@ -2,10 +2,11 @@ import path from 'node:path';
 import fs from 'node:fs';
 import logUpdate from 'log-update';
 import chalk from 'chalk';
-import {checkConfigFile, formatDir, getHighestVersion, mainActor, progressBar, readConfig, storageActor} from '../mops.js';
+import {checkConfigFile, formatDir, getHighestVersion, progressBar, readConfig, storageActor} from '../mops.js';
 import {parallel} from '../parallel.js';
 import {installFromGithub} from '../vessel.js';
 import {addCache, copyCache, isCached} from '../cache.js';
+import {downloadFile, getPackageFilesInfo} from '../helpers/download-package-files.js';
 
 export async function install(pkg: string, version = '', {verbose = false, silent = false, dep = false} = {}): Promise<Record<string, string> | false> {
 	if (!checkConfigFile()) {
@@ -31,7 +32,6 @@ export async function install(pkg: string, version = '', {verbose = false, silen
 	}
 
 	let dir = formatDir(pkg, version);
-	let actor = await mainActor();
 	let alreadyInstalled = false;
 
 	// already installed
@@ -46,28 +46,6 @@ export async function install(pkg: string, version = '', {verbose = false, silen
 	}
 	// download
 	else {
-		let [packageDetailsRes, filesIdsRes] = await Promise.all([
-			actor.getPackageDetails(pkg, version),
-			actor.getFileIds(pkg, version),
-		]);
-
-		if ('err' in packageDetailsRes) {
-			console.log(chalk.red('Error: ') + packageDetailsRes.err);
-			return false;
-		}
-		let packageDetails = packageDetailsRes.ok;
-
-		if ('err' in filesIdsRes) {
-			console.log(chalk.red('Error: ') + filesIdsRes.err);
-			return false;
-		}
-		let filesIds = filesIdsRes.ok;
-		total = filesIds.length + 2;
-
-		let storage = await storageActor(packageDetails.publication.storage);
-
-		// download files
-		let filesData = new Map;
 		let threads = 16;
 
 		// GitHub Actions fails with "fetch failed" if there are multiple concurrent actions
@@ -75,32 +53,29 @@ export async function install(pkg: string, version = '', {verbose = false, silen
 			threads = 4;
 		}
 
-		await parallel(threads, filesIds, async (fileId: string) => {
-			let fileMetaRes = await storage.getFileMeta(fileId);
-			if ('err' in fileMetaRes) {
-				console.log(chalk.red('ERR: ') + fileMetaRes.err);
-				return;
-			}
-			let fileMeta = fileMetaRes.ok;
-
-			let buffer = Buffer.from([]);
-			for (let i = 0n; i < fileMeta.chunkCount; i++) {
-				let chunkRes = await storage.downloadChunk(fileId, i);
-				if ('err' in chunkRes) {
-					console.log(chalk.red('ERR: ') + chunkRes.err);
-					return;
-				}
-				let chunk = chunkRes.ok;
-				buffer = Buffer.concat([buffer, Buffer.from(chunk)]);
+		try {
+			let {storageId, fileIds} = await getPackageFilesInfo(pkg, version);
+
+			total = fileIds.length + 2;
+
+			let filesData = new Map;
+			let storage = await storageActor(storageId);
+
+			await parallel(threads, fileIds, async (fileId: string) => {
+				let {path, data} = await downloadFile(storage, fileId);
+				filesData.set(path, data);
+				progress();
+			});
+
+			// write files to disk
+			for (let [filePath, data] of filesData.entries()) {
+				fs.mkdirSync(path.join(dir, path.dirname(filePath)), {recursive: true});
+				fs.writeFileSync(path.join(dir, filePath), Buffer.from(data));
 			}
-			filesData.set(fileMeta.path, buffer);
-			progress();
-		});
-
-		// write files to disk
-		for (let [filePath, buffer] of filesData.entries()) {
-			fs.mkdirSync(path.join(dir, path.dirname(filePath)), {recursive: true});
-			fs.writeFileSync(path.join(dir, filePath), buffer);
+		}
+		catch (err) {
+			console.error(chalk.red('Error: ') + err);
+			return false;
 		}
 
 		// add to cache

package/commands/search.ts

@@ -4,8 +4,7 @@ import {mainActor} from '../mops.js';
 
 export async function search(text: string) {
 	let actor = await mainActor();
-	let res = await actor.search(text, [], []);
-	let packages = res[0];
+	let [packages, _pageCount] = await actor.search(text, [], []);
 
 	if (!packages.length) {
 		console.log('Packages not found');

package/dist/commands/install.js

@@ -2,10 +2,11 @@ import path from 'node:path';
 import fs from 'node:fs';
 import logUpdate from 'log-update';
 import chalk from 'chalk';
-import { checkConfigFile, formatDir, getHighestVersion, mainActor, progressBar, readConfig, storageActor } from '../mops.js';
+import { checkConfigFile, formatDir, getHighestVersion, progressBar, readConfig, storageActor } from '../mops.js';
 import { parallel } from '../parallel.js';
 import { installFromGithub } from '../vessel.js';
 import { addCache, copyCache, isCached } from '../cache.js';
+import { downloadFile, getPackageFilesInfo } from '../helpers/download-package-files.js';
 export async function install(pkg, version = '', { verbose = false, silent = false, dep = false } = {}) {
     if (!checkConfigFile()) {
         return false;
@@ -27,7 +28,6 @@ export async function install(pkg, version = '', { verbose = false, silent = fal
         version = versionRes.ok;
     }
     let dir = formatDir(pkg, version);
-    let actor = await mainActor();
     let alreadyInstalled = false;
     // already installed
     if (fs.existsSync(dir)) {
@@ -41,53 +41,30 @@ export async function install(pkg, version = '', { verbose = false, silent = fal
     }
     // download
     else {
-        let [packageDetailsRes, filesIdsRes] = await Promise.all([
-            actor.getPackageDetails(pkg, version),
-            actor.getFileIds(pkg, version),
-        ]);
-        if ('err' in packageDetailsRes) {
-            console.log(chalk.red('Error: ') + packageDetailsRes.err);
-            return false;
-        }
-        let packageDetails = packageDetailsRes.ok;
-        if ('err' in filesIdsRes) {
-            console.log(chalk.red('Error: ') + filesIdsRes.err);
-            return false;
-        }
-        let filesIds = filesIdsRes.ok;
-        total = filesIds.length + 2;
-        let storage = await storageActor(packageDetails.publication.storage);
-        // download files
-        let filesData = new Map;
         let threads = 16;
         // GitHub Actions fails with "fetch failed" if there are multiple concurrent actions
         if (process.env.GITHUB_ENV) {
             threads = 4;
         }
-        await parallel(threads, filesIds, async (fileId) => {
-            let fileMetaRes = await storage.getFileMeta(fileId);
-            if ('err' in fileMetaRes) {
-                console.log(chalk.red('ERR: ') + fileMetaRes.err);
-                return;
+        try {
+            let { storageId, fileIds } = await getPackageFilesInfo(pkg, version);
+            total = fileIds.length + 2;
+            let filesData = new Map;
+            let storage = await storageActor(storageId);
+            await parallel(threads, fileIds, async (fileId) => {
+                let { path, data } = await downloadFile(storage, fileId);
+                filesData.set(path, data);
+                progress();
+            });
+            // write files to disk
+            for (let [filePath, data] of filesData.entries()) {
+                fs.mkdirSync(path.join(dir, path.dirname(filePath)), { recursive: true });
+                fs.writeFileSync(path.join(dir, filePath), Buffer.from(data));
             }
-            let fileMeta = fileMetaRes.ok;
-            let buffer = Buffer.from([]);
-            for (let i = 0n; i < fileMeta.chunkCount; i++) {
-                let chunkRes = await storage.downloadChunk(fileId, i);
-                if ('err' in chunkRes) {
-                    console.log(chalk.red('ERR: ') + chunkRes.err);
-                    return;
-                }
-                let chunk = chunkRes.ok;
-                buffer = Buffer.concat([buffer, Buffer.from(chunk)]);
-            }
-            filesData.set(fileMeta.path, buffer);
-            progress();
-        });
-        // write files to disk
-        for (let [filePath, buffer] of filesData.entries()) {
-            fs.mkdirSync(path.join(dir, path.dirname(filePath)), { recursive: true });
-            fs.writeFileSync(path.join(dir, filePath), buffer);
+        }
+        catch (err) {
+            console.error(chalk.red('Error: ') + err);
+            return false;
         }
         // add to cache
         await addCache(`${pkg}@${version}`, dir);

package/dist/commands/search.js

@@ -3,8 +3,7 @@ import chalk from 'chalk';
 import { mainActor } from '../mops.js';
 export async function search(text) {
     let actor = await mainActor();
-    let res = await actor.search(text, [], []);
-    let packages = res[0];
+    let [packages, _pageCount] = await actor.search(text, [], []);
     if (!packages.length) {
         console.log('Packages not found');
         return;

package/dist/helpers/download-package-files.d.ts

@@ -0,0 +1,12 @@
+import { Principal } from '@dfinity/principal';
+import { Storage } from '../declarations/storage/storage.did.js';
+export declare function downloadPackageFiles(pkg: string, version?: string, threads?: number, onLoad?: (_fileIds: string[], _fileId: string) => void): Promise<Map<string, string>>;
+export declare function getPackageFilesInfo(pkg: string, version: string): Promise<{
+    storageId: Principal;
+    fileIds: string[];
+}>;
+export declare function getFileIds(pkg: string, version: string): Promise<string[]>;
+export declare function downloadFile(storage: Storage, fileId: string): Promise<{
+    path: string;
+    data: Array<number>;
+}>;

package/dist/helpers/download-package-files.js

@@ -0,0 +1,62 @@
+import { mainActor, storageActor } from '../mops.js';
+import { parallel } from '../parallel.js';
+import { resolveVersion } from './resolve-version.js';
+export async function downloadPackageFiles(pkg, version = '', threads = 8, onLoad = (_fileIds, _fileId) => { }) {
+    version = await resolveVersion(pkg, version);
+    let { storageId, fileIds } = await getPackageFilesInfo(pkg, version);
+    let storage = await storageActor(storageId);
+    let filesData = new Map;
+    await parallel(threads, fileIds, async (fileId) => {
+        let { path, data } = await downloadFile(storage, fileId);
+        filesData.set(path, data);
+        onLoad(fileIds, fileId);
+    });
+    return filesData;
+}
+// get package files meta
+export async function getPackageFilesInfo(pkg, version) {
+    let actor = await mainActor();
+    let [packageDetailsRes, fileIds] = await Promise.all([
+        actor.getPackageDetails(pkg, version),
+        getFileIds(pkg, version),
+    ]);
+    if ('err' in packageDetailsRes) {
+        throw packageDetailsRes.err;
+    }
+    let packageDetails = packageDetailsRes.ok;
+    return {
+        storageId: packageDetails.publication.storage,
+        fileIds,
+    };
+}
+// get package files ids
+export async function getFileIds(pkg, version) {
+    let actor = await mainActor();
+    let fileIdsRes = await actor.getFileIds(pkg, version);
+    if ('err' in fileIdsRes) {
+        throw fileIdsRes.err;
+    }
+    let filesIds = fileIdsRes.ok;
+    return filesIds;
+}
+// download single file
+export async function downloadFile(storage, fileId) {
+    let fileMetaRes = await storage.getFileMeta(fileId);
+    if ('err' in fileMetaRes) {
+        throw fileMetaRes.err;
+    }
+    let fileMeta = fileMetaRes.ok;
+    let data = [];
+    for (let i = 0n; i < fileMeta.chunkCount; i++) {
+        let chunkRes = await storage.downloadChunk(fileId, i);
+        if ('err' in chunkRes) {
+            throw chunkRes.err;
+        }
+        let chunk = chunkRes.ok;
+        data = [...data, ...chunk];
+    }
+    return {
+        path: fileMeta.path,
+        data: data,
+    };
+}

package/dist/helpers/resolve-version.d.ts

@@ -0,0 +1 @@
+export declare function resolveVersion(pkg: string, version?: string): Promise<string>;

package/dist/helpers/resolve-version.js

@@ -0,0 +1,11 @@
+import { getHighestVersion } from '../mops.js';
+export async function resolveVersion(pkg, version = '') {
+    if (!version) {
+        let versionRes = await getHighestVersion(pkg);
+        if ('err' in versionRes) {
+            throw versionRes.err;
+        }
+        version = versionRes.ok;
+    }
+    return version;
+}

package/dist/mops.d.ts

@@ -3,6 +3,7 @@ import { _SERVICE } from './declarations/main/main.did.js';
 import { _SERVICE as _STORAGE_SERVICE } from './declarations/storage/storage.did.js';
 import { Config } from './types.js';
 import { Principal } from '@dfinity/principal';
+import { downloadPackageFiles } from './helpers/download-package-files.js';
 export declare let apiVersion: string;
 export declare let globalConfigDir: string;
 export declare let globalCacheDir: string;
@@ -34,3 +35,4 @@ export declare function formatDir(name: string, version: string): string;
 export declare function formatGithubDir(name: string, repo: string): string;
 export declare function readDfxJson(): any;
 export declare function checkApiCompatibility(): Promise<boolean>;
+export { downloadPackageFiles };

package/dist/mops.js

@@ -9,6 +9,7 @@ import fetch from 'node-fetch';
 import { idlFactory } from './declarations/main/index.js';
 import { idlFactory as storageIdlFactory } from './declarations/storage/index.js';
 import { decodeFile } from './pem.js';
+import { downloadPackageFiles } from './helpers/download-package-files.js';
 if (!global.fetch) {
     global.fetch = fetch;
 }
@@ -292,3 +293,4 @@ export async function checkApiCompatibility() {
     }
     return true;
 }
+export { downloadPackageFiles };

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "0.33.0",
+  "version": "0.34.0-0",
   "type": "module",
   "bin": {
     "mops": "dist/cli.js"

package/helpers/download-package-files.ts

@@ -0,0 +1,78 @@
+import {mainActor, storageActor} from '../mops.js';
+import {parallel} from '../parallel.js';
+import {Principal} from '@dfinity/principal';
+import {resolveVersion} from './resolve-version.js';
+import {Storage} from '../declarations/storage/storage.did.js';
+
+export async function downloadPackageFiles(pkg: string, version = '', threads = 8, onLoad = (_fileIds: string[], _fileId: string) => {}): Promise<Map<string, string>> {
+	version = await resolveVersion(pkg, version);
+
+	let {storageId, fileIds} = await getPackageFilesInfo(pkg, version);
+	let storage = await storageActor(storageId);
+
+	let filesData = new Map;
+	await parallel(threads, fileIds, async (fileId: string) => {
+		let {path, data} = await downloadFile(storage, fileId);
+		filesData.set(path, data);
+		onLoad(fileIds, fileId);
+	});
+
+	return filesData;
+}
+
+// get package files meta
+export async function getPackageFilesInfo(pkg: string, version: string): Promise<{ storageId: Principal, fileIds: string[] }> {
+	let actor = await mainActor();
+
+	let [packageDetailsRes, fileIds] = await Promise.all([
+		actor.getPackageDetails(pkg, version),
+		getFileIds(pkg, version),
+	]);
+
+	if ('err' in packageDetailsRes) {
+		throw packageDetailsRes.err;
+	}
+	let packageDetails = packageDetailsRes.ok;
+
+	return {
+		storageId: packageDetails.publication.storage,
+		fileIds,
+	};
+}
+
+// get package files ids
+export async function getFileIds(pkg: string, version: string): Promise<string[]> {
+	let actor = await mainActor();
+	let fileIdsRes = await actor.getFileIds(pkg, version);
+
+	if ('err' in fileIdsRes) {
+		throw fileIdsRes.err;
+	}
+	let filesIds = fileIdsRes.ok;
+
+	return filesIds;
+}
+
+// download single file
+export async function downloadFile(storage: Storage, fileId: string): Promise<{ path: string, data: Array<number> }> {
+	let fileMetaRes = await storage.getFileMeta(fileId);
+	if ('err' in fileMetaRes) {
+		throw fileMetaRes.err;
+	}
+	let fileMeta = fileMetaRes.ok;
+
+	let data: Array<number> = [];
+	for (let i = 0n; i < fileMeta.chunkCount; i++) {
+		let chunkRes = await storage.downloadChunk(fileId, i);
+		if ('err' in chunkRes) {
+			throw chunkRes.err;
+		}
+		let chunk = chunkRes.ok;
+		data = [...data, ...chunk];
+	}
+
+	return {
+		path: fileMeta.path,
+		data: data,
+	};
+}

package/helpers/resolve-version.ts

@@ -0,0 +1,12 @@
+import {getHighestVersion} from '../mops.js';
+
+export async function resolveVersion(pkg: string, version = ''): Promise<string> {
+	if (!version) {
+		let versionRes = await getHighestVersion(pkg);
+		if ('err' in versionRes) {
+			throw versionRes.err;
+		}
+		version = versionRes.ok;
+	}
+	return version;
+}

package/mops.ts

@@ -14,6 +14,7 @@ import {_SERVICE as _STORAGE_SERVICE} from './declarations/storage/storage.did.j
 import {decodeFile} from './pem.js';
 import {Config} from './types.js';
 import {Principal} from '@dfinity/principal';
+import {downloadPackageFiles} from './helpers/download-package-files.js';
 
 
 if (!global.fetch) {
@@ -338,4 +339,6 @@ export async function checkApiCompatibility() {
 		console.log('-'.repeat(50));
 	}
 	return true;
-}
+}
+
+export {downloadPackageFiles};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ic-mops",
-  "version": "0.33.0",
+  "version": "0.34.0-0",
   "type": "module",
   "bin": {
     "mops": "dist/cli.js"