@ohos-ports/azle 0.33.0-beta.0

package/src/experimental/build/commands/upload_assets/file_uploader/get_files_to_upload.ts

@@ -0,0 +1,21 @@
+import '#experimental/build/assert_experimental';
+
+import { getCanisterConfig } from '#utils/get_canister_config';
+
+export async function getFilesToUpload(
+    canisterName: string,
+    srcPath?: string,
+    destPath?: string
+): Promise<[string, string][]> {
+    if (srcPath !== undefined && destPath !== undefined) {
+        return [[srcPath, destPath]];
+    }
+    if (srcPath === undefined && destPath === undefined) {
+        // If both paths are undefined, look at the dfx.json for the assets to upload
+        const dfxJson = await getCanisterConfig(canisterName);
+        return dfxJson.custom?.assets ?? [];
+    }
+    throw new Error(
+        'The source path and destination path must be either both defined or both undefined'
+    );
+}

package/src/experimental/build/commands/upload_assets/file_uploader/incomplete_files.ts

@@ -0,0 +1,28 @@
+import '#experimental/build/assert_experimental';
+
+import { filterAsync } from '#experimental/utils/filter_async';
+
+import { Dest, Src } from '.';
+import { UploaderActor } from './uploader_actor';
+
+export async function getListOfIncompleteFiles(
+    paths: [Src, Dest][],
+    actor: UploaderActor
+): Promise<[Src, Dest][]> {
+    return filterAsync(
+        paths,
+        async ([_, path]) => !(await hasValidHash(path, actor))
+    );
+}
+
+async function hasValidHash(
+    path: string,
+    actor: UploaderActor
+): Promise<boolean> {
+    try {
+        const hashOption = await actor._azle_get_file_hash(path);
+        return hashOption.length === 1;
+    } catch {
+        return false;
+    }
+}

package/src/experimental/build/commands/upload_assets/file_uploader/index.ts

@@ -0,0 +1,43 @@
+import '#experimental/build/assert_experimental';
+
+import { getCanisterId } from '#utils/dfx';
+
+import { generateUploaderIdentity } from '../uploader_identity';
+import { expandPaths } from './expand_paths';
+import { onBeforeExit } from './on_before_exit';
+import { uploadFile } from './upload_file';
+import { createActor } from './uploader_actor';
+
+export type Src = string;
+export type Dest = string;
+
+export async function uploadFiles(
+    canisterName: string,
+    paths: [Src, Dest][]
+): Promise<void> {
+    if (
+        paths.length === 0 ||
+        process.env.AZLE_DISABLE_AUTO_FILE_UPLOAD === 'true'
+    ) {
+        return;
+    }
+
+    const canisterId = getCanisterId(canisterName);
+    const identityName = generateUploaderIdentity(canisterName);
+    const actor = await createActor(canisterId, identityName);
+
+    const chunkSize = 2_000_000; // The current message limit is about 2 MB
+
+    const expandedPaths = await expandPaths(paths);
+
+    for (const [srcPath, destPath] of expandedPaths) {
+        // Await each upload so the canister doesn't get overwhelmed by requests
+        await uploadFile(srcPath, destPath, chunkSize, actor);
+    }
+
+    console.info(
+        'Finished uploading files. Waiting for all chunks to finish uploading...' // TODO remove after https://github.com/demergent-labs/azle/issues/1996 is complete
+    );
+
+    onBeforeExit(expandedPaths, actor);
+}

package/src/experimental/build/commands/upload_assets/file_uploader/on_before_exit.ts

@@ -0,0 +1,34 @@
+import '#experimental/build/assert_experimental';
+
+import { Dest, Src } from '.';
+import { getListOfIncompleteFiles } from './incomplete_files';
+import { UploaderActor } from './uploader_actor';
+
+export function onBeforeExit(paths: [Src, Dest][], actor: UploaderActor): void {
+    let cleanUpComplete = false;
+
+    process.on('beforeExit', async () => {
+        if (cleanUpComplete === true) {
+            // If any async behavior happens in 'beforeExit' then 'beforeExit'
+            // will run again. This is need to prevent an infinite loop.
+            // Once clean up is complete we are ready to exit
+            console.info('File hashing finished');
+            return;
+        }
+
+        console.info('Cleaning up incomplete files');
+        await cleanup(paths, actor);
+        cleanUpComplete = true;
+        return;
+    });
+}
+
+async function cleanup(
+    paths: [Src, Dest][],
+    actor: UploaderActor
+): Promise<void> {
+    const incompleteFiles = await getListOfIncompleteFiles(paths, actor);
+    for (const [_, path] of incompleteFiles) {
+        await actor._azle_clear_file_and_info(path);
+    }
+}

package/src/experimental/build/commands/upload_assets/file_uploader/upload_file.ts

@@ -0,0 +1,91 @@
+import '#experimental/build/assert_experimental';
+
+import { open, stat } from 'fs/promises';
+
+import { hashFileByParts } from '../../../../../../.scripts/hash_file_by_parts';
+import { Dest, Src } from '.';
+import { bytesToHumanReadable } from './bytes_to_human_readable';
+import { UploaderActor } from './uploader_actor';
+
+export async function uploadFile(
+    srcPath: Src,
+    destPath: Dest,
+    chunkSize: number,
+    actor: UploaderActor
+): Promise<void> {
+    if (!(await shouldBeUploaded(srcPath, destPath, actor))) {
+        return;
+    }
+    const uploadStartTime = process.hrtime.bigint();
+    const fileSize = (await stat(srcPath)).size;
+    const file = await open(srcPath, 'r');
+    for (let startIndex = 0; startIndex <= fileSize; startIndex += chunkSize) {
+        let buffer = Buffer.alloc(chunkSize);
+        const { buffer: bytesToUpload, bytesRead } = await file.read(
+            buffer,
+            0,
+            chunkSize,
+            startIndex
+        );
+
+        await throttle();
+        const percentComplete = calculatePercentComplete(
+            startIndex + bytesRead,
+            fileSize
+        );
+        console.info(
+            `Uploading chunk: ${srcPath} | ${bytesToHumanReadable(
+                startIndex + bytesRead
+            )}/${bytesToHumanReadable(fileSize)} : ${percentComplete.toFixed(
+                2
+            )}%`
+        );
+        // Don't await here! Awaiting the agent will result in about a 4x increase in upload time.
+        // The above throttling is sufficient to manage the speed of uploads
+        actor
+            ._azle_upload_file_chunk(
+                destPath,
+                uploadStartTime,
+                BigInt(startIndex),
+                bytesToUpload.subarray(0, bytesRead),
+                BigInt(fileSize)
+            )
+            .catch((error) => {
+                console.error(error);
+            });
+    }
+    file.close();
+    console.info();
+}
+
+async function throttle(): Promise<void> {
+    // We can only process about 4Mib per second. So if chunks are about
+    // 2 MiB or less then we can only send off two per second.
+    if (process.env.DFX_NETWORK === 'ic') {
+        await new Promise((resolve) => setTimeout(resolve, 2_000)); // Mainnet requires more throttling. We found 2_000 by trial and error
+    } else {
+        await new Promise((resolve) => setTimeout(resolve, 500)); // Should be 500 (ie 1 every 1/2 second or 2 every second)
+    }
+}
+function calculatePercentComplete(
+    bytesComplete: number,
+    fileSize: number
+): number {
+    if (bytesComplete === 0 && fileSize === 0) {
+        return 100;
+    }
+    return (bytesComplete / Math.max(fileSize, 1)) * 100;
+}
+
+async function shouldBeUploaded(
+    srcPath: string,
+    destPath: string,
+    actor: UploaderActor
+): Promise<boolean> {
+    const localHash = (await hashFileByParts(srcPath)).toString('hex');
+    const canisterHashOption = await actor._azle_get_file_hash(destPath);
+    if (canisterHashOption.length === 0) {
+        return true;
+    }
+    return localHash !== canisterHashOption[0];
+}

package/src/experimental/build/commands/upload_assets/file_uploader/uploader_actor.ts

@@ -0,0 +1,51 @@
+import '#experimental/build/assert_experimental';
+
+import { Actor, ActorMethod, ActorSubclass } from '@dfinity/agent';
+
+import { createAuthenticatedAgent } from '#experimental/utils/dfx';
+
+export type UploaderActor = ActorSubclass<_SERVICE>;
+
+export async function createActor(
+    canisterId: string,
+    identityName: string
+): Promise<UploaderActor> {
+    const agent = await createAuthenticatedAgent(identityName);
+
+    return Actor.createActor<_SERVICE>(
+        ({ IDL }) => {
+            return IDL.Service({
+                _azle_clear_file_and_info: IDL.Func([IDL.Text], [], []),
+                _azle_get_file_hash: IDL.Func(
+                    [IDL.Text],
+                    [IDL.Opt(IDL.Text)],
+                    []
+                ),
+                _azle_upload_file_chunk: IDL.Func(
+                    [
+                        IDL.Text,
+                        IDL.Nat64,
+                        IDL.Nat64,
+                        IDL.Vec(IDL.Nat8),
+                        IDL.Nat64
+                    ],
+                    [],
+                    []
+                )
+            });
+        },
+        {
+            agent,
+            canisterId
+        }
+    );
+}
+
+interface _SERVICE {
+    _azle_clear_file_and_info: ActorMethod<[string], void>;
+    _azle_get_file_hash: ActorMethod<[string], [] | [string]>;
+    _azle_upload_file_chunk: ActorMethod<
+        [string, bigint, bigint, Uint8Array, bigint],
+        void
+    >;
+}

package/src/experimental/build/commands/upload_assets/index.ts

@@ -0,0 +1,16 @@
+import '#experimental/build/assert_experimental';
+
+import { uploadFiles } from './file_uploader';
+import { getFilesToUpload } from './file_uploader/get_files_to_upload';
+
+export async function runCommand(): Promise<void> {
+    const canisterName = process.argv[3];
+    const srcPath = process.argv[4];
+    const destPath = process.argv[5];
+    const filesToUpload = await getFilesToUpload(
+        canisterName,
+        srcPath,
+        destPath
+    );
+    await uploadFiles(canisterName, filesToUpload);
+}

package/src/experimental/build/commands/upload_assets/uploader_identity.ts

@@ -0,0 +1,23 @@
+import '#experimental/build/assert_experimental';
+
+import {
+    addController,
+    generateIdentity,
+    getPrincipal,
+    identityExists
+} from '#experimental/utils/dfx';
+
+export const AZLE_UPLOADER_IDENTITY_NAME =
+    process.env.AZLE_UPLOADER_IDENTITY_NAME ?? '_azle_file_uploader_identity';
+
+export function generateUploaderIdentity(canisterName: string): string {
+    if (!identityExists(AZLE_UPLOADER_IDENTITY_NAME)) {
+        generateIdentity(AZLE_UPLOADER_IDENTITY_NAME);
+    }
+
+    const principal = getPrincipal(AZLE_UPLOADER_IDENTITY_NAME);
+
+    addController(canisterName, AZLE_UPLOADER_IDENTITY_NAME, principal);
+
+    return AZLE_UPLOADER_IDENTITY_NAME;
+}

package/src/experimental/build/index.ts

@@ -0,0 +1,61 @@
+import '#experimental/build/assert_experimental';
+
+import { IOType } from 'child_process';
+
+import { build as stableBuild } from '#build/index';
+import { runCommand as runBuildCommand } from '#experimental/commands/build/index';
+import { runCommand as runUploadAssetsCommand } from '#experimental/commands/upload_assets/index';
+import { getCanisterConfig } from '#utils/get_canister_config';
+import { Command, ExperimentalCommand } from '#utils/types';
+
+export async function build(): Promise<void> {
+    const command = process.argv[2] as
+        | Command
+        | ExperimentalCommand
+        | undefined;
+
+    if (command === undefined) {
+        throw new Error(
+            `No command found when running azle. Running azle should start like this: azle [commandName]`
+        );
+    }
+
+    const ioType = process.env.AZLE_VERBOSE === 'true' ? 'inherit' : 'pipe';
+
+    if (command === 'post-install') {
+        await handlePostInstallCommand();
+
+        return;
+    }
+
+    if (command === 'upload-assets') {
+        await handleUploadAssetsCommand();
+
+        return;
+    }
+
+    if (command === 'build') {
+        await handleBuildCommand(ioType);
+
+        return;
+    }
+
+    stableBuild();
+}
+
+async function handlePostInstallCommand(): Promise<void> {
+    // Currently, the only post-install step is uploading assets.
+    // This function can be expanded if more steps are needed in the future.
+    await runUploadAssetsCommand();
+}
+
+async function handleUploadAssetsCommand(): Promise<void> {
+    await runUploadAssetsCommand();
+}
+
+async function handleBuildCommand(ioType: IOType): Promise<void> {
+    const canisterName = process.argv[3];
+    const canisterConfig = await getCanisterConfig(canisterName);
+
+    await runBuildCommand(canisterName, canisterConfig, ioType);
+}

package/src/experimental/build/utils/dfx.ts

@@ -0,0 +1,212 @@
+import '#experimental/build/assert_experimental';
+
+import { HttpAgent } from '@dfinity/agent';
+import { Secp256k1KeyIdentity } from '@dfinity/identity-secp256k1';
+import { execSync } from 'child_process';
+
+import { getCanisterId } from '#utils/dfx';
+import { getDfxRoot } from '#utils/dfx_root';
+
+type StorageMode = 'keyring' | 'password-protected' | 'plaintext';
+
+export function addController(
+    canisterName: string,
+    identityName: string,
+    principal: string
+): Buffer {
+    const currentIdentity = whoami();
+    console.info();
+    console.info(`Adding ${identityName} as a controller for ${canisterName}`);
+    console.info(`You may be prompted for ${currentIdentity}'s password`);
+    console.info();
+    const cmd = `dfx canister update-settings ${canisterName} --add-controller ${principal}`;
+    return execSync(cmd, {
+        cwd: getDfxRoot(),
+        stdio: ['inherit', 'pipe', 'inherit'] // TODO I would prefer it to pipe the stderr but pipe will cause this command to fail immediately
+    });
+}
+
+export async function createAnonymousAgent(): Promise<void> {
+    const agent = new HttpAgent({
+        host: getAgentHost()
+    });
+
+    if (process.env.DFX_NETWORK !== 'ic') {
+        await agent.fetchRootKey();
+    }
+}
+
+/**
+ * Returns an agent authenticated with the identity with the given name.
+ *
+ * Generates a new identity with the given name if shouldGenerateIdentity and
+ * the identity doesn't already exist.
+ *
+ * @param identityName
+ * @returns
+ */
+export async function createAuthenticatedAgent(
+    identityName: string,
+    shouldGenerateIdentity: boolean = false
+): Promise<HttpAgent> {
+    if (shouldGenerateIdentity && !identityExists(identityName)) {
+        generateIdentity(identityName);
+    }
+
+    const agent = new HttpAgent({
+        host: getAgentHost(),
+        identity: getSecp256k1KeyIdentity(identityName)
+    });
+
+    if (process.env.DFX_NETWORK !== 'ic') {
+        await agent.fetchRootKey();
+    }
+
+    return agent;
+}
+
+/**
+ * Returns an agent authenticated with the identity with the given name.
+ *
+ * Generates a new identity with the given name if shouldGenerateIdentity and
+ * the identity doesn't already exist.
+ *
+ * IMPORTANT: In order to be synchronous this call will not fetch the root key.
+ * If you are not on mainnet you will need to fetch the root key separately.
+ *
+ * @param identityName
+ * @returns
+ */
+export function createAuthenticatedAgentSync(
+    identityName: string,
+    shouldGenerateIdentity: boolean = false
+): HttpAgent {
+    if (shouldGenerateIdentity && !identityExists(identityName)) {
+        generateIdentity(identityName);
+    }
+
+    const agent = new HttpAgent({
+        host: getAgentHost(),
+        identity: getSecp256k1KeyIdentity(identityName)
+    });
+
+    return agent;
+}
+
+export function generateIdentity(name: string): Buffer {
+    console.info();
+    console.info(`Generating identity ${name}`);
+    const storageMode = determineStorageMode();
+    if (storageMode === undefined) {
+        console.info(`You may be prompted to create a password for ${name}`);
+        console.info();
+        return execSync(`dfx identity new ${name}`, {
+            cwd: getDfxRoot(),
+            stdio: ['inherit', 'pipe', 'inherit'] // TODO I would prefer it to pipe the stderr but pipe will cause this command to fail immediately
+        });
+    }
+    if (storageMode === 'password-protected') {
+        console.info(`You will be prompted to create a password for ${name}`);
+        console.info();
+    }
+    return execSync(`dfx identity new ${name} --storage-mode ${storageMode}`, {
+        cwd: getDfxRoot(),
+        stdio: ['inherit', 'pipe', 'inherit'] // TODO I would prefer it to pipe the stderr but pipe will cause this command to fail immediately
+    });
+}
+
+export function getAgentHost(): string {
+    return process.env.DFX_NETWORK === 'ic'
+        ? `https://icp-api.io`
+        : `http://127.0.0.1:${getWebServerPort()}`;
+}
+
+export function getCanisterOrigin(canisterName: string): string {
+    return `http://${getCanisterId(
+        canisterName
+    )}.raw.localhost:${getWebServerPort()}`;
+}
+
+export function getIdentities(): string[] {
+    const list = execSync(`dfx identity list`, {
+        cwd: getDfxRoot(),
+        encoding: 'utf-8'
+    }).trim();
+    const identities = list.split('\n');
+
+    return identities;
+}
+
+export function getPemKey(identityName: string): string {
+    console.info();
+    console.info(`Exporting PEM key for ${identityName}`);
+    console.info(`You may be prompted for ${identityName}'s password`);
+    console.info();
+    const cmd = `dfx identity export ${identityName}`;
+    const result = execSync(cmd, {
+        cwd: getDfxRoot(),
+        encoding: 'utf-8',
+        stdio: ['inherit', 'pipe', 'inherit'] // TODO I would prefer it to pipe the stderr but pipe will cause this command to fail immediately
+    }).trim();
+    return result;
+}
+
+export function getPrincipal(identityName: string): string {
+    console.info();
+    console.info(`Getting principal for ${identityName}`);
+    console.info(`You may be prompted for ${identityName}'s password`);
+    console.info();
+    const cmd = `dfx identity get-principal --identity ${identityName}`;
+    return execSync(cmd, {
+        cwd: getDfxRoot(),
+        encoding: 'utf-8',
+        stdio: ['inherit', 'pipe', 'inherit'] // TODO I would prefer it to pipe the stderr but pipe will cause this command to fail immediately
+    }).trim();
+}
+
+export function getSecp256k1KeyIdentity(
+    identityName: string
+): Secp256k1KeyIdentity {
+    return Secp256k1KeyIdentity.fromPem(getPemKey(identityName));
+}
+
+export function getWebServerPort(): string {
+    return execSync(`dfx info webserver-port`, {
+        cwd: getDfxRoot(),
+        encoding: 'utf-8'
+    }).trim();
+}
+
+export function identityExists(name: string): boolean {
+    const identities = getIdentities();
+    return identities.includes(name);
+}
+
+export function whoami(): string {
+    return execSync(`dfx identity whoami`, {
+        cwd: getDfxRoot(),
+        encoding: 'utf-8'
+    }).trim();
+}
+
+export function whoamiPrincipal(): string {
+    return execSync(`dfx identity get-principal`, {
+        cwd: getDfxRoot(),
+        encoding: 'utf-8'
+    }).trim();
+}
+
+function determineStorageMode(): StorageMode | undefined {
+    const mode = process.env.AZLE_IDENTITY_STORAGE_MODE;
+    if (
+        mode !== 'plaintext' &&
+        mode !== 'keyring' &&
+        mode !== 'password-protected' &&
+        mode !== undefined
+    ) {
+        throw new Error(
+            `AZLE_IDENTITY_STORAGE_MODE must be 'plaintext', 'keyring', 'password-protected'`
+        );
+    }
+    return mode;
+}

package/src/experimental/build/utils/filter_async.ts

@@ -0,0 +1,14 @@
+import '#experimental/build/assert_experimental';
+
+export async function filterAsync<T>(
+    elements: T[],
+    callback: (element: T) => Promise<boolean>
+): Promise<T[]> {
+    return await elements.reduce(async (accPromise: Promise<T[]>, element) => {
+        const acc = await accPromise;
+        if (await callback(element)) {
+            return [...acc, element];
+        }
+        return acc;
+    }, Promise.resolve([]));
+}

package/src/experimental/build/utils/find_project_root.ts

@@ -0,0 +1,67 @@
+import '#experimental/build/assert_experimental';
+
+import { existsSync } from 'fs';
+import { dirname, join, parse } from 'path';
+
+/**
+ * Finds the project root directory (the directory containing the package.json file)
+ * by checking common environment variables and searching parent directories.
+ * @returns The absolute path to the project root directory.
+ * @throws Throws an error if a `package.json` file cannot be found using any of the attempted methods.
+ *
+ * @remarks
+ * The function attempts to locate the project root using the following methods in order:
+ * 1. If the `npm_config_prefix` environment variable is set, it searches upwards from that path for the nearest directory containing `package.json`.
+ * 2. If the `npm_package_json` environment variable is set and points to an existing `package.json` file, it returns the directory containing that file.
+ * 3. It searches upwards from the current working directory (`process.cwd()`) for the nearest directory containing `package.json`.
+ *
+ * If none of these methods succeed in finding a `package.json` file, the function throws an error indicating that it could not determine the project root.
+ */
+export async function findProjectRoot(): Promise<string> {
+    // Method 1: Check if the script was called with an explicit prefix
+    if (process.env.npm_config_prefix !== undefined) {
+        const prefixPath = process.env.npm_config_prefix;
+
+        const projectRoot = findNearestProjectRoot(prefixPath);
+
+        if (projectRoot !== undefined) {
+            return projectRoot;
+        }
+    }
+
+    // Method 2: Try to use npm_package_json which points to package.json
+    if (process.env.npm_package_json !== undefined) {
+        if (existsSync(process.env.npm_package_json)) {
+            return dirname(process.env.npm_package_json);
+        }
+    }
+
+    // Method 3: Look for package.json in parent directories
+    const projectRoot = findNearestProjectRoot(process.cwd());
+
+    if (projectRoot !== undefined) {
+        return projectRoot;
+    }
+
+    throw new Error(
+        `Could not find a package.json file in the current directory or any parent directory.\nPlease ensure you are running this command from within a valid Azle project.`
+    );
+}
+
+/**
+ * Finds the nearest directory containing a package.json file by traversing up the directory tree.
+ * @param startPath - Path to start the search from (file or directory)
+ * @returns The directory containing package.json, or undefined if not found
+ */
+function findNearestProjectRoot(startPath: string): string | undefined {
+    if (existsSync(join(startPath, 'package.json'))) {
+        return startPath;
+    }
+
+    const rootDir = parse(startPath).root;
+    if (startPath === rootDir) {
+        return undefined;
+    }
+
+    return findNearestProjectRoot(dirname(startPath));
+}

package/src/experimental/build/utils/global_paths.ts

@@ -0,0 +1,8 @@
+import '#experimental/build/assert_experimental';
+
+import { createRequire } from 'module';
+import { dirname } from 'path';
+
+export const WASMEDGE_QUICKJS_PATH = dirname(
+    createRequire(import.meta.url).resolve('wasmedge_quickjs/package.json')
+);