@ohos-ports/azle 0.33.0-beta.0

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/init_and_post_upgrade.rs

@@ -0,0 +1,212 @@
+use core::time::Duration;
+use std::error::Error;
+
+use candid::Principal;
+use ic_cdk::{call::Call, trap};
+use ic_cdk_timers::set_timer;
+use ic_stable_structures::memory_manager::MemoryId;
+use wasmedge_quickjs::AsObject;
+
+use crate::{
+    MEMORY_MANAGER_REF_CELL, RUNTIME, WASM_DATA_REF_CELL, execute_method_js, ic,
+    ic::{drain_microtasks, rand_seed::rand_seed},
+    wasm_binary_manipulation::{WasmData, get_js_code, get_wasm_data},
+};
+
+use crate::{upload_file, web_assembly};
+
+#[inline(never)]
+#[unsafe(no_mangle)]
+pub extern "C" fn init(function_index: i32) {
+    // Without something like this the init and post_upgrade functions
+    // seem to be optimized into the same function in the Wasm binary
+    // This causes problems during Wasm binary manipulation
+    let _ = format!("prevent init and post_upgrade optimization");
+
+    initialize(true, function_index);
+
+    upload_file::init_hashes().unwrap();
+}
+
+#[inline(never)]
+#[unsafe(no_mangle)]
+pub extern "C" fn post_upgrade(function_index: i32) {
+    initialize(false, function_index);
+}
+
+fn initialize(init: bool, function_index: i32) {
+    std::panic::set_hook(Box::new(|panic_info| {
+        let msg = if let Some(s) = panic_info.payload().downcast_ref::<&str>() {
+            *s
+        } else if let Some(s) = panic_info.payload().downcast_ref::<String>() {
+            s.as_str()
+        } else {
+            "Unknown panic message"
+        };
+
+        let location = if let Some(location) = panic_info.location() {
+            format!(" at {}:{}", location.file(), location.line())
+        } else {
+            " (unknown location)".to_string()
+        };
+
+        let message = &format!("Panic occurred: {}{}", msg, location);
+
+        ic_cdk::println!("{}", message);
+
+        ic_cdk::trap(message);
+    }));
+
+    let wasm_data = get_wasm_data();
+
+    WASM_DATA_REF_CELL.with(|wasm_data_ref_cell| {
+        *wasm_data_ref_cell.borrow_mut() = Some(wasm_data.clone());
+    });
+
+    let env_vars: Vec<(&str, &str)> = wasm_data
+        .env_vars
+        .iter()
+        .map(|(key, value)| (key.as_str(), value.as_str()))
+        .collect();
+
+    let polyfill_memory =
+        MEMORY_MANAGER_REF_CELL.with(|manager| manager.borrow().get(MemoryId::new(254)));
+    ic_wasi_polyfill::init_with_memory(&[], &env_vars, polyfill_memory);
+
+    std::fs::write("/candid/icp/management.did", &wasm_data.management_did).unwrap();
+
+    let js = get_js_code();
+
+    initialize_js(
+        &wasm_data,
+        std::str::from_utf8(&js).unwrap(),
+        init,
+        function_index,
+    );
+
+    seed_from_raw_rand();
+}
+
+pub fn initialize_js(wasm_data: &WasmData, js: &str, init: bool, function_index: i32) {
+    let mut rt = wasmedge_quickjs::Runtime::new();
+
+    rt.run_with_context(|context| {
+        ic::register(context);
+
+        web_assembly::register(context);
+
+        let mut env = context.new_object();
+
+        for (key, value) in std::env::vars() {
+            env.set(&key, context.new_string(&value).into());
+        }
+
+        let mut process = context.new_object();
+
+        process.set("env", env.into());
+
+        context.get_global().set("process", process.into());
+
+        context
+            .get_global()
+            .set("_azleActions", context.new_array().into());
+
+        context.get_global().set(
+            "_azleNodejsWasmEnvironment",
+            wasmedge_quickjs::JsValue::Bool(false),
+        );
+
+        context
+            .get_global()
+            .set("_azleCanisterMethodNames", context.new_object().into());
+
+        context.get_global().set(
+            "_azleIcpReplicaWasmEnvironment",
+            wasmedge_quickjs::JsValue::Bool(true),
+        );
+
+        context
+            .get_global()
+            .set("_azleTimerCallbacks", context.new_object().into());
+
+        context
+            .get_global()
+            .set("_azleRejectCallbacks", context.new_object().into());
+
+        context
+            .get_global()
+            .set("_azleResolveCallbacks", context.new_object().into());
+
+        // TODO what do we do if there is an error in here?
+        context.eval_global_str("globalThis.exports = {};".to_string());
+        context.eval_global_str(format!("globalThis._azleExperimental = true;"));
+
+        context.eval_module_str(js.to_string(), &wasm_data.main_js_path);
+
+        drain_microtasks(context);
+
+        // let temp = context.eval_module_str(std::str::from_utf8(MAIN_JS).unwrap().to_string(), "main");
+
+        // match &temp {
+        //     wasmedge_quickjs::JsValue::Exception(js_exception) => {
+        //         js_exception.dump_error();
+        //         panic!("we had an error");
+        //     },
+        //     _ => {}
+        // };
+
+        // ic_cdk::println!("temp: {:#?}", temp);
+    });
+
+    RUNTIME.with(|runtime| {
+        let mut runtime = runtime.borrow_mut();
+        *runtime = Some(rt);
+    });
+
+    if function_index != -1 {
+        execute_method_js::execute_method_js(function_index);
+    }
+
+    // _azleInitCalled and _azlePostUpgradeCalled refer to Azle's own init/post_upgrade methods being called
+    // these variables do not indicate if the developer's own init/post_upgrade methods were called
+    RUNTIME.with(|runtime| {
+        let mut runtime = runtime.borrow_mut();
+        let runtime = runtime.as_mut().unwrap();
+
+        runtime.run_with_context(|context| {
+            let assignment = if init {
+                "globalThis._azleInitCalled = true;\nglobalThis._azlePostUpgradeCalled = false;"
+            } else {
+                "globalThis._azleInitCalled = false;\nglobalThis._azlePostUpgradeCalled = true;"
+            };
+
+            context.eval_global_str(assignment.to_string());
+        });
+    });
+}
+
+fn seed_from_raw_rand() {
+    set_timer(Duration::new(0, 0), || {
+        ic_cdk::futures::spawn(async {
+            let result: Result<(), Box<dyn Error>> = async {
+                let randomness: Vec<u8> =
+                    Call::unbounded_wait(Principal::management_canister(), "raw_rand")
+                        .await?
+                        .candid()?;
+
+                rand_seed(
+                    randomness
+                        .try_into()
+                        .map_err(|_| "seed must be exactly 32 bytes in length")?,
+                );
+
+                Ok(())
+            }
+            .await;
+
+            if let Err(e) = result {
+                trap(&format!("Azle SeedFromRawRandError: {}", e));
+            }
+        });
+    });
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/internal_canister_methods.rs

@@ -0,0 +1,142 @@
+use wasmedge_quickjs::JsValue;
+
+use crate::{
+    PEAK_ALLOC, RUNTIME,
+    autoreload::reload_js,
+    benchmarking::{BENCHMARKS_REF_CELL, BenchmarkEntry},
+    guards::guard_against_non_controllers,
+    upload_file::{get_file_hash, reset_for_new_upload, upload_file_chunk},
+};
+
+#[ic_cdk::update]
+pub fn _azle_chunk() {}
+
+#[ic_cdk::update(guard = "guard_against_non_controllers")]
+fn _azle_reload_js(
+    timestamp: u64,
+    chunk_number: u64,
+    js_bytes: Vec<u8>,
+    total_len: u64,
+    function_index: i32,
+) {
+    reload_js(timestamp, chunk_number, js_bytes, total_len, function_index);
+}
+
+#[ic_cdk::update(guard = "guard_against_non_controllers")]
+pub async fn _azle_upload_file_chunk(
+    dest_path: String,
+    timestamp: u64,
+    start_index: u64,
+    file_bytes: Vec<u8>,
+    total_file_len: u64,
+) {
+    upload_file_chunk(
+        dest_path,
+        timestamp,
+        start_index,
+        file_bytes,
+        total_file_len,
+    )
+    .await
+}
+
+#[ic_cdk::update(guard = "guard_against_non_controllers")]
+pub fn _azle_clear_file_and_info(path: String) {
+    reset_for_new_upload(&path, 0).unwrap()
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+pub fn _azle_get_file_hash(path: String) -> Option<String> {
+    get_file_hash(path)
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+pub fn _azle_get_benchmarks() -> Vec<BenchmarkEntry> {
+    BENCHMARKS_REF_CELL.with(|benchmarks_ref_cell| benchmarks_ref_cell.borrow().clone())
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_reject_callbacks_len() -> i32 {
+    RUNTIME.with(|runtime| {
+        let mut runtime = runtime.borrow_mut();
+        let runtime = runtime.as_mut().unwrap();
+
+        runtime.run_with_context(|context| {
+            match context
+                .eval_global_str("Object.keys(globalThis._azleRejectCallbacks).length".to_string())
+            {
+                JsValue::Int(length) => length,
+                _ => panic!("Could not convert JsValue to i32"),
+            }
+        })
+    })
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_resolve_callbacks_len() -> i32 {
+    RUNTIME.with(|runtime| {
+        let mut runtime = runtime.borrow_mut();
+        let runtime = runtime.as_mut().unwrap();
+
+        runtime.run_with_context(|context| {
+            match context
+                .eval_global_str("Object.keys(globalThis._azleResolveCallbacks).length".to_string())
+            {
+                JsValue::Int(length) => length,
+                _ => panic!("Could not convert JsValue to i32"),
+            }
+        })
+    })
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_timer_callbacks_len() -> i32 {
+    RUNTIME.with(|runtime| {
+        let mut runtime = runtime.borrow_mut();
+        let runtime = runtime.as_mut().unwrap();
+
+        runtime.run_with_context(|context| {
+            match context
+                .eval_global_str("Object.keys(globalThis._azleTimerCallbacks).length".to_string())
+            {
+                JsValue::Int(length) => length,
+                _ => panic!("Could not convert JsValue to i32"),
+            }
+        })
+    })
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_actions_len() -> i32 {
+    RUNTIME.with(|runtime| {
+        let mut runtime = runtime.borrow_mut();
+        let runtime = runtime.as_mut().unwrap();
+
+        runtime.run_with_context(|context| {
+            match context.eval_global_str("globalThis._azleActions.length".to_string()) {
+                JsValue::Int(length) => length,
+                _ => panic!("Could not convert JsValue to i32"),
+            }
+        })
+    })
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_inter_canister_call_futures_len() -> u32 {
+    // Hard-coded to 0 because we don't use the inter-canister call futures queue in experimental mode
+    0
+}
+
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_is_job_queue_empty() -> bool {
+    // Hard-coded to true because wasmedge-quickjs doesn't seem to expose an ideal
+    // way to get to the length of the job queue, and we would like to
+    // remove wasmedge-quickjs soon after 1.0 is released
+    true
+}
+
+/// Returns the current size of the heap in bytes to aid in testing for memory leaks and other global state issues.
+#[ic_cdk::query(guard = "guard_against_non_controllers")]
+fn _azle_heap_allocation() -> u32 {
+    PEAK_ALLOC.current_usage() as u32
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/lib.rs

@@ -0,0 +1,35 @@
+use std::cell::RefCell;
+
+use ic_stable_structures::{
+    DefaultMemoryImpl,
+    memory_manager::{MemoryManager, VirtualMemory},
+};
+use peak_alloc::PeakAlloc;
+use rand::{SeedableRng, rngs::StdRng};
+
+mod autoreload;
+mod benchmarking;
+mod candid;
+mod chunk;
+mod execute_method_js;
+mod guards;
+mod ic;
+mod init_and_post_upgrade;
+mod internal_canister_methods;
+mod stable_b_tree_map;
+mod upload_file;
+mod wasm_binary_manipulation;
+mod web_assembly;
+
+#[global_allocator]
+static PEAK_ALLOC: PeakAlloc = PeakAlloc;
+
+#[allow(unused)]
+type Memory = VirtualMemory<DefaultMemoryImpl>;
+
+thread_local! {
+    static CSPRNG: RefCell<StdRng> = RefCell::new(StdRng::from_seed([0;32]));
+    static MEMORY_MANAGER_REF_CELL: RefCell<MemoryManager<DefaultMemoryImpl>> = RefCell::new(MemoryManager::init(DefaultMemoryImpl::default()));
+    static RUNTIME: RefCell<Option<wasmedge_quickjs::Runtime>> = RefCell::new(None);
+    static WASM_DATA_REF_CELL: RefCell<Option<wasm_binary_manipulation::WasmData>> = RefCell::new(None);
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/stable_b_tree_map.rs

@@ -0,0 +1,50 @@
+use std::{cell::RefCell, collections::BTreeMap};
+
+use ic_stable_structures::{storable::Bound, StableBTreeMap, Storable};
+
+use crate::Memory;
+
+#[allow(unused)]
+type AzleStableBTreeMap = StableBTreeMap<AzleStableBTreeMapKey, AzleStableBTreeMapValue, Memory>;
+
+thread_local! {
+    pub static STABLE_B_TREE_MAPS: RefCell<BTreeMap<u8, AzleStableBTreeMap>> = RefCell::new(BTreeMap::new());
+}
+
+#[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
+pub struct AzleStableBTreeMapKey {
+    pub bytes: Vec<u8>,
+}
+
+impl Storable for AzleStableBTreeMapKey {
+    fn to_bytes(&self) -> std::borrow::Cow<[u8]> {
+        std::borrow::Cow::Borrowed(&self.bytes)
+    }
+
+    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
+        AzleStableBTreeMapKey {
+            bytes: bytes.to_vec(),
+        }
+    }
+
+    const BOUND: Bound = Bound::Unbounded;
+}
+
+#[derive(Ord, PartialOrd, Eq, PartialEq, Clone)]
+pub struct AzleStableBTreeMapValue {
+    pub bytes: Vec<u8>,
+}
+
+impl Storable for AzleStableBTreeMapValue {
+    fn to_bytes(&self) -> std::borrow::Cow<[u8]> {
+        std::borrow::Cow::Borrowed(&self.bytes)
+    }
+
+    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
+        AzleStableBTreeMapValue {
+            bytes: bytes.to_vec(),
+        }
+    }
+
+    const BOUND: Bound = Bound::Unbounded;
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/upload_file/bytes_to_human_readable.rs

@@ -0,0 +1,20 @@
+pub fn bytes_to_human_readable(size_in_bytes: u64) -> String {
+    let suffixes = ["B", "KiB", "MiB", "GiB"];
+    let size = size_in_bytes as f64;
+
+    let result = suffixes.iter().fold(
+        (size, suffixes[0], false),
+        |(remaining_size, selected_suffix, done), suffix| {
+            if done {
+                return (remaining_size, selected_suffix, done);
+            }
+            if remaining_size < 1024.0 {
+                (remaining_size, suffix, true)
+            } else {
+                (remaining_size / 1024.0, suffix, false)
+            }
+        },
+    );
+
+    format!("{:.2} {}", result.0, result.1)
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/upload_file/file_info.rs

@@ -0,0 +1,60 @@
+use crate::upload_file::{Timestamp, FILE_INFO};
+
+pub fn initialize_file_info(path: &str, timestamp: Timestamp) {
+    FILE_INFO.with(|file_info| {
+        let mut file_info_mut = file_info.borrow_mut();
+
+        file_info_mut.insert(path.to_string(), (timestamp, 0, None, 0));
+    });
+}
+
+pub fn get_partial_file_hash(path: &str) -> Option<Vec<u8>> {
+    FILE_INFO
+        .with(|file_info| Some(file_info.borrow().get(path)?.2.clone()))
+        .flatten()
+}
+
+pub fn set_partial_hash(path: &str, hash: Vec<u8>) {
+    FILE_INFO.with(|file_hashes| {
+        if let Some(entry) = file_hashes.borrow_mut().get_mut(path) {
+            entry.2 = Some(hash);
+        } else {
+            panic!("Couldn't find file info for {}", path)
+        }
+    });
+}
+
+pub fn clear_file_info(path: &str) {
+    FILE_INFO.with(|file_info| file_info.borrow_mut().remove(path));
+}
+
+pub fn set_bytes_hashed(path: &str, bytes_hashed: u64) {
+    FILE_INFO.with(|file_info| {
+        let mut file_info_mut = file_info.borrow_mut();
+        if let Some(file_info_entry) = file_info_mut.get_mut(path) {
+            file_info_entry.3 = bytes_hashed;
+        }
+    });
+}
+
+pub fn set_bytes_received(dest_path: &str, bytes_in_chunk: usize) -> u64 {
+    FILE_INFO.with(|total_bytes_received| {
+        let mut total_bytes_received_mut = total_bytes_received.borrow_mut();
+        match total_bytes_received_mut.get_mut(dest_path) {
+            Some((_, total_bytes, _, _)) => {
+                *total_bytes += bytes_in_chunk as u64;
+                *total_bytes
+            }
+            None => panic!("Couldn't find file info for {}", dest_path),
+        }
+    })
+}
+
+pub fn get_timestamp(path: &str) -> Timestamp {
+    FILE_INFO.with(|uploaded_file_timestamps_map| {
+        match uploaded_file_timestamps_map.borrow().get(path) {
+            Some((timestamp, _, _, _)) => timestamp.clone(),
+            None => 0,
+        }
+    })
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/upload_file/hash.rs

@@ -0,0 +1,111 @@
+use std::collections::HashMap;
+
+use crate::chunk;
+use crate::upload_file::bytes_to_human_readable;
+use crate::upload_file::file_info;
+
+pub fn init_hashes() -> Result<(), std::io::Error> {
+    save_hashes(&HashMap::new())
+}
+
+pub fn get_file_hash(path: String) -> Option<String> {
+    Some(
+        load_hashes()
+            .unwrap()
+            .get(&path)?
+            .iter()
+            .map(|bytes| format!("{:02x}", bytes))
+            .collect(),
+    )
+}
+
+pub async fn hash_file(path: &str) {
+    clear_file_hash(path);
+
+    let file_length = std::fs::metadata(path).unwrap().len();
+    let limit = 75 * 1024 * 1024; // This limit will be determine by how much hashing an update method can do without running out of cycles. It runs out somewhere between 75 and 80
+                                  // This limit must be the same as on the node side or else the hashes will not match
+    let mut file = std::fs::File::open(path).unwrap();
+
+    let mut position = 0;
+
+    loop {
+        let percentage = position / file_length.max(1) * 100;
+        ic_cdk::println!(
+            "Hashing: {} | {}/{} : {:.2}%",
+            path,
+            bytes_to_human_readable::bytes_to_human_readable(position),
+            bytes_to_human_readable::bytes_to_human_readable(file_length),
+            percentage
+        );
+
+        std::io::Seek::seek(&mut file, std::io::SeekFrom::Start(position)).unwrap();
+
+        let mut buffer = vec![0; limit];
+        let bytes_read = std::io::Read::read(&mut file, &mut buffer).unwrap();
+
+        let previous_hash = file_info::get_partial_file_hash(path);
+
+        if bytes_read != 0 {
+            let new_hash = hash_chunk_with(&buffer, previous_hash.as_ref());
+            file_info::set_partial_hash(path, new_hash);
+            file_info::set_bytes_hashed(path, position + bytes_read as u64);
+            position += bytes_read as u64;
+            chunk::chunk().await;
+            continue;
+        } else {
+            // No more bytes to hash, set as final hash for this file
+            let final_hash = match previous_hash {
+                Some(hash) => hash,
+                None => {
+                    let empty_file_hash = hash_chunk_with(&[], None);
+                    empty_file_hash
+                }
+            };
+            chunk::chunk().await;
+            set_file_hash(path, final_hash);
+            file_info::clear_file_info(path);
+            break;
+        }
+    }
+}
+
+fn clear_file_hash(path: &str) {
+    let mut file_hashes = load_hashes().unwrap();
+    file_hashes.remove(path);
+    save_hashes(&file_hashes).unwrap();
+}
+
+fn set_file_hash(path: &str, hash: Vec<u8>) {
+    let mut file_hashes = load_hashes().unwrap();
+    file_hashes.insert(path.to_string(), hash);
+    save_hashes(&file_hashes).unwrap();
+}
+
+fn get_file_hash_path() -> std::path::PathBuf {
+    std::path::Path::new(".config")
+        .join("azle")
+        .join("file_hashes.json")
+}
+
+fn hash_chunk_with(data: &[u8], previous_hash: Option<&Vec<u8>>) -> Vec<u8> {
+    let mut h: sha2::Sha256 = sha2::Digest::new();
+    sha2::Digest::update(&mut h, data);
+    if let Some(hash) = previous_hash {
+        sha2::Digest::update(&mut h, hash);
+    }
+    sha2::Digest::finalize(h).to_vec()
+}
+
+fn load_hashes() -> Result<HashMap<String, Vec<u8>>, std::io::Error> {
+    let file_hash_path = get_file_hash_path();
+    let buffer = std::fs::read(file_hash_path)?;
+
+    Ok(serde_json::from_slice(&buffer)?)
+}
+
+fn save_hashes(file_hashes: &HashMap<String, Vec<u8>>) -> Result<(), std::io::Error> {
+    let data = serde_json::to_vec(file_hashes)?;
+
+    std::fs::write(get_file_hash_path(), data)
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/upload_file/mod.rs

@@ -0,0 +1,20 @@
+use std::{cell::RefCell, collections::BTreeMap};
+
+mod bytes_to_human_readable;
+mod file_info;
+mod hash;
+mod reset;
+mod upload_file_chunk;
+
+pub use hash::{get_file_hash, init_hashes};
+pub use reset::reset_for_new_upload;
+pub use upload_file_chunk::upload_file_chunk;
+
+type BytesHashed = u64;
+type BytesReceived = u64;
+type Hash = Option<Vec<u8>>;
+type Timestamp = u64;
+
+thread_local! {
+    static FILE_INFO: RefCell<BTreeMap<String, (Timestamp, BytesReceived, Hash, BytesHashed)>> = RefCell::new(BTreeMap::new());
+}

package/src/experimental/build/commands/build/wasm_binary/rust/experimental_canister_template/src/upload_file/reset.rs

@@ -0,0 +1,15 @@
+use crate::upload_file::file_info;
+use crate::upload_file::Timestamp;
+
+pub fn reset_for_new_upload(path: &str, timestamp: Timestamp) -> std::io::Result<()> {
+    delete_if_exists(path)?;
+    file_info::initialize_file_info(path, timestamp);
+    Ok(())
+}
+
+fn delete_if_exists(path: &str) -> std::io::Result<()> {
+    if std::fs::metadata(path).is_ok() {
+        std::fs::remove_file(path)?;
+    }
+    Ok(())
+}