wasmtime 20.0.2 → 21.0.1

data/ext/cargo-vendor/cranelift-codegen-0.108.1/src/context.rs

@@ -0,0 +1,395 @@
+//! Cranelift compilation context and main entry point.
+//!
+//! When compiling many small functions, it is important to avoid repeatedly allocating and
+//! deallocating the data structures needed for compilation. The `Context` struct is used to hold
+//! on to memory allocations between function compilations.
+//!
+//! The context does not hold a `TargetIsa` instance which has to be provided as an argument
+//! instead. This is because an ISA instance is immutable and can be used by multiple compilation
+//! contexts concurrently. Typically, you would have one context per compilation thread and only a
+//! single ISA instance.
+
+use crate::alias_analysis::AliasAnalysis;
+use crate::dce::do_dce;
+use crate::dominator_tree::DominatorTree;
+use crate::egraph::EgraphPass;
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::Function;
+use crate::isa::TargetIsa;
+use crate::legalizer::simple_legalize;
+use crate::loop_analysis::LoopAnalysis;
+use crate::machinst::{CompiledCode, CompiledCodeStencil};
+use crate::nan_canonicalization::do_nan_canonicalization;
+use crate::remove_constant_phis::do_remove_constant_phis;
+use crate::result::{CodegenResult, CompileResult};
+use crate::settings::{FlagsOrIsa, OptLevel};
+use crate::trace;
+use crate::unreachable_code::eliminate_unreachable_code;
+use crate::verifier::{verify_context, VerifierErrors, VerifierResult};
+use crate::{timing, CompileError};
+#[cfg(feature = "souper-harvest")]
+use alloc::string::String;
+use alloc::vec::Vec;
+use cranelift_control::ControlPlane;
+use target_lexicon::Architecture;
+
+#[cfg(feature = "souper-harvest")]
+use crate::souper_harvest::do_souper_harvest;
+
+/// Persistent data structures and compilation pipeline.
+pub struct Context {
+    /// The function we're compiling.
+    pub func: Function,
+
+    /// The control flow graph of `func`.
+    pub cfg: ControlFlowGraph,
+
+    /// Dominator tree for `func`.
+    pub domtree: DominatorTree,
+
+    /// Loop analysis of `func`.
+    pub loop_analysis: LoopAnalysis,
+
+    /// Result of MachBackend compilation, if computed.
+    pub(crate) compiled_code: Option<CompiledCode>,
+
+    /// Flag: do we want a disassembly with the CompiledCode?
+    pub want_disasm: bool,
+}
+
+impl Context {
+    /// Allocate a new compilation context.
+    ///
+    /// The returned instance should be reused for compiling multiple functions in order to avoid
+    /// needless allocator thrashing.
+    pub fn new() -> Self {
+        Self::for_function(Function::new())
+    }
+
+    /// Allocate a new compilation context with an existing Function.
+    ///
+    /// The returned instance should be reused for compiling multiple functions in order to avoid
+    /// needless allocator thrashing.
+    pub fn for_function(func: Function) -> Self {
+        Self {
+            func,
+            cfg: ControlFlowGraph::new(),
+            domtree: DominatorTree::new(),
+            loop_analysis: LoopAnalysis::new(),
+            compiled_code: None,
+            want_disasm: false,
+        }
+    }
+
+    /// Clear all data structures in this context.
+    pub fn clear(&mut self) {
+        self.func.clear();
+        self.cfg.clear();
+        self.domtree.clear();
+        self.loop_analysis.clear();
+        self.compiled_code = None;
+        self.want_disasm = false;
+    }
+
+    /// Returns the compilation result for this function, available after any `compile` function
+    /// has been called.
+    pub fn compiled_code(&self) -> Option<&CompiledCode> {
+        self.compiled_code.as_ref()
+    }
+
+    /// Set the flag to request a disassembly when compiling with a
+    /// `MachBackend` backend.
+    pub fn set_disasm(&mut self, val: bool) {
+        self.want_disasm = val;
+    }
+
+    /// Compile the function, and emit machine code into a `Vec<u8>`.
+    ///
+    /// Run the function through all the passes necessary to generate
+    /// code for the target ISA represented by `isa`, as well as the
+    /// final step of emitting machine code into a `Vec<u8>`. The
+    /// machine code is not relocated. Instead, any relocations can be
+    /// obtained from `compiled_code()`.
+    ///
+    /// Performs any optimizations that are enabled, unless
+    /// `optimize()` was already invoked.
+    ///
+    /// This function calls `compile`, taking care to resize `mem` as
+    /// needed.
+    ///
+    /// Returns information about the function's code and read-only
+    /// data.
+    pub fn compile_and_emit(
+        &mut self,
+        isa: &dyn TargetIsa,
+        mem: &mut Vec<u8>,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CompileResult<&CompiledCode> {
+        let compiled_code = self.compile(isa, ctrl_plane)?;
+        mem.extend_from_slice(compiled_code.code_buffer());
+        Ok(compiled_code)
+    }
+
+    /// Internally compiles the function into a stencil.
+    ///
+    /// Public only for testing and fuzzing purposes.
+    pub fn compile_stencil(
+        &mut self,
+        isa: &dyn TargetIsa,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CodegenResult<CompiledCodeStencil> {
+        let _tt = timing::compile();
+
+        self.verify_if(isa)?;
+
+        self.optimize(isa, ctrl_plane)?;
+
+        isa.compile_function(&self.func, &self.domtree, self.want_disasm, ctrl_plane)
+    }
+
+    /// Optimize the function, performing all compilation steps up to
+    /// but not including machine-code lowering and register
+    /// allocation.
+    ///
+    /// Public only for testing purposes.
+    pub fn optimize(
+        &mut self,
+        isa: &dyn TargetIsa,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CodegenResult<()> {
+        log::debug!(
+            "Number of CLIF instructions to optimize: {}",
+            self.func.dfg.num_insts()
+        );
+        log::debug!(
+            "Number of CLIF blocks to optimize: {}",
+            self.func.dfg.num_blocks()
+        );
+
+        let opt_level = isa.flags().opt_level();
+        crate::trace!(
+            "Optimizing (opt level {:?}):\n{}",
+            opt_level,
+            self.func.display()
+        );
+
+        self.compute_cfg();
+        if isa.flags().enable_nan_canonicalization() {
+            self.canonicalize_nans(isa)?;
+        }
+
+        self.legalize(isa)?;
+
+        self.compute_domtree();
+        self.eliminate_unreachable_code(isa)?;
+
+        if opt_level != OptLevel::None {
+            self.dce(isa)?;
+        }
+
+        self.remove_constant_phis(isa)?;
+
+        if opt_level != OptLevel::None {
+            self.egraph_pass(isa, ctrl_plane)?;
+        }
+
+        Ok(())
+    }
+
+    /// Compile the function.
+    ///
+    /// Run the function through all the passes necessary to generate code for the target ISA
+    /// represented by `isa`. This does not include the final step of emitting machine code into a
+    /// code sink.
+    ///
+    /// Returns information about the function's code and read-only data.
+    pub fn compile(
+        &mut self,
+        isa: &dyn TargetIsa,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CompileResult<&CompiledCode> {
+        let stencil = self
+            .compile_stencil(isa, ctrl_plane)
+            .map_err(|error| CompileError {
+                inner: error,
+                func: &self.func,
+            })?;
+        Ok(self
+            .compiled_code
+            .insert(stencil.apply_params(&self.func.params)))
+    }
+
+    /// If available, return information about the code layout in the
+    /// final machine code: the offsets (in bytes) of each basic-block
+    /// start, and all basic-block edges.
+    #[deprecated = "use CompiledCode::get_code_bb_layout"]
+    pub fn get_code_bb_layout(&self) -> Option<(Vec<usize>, Vec<(usize, usize)>)> {
+        self.compiled_code().map(CompiledCode::get_code_bb_layout)
+    }
+
+    /// Creates unwind information for the function.
+    ///
+    /// Returns `None` if the function has no unwind information.
+    #[cfg(feature = "unwind")]
+    #[deprecated = "use CompiledCode::create_unwind_info"]
+    pub fn create_unwind_info(
+        &self,
+        isa: &dyn TargetIsa,
+    ) -> CodegenResult<Option<crate::isa::unwind::UnwindInfo>> {
+        self.compiled_code().unwrap().create_unwind_info(isa)
+    }
+
+    /// Run the verifier on the function.
+    ///
+    /// Also check that the dominator tree and control flow graph are consistent with the function.
+    ///
+    /// TODO: rename to "CLIF validate" or similar.
+    pub fn verify<'a, FOI: Into<FlagsOrIsa<'a>>>(&self, fisa: FOI) -> VerifierResult<()> {
+        let mut errors = VerifierErrors::default();
+        let _ = verify_context(&self.func, &self.cfg, &self.domtree, fisa, &mut errors);
+
+        if errors.is_empty() {
+            Ok(())
+        } else {
+            Err(errors)
+        }
+    }
+
+    /// Run the verifier only if the `enable_verifier` setting is true.
+    pub fn verify_if<'a, FOI: Into<FlagsOrIsa<'a>>>(&self, fisa: FOI) -> CodegenResult<()> {
+        let fisa = fisa.into();
+        if fisa.flags.enable_verifier() {
+            self.verify(fisa)?;
+        }
+        Ok(())
+    }
+
+    /// Perform dead-code elimination on the function.
+    pub fn dce<'a, FOI: Into<FlagsOrIsa<'a>>>(&mut self, fisa: FOI) -> CodegenResult<()> {
+        do_dce(&mut self.func, &mut self.domtree);
+        self.verify_if(fisa)?;
+        Ok(())
+    }
+
+    /// Perform constant-phi removal on the function.
+    pub fn remove_constant_phis<'a, FOI: Into<FlagsOrIsa<'a>>>(
+        &mut self,
+        fisa: FOI,
+    ) -> CodegenResult<()> {
+        do_remove_constant_phis(&mut self.func, &mut self.domtree);
+        self.verify_if(fisa)?;
+        Ok(())
+    }
+
+    /// Perform NaN canonicalizing rewrites on the function.
+    pub fn canonicalize_nans(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        // Currently only RiscV64 is the only arch that may not have vector support.
+        let has_vector_support = match isa.triple().architecture {
+            Architecture::Riscv64(_) => match isa.isa_flags().iter().find(|f| f.name == "has_v") {
+                Some(value) => value.as_bool().unwrap_or(false),
+                None => false,
+            },
+            _ => true,
+        };
+        do_nan_canonicalization(&mut self.func, has_vector_support);
+        self.verify_if(isa)
+    }
+
+    /// Run the legalizer for `isa` on the function.
+    pub fn legalize(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
+        // Legalization invalidates the domtree and loop_analysis by mutating the CFG.
+        // TODO: Avoid doing this when legalization doesn't actually mutate the CFG.
+        self.domtree.clear();
+        self.loop_analysis.clear();
+
+        // Run some specific legalizations only.
+        simple_legalize(&mut self.func, &mut self.cfg, isa);
+        self.verify_if(isa)
+    }
+
+    /// Compute the control flow graph.
+    pub fn compute_cfg(&mut self) {
+        self.cfg.compute(&self.func)
+    }
+
+    /// Compute dominator tree.
+    pub fn compute_domtree(&mut self) {
+        self.domtree.compute(&self.func, &self.cfg)
+    }
+
+    /// Compute the loop analysis.
+    pub fn compute_loop_analysis(&mut self) {
+        self.loop_analysis
+            .compute(&self.func, &self.cfg, &self.domtree)
+    }
+
+    /// Compute the control flow graph and dominator tree.
+    pub fn flowgraph(&mut self) {
+        self.compute_cfg();
+        self.compute_domtree()
+    }
+
+    /// Perform unreachable code elimination.
+    pub fn eliminate_unreachable_code<'a, FOI>(&mut self, fisa: FOI) -> CodegenResult<()>
+    where
+        FOI: Into<FlagsOrIsa<'a>>,
+    {
+        eliminate_unreachable_code(&mut self.func, &mut self.cfg, &self.domtree);
+        self.verify_if(fisa)
+    }
+
+    /// Replace all redundant loads with the known values in
+    /// memory. These are loads whose values were already loaded by
+    /// other loads earlier, as well as loads whose values were stored
+    /// by a store instruction to the same instruction (so-called
+    /// "store-to-load forwarding").
+    pub fn replace_redundant_loads(&mut self) -> CodegenResult<()> {
+        let mut analysis = AliasAnalysis::new(&self.func, &self.domtree);
+        analysis.compute_and_update_aliases(&mut self.func);
+        Ok(())
+    }
+
+    /// Harvest candidate left-hand sides for superoptimization with Souper.
+    #[cfg(feature = "souper-harvest")]
+    pub fn souper_harvest(
+        &mut self,
+        out: &mut std::sync::mpsc::Sender<String>,
+    ) -> CodegenResult<()> {
+        do_souper_harvest(&self.func, out);
+        Ok(())
+    }
+
+    /// Run optimizations via the egraph infrastructure.
+    pub fn egraph_pass<'a, FOI>(
+        &mut self,
+        fisa: FOI,
+        ctrl_plane: &mut ControlPlane,
+    ) -> CodegenResult<()>
+    where
+        FOI: Into<FlagsOrIsa<'a>>,
+    {
+        let _tt = timing::egraph();
+
+        trace!(
+            "About to optimize with egraph phase:\n{}",
+            self.func.display()
+        );
+        let fisa = fisa.into();
+        self.compute_loop_analysis();
+        let mut alias_analysis = AliasAnalysis::new(&self.func, &self.domtree);
+        let mut pass = EgraphPass::new(
+            &mut self.func,
+            &self.domtree,
+            &self.loop_analysis,
+            &mut alias_analysis,
+            &fisa.flags,
+            ctrl_plane,
+        );
+        pass.run();
+        log::debug!("egraph stats: {:?}", pass.stats);
+        trace!("pinned_union_count: {}", pass.eclasses.pinned_union_count);
+        trace!("After egraph optimization:\n{}", self.func.display());
+
+        self.verify_if(fisa)
+    }
+}

data/ext/cargo-vendor/cranelift-codegen-0.108.1/src/ctxhash.rs

@@ -0,0 +1,167 @@
+//! A hashmap with "external hashing": nodes are hashed or compared for
+//! equality only with some external context provided on lookup/insert.
+//! This allows very memory-efficient data structures where
+//! node-internal data references some other storage (e.g., offsets into
+//! an array or pool of shared data).
+
+use hashbrown::raw::RawTable;
+use std::hash::{Hash, Hasher};
+
+/// Trait that allows for equality comparison given some external
+/// context.
+///
+/// Note that this trait is implemented by the *context*, rather than
+/// the item type, for somewhat complex lifetime reasons (lack of GATs
+/// to allow `for<'ctx> Ctx<'ctx>`-like associated types in traits on
+/// the value type).
+pub trait CtxEq<V1: ?Sized, V2: ?Sized> {
+    /// Determine whether `a` and `b` are equal, given the context in
+    /// `self` and the union-find data structure `uf`.
+    fn ctx_eq(&self, a: &V1, b: &V2) -> bool;
+}
+
+/// Trait that allows for hashing given some external context.
+pub trait CtxHash<Value: ?Sized>: CtxEq<Value, Value> {
+    /// Compute the hash of `value`, given the context in `self` and
+    /// the union-find data structure `uf`.
+    fn ctx_hash<H: Hasher>(&self, state: &mut H, value: &Value);
+}
+
+/// A null-comparator context type for underlying value types that
+/// already have `Eq` and `Hash`.
+#[derive(Default)]
+pub struct NullCtx;
+
+impl<V: Eq + Hash> CtxEq<V, V> for NullCtx {
+    fn ctx_eq(&self, a: &V, b: &V) -> bool {
+        a.eq(b)
+    }
+}
+impl<V: Eq + Hash> CtxHash<V> for NullCtx {
+    fn ctx_hash<H: Hasher>(&self, state: &mut H, value: &V) {
+        value.hash(state);
+    }
+}
+
+/// A bucket in the hash table.
+///
+/// Some performance-related design notes: we cache the hashcode for
+/// speed, as this often buys a few percent speed in
+/// interning-table-heavy workloads. We only keep the low 32 bits of
+/// the hashcode, for memory efficiency: in common use, `K` and `V`
+/// are often 32 bits also, and a 12-byte bucket is measurably better
+/// than a 16-byte bucket.
+struct BucketData<K, V> {
+    hash: u32,
+    k: K,
+    v: V,
+}
+
+/// A HashMap that takes external context for all operations.
+pub struct CtxHashMap<K, V> {
+    raw: RawTable<BucketData<K, V>>,
+}
+
+impl<K, V> CtxHashMap<K, V> {
+    /// Create an empty hashmap with pre-allocated space for the given
+    /// capacity.
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            raw: RawTable::with_capacity(capacity),
+        }
+    }
+}
+
+fn compute_hash<Ctx, K>(ctx: &Ctx, k: &K) -> u32
+where
+    Ctx: CtxHash<K>,
+{
+    let mut hasher = rustc_hash::FxHasher::default();
+    ctx.ctx_hash(&mut hasher, k);
+    hasher.finish() as u32
+}
+
+impl<K, V> CtxHashMap<K, V> {
+    /// Insert a new key-value pair, returning the old value associated
+    /// with this key (if any).
+    pub fn insert<Ctx>(&mut self, k: K, v: V, ctx: &Ctx) -> Option<V>
+    where
+        Ctx: CtxEq<K, K> + CtxHash<K>,
+    {
+        let hash = compute_hash(ctx, &k);
+        match self.raw.find(hash as u64, |bucket| {
+            hash == bucket.hash && ctx.ctx_eq(&bucket.k, &k)
+        }) {
+            Some(bucket) => {
+                let data = unsafe { bucket.as_mut() };
+                Some(std::mem::replace(&mut data.v, v))
+            }
+            None => {
+                let data = BucketData { hash, k, v };
+                self.raw
+                    .insert_entry(hash as u64, data, |bucket| bucket.hash as u64);
+                None
+            }
+        }
+    }
+
+    /// Look up a key, returning a borrow of the value if present.
+    pub fn get<'a, Q, Ctx>(&'a self, k: &Q, ctx: &Ctx) -> Option<&'a V>
+    where
+        Ctx: CtxEq<K, Q> + CtxHash<Q> + CtxHash<K>,
+    {
+        let hash = compute_hash(ctx, k);
+        self.raw
+            .find(hash as u64, |bucket| {
+                hash == bucket.hash && ctx.ctx_eq(&bucket.k, k)
+            })
+            .map(|bucket| {
+                let data = unsafe { bucket.as_ref() };
+                &data.v
+            })
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[derive(Clone, Copy, Debug)]
+    struct Key {
+        index: u32,
+    }
+    struct Ctx {
+        vals: &'static [&'static str],
+    }
+    impl CtxEq<Key, Key> for Ctx {
+        fn ctx_eq(&self, a: &Key, b: &Key) -> bool {
+            self.vals[a.index as usize].eq(self.vals[b.index as usize])
+        }
+    }
+    impl CtxHash<Key> for Ctx {
+        fn ctx_hash<H: Hasher>(&self, state: &mut H, value: &Key) {
+            self.vals[value.index as usize].hash(state);
+        }
+    }
+
+    #[test]
+    fn test_basic() {
+        let ctx = Ctx {
+            vals: &["a", "b", "a"],
+        };
+
+        let k0 = Key { index: 0 };
+        let k1 = Key { index: 1 };
+        let k2 = Key { index: 2 };
+
+        assert!(ctx.ctx_eq(&k0, &k2));
+        assert!(!ctx.ctx_eq(&k0, &k1));
+        assert!(!ctx.ctx_eq(&k2, &k1));
+
+        let mut map: CtxHashMap<Key, u64> = CtxHashMap::with_capacity(4);
+        assert_eq!(map.insert(k0, 42, &ctx), None);
+        assert_eq!(map.insert(k2, 84, &ctx), Some(42));
+        assert_eq!(map.get(&k1, &ctx), None);
+        assert_eq!(*map.get(&k0, &ctx).unwrap(), 84);
+    }
+}