wasmtime 9.0.4 → 10.0.1

data/ext/cargo-vendor/winch-codegen-0.8.1/src/isa/x64/masm.rs

@@ -0,0 +1,288 @@
+use super::{
+    abi::X64ABI,
+    address::Address,
+    asm::{Assembler, Operand},
+    regs::{self, rbp, rsp},
+};
+use crate::masm::{CmpKind, DivKind, MacroAssembler as Masm, OperandSize, RegImm, RemKind};
+use crate::{
+    abi::{self, align_to, calculate_frame_adjustment, LocalSlot},
+    codegen::CodeGenContext,
+    stack::Val,
+};
+use crate::{isa::reg::Reg, masm::CalleeKind};
+use cranelift_codegen::{isa::x64::settings as x64_settings, settings, Final, MachBufferFinalized};
+
+/// x64 MacroAssembler.
+pub(crate) struct MacroAssembler {
+    /// Stack pointer offset.
+    sp_offset: u32,
+    /// Low level assembler.
+    asm: Assembler,
+}
+
+// Conversions between generic masm arguments and x64 operands.
+
+impl From<RegImm> for Operand {
+    fn from(rimm: RegImm) -> Self {
+        match rimm {
+            RegImm::Reg(r) => r.into(),
+            RegImm::Imm(imm) => Operand::Imm(imm),
+        }
+    }
+}
+
+impl From<Reg> for Operand {
+    fn from(reg: Reg) -> Self {
+        Operand::Reg(reg)
+    }
+}
+
+impl From<Address> for Operand {
+    fn from(addr: Address) -> Self {
+        Operand::Mem(addr)
+    }
+}
+
+impl Masm for MacroAssembler {
+    type Address = Address;
+    type Ptr = u8;
+    type ABI = X64ABI;
+
+    fn prologue(&mut self) {
+        let frame_pointer = rbp();
+        let stack_pointer = rsp();
+
+        self.asm.push_r(frame_pointer);
+        self.asm
+            .mov_rr(stack_pointer, frame_pointer, OperandSize::S64);
+    }
+
+    fn push(&mut self, reg: Reg) -> u32 {
+        self.asm.push_r(reg);
+        self.increment_sp(<Self::ABI as abi::ABI>::word_bytes());
+
+        self.sp_offset
+    }
+
+    fn reserve_stack(&mut self, bytes: u32) {
+        if bytes == 0 {
+            return;
+        }
+
+        self.asm.sub_ir(bytes as i32, rsp(), OperandSize::S64);
+        self.increment_sp(bytes);
+    }
+
+    fn free_stack(&mut self, bytes: u32) {
+        if bytes == 0 {
+            return;
+        }
+        self.asm.add_ir(bytes as i32, rsp(), OperandSize::S64);
+        self.decrement_sp(bytes);
+    }
+
+    fn local_address(&mut self, local: &LocalSlot) -> Address {
+        let (reg, offset) = local
+            .addressed_from_sp()
+            .then(|| {
+                let offset = self.sp_offset.checked_sub(local.offset).expect(&format!(
+                    "Invalid local offset = {}; sp offset = {}",
+                    local.offset, self.sp_offset
+                ));
+                (rsp(), offset)
+            })
+            .unwrap_or((rbp(), local.offset));
+
+        Address::offset(reg, offset)
+    }
+
+    fn address_from_sp(&self, offset: u32) -> Self::Address {
+        Address::offset(regs::rsp(), self.sp_offset - offset)
+    }
+
+    fn address_at_sp(&self, offset: u32) -> Self::Address {
+        Address::offset(regs::rsp(), offset)
+    }
+
+    fn store(&mut self, src: RegImm, dst: Address, size: OperandSize) {
+        let src: Operand = src.into();
+        let dst: Operand = dst.into();
+
+        self.asm.mov(src, dst, size);
+    }
+
+    fn pop(&mut self, dst: Reg) {
+        self.asm.pop_r(dst);
+        self.decrement_sp(<Self::ABI as abi::ABI>::word_bytes());
+    }
+
+    fn call(
+        &mut self,
+        stack_args_size: u32,
+        mut load_callee: impl FnMut(&mut Self) -> CalleeKind,
+    ) -> u32 {
+        let alignment: u32 = <Self::ABI as abi::ABI>::call_stack_align().into();
+        let addend: u32 = <Self::ABI as abi::ABI>::arg_base_offset().into();
+        let delta = calculate_frame_adjustment(self.sp_offset(), addend, alignment);
+        let aligned_args_size = align_to(stack_args_size, alignment);
+        let total_stack = delta + aligned_args_size;
+        self.reserve_stack(total_stack);
+        let callee = load_callee(self);
+        self.asm.call(callee);
+        total_stack
+    }
+
+    fn load(&mut self, src: Address, dst: Reg, size: OperandSize) {
+        let src = src.into();
+        let dst = dst.into();
+        self.asm.mov(src, dst, size);
+    }
+
+    fn sp_offset(&self) -> u32 {
+        self.sp_offset
+    }
+
+    fn zero(&mut self, reg: Reg) {
+        self.asm.xor_rr(reg, reg, OperandSize::S32);
+    }
+
+    fn mov(&mut self, src: RegImm, dst: RegImm, size: OperandSize) {
+        let src: Operand = src.into();
+        let dst: Operand = dst.into();
+
+        self.asm.mov(src, dst, size);
+    }
+
+    fn add(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize) {
+        let (src, dst): (Operand, Operand) = if dst == lhs {
+            (rhs.into(), dst.into())
+        } else {
+            panic!(
+                "the destination and first source argument must be the same, dst={:?}, lhs={:?}",
+                dst, lhs
+            );
+        };
+
+        self.asm.add(src, dst, size);
+    }
+
+    fn sub(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize) {
+        let (src, dst): (Operand, Operand) = if dst == lhs {
+            (rhs.into(), dst.into())
+        } else {
+            panic!(
+                "the destination and first source argument must be the same, dst={:?}, lhs={:?}",
+                dst, lhs
+            );
+        };
+
+        self.asm.sub(src, dst, size);
+    }
+
+    fn mul(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize) {
+        let (src, dst): (Operand, Operand) = if dst == lhs {
+            (rhs.into(), dst.into())
+        } else {
+            panic!(
+                "the destination and first source argument must be the same, dst={:?}, lhs={:?}",
+                dst, lhs
+            );
+        };
+
+        self.asm.mul(src, dst, size);
+    }
+
+    fn div(&mut self, context: &mut CodeGenContext, kind: DivKind, size: OperandSize) {
+        // Allocate rdx:rax.
+        let rdx = context.gpr(regs::rdx(), self);
+        let rax = context.gpr(regs::rax(), self);
+
+        // Allocate the divisor, which can be any gpr.
+        let divisor = context.pop_to_reg(self, None, size);
+
+        // Mark rax as allocatable.
+        context.regalloc.free_gpr(rax);
+        // Move the top value to rax.
+        let rax = context.pop_to_reg(self, Some(rax), size);
+        self.asm.div(divisor, (rax, rdx), kind, size);
+
+        // Free the divisor and rdx.
+        context.free_gpr(divisor);
+        context.free_gpr(rdx);
+
+        // Push the quotient.
+        context.stack.push(Val::reg(rax));
+    }
+
+    fn rem(&mut self, context: &mut CodeGenContext, kind: RemKind, size: OperandSize) {
+        // Allocate rdx:rax.
+        let rdx = context.gpr(regs::rdx(), self);
+        let rax = context.gpr(regs::rax(), self);
+
+        // Allocate the divisor, which can be any gpr.
+        let divisor = context.pop_to_reg(self, None, size);
+
+        // Mark rax as allocatable.
+        context.regalloc.free_gpr(rax);
+        // Move the top value to rax.
+        let rax = context.pop_to_reg(self, Some(rax), size);
+        self.asm.rem(divisor, (rax, rdx), kind, size);
+
+        // Free the divisor and rax.
+        context.free_gpr(divisor);
+        context.free_gpr(rax);
+
+        // Push the remainder.
+        context.stack.push(Val::reg(rdx));
+    }
+
+    fn epilogue(&mut self, locals_size: u32) {
+        assert!(self.sp_offset == locals_size);
+
+        let rsp = rsp();
+        if locals_size > 0 {
+            self.asm.add_ir(locals_size as i32, rsp, OperandSize::S64);
+        }
+        self.asm.pop_r(rbp());
+        self.asm.ret();
+    }
+
+    fn finalize(self) -> MachBufferFinalized<Final> {
+        self.asm.finalize()
+    }
+
+    fn address_at_reg(&self, reg: Reg, offset: u32) -> Self::Address {
+        Address::offset(reg, offset)
+    }
+
+    fn cmp_with_set(&mut self, src: RegImm, dst: RegImm, kind: CmpKind, size: OperandSize) {
+        let dst = dst.into();
+        self.asm.cmp(src.into(), dst, size);
+        self.asm.setcc(kind, dst);
+    }
+}
+
+impl MacroAssembler {
+    /// Create an x64 MacroAssembler.
+    pub fn new(shared_flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
+        Self {
+            sp_offset: 0,
+            asm: Assembler::new(shared_flags, isa_flags),
+        }
+    }
+
+    fn increment_sp(&mut self, bytes: u32) {
+        self.sp_offset += bytes;
+    }
+
+    fn decrement_sp(&mut self, bytes: u32) {
+        assert!(
+            self.sp_offset >= bytes,
+            "sp offset = {}; bytes = {}",
+            self.sp_offset,
+            bytes
+        );
+        self.sp_offset -= bytes;
+    }
+}

data/ext/cargo-vendor/winch-codegen-0.8.1/src/isa/x64/mod.rs

@@ -0,0 +1,149 @@
+use crate::{
+    abi::ABI,
+    codegen::{CodeGen, CodeGenContext, FuncEnv},
+};
+
+use crate::frame::{DefinedLocals, Frame};
+use crate::isa::{x64::masm::MacroAssembler as X64Masm, CallingConvention};
+use crate::masm::MacroAssembler;
+use crate::regalloc::RegAlloc;
+use crate::stack::Stack;
+use crate::trampoline::{Trampoline, TrampolineKind};
+use crate::{
+    isa::{Builder, TargetIsa},
+    regset::RegSet,
+};
+use anyhow::Result;
+use cranelift_codegen::settings::{self, Flags};
+use cranelift_codegen::{isa::x64::settings as x64_settings, Final, MachBufferFinalized};
+use cranelift_codegen::{MachTextSectionBuilder, TextSectionBuilder};
+use target_lexicon::Triple;
+use wasmparser::{FuncValidator, FunctionBody, ValidatorResources};
+use wasmtime_environ::{ModuleTranslation, WasmFuncType};
+
+use self::regs::ALL_GPR;
+
+mod abi;
+mod address;
+mod asm;
+mod masm;
+// Not all the fpr and gpr constructors are used at the moment;
+// in that sense, this directive is a temporary measure to avoid
+// dead code warnings.
+#[allow(dead_code)]
+mod regs;
+
+/// Create an ISA builder.
+pub(crate) fn isa_builder(triple: Triple) -> Builder {
+    Builder::new(
+        triple,
+        x64_settings::builder(),
+        |triple, shared_flags, settings| {
+            // TODO: Once enabling/disabling flags is allowed, and once features like SIMD are supported
+            // ensure compatibility between shared flags and ISA flags.
+            let isa_flags = x64_settings::Flags::new(&shared_flags, settings);
+            let isa = X64::new(triple, shared_flags, isa_flags);
+            Ok(Box::new(isa))
+        },
+    )
+}
+
+/// x64 ISA.
+pub(crate) struct X64 {
+    /// The target triple.
+    triple: Triple,
+    /// ISA specific flags.
+    isa_flags: x64_settings::Flags,
+    /// Shared flags.
+    shared_flags: Flags,
+}
+
+impl X64 {
+    /// Create a x64 ISA.
+    pub fn new(triple: Triple, shared_flags: Flags, isa_flags: x64_settings::Flags) -> Self {
+        Self {
+            isa_flags,
+            shared_flags,
+            triple,
+        }
+    }
+}
+
+impl TargetIsa for X64 {
+    fn name(&self) -> &'static str {
+        "x64"
+    }
+
+    fn triple(&self) -> &Triple {
+        &self.triple
+    }
+
+    fn flags(&self) -> &settings::Flags {
+        &self.shared_flags
+    }
+
+    fn isa_flags(&self) -> Vec<settings::Value> {
+        self.isa_flags.iter().collect()
+    }
+
+    fn compile_function(
+        &self,
+        sig: &WasmFuncType,
+        body: &FunctionBody,
+        translation: &ModuleTranslation,
+        validator: &mut FuncValidator<ValidatorResources>,
+    ) -> Result<MachBufferFinalized<Final>> {
+        let mut body = body.get_binary_reader();
+        let mut masm = X64Masm::new(self.shared_flags.clone(), self.isa_flags.clone());
+        let stack = Stack::new();
+        let abi_sig = abi::X64ABI::sig(sig, &CallingConvention::Default);
+
+        let defined_locals = DefinedLocals::new(translation, &mut body, validator)?;
+        let frame = Frame::new::<abi::X64ABI>(&abi_sig, &defined_locals)?;
+        // TODO Add in floating point bitmask
+        let regalloc = RegAlloc::new(RegSet::new(ALL_GPR, 0), regs::scratch());
+        let codegen_context = CodeGenContext::new(regalloc, stack, &frame);
+        let env = FuncEnv::new(self.pointer_bytes(), translation);
+        let mut codegen = CodeGen::new(&mut masm, codegen_context, env, abi_sig);
+
+        codegen.emit(&mut body, validator)?;
+
+        Ok(masm.finalize())
+    }
+
+    fn text_section_builder(&self, num_funcs: usize) -> Box<dyn TextSectionBuilder> {
+        Box::new(MachTextSectionBuilder::<cranelift_codegen::isa::x64::Inst>::new(num_funcs))
+    }
+
+    fn function_alignment(&self) -> u32 {
+        // See `cranelift_codegen`'s value of this for more information.
+        16
+    }
+
+    fn compile_trampoline(
+        &self,
+        ty: &WasmFuncType,
+        kind: TrampolineKind,
+    ) -> Result<MachBufferFinalized<Final>> {
+        use TrampolineKind::*;
+
+        let mut masm = X64Masm::new(self.shared_flags.clone(), self.isa_flags.clone());
+        let call_conv = self.wasmtime_call_conv();
+
+        let mut trampoline = Trampoline::new(
+            &mut masm,
+            regs::scratch(),
+            regs::argv(),
+            &call_conv,
+            self.pointer_bytes(),
+        );
+
+        match kind {
+            ArrayToWasm(idx) => trampoline.emit_array_to_wasm(ty, idx)?,
+            NativeToWasm(idx) => trampoline.emit_native_to_wasm(ty, idx)?,
+            WasmToNative => trampoline.emit_wasm_to_native(ty)?,
+        }
+
+        Ok(masm.finalize())
+    }
+}

data/ext/cargo-vendor/winch-codegen-0.8.1/src/isa/x64/regs.rs

@@ -0,0 +1,192 @@
+//! X64 register definition.
+
+use crate::isa::{reg::Reg, CallingConvention};
+use regalloc2::{PReg, RegClass};
+use smallvec::{smallvec, SmallVec};
+
+const ENC_RAX: u8 = 0;
+const ENC_RCX: u8 = 1;
+const ENC_RDX: u8 = 2;
+const ENC_RBX: u8 = 3;
+const ENC_RSP: u8 = 4;
+const ENC_RBP: u8 = 5;
+const ENC_RSI: u8 = 6;
+const ENC_RDI: u8 = 7;
+const ENC_R8: u8 = 8;
+const ENC_R9: u8 = 9;
+const ENC_R10: u8 = 10;
+const ENC_R11: u8 = 11;
+const ENC_R12: u8 = 12;
+const ENC_R13: u8 = 13;
+const ENC_R14: u8 = 14;
+const ENC_R15: u8 = 15;
+
+fn gpr(enc: u8) -> Reg {
+    Reg::new(PReg::new(enc as usize, RegClass::Int))
+}
+
+/// Constructors for GPR.
+
+pub(crate) fn rsi() -> Reg {
+    gpr(ENC_RSI)
+}
+pub(crate) fn rdi() -> Reg {
+    gpr(ENC_RDI)
+}
+pub(crate) fn rax() -> Reg {
+    gpr(ENC_RAX)
+}
+pub(crate) fn rcx() -> Reg {
+    gpr(ENC_RCX)
+}
+pub(crate) fn rdx() -> Reg {
+    gpr(ENC_RDX)
+}
+pub(crate) fn r8() -> Reg {
+    gpr(ENC_R8)
+}
+pub(crate) fn r9() -> Reg {
+    gpr(ENC_R9)
+}
+pub(crate) fn r10() -> Reg {
+    gpr(ENC_R10)
+}
+pub(crate) fn r12() -> Reg {
+    gpr(ENC_R12)
+}
+pub(crate) fn r13() -> Reg {
+    gpr(ENC_R13)
+}
+/// Used as a pinned register to hold
+/// the `VMContext`.
+/// Non-allocatable in Winch's default
+/// ABI, and callee-saved in SystemV and
+/// Fastcall.
+pub(crate) fn r14() -> Reg {
+    gpr(ENC_R14)
+}
+
+pub(crate) fn vmctx() -> Reg {
+    r14()
+}
+
+pub(crate) fn rbx() -> Reg {
+    gpr(ENC_RBX)
+}
+
+pub(crate) fn r15() -> Reg {
+    gpr(ENC_R15)
+}
+
+pub(crate) fn rsp() -> Reg {
+    gpr(ENC_RSP)
+}
+pub(crate) fn rbp() -> Reg {
+    gpr(ENC_RBP)
+}
+
+/// Used as the scratch register.
+/// Non-allocatable in Winch's default
+/// ABI.
+pub(crate) fn r11() -> Reg {
+    gpr(ENC_R11)
+}
+
+pub(crate) fn scratch() -> Reg {
+    r11()
+}
+
+/// This register is used as a scratch register, in the context of trampolines only,
+/// where we assume that callee-saved registers are given the correct handling
+/// according to the system ABI. r12 is chosen given that it's a callee-saved,
+/// non-argument register.
+///
+/// In the context of all other internal functions, this register is not excluded
+/// from register allocation, so no extra assumptions should be made regarding
+/// its availability.
+pub(crate) fn argv() -> Reg {
+    r12()
+}
+
+fn fpr(enc: u8) -> Reg {
+    Reg::new(PReg::new(enc as usize, RegClass::Float))
+}
+
+/// Constructors for FPR.
+
+pub(crate) fn xmm0() -> Reg {
+    fpr(0)
+}
+pub(crate) fn xmm1() -> Reg {
+    fpr(1)
+}
+pub(crate) fn xmm2() -> Reg {
+    fpr(2)
+}
+pub(crate) fn xmm3() -> Reg {
+    fpr(3)
+}
+pub(crate) fn xmm4() -> Reg {
+    fpr(4)
+}
+pub(crate) fn xmm5() -> Reg {
+    fpr(5)
+}
+pub(crate) fn xmm6() -> Reg {
+    fpr(6)
+}
+pub(crate) fn xmm7() -> Reg {
+    fpr(7)
+}
+pub(crate) fn xmm8() -> Reg {
+    fpr(8)
+}
+pub(crate) fn xmm9() -> Reg {
+    fpr(9)
+}
+pub(crate) fn xmm10() -> Reg {
+    fpr(10)
+}
+pub(crate) fn xmm11() -> Reg {
+    fpr(11)
+}
+pub(crate) fn xmm12() -> Reg {
+    fpr(12)
+}
+pub(crate) fn xmm13() -> Reg {
+    fpr(13)
+}
+pub(crate) fn xmm14() -> Reg {
+    fpr(14)
+}
+pub(crate) fn xmm15() -> Reg {
+    fpr(15)
+}
+
+const GPR: u32 = 16;
+const ALLOCATABLE_GPR: u32 = (1 << GPR) - 1;
+const NON_ALLOCATABLE_GPR: u32 = (1 << ENC_RBP) | (1 << ENC_RSP) | (1 << ENC_R11) | (1 << ENC_R14);
+
+/// Bitmask to represent the available general purpose registers.
+pub(crate) const ALL_GPR: u32 = ALLOCATABLE_GPR & !NON_ALLOCATABLE_GPR;
+
+/// Returns the callee-saved registers according to a particular calling
+/// convention.
+///
+/// This function will return the set of registers that need to be saved
+/// according to the system ABI and that are known not to be saved during the
+/// prologue emission.
+pub(crate) fn callee_saved(call_conv: &CallingConvention) -> SmallVec<[Reg; 9]> {
+    use CallingConvention::*;
+    match call_conv {
+        WasmtimeSystemV => {
+            smallvec![rbx(), r12(), r13(), r14(), r15(),]
+        }
+        // TODO: Once float registers are supported,
+        // account for callee-saved float registers.
+        WasmtimeFastcall => {
+            smallvec![rbx(), rdi(), rsi(), r12(), r13(), r14(), r15(),]
+        }
+        _ => unreachable!(),
+    }
+}

data/ext/cargo-vendor/winch-codegen-0.8.1/src/lib.rs

@@ -0,0 +1,21 @@
+//! Code generation library for Winch.
+
+// Unless this library is compiled with `all-arch`, the rust compiler
+// is going to emit dead code warnings. This directive is fine as long
+// as we configure to run CI at least once with the `all-arch` feature
+// enabled.
+#![cfg_attr(not(feature = "all-arch"), allow(dead_code))]
+
+mod abi;
+pub use codegen::FuncEnv;
+mod codegen;
+mod frame;
+pub mod isa;
+pub use isa::*;
+mod masm;
+mod regalloc;
+mod regset;
+mod stack;
+mod trampoline;
+pub use trampoline::TrampolineKind;
+mod visitor;