wasmtime 12.0.1 → 13.0.0

data/ext/cargo-vendor/winch-codegen-0.11.0/src/isa/aarch64/masm.rs

@@ -0,0 +1,357 @@
+use super::{abi::Aarch64ABI, address::Address, asm::Assembler, regs};
+use crate::{
+    abi::{self, local::LocalSlot},
+    codegen::CodeGenContext,
+    isa::reg::Reg,
+    masm::{
+        CalleeKind, CmpKind, DivKind, Imm as I, MacroAssembler as Masm, OperandSize, RegImm,
+        RemKind, ShiftKind, StackSlot,
+    },
+};
+use cranelift_codegen::{settings, Final, MachBufferFinalized, MachLabel};
+
+/// Aarch64 MacroAssembler.
+pub(crate) struct MacroAssembler {
+    /// Low level assembler.
+    asm: Assembler,
+    /// Stack pointer offset.
+    sp_offset: u32,
+}
+
+impl MacroAssembler {
+    /// Create an Aarch64 MacroAssembler.
+    pub fn new(shared_flags: settings::Flags) -> Self {
+        Self {
+            asm: Assembler::new(shared_flags),
+            sp_offset: 0u32,
+        }
+    }
+}
+
+impl Masm for MacroAssembler {
+    type Address = Address;
+    type Ptr = u8;
+    type ABI = Aarch64ABI;
+
+    fn prologue(&mut self) {
+        let lr = regs::lr();
+        let fp = regs::fp();
+        let sp = regs::sp();
+        let addr = Address::pre_indexed_from_sp(-16);
+
+        self.asm.stp(fp, lr, addr);
+        self.asm.mov_rr(sp, fp, OperandSize::S64);
+        self.move_sp_to_shadow_sp();
+    }
+
+    fn epilogue(&mut self, locals_size: u32) {
+        assert!(self.sp_offset == locals_size);
+
+        let sp = regs::sp();
+        if locals_size > 0 {
+            self.asm
+                .add_ir(locals_size as u64, sp, sp, OperandSize::S64);
+            self.move_sp_to_shadow_sp();
+        }
+
+        let lr = regs::lr();
+        let fp = regs::fp();
+        let addr = Address::post_indexed_from_sp(16);
+
+        self.asm.ldp(fp, lr, addr);
+        self.asm.ret();
+    }
+
+    fn reserve_stack(&mut self, bytes: u32) {
+        if bytes == 0 {
+            return;
+        }
+
+        let sp = regs::sp();
+        self.asm.sub_ir(bytes as u64, sp, sp, OperandSize::S64);
+        self.move_sp_to_shadow_sp();
+
+        self.increment_sp(bytes);
+    }
+
+    fn free_stack(&mut self, _bytes: u32) {
+        todo!()
+    }
+
+    fn reset_stack_pointer(&mut self, offset: u32) {
+        self.sp_offset = offset;
+    }
+
+    fn local_address(&mut self, local: &LocalSlot) -> Address {
+        let (reg, offset) = local
+            .addressed_from_sp()
+            .then(|| {
+                let offset = self.sp_offset.checked_sub(local.offset).expect(&format!(
+                    "Invalid local offset = {}; sp offset = {}",
+                    local.offset, self.sp_offset
+                ));
+                (regs::shadow_sp(), offset)
+            })
+            .unwrap_or((regs::fp(), local.offset));
+
+        Address::offset(reg, offset as i64)
+    }
+
+    fn address_from_sp(&self, _offset: u32) -> Self::Address {
+        todo!()
+    }
+
+    fn address_at_sp(&self, _offset: u32) -> Self::Address {
+        todo!()
+    }
+
+    fn store(&mut self, src: RegImm, dst: Address, size: OperandSize) {
+        let src = match src {
+            RegImm::Imm(v) => {
+                let imm = match v {
+                    I::I32(v) => v as u64,
+                    I::I64(v) => v,
+                    _ => unreachable!(),
+                };
+                let scratch = regs::scratch();
+                self.asm.load_constant(imm, scratch);
+                scratch
+            }
+            RegImm::Reg(reg) => reg,
+        };
+
+        self.asm.str(src, dst, size);
+    }
+
+    fn call(
+        &mut self,
+        _stack_args_size: u32,
+        _load_callee: impl FnMut(&mut Self) -> CalleeKind,
+    ) -> u32 {
+        todo!()
+    }
+
+    fn load(&mut self, src: Address, dst: Reg, size: OperandSize) {
+        self.asm.ldr(src, dst, size);
+    }
+
+    fn pop(&mut self, _dst: Reg, _size: OperandSize) {
+        todo!()
+    }
+
+    fn sp_offset(&self) -> u32 {
+        self.sp_offset
+    }
+
+    fn finalize(self) -> MachBufferFinalized<Final> {
+        self.asm.finalize()
+    }
+
+    fn mov(&mut self, src: RegImm, dst: RegImm, size: OperandSize) {
+        match (src, dst) {
+            (RegImm::Imm(v), RegImm::Reg(rd)) => {
+                let imm = match v {
+                    I::I32(v) => v as u64,
+                    I::I64(v) => v,
+                    _ => panic!(),
+                };
+
+                let scratch = regs::scratch();
+                self.asm.load_constant(imm as u64, scratch);
+                self.asm.mov_rr(scratch, rd, size);
+            }
+            (RegImm::Reg(rs), RegImm::Reg(rd)) => {
+                self.asm.mov_rr(rs, rd, size);
+            }
+            _ => Self::handle_invalid_two_form_operand_combination(src, dst),
+        }
+    }
+
+    fn cmov(&mut self, _src: Reg, _dst: Reg, _cc: CmpKind, _size: OperandSize) {
+        todo!()
+    }
+
+    fn add(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize) {
+        match (rhs, lhs, dst) {
+            (RegImm::Imm(v), RegImm::Reg(rn), RegImm::Reg(rd)) => {
+                let imm = match v {
+                    I::I32(v) => v as u64,
+                    I::I64(v) => v,
+                    _ => unreachable!(),
+                };
+
+                self.asm.add_ir(imm, rn, rd, size);
+            }
+
+            (RegImm::Reg(rm), RegImm::Reg(rn), RegImm::Reg(rd)) => {
+                self.asm.add_rrr(rm, rn, rd, size);
+            }
+            _ => Self::handle_invalid_three_form_operand_combination(dst, lhs, rhs),
+        }
+    }
+
+    fn sub(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize) {
+        match (rhs, lhs, dst) {
+            (RegImm::Imm(v), RegImm::Reg(rn), RegImm::Reg(rd)) => {
+                let imm = match v {
+                    I::I32(v) => v as u64,
+                    I::I64(v) => v,
+                    _ => unreachable!(),
+                };
+
+                self.asm.sub_ir(imm, rn, rd, size);
+            }
+
+            (RegImm::Reg(rm), RegImm::Reg(rn), RegImm::Reg(rd)) => {
+                self.asm.sub_rrr(rm, rn, rd, size);
+            }
+            _ => Self::handle_invalid_three_form_operand_combination(dst, lhs, rhs),
+        }
+    }
+
+    fn mul(&mut self, dst: RegImm, lhs: RegImm, rhs: RegImm, size: OperandSize) {
+        match (rhs, lhs, dst) {
+            (RegImm::Imm(v), RegImm::Reg(rn), RegImm::Reg(rd)) => {
+                let imm = match v {
+                    I::I32(v) => v as u64,
+                    I::I64(v) => v,
+                    _ => unreachable!(),
+                };
+
+                self.asm.mul_ir(imm, rn, rd, size);
+            }
+
+            (RegImm::Reg(rm), RegImm::Reg(rn), RegImm::Reg(rd)) => {
+                self.asm.mul_rrr(rm, rn, rd, size);
+            }
+            _ => Self::handle_invalid_three_form_operand_combination(dst, lhs, rhs),
+        }
+    }
+
+    fn and(&mut self, _dst: RegImm, _lhs: RegImm, _rhs: RegImm, _size: OperandSize) {
+        todo!()
+    }
+
+    fn or(&mut self, _dst: RegImm, _lhs: RegImm, _rhs: RegImm, _size: OperandSize) {
+        todo!()
+    }
+
+    fn xor(&mut self, _dst: RegImm, _lhs: RegImm, _rhs: RegImm, _size: OperandSize) {
+        todo!()
+    }
+
+    fn shift(&mut self, _context: &mut CodeGenContext, _kind: ShiftKind, _size: OperandSize) {
+        todo!()
+    }
+
+    fn div(&mut self, _context: &mut CodeGenContext, _kind: DivKind, _size: OperandSize) {
+        todo!()
+    }
+
+    fn rem(&mut self, _context: &mut CodeGenContext, _kind: RemKind, _size: OperandSize) {
+        todo!()
+    }
+
+    fn zero(&mut self, reg: Reg) {
+        self.asm.load_constant(0, reg);
+    }
+
+    fn popcnt(&mut self, _context: &mut CodeGenContext, _size: OperandSize) {
+        todo!()
+    }
+
+    fn push(&mut self, reg: Reg, _size: OperandSize) -> StackSlot {
+        let size = <Self::ABI as abi::ABI>::word_bytes();
+        self.reserve_stack(size);
+        let address = Address::from_shadow_sp(size as i64);
+        self.asm.str(reg, address, OperandSize::S64);
+
+        StackSlot {
+            offset: self.sp_offset,
+            size,
+        }
+    }
+
+    fn address_at_reg(&self, reg: Reg, offset: u32) -> Self::Address {
+        Address::offset(reg, offset as i64)
+    }
+
+    fn cmp_with_set(&mut self, _src: RegImm, _dst: Reg, _kind: CmpKind, _size: OperandSize) {
+        todo!()
+    }
+
+    fn cmp(&mut self, _src: RegImm, _dest: Reg, _size: OperandSize) {
+        todo!()
+    }
+
+    fn clz(&mut self, _src: Reg, _dst: Reg, _size: OperandSize) {
+        todo!()
+    }
+
+    fn ctz(&mut self, _src: Reg, _dst: Reg, _size: OperandSize) {
+        todo!()
+    }
+
+    fn get_label(&mut self) -> MachLabel {
+        self.asm.get_label()
+    }
+
+    fn bind(&mut self, label: MachLabel) {
+        let buffer = self.asm.buffer_mut();
+        buffer.bind_label(label, &mut Default::default());
+    }
+
+    fn branch(
+        &mut self,
+        _kind: CmpKind,
+        _lhs: RegImm,
+        _rhs: RegImm,
+        _taken: MachLabel,
+        _size: OperandSize,
+    ) {
+        todo!()
+    }
+
+    fn jmp(&mut self, _target: MachLabel) {
+        todo!()
+    }
+
+    fn unreachable(&mut self) {
+        todo!()
+    }
+
+    fn jmp_table(&mut self, _targets: &[MachLabel], _index: Reg, _tmp: Reg) {
+        todo!()
+    }
+}
+
+impl MacroAssembler {
+    fn increment_sp(&mut self, bytes: u32) {
+        self.sp_offset += bytes;
+    }
+
+    // Copies the value of the stack pointer to the shadow stack
+    // pointer: mov x28, sp
+
+    // This function is usually called whenever the real stack pointer
+    // changes, for example after allocating or deallocating stack
+    // space, or after performing a push or pop.
+    // For more details around the stack pointer and shadow stack
+    // pointer see the docs at regs::shadow_sp().
+    fn move_sp_to_shadow_sp(&mut self) {
+        let sp = regs::sp();
+        let shadow_sp = regs::shadow_sp();
+        self.asm.mov_rr(sp, shadow_sp, OperandSize::S64);
+    }
+
+    fn handle_invalid_two_form_operand_combination(src: RegImm, dst: RegImm) {
+        panic!("Invalid operand combination; src={:?}, dst={:?}", src, dst);
+    }
+
+    fn handle_invalid_three_form_operand_combination(dst: RegImm, lhs: RegImm, rhs: RegImm) {
+        panic!(
+            "Invalid operand combination; dst={:?}, lhs={:?}, rhs={:?}",
+            dst, lhs, rhs
+        );
+    }
+}

data/ext/cargo-vendor/winch-codegen-0.11.0/src/isa/aarch64/regs.rs

@@ -0,0 +1,177 @@
+//! AArch64 register definition.
+
+use crate::{isa::reg::Reg, masm::OperandSize};
+use regalloc2::{PReg, RegClass};
+use smallvec::{smallvec, SmallVec};
+
+/// Construct a X-register from an index.
+pub(crate) const fn xreg(num: u8) -> Reg {
+    assert!(num < 32);
+    Reg::new(PReg::new(num as usize, RegClass::Int))
+}
+
+/// Construct a V-register from an index.
+pub(crate) const fn vreg(num: u8) -> Reg {
+    assert!(num < 32);
+    Reg::new(PReg::new(num as usize, RegClass::Float))
+}
+
+/// Scratch register.
+/// Intra-procedure-call corruptible register.
+pub(crate) const fn ip0() -> Reg {
+    xreg(16)
+}
+
+/// Alias to the IP0 register.
+pub(crate) const fn scratch() -> Reg {
+    ip0()
+}
+
+/// Scratch register.
+/// Intra-procedure-call corruptible register.
+pub(crate) const fn ip1() -> Reg {
+    xreg(17)
+}
+
+/// Register used to carry platform state.
+const fn platform() -> Reg {
+    xreg(18)
+}
+
+/// Frame pointer register.
+pub(crate) const fn fp() -> Reg {
+    xreg(29)
+}
+
+/// Link register for function calls.
+pub(crate) const fn lr() -> Reg {
+    xreg(30)
+}
+
+/// Zero register.
+pub(crate) const fn zero() -> Reg {
+    xreg(31)
+}
+
+/// The VM context register.
+pub(crate) const fn vmctx() -> Reg {
+    xreg(9)
+}
+
+/// Stack pointer register.
+///
+/// In aarch64 the zero and stack pointer registers are contextually
+/// different but have the same hardware encoding; to differentiate
+/// them, we are following Cranelift's encoding and representing it as
+/// 31 + 32.  Ref:
+/// https://github.com/bytecodealliance/wasmtime/blob/main/cranelift/codegen/src/isa/aarch64/inst/regs.rs#L70
+pub(crate) const fn sp() -> Reg {
+    Reg::new(PReg::new(31 + 32, RegClass::Int))
+}
+
+/// Shadow stack pointer register.
+///
+/// The shadow stack pointer is used as the base for memory addressing
+/// to workaround Aarch64's constraint on the stack pointer 16-byte
+/// alignment for memory addressing. This allows word-size loads and
+/// stores.  It's always assumed that the real stack pointer is
+/// 16-byte unaligned; the only exceptions to this assumption are the function
+/// prologue and epilogue in which we use the real stack pointer for
+/// addressing, assuming that the 16-byte alignment is respected.
+///
+/// The fact that the shadow stack pointer is used for memory
+/// addressing, doesn't change the meaning of the real stack pointer,
+/// which should always be used to allocate and deallocate stack
+/// space. The real stack pointer is always treated as "primary".
+/// Throughout the code generation any change to the stack pointer is
+/// reflected in the shadow stack pointer via the
+/// [MacroAssembler::move_sp_to_shadow_sp] function.
+///
+/// This approach, requires copying the real stack pointer value into
+/// x28 everytime the real stack pointer moves, which involves
+/// emitting one more instruction. For example, this is generally how
+/// the real stack pointer and x28 will look like during a function:
+///
+/// +-----------+
+/// |           |      Save x28 (callee-saved)
+/// +-----------+----- SP at function entry (after epilogue, slots for FP and LR)
+/// |           |      Copy the value of SP to x28
+/// |           |
+/// +-----------+----- SP after reserving stack space for locals and arguments
+/// |           |      Copy the value of SP to x28
+/// |           |
+/// +-----------+----- SP after a push
+/// |           |      Copy the value of SP to x28 (similar after a pop)
+/// |           |      
+/// |           |       
+/// |           |
+/// |           |
+/// +-----------+----- At epilogue restore x28 (callee-saved)
+/// +-----------+
+///
+/// In summary, the following invariants must be respected:
+///
+/// * The real stack pointer is always primary, and must be used to
+///   allocate and deallocate stack space(e.g. push, pop). This
+///   operation must always be followed by a copy of the real stack
+///   pointer to x28.
+/// * The real stack pointer must never be used to
+///   address memory except when we are certain that the required
+///   alignment is respected (e.g.  during the prologue and epilogue)
+/// * The value of the real stack pointer is copied to x28 when
+///   entering a function.
+/// * The value of x28 doesn't change between
+///   function calls (as it's callee saved), compliant with
+///   Aarch64's ABI.
+/// * x28 is not available during register allocation.
+/// * Since the real stack pointer is always primary, there's no need
+///   to copy the shadow stack pointer into the real stack
+///   pointer. The copy is only done SP -> Shadow SP direction.
+pub(crate) const fn shadow_sp() -> Reg {
+    xreg(28)
+}
+
+const NON_ALLOCATABLE_GPR: u32 = (1 << ip0().hw_enc())
+    | (1 << ip1().hw_enc())
+    | (1 << platform().hw_enc())
+    | (1 << fp().hw_enc())
+    | (1 << lr().hw_enc())
+    | (1 << zero().hw_enc())
+    | (1 << shadow_sp().hw_enc())
+    | (1 << vmctx().hw_enc());
+
+/// Bitmask to represent the available general purpose registers.
+pub(crate) const ALL_GPR: u32 = u32::MAX & !NON_ALLOCATABLE_GPR;
+
+/// Returns the callee-saved registers.
+///
+/// This function will return the set of registers that need to be saved
+/// according to the system ABI and that are known not to be saved during the
+/// prologue emission.
+pub(crate) fn callee_saved() -> SmallVec<[(Reg, OperandSize); 18]> {
+    use OperandSize::*;
+    let regs: SmallVec<[_; 18]> = smallvec![
+        xreg(19),
+        xreg(20),
+        xreg(21),
+        xreg(22),
+        xreg(23),
+        xreg(24),
+        xreg(25),
+        xreg(26),
+        xreg(27),
+        xreg(28),
+        vreg(8),
+        vreg(9),
+        vreg(10),
+        vreg(11),
+        vreg(12),
+        vreg(13),
+        vreg(14),
+        vreg(15),
+    ];
+    // Aarch64's calling convention states that for VReg's only
+    // the lower 64 bits are callee-saved (D8-D15).  See
+    // https://developer.arm.com/documentation/102374/0101/Procedure-Call-Standard
+    regs.into_iter().map(|reg| (reg, S64)).collect()
+}

data/ext/cargo-vendor/winch-codegen-0.11.0/src/isa/reg.rs

@@ -0,0 +1,73 @@
+use regalloc2::PReg;
+pub use regalloc2::RegClass;
+
+/// A newtype abstraction on top of a physical register.
+//
+// NOTE
+// This is temporary; the intention behind this newtype
+// is to keep the usage of PReg contained to this module
+// so that the rest of Winch should only need to operate
+// on top of the concept of `Reg`.
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Reg(PReg);
+
+impl Reg {
+    /// Create a register from its encoding and class.
+    pub fn from(class: RegClass, enc: usize) -> Self {
+        Self::new(PReg::new(enc, class))
+    }
+
+    /// Create a new register from a physical register.
+    pub const fn new(raw: PReg) -> Self {
+        Reg(raw)
+    }
+
+    /// Create a new general purpose register from encoding.
+    pub fn int(enc: usize) -> Self {
+        Self::new(PReg::new(enc, RegClass::Int))
+    }
+
+    /// Create a new floating point register from encoding.
+    #[allow(dead_code)]
+    pub fn float(enc: usize) -> Self {
+        Self::new(PReg::new(enc, RegClass::Float))
+    }
+
+    /// Get the encoding of the underlying register.
+    pub const fn hw_enc(self) -> u8 {
+        self.0.hw_enc() as u8
+    }
+
+    /// Get the physical register representation.
+    pub(super) fn inner(&self) -> PReg {
+        self.0
+    }
+
+    /// Get the register class.
+    pub fn class(&self) -> RegClass {
+        self.0.class()
+    }
+
+    /// Returns true if the registers is a general purpose
+    /// integer register.
+    pub fn is_int(&self) -> bool {
+        self.class() == RegClass::Int
+    }
+
+    /// Returns true if the registers is a float register.
+    pub fn is_float(&self) -> bool {
+        self.class() == RegClass::Float
+    }
+}
+
+impl From<Reg> for cranelift_codegen::Reg {
+    fn from(reg: Reg) -> Self {
+        reg.inner().into()
+    }
+}
+
+impl std::fmt::Debug for Reg {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}