wasmtime 9.0.4 → 10.0.1

data/ext/cargo-vendor/cranelift-codegen-0.97.1/src/isa/riscv64/lower/isle.rs

@@ -0,0 +1,553 @@
+//! ISLE integration glue code for riscv64 lowering.
+
+// Pull in the ISLE generated code.
+#[allow(unused)]
+pub mod generated_code;
+use generated_code::{Context, ExtendOp, MInst};
+
+// Types that the generated ISLE code uses via `use super::*`.
+use self::generated_code::VecAluOpRR;
+use super::{writable_zero_reg, zero_reg};
+use crate::isa::riscv64::abi::Riscv64ABICallSite;
+use crate::isa::riscv64::lower::args::{
+    FReg, VReg, WritableFReg, WritableVReg, WritableXReg, XReg,
+};
+use crate::isa::riscv64::Riscv64Backend;
+use crate::machinst::Reg;
+use crate::machinst::{isle::*, MachInst, SmallInstVec};
+use crate::machinst::{VCodeConstant, VCodeConstantData};
+use crate::{
+    ir::{
+        immediates::*, types::*, AtomicRmwOp, BlockCall, ExternalName, Inst, InstructionData,
+        MemFlags, StackSlot, TrapCode, Value, ValueList,
+    },
+    isa::riscv64::inst::*,
+    machinst::{ArgPair, InstOutput, Lower},
+};
+use crate::{isle_common_prelude_methods, isle_lower_prelude_methods};
+use regalloc2::PReg;
+use std::boxed::Box;
+use std::convert::TryFrom;
+use std::vec::Vec;
+
+type BoxCallInfo = Box<CallInfo>;
+type BoxCallIndInfo = Box<CallIndInfo>;
+type BoxExternalName = Box<ExternalName>;
+type VecMachLabel = Vec<MachLabel>;
+type VecArgPair = Vec<ArgPair>;
+use crate::machinst::valueregs;
+
+pub(crate) struct RV64IsleContext<'a, 'b, I, B>
+where
+    I: VCodeInst,
+    B: LowerBackend,
+{
+    pub lower_ctx: &'a mut Lower<'b, I>,
+    pub backend: &'a B,
+    /// Precalucated value for the minimum vector register size. Will be 0 if
+    /// vectors are not supported.
+    min_vec_reg_size: u64,
+}
+
+impl<'a, 'b> RV64IsleContext<'a, 'b, MInst, Riscv64Backend> {
+    isle_prelude_method_helpers!(Riscv64ABICallSite);
+
+    fn new(lower_ctx: &'a mut Lower<'b, MInst>, backend: &'a Riscv64Backend) -> Self {
+        Self {
+            lower_ctx,
+            backend,
+            min_vec_reg_size: backend.isa_flags.min_vec_reg_size(),
+        }
+    }
+
+    #[inline]
+    fn emit_list(&mut self, list: &SmallInstVec<MInst>) {
+        for i in list {
+            self.lower_ctx.emit(i.clone());
+        }
+    }
+}
+
+impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend> {
+    isle_lower_prelude_methods!();
+    isle_prelude_caller_methods!(Riscv64MachineDeps, Riscv64ABICallSite);
+
+    fn vreg_new(&mut self, r: Reg) -> VReg {
+        VReg::new(r).unwrap()
+    }
+    fn writable_vreg_new(&mut self, r: WritableReg) -> WritableVReg {
+        r.map(|wr| VReg::new(wr).unwrap())
+    }
+    fn writable_vreg_to_vreg(&mut self, arg0: WritableVReg) -> VReg {
+        arg0.to_reg()
+    }
+    fn writable_vreg_to_writable_reg(&mut self, arg0: WritableVReg) -> WritableReg {
+        arg0.map(|vr| vr.to_reg())
+    }
+    fn vreg_to_reg(&mut self, arg0: VReg) -> Reg {
+        *arg0
+    }
+    fn xreg_new(&mut self, r: Reg) -> XReg {
+        XReg::new(r).unwrap()
+    }
+    fn writable_xreg_new(&mut self, r: WritableReg) -> WritableXReg {
+        r.map(|wr| XReg::new(wr).unwrap())
+    }
+    fn writable_xreg_to_xreg(&mut self, arg0: WritableXReg) -> XReg {
+        arg0.to_reg()
+    }
+    fn writable_xreg_to_writable_reg(&mut self, arg0: WritableXReg) -> WritableReg {
+        arg0.map(|xr| xr.to_reg())
+    }
+    fn xreg_to_reg(&mut self, arg0: XReg) -> Reg {
+        *arg0
+    }
+    fn freg_new(&mut self, r: Reg) -> FReg {
+        FReg::new(r).unwrap()
+    }
+    fn writable_freg_new(&mut self, r: WritableReg) -> WritableFReg {
+        r.map(|wr| FReg::new(wr).unwrap())
+    }
+    fn writable_freg_to_freg(&mut self, arg0: WritableFReg) -> FReg {
+        arg0.to_reg()
+    }
+    fn writable_freg_to_writable_reg(&mut self, arg0: WritableFReg) -> WritableReg {
+        arg0.map(|fr| fr.to_reg())
+    }
+    fn freg_to_reg(&mut self, arg0: FReg) -> Reg {
+        *arg0
+    }
+
+    fn vec_writable_to_regs(&mut self, val: &VecWritableReg) -> ValueRegs {
+        match val.len() {
+            1 => ValueRegs::one(val[0].to_reg()),
+            2 => ValueRegs::two(val[0].to_reg(), val[1].to_reg()),
+            _ => unreachable!(),
+        }
+    }
+    fn intcc_to_extend_op(&mut self, cc: &IntCC) -> ExtendOp {
+        use IntCC::*;
+        match *cc {
+            Equal
+            | NotEqual
+            | UnsignedLessThan
+            | UnsignedGreaterThanOrEqual
+            | UnsignedGreaterThan
+            | UnsignedLessThanOrEqual => ExtendOp::Zero,
+
+            SignedLessThan
+            | SignedGreaterThanOrEqual
+            | SignedGreaterThan
+            | SignedLessThanOrEqual => ExtendOp::Signed,
+        }
+    }
+    fn lower_cond_br(
+        &mut self,
+        cc: &IntCC,
+        a: ValueRegs,
+        targets: &VecMachLabel,
+        ty: Type,
+    ) -> Unit {
+        MInst::lower_br_icmp(
+            *cc,
+            a,
+            self.int_zero_reg(ty),
+            BranchTarget::Label(targets[0]),
+            BranchTarget::Label(targets[1]),
+            ty,
+        )
+        .iter()
+        .for_each(|i| self.emit(i));
+    }
+    fn lower_br_icmp(
+        &mut self,
+        cc: &IntCC,
+        a: ValueRegs,
+        b: ValueRegs,
+        targets: &VecMachLabel,
+        ty: Type,
+    ) -> Unit {
+        let test = generated_code::constructor_lower_icmp(self, cc, a, b, ty);
+        self.emit(&MInst::CondBr {
+            taken: BranchTarget::Label(targets[0]),
+            not_taken: BranchTarget::Label(targets[1]),
+            kind: IntegerCompare {
+                kind: IntCC::NotEqual,
+                rs1: test,
+                rs2: zero_reg(),
+            },
+        });
+    }
+    fn load_ra(&mut self) -> Reg {
+        if self.backend.flags.preserve_frame_pointers() {
+            let tmp = self.temp_writable_reg(I64);
+            self.emit(&MInst::Load {
+                rd: tmp,
+                op: LoadOP::Ld,
+                flags: MemFlags::trusted(),
+                from: AMode::FPOffset(8, I64),
+            });
+            tmp.to_reg()
+        } else {
+            link_reg()
+        }
+    }
+    fn int_zero_reg(&mut self, ty: Type) -> ValueRegs {
+        assert!(ty.is_int(), "{:?}", ty);
+        if ty.bits() == 128 {
+            ValueRegs::two(self.zero_reg(), self.zero_reg())
+        } else {
+            ValueRegs::one(self.zero_reg())
+        }
+    }
+
+    fn vec_label_get(&mut self, val: &VecMachLabel, x: u8) -> MachLabel {
+        val[x as usize]
+    }
+
+    fn label_to_br_target(&mut self, label: MachLabel) -> BranchTarget {
+        BranchTarget::Label(label)
+    }
+
+    fn vec_writable_clone(&mut self, v: &VecWritableReg) -> VecWritableReg {
+        v.clone()
+    }
+
+    fn imm12_and(&mut self, imm: Imm12, x: u64) -> Imm12 {
+        Imm12::from_bits(imm.as_i16() & (x as i16))
+    }
+
+    fn alloc_vec_writable(&mut self, ty: Type) -> VecWritableReg {
+        if ty.is_int() || ty == R32 || ty == R64 {
+            if ty.bits() <= 64 {
+                vec![self.temp_writable_reg(I64)]
+            } else {
+                vec![self.temp_writable_reg(I64), self.temp_writable_reg(I64)]
+            }
+        } else if ty.is_float() {
+            vec![self.temp_writable_reg(ty)]
+        } else {
+            unimplemented!("ty:{:?}", ty)
+        }
+    }
+
+    fn imm(&mut self, ty: Type, val: u64) -> Reg {
+        let tmp = self.temp_writable_reg(ty);
+        let alloc_tmp = &mut |ty| self.temp_writable_reg(ty);
+        let insts = match ty {
+            F32 => MInst::load_fp_constant32(tmp, val as u32, alloc_tmp),
+            F64 => MInst::load_fp_constant64(tmp, val, alloc_tmp),
+            _ => MInst::load_constant_u64(tmp, val, alloc_tmp),
+        };
+        self.emit_list(&insts);
+        tmp.to_reg()
+    }
+    #[inline]
+    fn emit(&mut self, arg0: &MInst) -> Unit {
+        self.lower_ctx.emit(arg0.clone());
+    }
+    #[inline]
+    fn imm12_from_u64(&mut self, arg0: u64) -> Option<Imm12> {
+        Imm12::maybe_from_u64(arg0)
+    }
+    #[inline]
+    fn imm5_from_u64(&mut self, arg0: u64) -> Option<Imm5> {
+        Imm5::maybe_from_i8(i8::try_from(arg0 as i64).ok()?)
+    }
+    #[inline]
+    fn imm5_from_i8(&mut self, arg0: i8) -> Option<Imm5> {
+        Imm5::maybe_from_i8(arg0)
+    }
+    #[inline]
+    fn uimm5_bitcast_to_imm5(&mut self, arg0: UImm5) -> Imm5 {
+        Imm5::from_bits(arg0.bits() as u8)
+    }
+    #[inline]
+    fn uimm5_from_u8(&mut self, arg0: u8) -> Option<UImm5> {
+        UImm5::maybe_from_u8(arg0)
+    }
+    #[inline]
+    fn uimm5_from_u64(&mut self, arg0: u64) -> Option<UImm5> {
+        arg0.try_into().ok().and_then(UImm5::maybe_from_u8)
+    }
+    #[inline]
+    fn writable_zero_reg(&mut self) -> WritableReg {
+        writable_zero_reg()
+    }
+    #[inline]
+    fn neg_imm12(&mut self, arg0: Imm12) -> Imm12 {
+        -arg0
+    }
+    #[inline]
+    fn zero_reg(&mut self) -> Reg {
+        zero_reg()
+    }
+    #[inline]
+    fn imm_from_bits(&mut self, val: u64) -> Imm12 {
+        Imm12::maybe_from_u64(val).unwrap()
+    }
+    #[inline]
+    fn imm_from_neg_bits(&mut self, val: i64) -> Imm12 {
+        Imm12::maybe_from_u64(val as u64).unwrap()
+    }
+
+    fn gen_default_frm(&mut self) -> OptionFloatRoundingMode {
+        None
+    }
+    fn gen_select_reg(&mut self, cc: &IntCC, a: XReg, b: XReg, rs1: Reg, rs2: Reg) -> Reg {
+        let rd = self.temp_writable_reg(MInst::canonical_type_for_rc(rs1.class()));
+        self.emit(&MInst::SelectReg {
+            rd,
+            rs1,
+            rs2,
+            condition: IntegerCompare {
+                kind: *cc,
+                rs1: a.to_reg(),
+                rs2: b.to_reg(),
+            },
+        });
+        rd.to_reg()
+    }
+    fn load_u64_constant(&mut self, val: u64) -> Reg {
+        let rd = self.temp_writable_reg(I64);
+        MInst::load_constant_u64(rd, val, &mut |ty| self.temp_writable_reg(ty))
+            .iter()
+            .for_each(|i| self.emit(i));
+        rd.to_reg()
+    }
+    fn u8_as_i32(&mut self, x: u8) -> i32 {
+        x as i32
+    }
+
+    fn imm12_const(&mut self, val: i32) -> Imm12 {
+        if let Some(res) = Imm12::maybe_from_u64(val as u64) {
+            res
+        } else {
+            panic!("Unable to make an Imm12 value from {}", val)
+        }
+    }
+    fn imm12_const_add(&mut self, val: i32, add: i32) -> Imm12 {
+        Imm12::maybe_from_u64((val + add) as u64).unwrap()
+    }
+
+    //
+    fn gen_shamt(&mut self, ty: Type, shamt: XReg) -> ValueRegs {
+        let ty_bits = if ty.bits() > 64 { 64 } else { ty.bits() };
+        let shamt = {
+            let tmp = self.temp_writable_reg(I64);
+            self.emit(&MInst::AluRRImm12 {
+                alu_op: AluOPRRI::Andi,
+                rd: tmp,
+                rs: shamt.to_reg(),
+                imm12: Imm12::from_bits((ty_bits - 1) as i16),
+            });
+            tmp.to_reg()
+        };
+        let len_sub_shamt = {
+            let tmp = self.temp_writable_reg(I64);
+            self.emit(&MInst::load_imm12(tmp, Imm12::from_bits(ty_bits as i16)));
+            let len_sub_shamt = self.temp_writable_reg(I64);
+            self.emit(&MInst::AluRRR {
+                alu_op: AluOPRRR::Sub,
+                rd: len_sub_shamt,
+                rs1: tmp.to_reg(),
+                rs2: shamt,
+            });
+            len_sub_shamt.to_reg()
+        };
+        ValueRegs::two(shamt, len_sub_shamt)
+    }
+
+    fn has_v(&mut self) -> bool {
+        self.backend.isa_flags.has_v()
+    }
+
+    fn has_zbkb(&mut self) -> bool {
+        self.backend.isa_flags.has_zbkb()
+    }
+
+    fn has_zba(&mut self) -> bool {
+        self.backend.isa_flags.has_zba()
+    }
+
+    fn has_zbb(&mut self) -> bool {
+        self.backend.isa_flags.has_zbb()
+    }
+
+    fn has_zbc(&mut self) -> bool {
+        self.backend.isa_flags.has_zbc()
+    }
+
+    fn has_zbs(&mut self) -> bool {
+        self.backend.isa_flags.has_zbs()
+    }
+
+    fn offset32_imm(&mut self, offset: i32) -> Offset32 {
+        Offset32::new(offset)
+    }
+    fn default_memflags(&mut self) -> MemFlags {
+        MemFlags::new()
+    }
+
+    fn pack_float_rounding_mode(&mut self, f: &FRM) -> OptionFloatRoundingMode {
+        Some(*f)
+    }
+
+    fn int_convert_2_float_op(&mut self, from: Type, is_signed: bool, to: Type) -> FpuOPRR {
+        FpuOPRR::int_convert_2_float_op(from, is_signed, to)
+    }
+
+    fn gen_amode(&mut self, base: Reg, offset: Offset32, ty: Type) -> AMode {
+        AMode::RegOffset(base, i64::from(offset), ty)
+    }
+
+    fn gen_const_amode(&mut self, c: VCodeConstant) -> AMode {
+        AMode::Const(c)
+    }
+
+    fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {
+        if ty.is_int() && ty.bits() <= 64 {
+            Some(ty)
+        } else {
+            None
+        }
+    }
+    fn is_atomic_rmw_max_etc(&mut self, op: &AtomicRmwOp) -> Option<(AtomicRmwOp, bool)> {
+        let op = *op;
+        match op {
+            crate::ir::AtomicRmwOp::Umin => Some((op, false)),
+            crate::ir::AtomicRmwOp::Umax => Some((op, false)),
+            crate::ir::AtomicRmwOp::Smin => Some((op, true)),
+            crate::ir::AtomicRmwOp::Smax => Some((op, true)),
+            _ => None,
+        }
+    }
+    fn load_op(&mut self, ty: Type) -> LoadOP {
+        LoadOP::from_type(ty)
+    }
+    fn store_op(&mut self, ty: Type) -> StoreOP {
+        StoreOP::from_type(ty)
+    }
+    fn load_ext_name(&mut self, name: ExternalName, offset: i64) -> Reg {
+        let tmp = self.temp_writable_reg(I64);
+        self.emit(&MInst::LoadExtName {
+            rd: tmp,
+            name: Box::new(name),
+            offset,
+        });
+        tmp.to_reg()
+    }
+
+    fn offset32_add(&mut self, a: Offset32, adden: i64) -> Offset32 {
+        a.try_add_i64(adden).expect("offset exceed range.")
+    }
+
+    fn gen_stack_addr(&mut self, slot: StackSlot, offset: Offset32) -> Reg {
+        let result = self.temp_writable_reg(I64);
+        let i = self
+            .lower_ctx
+            .abi()
+            .sized_stackslot_addr(slot, i64::from(offset) as u32, result);
+        self.emit(&i);
+        result.to_reg()
+    }
+    fn atomic_amo(&mut self) -> AMO {
+        AMO::SeqCst
+    }
+
+    fn lower_br_table(&mut self, index: Reg, targets: &VecMachLabel) -> Unit {
+        let tmp1 = self.temp_writable_reg(I64);
+        let tmp2 = self.temp_writable_reg(I64);
+        let targets: Vec<BranchTarget> = targets
+            .into_iter()
+            .copied()
+            .map(BranchTarget::Label)
+            .collect();
+        self.emit(&MInst::BrTable {
+            index,
+            tmp1,
+            tmp2,
+            targets,
+        });
+    }
+
+    fn fp_reg(&mut self) -> PReg {
+        px_reg(8)
+    }
+
+    fn sp_reg(&mut self) -> PReg {
+        px_reg(2)
+    }
+
+    fn shift_int_to_most_significant(&mut self, v: XReg, ty: Type) -> XReg {
+        assert!(ty.is_int() && ty.bits() <= 64);
+        if ty == I64 {
+            return v;
+        }
+        let tmp = self.temp_writable_reg(I64);
+        self.emit(&MInst::AluRRImm12 {
+            alu_op: AluOPRRI::Slli,
+            rd: tmp,
+            rs: v.to_reg(),
+            imm12: Imm12::from_bits((64 - ty.bits()) as i16),
+        });
+
+        self.xreg_new(tmp.to_reg())
+    }
+
+    #[inline]
+    fn int_compare(&mut self, kind: &IntCC, rs1: XReg, rs2: XReg) -> IntegerCompare {
+        IntegerCompare {
+            kind: *kind,
+            rs1: rs1.to_reg(),
+            rs2: rs2.to_reg(),
+        }
+    }
+
+    #[inline]
+    fn vstate_from_type(&mut self, ty: Type) -> VState {
+        VState::from_type(ty)
+    }
+
+    fn min_vec_reg_size(&mut self) -> u64 {
+        self.min_vec_reg_size
+    }
+
+    #[inline]
+    fn ty_vec_fits_in_register(&mut self, ty: Type) -> Option<Type> {
+        if ty.is_vector() && (ty.bits() as u64) <= self.min_vec_reg_size() {
+            Some(ty)
+        } else {
+            None
+        }
+    }
+
+    fn vec_alu_rr_dst_type(&mut self, op: &VecAluOpRR) -> Type {
+        MInst::canonical_type_for_rc(op.dst_regclass())
+    }
+}
+
+/// The main entry point for lowering with ISLE.
+pub(crate) fn lower(
+    lower_ctx: &mut Lower<MInst>,
+    backend: &Riscv64Backend,
+    inst: Inst,
+) -> Option<InstOutput> {
+    // TODO: reuse the ISLE context across lowerings so we can reuse its
+    // internal heap allocations.
+    let mut isle_ctx = RV64IsleContext::new(lower_ctx, backend);
+    generated_code::constructor_lower(&mut isle_ctx, inst)
+}
+
+/// The main entry point for branch lowering with ISLE.
+pub(crate) fn lower_branch(
+    lower_ctx: &mut Lower<MInst>,
+    backend: &Riscv64Backend,
+    branch: Inst,
+    targets: &[MachLabel],
+) -> Option<()> {
+    // TODO: reuse the ISLE context across lowerings so we can reuse its
+    // internal heap allocations.
+    let mut isle_ctx = RV64IsleContext::new(lower_ctx, backend);
+    generated_code::constructor_lower_branch(&mut isle_ctx, branch, &targets.to_vec())
+}