wasmtime 20.0.2 → 21.0.1

data/ext/cargo-vendor/wasmtime-20.0.2/src/runtime/component/values.rs

@@ -1,978 +0,0 @@
-use crate::component::func::{desc, Lift, LiftContext, Lower, LowerContext};
-use crate::component::ResourceAny;
-use crate::ValRaw;
-use anyhow::{anyhow, bail, Result};
-use std::mem::MaybeUninit;
-use wasmtime_component_util::{DiscriminantSize, FlagsSize};
-use wasmtime_environ::component::{
-    CanonicalAbiInfo, InterfaceType, TypeEnum, TypeFlags, TypeListIndex, TypeOption, TypeResult,
-    TypeVariant, VariantInfo,
-};
-
-/// Represents possible runtime values which a component function can either
-/// consume or produce
-///
-/// This is a dynamic representation of possible values in the component model.
-/// Note that this is not an efficient representation but is instead intended to
-/// be a flexible and somewhat convenient representation. The most efficient
-/// representation of component model types is to use the `bindgen!` macro to
-/// generate native Rust types with specialized liftings and lowerings.
-///
-/// This type is used in conjunction with [`Func::call`] for example if the
-/// signature of a component is not statically known ahead of time.
-///
-/// # Equality and `Val`
-///
-/// This type implements both the Rust `PartialEq` and `Eq` traits. This type
-/// additionally contains values which are not necessarily easily equated,
-/// however, such as floats (`Float32` and `Float64`) and resources. Equality
-/// does require that two values have the same type, and then these cases are
-/// handled as:
-///
-/// * Floats are tested if they are "semantically the same" meaning all NaN
-///   values are equal to all other NaN values. Additionally zero values must be
-///   exactly the same, so positive zero is not equal to negative zero. The
-///   primary use case at this time is fuzzing-related equality which this is
-///   sufficient for.
-///
-/// * Resources are tested if their types and indices into the host table are
-///   equal. This does not compare the underlying representation so borrows of
-///   the same guest resource are not considered equal. This additionally
-///   doesn't go further and test for equality in the guest itself (for example
-///   two different heap allocations of `Box<u32>` can be equal in normal Rust
-///   if they contain the same value, but will never be considered equal when
-///   compared as `Val::Resource`s).
-///
-/// In general if a strict guarantee about equality is required here it's
-/// recommended to "build your own" as this equality intended for fuzzing
-/// Wasmtime may not be suitable for you.
-///
-/// # Component model types and `Val`
-///
-/// The `Val` type here does not contain enough information to say what the
-/// component model type of a `Val` is. This is instead more of an AST of sorts.
-/// For example the `Val::Enum` only carries information about a single
-/// discriminant, not the entire enumeration or what it's a discriminant of.
-///
-/// This means that when a `Val` is passed to Wasmtime, for example as a
-/// function parameter when calling a function or as a return value from an
-/// host-defined imported function, then it must pass a type-check. Instances of
-/// `Val` are type-checked against what's required by the component itself.
-///
-/// [`Func::call`]: crate::component::Func::call
-#[derive(Debug, Clone)]
-#[allow(missing_docs)]
-pub enum Val {
-    Bool(bool),
-    S8(i8),
-    U8(u8),
-    S16(i16),
-    U16(u16),
-    S32(i32),
-    U32(u32),
-    S64(i64),
-    U64(u64),
-    Float32(f32),
-    Float64(f64),
-    Char(char),
-    String(String),
-    List(Vec<Val>),
-    Record(Vec<(String, Val)>),
-    Tuple(Vec<Val>),
-    Variant(String, Option<Box<Val>>),
-    Enum(String),
-    Option(Option<Box<Val>>),
-    Result(Result<Option<Box<Val>>, Option<Box<Val>>>),
-    Flags(Vec<String>),
-    Resource(ResourceAny),
-}
-
-impl Val {
-    /// Deserialize a value of this type from core Wasm stack values.
-    pub(crate) fn lift(
-        cx: &mut LiftContext<'_>,
-        ty: InterfaceType,
-        src: &mut std::slice::Iter<'_, ValRaw>,
-    ) -> Result<Val> {
-        Ok(match ty {
-            InterfaceType::Bool => Val::Bool(bool::lift(cx, ty, next(src))?),
-            InterfaceType::S8 => Val::S8(i8::lift(cx, ty, next(src))?),
-            InterfaceType::U8 => Val::U8(u8::lift(cx, ty, next(src))?),
-            InterfaceType::S16 => Val::S16(i16::lift(cx, ty, next(src))?),
-            InterfaceType::U16 => Val::U16(u16::lift(cx, ty, next(src))?),
-            InterfaceType::S32 => Val::S32(i32::lift(cx, ty, next(src))?),
-            InterfaceType::U32 => Val::U32(u32::lift(cx, ty, next(src))?),
-            InterfaceType::S64 => Val::S64(i64::lift(cx, ty, next(src))?),
-            InterfaceType::U64 => Val::U64(u64::lift(cx, ty, next(src))?),
-            InterfaceType::Float32 => Val::Float32(f32::lift(cx, ty, next(src))?),
-            InterfaceType::Float64 => Val::Float64(f64::lift(cx, ty, next(src))?),
-            InterfaceType::Char => Val::Char(char::lift(cx, ty, next(src))?),
-            InterfaceType::Own(_) | InterfaceType::Borrow(_) => {
-                Val::Resource(ResourceAny::lift(cx, ty, next(src))?)
-            }
-            InterfaceType::String => Val::String(<_>::lift(cx, ty, &[*next(src), *next(src)])?),
-            InterfaceType::List(i) => {
-                // FIXME: needs memory64 treatment
-                let ptr = u32::lift(cx, InterfaceType::U32, next(src))? as usize;
-                let len = u32::lift(cx, InterfaceType::U32, next(src))? as usize;
-                load_list(cx, i, ptr, len)?
-            }
-            InterfaceType::Record(i) => Val::Record(
-                cx.types[i]
-                    .fields
-                    .iter()
-                    .map(|field| {
-                        let val = Self::lift(cx, field.ty, src)?;
-                        Ok((field.name.to_string(), val))
-                    })
-                    .collect::<Result<_>>()?,
-            ),
-            InterfaceType::Tuple(i) => Val::Tuple(
-                cx.types[i]
-                    .types
-                    .iter()
-                    .map(|ty| Self::lift(cx, *ty, src))
-                    .collect::<Result<_>>()?,
-            ),
-            InterfaceType::Variant(i) => {
-                let vty = &cx.types[i];
-                let (discriminant, value) = lift_variant(
-                    cx,
-                    cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap(),
-                    vty.cases.values().copied(),
-                    src,
-                )?;
-
-                let (k, _) = vty.cases.get_index(discriminant as usize).unwrap();
-                Val::Variant(k.clone(), value)
-            }
-            InterfaceType::Enum(i) => {
-                let ety = &cx.types[i];
-                let (discriminant, _) = lift_variant(
-                    cx,
-                    cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap(),
-                    ety.names.iter().map(|_| None),
-                    src,
-                )?;
-
-                Val::Enum(ety.names[discriminant as usize].clone())
-            }
-            InterfaceType::Option(i) => {
-                let (_discriminant, value) = lift_variant(
-                    cx,
-                    cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap(),
-                    [None, Some(cx.types[i].ty)].into_iter(),
-                    src,
-                )?;
-
-                Val::Option(value)
-            }
-            InterfaceType::Result(i) => {
-                let result_ty = &cx.types[i];
-                let (discriminant, value) = lift_variant(
-                    cx,
-                    cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap(),
-                    [result_ty.ok, result_ty.err].into_iter(),
-                    src,
-                )?;
-
-                Val::Result(if discriminant == 0 {
-                    Ok(value)
-                } else {
-                    Err(value)
-                })
-            }
-            InterfaceType::Flags(i) => {
-                let u32_count = cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap();
-                let ty = &cx.types[i];
-                let mut flags = Vec::new();
-                for i in 0..u32_count {
-                    push_flags(
-                        ty,
-                        &mut flags,
-                        (i as u32) * 32,
-                        u32::lift(cx, InterfaceType::U32, next(src))?,
-                    );
-                }
-
-                Val::Flags(flags.into())
-            }
-        })
-    }
-
-    /// Deserialize a value of this type from the heap.
-    pub(crate) fn load(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Val> {
-        Ok(match ty {
-            InterfaceType::Bool => Val::Bool(bool::load(cx, ty, bytes)?),
-            InterfaceType::S8 => Val::S8(i8::load(cx, ty, bytes)?),
-            InterfaceType::U8 => Val::U8(u8::load(cx, ty, bytes)?),
-            InterfaceType::S16 => Val::S16(i16::load(cx, ty, bytes)?),
-            InterfaceType::U16 => Val::U16(u16::load(cx, ty, bytes)?),
-            InterfaceType::S32 => Val::S32(i32::load(cx, ty, bytes)?),
-            InterfaceType::U32 => Val::U32(u32::load(cx, ty, bytes)?),
-            InterfaceType::S64 => Val::S64(i64::load(cx, ty, bytes)?),
-            InterfaceType::U64 => Val::U64(u64::load(cx, ty, bytes)?),
-            InterfaceType::Float32 => Val::Float32(f32::load(cx, ty, bytes)?),
-            InterfaceType::Float64 => Val::Float64(f64::load(cx, ty, bytes)?),
-            InterfaceType::Char => Val::Char(char::load(cx, ty, bytes)?),
-            InterfaceType::String => Val::String(<_>::load(cx, ty, bytes)?),
-            InterfaceType::Own(_) | InterfaceType::Borrow(_) => {
-                Val::Resource(ResourceAny::load(cx, ty, bytes)?)
-            }
-            InterfaceType::List(i) => {
-                // FIXME: needs memory64 treatment
-                let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap()) as usize;
-                let len = u32::from_le_bytes(bytes[4..].try_into().unwrap()) as usize;
-                load_list(cx, i, ptr, len)?
-            }
-
-            InterfaceType::Record(i) => {
-                let mut offset = 0;
-                let fields = cx.types[i].fields.iter();
-                Val::Record(
-                    fields
-                        .map(|field| -> Result<(String, Val)> {
-                            let abi = cx.types.canonical_abi(&field.ty);
-                            let offset = abi.next_field32(&mut offset);
-                            let offset = usize::try_from(offset).unwrap();
-                            let size = usize::try_from(abi.size32).unwrap();
-                            Ok((
-                                field.name.to_string(),
-                                Val::load(cx, field.ty, &bytes[offset..][..size])?,
-                            ))
-                        })
-                        .collect::<Result<_>>()?,
-                )
-            }
-            InterfaceType::Tuple(i) => {
-                let types = cx.types[i].types.iter().copied();
-                let mut offset = 0;
-                Val::Tuple(
-                    types
-                        .map(|ty| {
-                            let abi = cx.types.canonical_abi(&ty);
-                            let offset = abi.next_field32(&mut offset);
-                            let offset = usize::try_from(offset).unwrap();
-                            let size = usize::try_from(abi.size32).unwrap();
-                            Val::load(cx, ty, &bytes[offset..][..size])
-                        })
-                        .collect::<Result<_>>()?,
-                )
-            }
-            InterfaceType::Variant(i) => {
-                let ty = &cx.types[i];
-                let (discriminant, value) =
-                    load_variant(cx, &ty.info, ty.cases.values().copied(), bytes)?;
-
-                let (k, _) = ty.cases.get_index(discriminant as usize).unwrap();
-                Val::Variant(k.clone(), value)
-            }
-            InterfaceType::Enum(i) => {
-                let ty = &cx.types[i];
-                let (discriminant, _) =
-                    load_variant(cx, &ty.info, ty.names.iter().map(|_| None), bytes)?;
-
-                Val::Enum(ty.names[discriminant as usize].clone())
-            }
-            InterfaceType::Option(i) => {
-                let ty = &cx.types[i];
-                let (_discriminant, value) =
-                    load_variant(cx, &ty.info, [None, Some(ty.ty)].into_iter(), bytes)?;
-
-                Val::Option(value)
-            }
-            InterfaceType::Result(i) => {
-                let ty = &cx.types[i];
-                let (discriminant, value) =
-                    load_variant(cx, &ty.info, [ty.ok, ty.err].into_iter(), bytes)?;
-
-                Val::Result(if discriminant == 0 {
-                    Ok(value)
-                } else {
-                    Err(value)
-                })
-            }
-            InterfaceType::Flags(i) => {
-                let ty = &cx.types[i];
-                let mut flags = Vec::new();
-                match FlagsSize::from_count(ty.names.len()) {
-                    FlagsSize::Size0 => {}
-                    FlagsSize::Size1 => {
-                        let bits = u8::load(cx, InterfaceType::U8, bytes)?;
-                        push_flags(ty, &mut flags, 0, u32::from(bits));
-                    }
-                    FlagsSize::Size2 => {
-                        let bits = u16::load(cx, InterfaceType::U16, bytes)?;
-                        push_flags(ty, &mut flags, 0, u32::from(bits));
-                    }
-                    FlagsSize::Size4Plus(n) => {
-                        for i in 0..n {
-                            let bits = u32::load(
-                                cx,
-                                InterfaceType::U32,
-                                &bytes[usize::from(i) * 4..][..4],
-                            )?;
-                            push_flags(ty, &mut flags, u32::from(i) * 32, bits);
-                        }
-                    }
-                }
-                Val::Flags(flags.into())
-            }
-        })
-    }
-
-    /// Serialize this value as core Wasm stack values.
-    pub(crate) fn lower<T>(
-        &self,
-        cx: &mut LowerContext<'_, T>,
-        ty: InterfaceType,
-        dst: &mut std::slice::IterMut<'_, MaybeUninit<ValRaw>>,
-    ) -> Result<()> {
-        match (ty, self) {
-            (InterfaceType::Bool, Val::Bool(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::Bool, _) => unexpected(ty, self),
-            (InterfaceType::S8, Val::S8(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::S8, _) => unexpected(ty, self),
-            (InterfaceType::U8, Val::U8(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::U8, _) => unexpected(ty, self),
-            (InterfaceType::S16, Val::S16(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::S16, _) => unexpected(ty, self),
-            (InterfaceType::U16, Val::U16(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::U16, _) => unexpected(ty, self),
-            (InterfaceType::S32, Val::S32(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::S32, _) => unexpected(ty, self),
-            (InterfaceType::U32, Val::U32(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::U32, _) => unexpected(ty, self),
-            (InterfaceType::S64, Val::S64(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::S64, _) => unexpected(ty, self),
-            (InterfaceType::U64, Val::U64(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::U64, _) => unexpected(ty, self),
-            (InterfaceType::Float32, Val::Float32(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::Float32, _) => unexpected(ty, self),
-            (InterfaceType::Float64, Val::Float64(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::Float64, _) => unexpected(ty, self),
-            (InterfaceType::Char, Val::Char(value)) => value.lower(cx, ty, next_mut(dst)),
-            (InterfaceType::Char, _) => unexpected(ty, self),
-            // NB: `lower` on `ResourceAny` does its own type-checking, so skip
-            // looking at it here.
-            (InterfaceType::Borrow(_) | InterfaceType::Own(_), Val::Resource(value)) => {
-                value.lower(cx, ty, next_mut(dst))
-            }
-            (InterfaceType::Borrow(_) | InterfaceType::Own(_), _) => unexpected(ty, self),
-            (InterfaceType::String, Val::String(value)) => {
-                let my_dst = &mut MaybeUninit::<[ValRaw; 2]>::uninit();
-                value.lower(cx, ty, my_dst)?;
-                let my_dst = unsafe { my_dst.assume_init() };
-                next_mut(dst).write(my_dst[0]);
-                next_mut(dst).write(my_dst[1]);
-                Ok(())
-            }
-            (InterfaceType::String, _) => unexpected(ty, self),
-            (InterfaceType::List(ty), Val::List(values)) => {
-                let ty = &cx.types[ty];
-                let (ptr, len) = lower_list(cx, ty.element, values)?;
-                next_mut(dst).write(ValRaw::i64(ptr as i64));
-                next_mut(dst).write(ValRaw::i64(len as i64));
-                Ok(())
-            }
-            (InterfaceType::List(_), _) => unexpected(ty, self),
-            (InterfaceType::Record(ty), Val::Record(values)) => {
-                let ty = &cx.types[ty];
-                if ty.fields.len() != values.len() {
-                    bail!("expected {} fields, got {}", ty.fields.len(), values.len());
-                }
-                for ((name, value), field) in values.iter().zip(ty.fields.iter()) {
-                    if *name != field.name {
-                        bail!("expected field `{}`, got `{name}`", field.name);
-                    }
-                    value.lower(cx, field.ty, dst)?;
-                }
-                Ok(())
-            }
-            (InterfaceType::Record(_), _) => unexpected(ty, self),
-            (InterfaceType::Tuple(ty), Val::Tuple(values)) => {
-                let ty = &cx.types[ty];
-                if ty.types.len() != values.len() {
-                    bail!("expected {} types, got {}", ty.types.len(), values.len());
-                }
-                for (value, ty) in values.iter().zip(ty.types.iter()) {
-                    value.lower(cx, *ty, dst)?;
-                }
-                Ok(())
-            }
-            (InterfaceType::Tuple(_), _) => unexpected(ty, self),
-            (InterfaceType::Variant(ty), Val::Variant(n, v)) => {
-                GenericVariant::variant(&cx.types[ty], n, v)?.lower(cx, dst)
-            }
-            (InterfaceType::Variant(_), _) => unexpected(ty, self),
-            (InterfaceType::Option(ty), Val::Option(v)) => {
-                GenericVariant::option(&cx.types[ty], v).lower(cx, dst)
-            }
-            (InterfaceType::Option(_), _) => unexpected(ty, self),
-            (InterfaceType::Result(ty), Val::Result(v)) => {
-                GenericVariant::result(&cx.types[ty], v)?.lower(cx, dst)
-            }
-            (InterfaceType::Result(_), _) => unexpected(ty, self),
-            (InterfaceType::Enum(ty), Val::Enum(discriminant)) => {
-                let discriminant = get_enum_discriminant(&cx.types[ty], discriminant)?;
-                next_mut(dst).write(ValRaw::u32(discriminant));
-                Ok(())
-            }
-            (InterfaceType::Enum(_), _) => unexpected(ty, self),
-            (InterfaceType::Flags(ty), Val::Flags(value)) => {
-                let ty = &cx.types[ty];
-                let storage = flags_to_storage(ty, value)?;
-                for value in storage {
-                    next_mut(dst).write(ValRaw::u32(value));
-                }
-                Ok(())
-            }
-            (InterfaceType::Flags(_), _) => unexpected(ty, self),
-        }
-    }
-
-    /// Serialize this value to the heap at the specified memory location.
-    pub(crate) fn store<T>(
-        &self,
-        cx: &mut LowerContext<'_, T>,
-        ty: InterfaceType,
-        offset: usize,
-    ) -> Result<()> {
-        debug_assert!(offset % usize::try_from(cx.types.canonical_abi(&ty).align32)? == 0);
-
-        match (ty, self) {
-            (InterfaceType::Bool, Val::Bool(value)) => value.store(cx, ty, offset),
-            (InterfaceType::Bool, _) => unexpected(ty, self),
-            (InterfaceType::U8, Val::U8(value)) => value.store(cx, ty, offset),
-            (InterfaceType::U8, _) => unexpected(ty, self),
-            (InterfaceType::S8, Val::S8(value)) => value.store(cx, ty, offset),
-            (InterfaceType::S8, _) => unexpected(ty, self),
-            (InterfaceType::U16, Val::U16(value)) => value.store(cx, ty, offset),
-            (InterfaceType::U16, _) => unexpected(ty, self),
-            (InterfaceType::S16, Val::S16(value)) => value.store(cx, ty, offset),
-            (InterfaceType::S16, _) => unexpected(ty, self),
-            (InterfaceType::U32, Val::U32(value)) => value.store(cx, ty, offset),
-            (InterfaceType::U32, _) => unexpected(ty, self),
-            (InterfaceType::S32, Val::S32(value)) => value.store(cx, ty, offset),
-            (InterfaceType::S32, _) => unexpected(ty, self),
-            (InterfaceType::U64, Val::U64(value)) => value.store(cx, ty, offset),
-            (InterfaceType::U64, _) => unexpected(ty, self),
-            (InterfaceType::S64, Val::S64(value)) => value.store(cx, ty, offset),
-            (InterfaceType::S64, _) => unexpected(ty, self),
-            (InterfaceType::Float32, Val::Float32(value)) => value.store(cx, ty, offset),
-            (InterfaceType::Float32, _) => unexpected(ty, self),
-            (InterfaceType::Float64, Val::Float64(value)) => value.store(cx, ty, offset),
-            (InterfaceType::Float64, _) => unexpected(ty, self),
-            (InterfaceType::Char, Val::Char(value)) => value.store(cx, ty, offset),
-            (InterfaceType::Char, _) => unexpected(ty, self),
-            (InterfaceType::String, Val::String(value)) => value.store(cx, ty, offset),
-            (InterfaceType::String, _) => unexpected(ty, self),
-
-            // NB: resources do type-checking when they lower.
-            (InterfaceType::Borrow(_) | InterfaceType::Own(_), Val::Resource(value)) => {
-                value.store(cx, ty, offset)
-            }
-            (InterfaceType::Borrow(_) | InterfaceType::Own(_), _) => unexpected(ty, self),
-            (InterfaceType::List(ty), Val::List(values)) => {
-                let ty = &cx.types[ty];
-                let (ptr, len) = lower_list(cx, ty.element, values)?;
-                // FIXME: needs memory64 handling
-                *cx.get(offset + 0) = (ptr as i32).to_le_bytes();
-                *cx.get(offset + 4) = (len as i32).to_le_bytes();
-                Ok(())
-            }
-            (InterfaceType::List(_), _) => unexpected(ty, self),
-            (InterfaceType::Record(ty), Val::Record(values)) => {
-                let ty = &cx.types[ty];
-                if ty.fields.len() != values.len() {
-                    bail!("expected {} fields, got {}", ty.fields.len(), values.len());
-                }
-                let mut offset = offset;
-                for ((name, value), field) in values.iter().zip(ty.fields.iter()) {
-                    if *name != field.name {
-                        bail!("expected field `{}`, got `{name}`", field.name);
-                    }
-                    value.store(
-                        cx,
-                        field.ty,
-                        cx.types
-                            .canonical_abi(&field.ty)
-                            .next_field32_size(&mut offset),
-                    )?;
-                }
-                Ok(())
-            }
-            (InterfaceType::Record(_), _) => unexpected(ty, self),
-            (InterfaceType::Tuple(ty), Val::Tuple(values)) => {
-                let ty = &cx.types[ty];
-                if ty.types.len() != values.len() {
-                    bail!("expected {} types, got {}", ty.types.len(), values.len());
-                }
-                let mut offset = offset;
-                for (value, ty) in values.iter().zip(ty.types.iter()) {
-                    value.store(
-                        cx,
-                        *ty,
-                        cx.types.canonical_abi(ty).next_field32_size(&mut offset),
-                    )?;
-                }
-                Ok(())
-            }
-            (InterfaceType::Tuple(_), _) => unexpected(ty, self),
-
-            (InterfaceType::Variant(ty), Val::Variant(n, v)) => {
-                GenericVariant::variant(&cx.types[ty], n, v)?.store(cx, offset)
-            }
-            (InterfaceType::Variant(_), _) => unexpected(ty, self),
-            (InterfaceType::Enum(ty), Val::Enum(v)) => {
-                GenericVariant::enum_(&cx.types[ty], v)?.store(cx, offset)
-            }
-            (InterfaceType::Enum(_), _) => unexpected(ty, self),
-            (InterfaceType::Option(ty), Val::Option(v)) => {
-                GenericVariant::option(&cx.types[ty], v).store(cx, offset)
-            }
-            (InterfaceType::Option(_), _) => unexpected(ty, self),
-            (InterfaceType::Result(ty), Val::Result(v)) => {
-                GenericVariant::result(&cx.types[ty], v)?.store(cx, offset)
-            }
-            (InterfaceType::Result(_), _) => unexpected(ty, self),
-
-            (InterfaceType::Flags(ty), Val::Flags(flags)) => {
-                let ty = &cx.types[ty];
-                let storage = flags_to_storage(ty, flags)?;
-                match FlagsSize::from_count(ty.names.len()) {
-                    FlagsSize::Size0 => {}
-                    FlagsSize::Size1 => {
-                        u8::try_from(storage[0])
-                            .unwrap()
-                            .store(cx, InterfaceType::U8, offset)?
-                    }
-                    FlagsSize::Size2 => {
-                        u16::try_from(storage[0])
-                            .unwrap()
-                            .store(cx, InterfaceType::U16, offset)?
-                    }
-                    FlagsSize::Size4Plus(_) => {
-                        let mut offset = offset;
-                        for value in storage {
-                            value.store(cx, InterfaceType::U32, offset)?;
-                            offset += 4;
-                        }
-                    }
-                }
-                Ok(())
-            }
-            (InterfaceType::Flags(_), _) => unexpected(ty, self),
-        }
-    }
-
-    fn desc(&self) -> &'static str {
-        match self {
-            Val::Bool(_) => "bool",
-            Val::U8(_) => "u8",
-            Val::S8(_) => "s8",
-            Val::U16(_) => "u16",
-            Val::S16(_) => "s16",
-            Val::U32(_) => "u32",
-            Val::S32(_) => "s32",
-            Val::U64(_) => "u64",
-            Val::S64(_) => "s64",
-            Val::Float32(_) => "f32",
-            Val::Float64(_) => "f64",
-            Val::Char(_) => "char",
-            Val::List(_) => "list",
-            Val::String(_) => "string",
-            Val::Record(_) => "record",
-            Val::Enum(_) => "enum",
-            Val::Variant(..) => "variant",
-            Val::Tuple(_) => "tuple",
-            Val::Option(_) => "option",
-            Val::Result(_) => "result",
-            Val::Resource(_) => "resource",
-            Val::Flags(_) => "flags",
-        }
-    }
-}
-
-impl PartialEq for Val {
-    fn eq(&self, other: &Self) -> bool {
-        match (self, other) {
-            // IEEE 754 equality considers NaN inequal to NaN and negative zero
-            // equal to positive zero, however we do the opposite here, because
-            // this logic is used by testing and fuzzing, which want to know
-            // whether two values are semantically the same, rather than
-            // numerically equal.
-            (Self::Float32(l), Self::Float32(r)) => {
-                (*l != 0.0 && l == r)
-                    || (*l == 0.0 && l.to_bits() == r.to_bits())
-                    || (l.is_nan() && r.is_nan())
-            }
-            (Self::Float32(_), _) => false,
-            (Self::Float64(l), Self::Float64(r)) => {
-                (*l != 0.0 && l == r)
-                    || (*l == 0.0 && l.to_bits() == r.to_bits())
-                    || (l.is_nan() && r.is_nan())
-            }
-            (Self::Float64(_), _) => false,
-
-            (Self::Bool(l), Self::Bool(r)) => l == r,
-            (Self::Bool(_), _) => false,
-            (Self::S8(l), Self::S8(r)) => l == r,
-            (Self::S8(_), _) => false,
-            (Self::U8(l), Self::U8(r)) => l == r,
-            (Self::U8(_), _) => false,
-            (Self::S16(l), Self::S16(r)) => l == r,
-            (Self::S16(_), _) => false,
-            (Self::U16(l), Self::U16(r)) => l == r,
-            (Self::U16(_), _) => false,
-            (Self::S32(l), Self::S32(r)) => l == r,
-            (Self::S32(_), _) => false,
-            (Self::U32(l), Self::U32(r)) => l == r,
-            (Self::U32(_), _) => false,
-            (Self::S64(l), Self::S64(r)) => l == r,
-            (Self::S64(_), _) => false,
-            (Self::U64(l), Self::U64(r)) => l == r,
-            (Self::U64(_), _) => false,
-            (Self::Char(l), Self::Char(r)) => l == r,
-            (Self::Char(_), _) => false,
-            (Self::String(l), Self::String(r)) => l == r,
-            (Self::String(_), _) => false,
-            (Self::List(l), Self::List(r)) => l == r,
-            (Self::List(_), _) => false,
-            (Self::Record(l), Self::Record(r)) => l == r,
-            (Self::Record(_), _) => false,
-            (Self::Tuple(l), Self::Tuple(r)) => l == r,
-            (Self::Tuple(_), _) => false,
-            (Self::Variant(ln, lv), Self::Variant(rn, rv)) => ln == rn && lv == rv,
-            (Self::Variant(..), _) => false,
-            (Self::Enum(l), Self::Enum(r)) => l == r,
-            (Self::Enum(_), _) => false,
-            (Self::Option(l), Self::Option(r)) => l == r,
-            (Self::Option(_), _) => false,
-            (Self::Result(l), Self::Result(r)) => l == r,
-            (Self::Result(_), _) => false,
-            (Self::Flags(l), Self::Flags(r)) => l == r,
-            (Self::Flags(_), _) => false,
-            (Self::Resource(l), Self::Resource(r)) => l == r,
-            (Self::Resource(_), _) => false,
-        }
-    }
-}
-
-impl Eq for Val {}
-
-struct GenericVariant<'a> {
-    discriminant: u32,
-    payload: Option<(&'a Val, InterfaceType)>,
-    abi: &'a CanonicalAbiInfo,
-    info: &'a VariantInfo,
-}
-
-impl GenericVariant<'_> {
-    fn result<'a>(
-        ty: &'a TypeResult,
-        r: &'a Result<Option<Box<Val>>, Option<Box<Val>>>,
-    ) -> Result<GenericVariant<'a>> {
-        let (discriminant, payload) = match r {
-            Ok(val) => {
-                let payload = match (val, ty.ok) {
-                    (Some(val), Some(ty)) => Some((&**val, ty)),
-                    (None, None) => None,
-                    (Some(_), None) => {
-                        bail!("payload provided to `ok` but not expected");
-                    }
-                    (None, Some(_)) => {
-                        bail!("payload expected to `ok` but not provided");
-                    }
-                };
-                (0, payload)
-            }
-            Err(val) => {
-                let payload = match (val, ty.err) {
-                    (Some(val), Some(ty)) => Some((&**val, ty)),
-                    (None, None) => None,
-                    (Some(_), None) => {
-                        bail!("payload provided to `err` but not expected");
-                    }
-                    (None, Some(_)) => {
-                        bail!("payload expected to `err` but not provided");
-                    }
-                };
-                (1, payload)
-            }
-        };
-        Ok(GenericVariant {
-            discriminant,
-            payload,
-            abi: &ty.abi,
-            info: &ty.info,
-        })
-    }
-
-    fn option<'a>(ty: &'a TypeOption, r: &'a Option<Box<Val>>) -> GenericVariant<'a> {
-        let (discriminant, payload) = match r {
-            None => (0, None),
-            Some(val) => (1, Some((&**val, ty.ty))),
-        };
-        GenericVariant {
-            discriminant,
-            payload,
-            abi: &ty.abi,
-            info: &ty.info,
-        }
-    }
-
-    fn enum_<'a>(ty: &'a TypeEnum, discriminant: &str) -> Result<GenericVariant<'a>> {
-        let discriminant = get_enum_discriminant(ty, discriminant)?;
-
-        Ok(GenericVariant {
-            discriminant,
-            payload: None,
-            abi: &ty.abi,
-            info: &ty.info,
-        })
-    }
-
-    fn variant<'a>(
-        ty: &'a TypeVariant,
-        discriminant_name: &str,
-        payload: &'a Option<Box<Val>>,
-    ) -> Result<GenericVariant<'a>> {
-        let (discriminant, payload_ty) = get_variant_discriminant(ty, discriminant_name)?;
-
-        let payload = match (payload, payload_ty) {
-            (Some(val), Some(ty)) => Some((&**val, *ty)),
-            (None, None) => None,
-            (Some(_), None) => bail!("did not expect a payload for case `{discriminant_name}`"),
-            (None, Some(_)) => bail!("expected a payload for case `{discriminant_name}`"),
-        };
-
-        Ok(GenericVariant {
-            discriminant,
-            payload,
-            abi: &ty.abi,
-            info: &ty.info,
-        })
-    }
-
-    fn lower<T>(
-        &self,
-        cx: &mut LowerContext<'_, T>,
-        dst: &mut std::slice::IterMut<'_, MaybeUninit<ValRaw>>,
-    ) -> Result<()> {
-        next_mut(dst).write(ValRaw::u32(self.discriminant));
-
-        // For the remaining lowered representation of this variant that
-        // the payload didn't write we write out zeros here to ensure
-        // the entire variant is written.
-        let value_flat = match self.payload {
-            Some((value, ty)) => {
-                value.lower(cx, ty, dst)?;
-                cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap()
-            }
-            None => 0,
-        };
-        let variant_flat = self.abi.flat_count(usize::MAX).unwrap();
-        for _ in (1 + value_flat)..variant_flat {
-            next_mut(dst).write(ValRaw::u64(0));
-        }
-        Ok(())
-    }
-
-    fn store<T>(&self, cx: &mut LowerContext<'_, T>, offset: usize) -> Result<()> {
-        match self.info.size {
-            DiscriminantSize::Size1 => {
-                u8::try_from(self.discriminant)
-                    .unwrap()
-                    .store(cx, InterfaceType::U8, offset)?
-            }
-            DiscriminantSize::Size2 => {
-                u16::try_from(self.discriminant)
-                    .unwrap()
-                    .store(cx, InterfaceType::U16, offset)?
-            }
-            DiscriminantSize::Size4 => self.discriminant.store(cx, InterfaceType::U32, offset)?,
-        }
-
-        if let Some((value, ty)) = self.payload {
-            let offset = offset + usize::try_from(self.info.payload_offset32).unwrap();
-            value.store(cx, ty, offset)?;
-        }
-
-        Ok(())
-    }
-}
-
-fn load_list(cx: &mut LiftContext<'_>, ty: TypeListIndex, ptr: usize, len: usize) -> Result<Val> {
-    let elem = cx.types[ty].element;
-    let abi = cx.types.canonical_abi(&elem);
-    let element_size = usize::try_from(abi.size32).unwrap();
-    let element_alignment = abi.align32;
-
-    match len
-        .checked_mul(element_size)
-        .and_then(|len| ptr.checked_add(len))
-    {
-        Some(n) if n <= cx.memory().len() => {}
-        _ => bail!("list pointer/length out of bounds of memory"),
-    }
-    if ptr % usize::try_from(element_alignment)? != 0 {
-        bail!("list pointer is not aligned")
-    }
-
-    Ok(Val::List(
-        (0..len)
-            .map(|index| {
-                Val::load(
-                    cx,
-                    elem,
-                    &cx.memory()[ptr + (index * element_size)..][..element_size],
-                )
-            })
-            .collect::<Result<_>>()?,
-    ))
-}
-
-fn load_variant(
-    cx: &mut LiftContext<'_>,
-    info: &VariantInfo,
-    mut types: impl ExactSizeIterator<Item = Option<InterfaceType>>,
-    bytes: &[u8],
-) -> Result<(u32, Option<Box<Val>>)> {
-    let discriminant = match info.size {
-        DiscriminantSize::Size1 => u32::from(u8::load(cx, InterfaceType::U8, &bytes[..1])?),
-        DiscriminantSize::Size2 => u32::from(u16::load(cx, InterfaceType::U16, &bytes[..2])?),
-        DiscriminantSize::Size4 => u32::load(cx, InterfaceType::U32, &bytes[..4])?,
-    };
-    let case_ty = types.nth(discriminant as usize).ok_or_else(|| {
-        anyhow!(
-            "discriminant {} out of range [0..{})",
-            discriminant,
-            types.len()
-        )
-    })?;
-    let value = match case_ty {
-        Some(case_ty) => {
-            let payload_offset = usize::try_from(info.payload_offset32).unwrap();
-            let case_abi = cx.types.canonical_abi(&case_ty);
-            let case_size = usize::try_from(case_abi.size32).unwrap();
-            Some(Box::new(Val::load(
-                cx,
-                case_ty,
-                &bytes[payload_offset..][..case_size],
-            )?))
-        }
-        None => None,
-    };
-    Ok((discriminant, value))
-}
-
-fn lift_variant(
-    cx: &mut LiftContext<'_>,
-    flatten_count: usize,
-    mut types: impl ExactSizeIterator<Item = Option<InterfaceType>>,
-    src: &mut std::slice::Iter<'_, ValRaw>,
-) -> Result<(u32, Option<Box<Val>>)> {
-    let len = types.len();
-    let discriminant = next(src).get_u32();
-    let ty = types
-        .nth(discriminant as usize)
-        .ok_or_else(|| anyhow!("discriminant {} out of range [0..{})", discriminant, len))?;
-    let (value, value_flat) = match ty {
-        Some(ty) => (
-            Some(Box::new(Val::lift(cx, ty, src)?)),
-            cx.types.canonical_abi(&ty).flat_count(usize::MAX).unwrap(),
-        ),
-        None => (None, 0),
-    };
-    for _ in (1 + value_flat)..flatten_count {
-        next(src);
-    }
-    Ok((discriminant, value))
-}
-
-/// Lower a list with the specified element type and values.
-fn lower_list<T>(
-    cx: &mut LowerContext<'_, T>,
-    element_type: InterfaceType,
-    items: &[Val],
-) -> Result<(usize, usize)> {
-    let abi = cx.types.canonical_abi(&element_type);
-    let elt_size = usize::try_from(abi.size32)?;
-    let elt_align = abi.align32;
-    let size = items
-        .len()
-        .checked_mul(elt_size)
-        .ok_or_else(|| anyhow::anyhow!("size overflow copying a list"))?;
-    let ptr = cx.realloc(0, 0, elt_align, size)?;
-    let mut element_ptr = ptr;
-    for item in items {
-        item.store(cx, element_type, element_ptr)?;
-        element_ptr += elt_size;
-    }
-    Ok((ptr, items.len()))
-}
-
-fn push_flags(ty: &TypeFlags, flags: &mut Vec<String>, mut offset: u32, mut bits: u32) {
-    while bits > 0 {
-        if bits & 1 != 0 {
-            flags.push(ty.names[offset as usize].clone());
-        }
-        bits >>= 1;
-        offset += 1;
-    }
-}
-
-fn flags_to_storage(ty: &TypeFlags, flags: &[String]) -> Result<Vec<u32>> {
-    let mut storage = match FlagsSize::from_count(ty.names.len()) {
-        FlagsSize::Size0 => Vec::new(),
-        FlagsSize::Size1 | FlagsSize::Size2 => vec![0],
-        FlagsSize::Size4Plus(n) => vec![0; n.into()],
-    };
-
-    for flag in flags {
-        let bit = ty
-            .names
-            .get_index_of(flag)
-            .ok_or_else(|| anyhow::anyhow!("unknown flag: `{flag}`"))?;
-        storage[bit / 32] |= 1 << (bit % 32);
-    }
-    Ok(storage)
-}
-
-fn get_enum_discriminant(ty: &TypeEnum, n: &str) -> Result<u32> {
-    ty.names
-        .get_index_of(n)
-        .ok_or_else(|| anyhow::anyhow!("enum variant name `{n}` is not valid"))
-        .map(|i| i as u32)
-}
-
-fn get_variant_discriminant<'a>(
-    ty: &'a TypeVariant,
-    name: &str,
-) -> Result<(u32, &'a Option<InterfaceType>)> {
-    let (i, _, ty) = ty
-        .cases
-        .get_full(name)
-        .ok_or_else(|| anyhow::anyhow!("unknown variant case: `{name}`"))?;
-    Ok((i as u32, ty))
-}
-
-fn next<'a>(src: &mut std::slice::Iter<'a, ValRaw>) -> &'a ValRaw {
-    src.next().unwrap()
-}
-
-fn next_mut<'a>(
-    dst: &mut std::slice::IterMut<'a, MaybeUninit<ValRaw>>,
-) -> &'a mut MaybeUninit<ValRaw> {
-    dst.next().unwrap()
-}
-
-#[cold]
-fn unexpected<T>(ty: InterfaceType, val: &Val) -> Result<T> {
-    bail!(
-        "type mismatch: expected {}, found {}",
-        desc(&ty),
-        val.desc()
-    )
-}