wasmtime 37.0.2 → 38.0.4

data/ext/cargo-vendor/wasmtime-38.0.4/src/compile.rs

@@ -0,0 +1,1052 @@
+//! Wasm compilation orchestration.
+//!
+//! It works roughly like this:
+//!
+//! * We walk over the Wasm module/component and make a list of all the things
+//!   we need to compile. This is a `CompileInputs`.
+//!
+//! * The `CompileInputs::compile` method compiles each of these in parallel,
+//!   producing a `UnlinkedCompileOutputs`. This is an unlinked set of compiled
+//!   functions, bucketed by type of function.
+//!
+//! * The `UnlinkedCompileOutputs::pre_link` method re-arranges the compiled
+//!   functions into a flat list. This is the order we will place them within
+//!   the ELF file, so we must also keep track of all the functions' indices
+//!   within this list, because we will need them for resolving
+//!   relocations. These indices are kept track of in the resulting
+//!   `FunctionIndices`.
+//!
+//! * The `FunctionIndices::link_and_append_code` method appends the functions
+//!   to the given ELF file and resolves relocations. It produces an `Artifacts`
+//!   which contains the data needed at runtime to find and call Wasm
+//!   functions. It is up to the caller to serialize the relevant parts of the
+//!   `Artifacts` into the ELF file.
+
+use crate::Engine;
+use crate::hash_map::HashMap;
+use crate::hash_set::HashSet;
+use crate::prelude::*;
+use std::{any::Any, borrow::Cow, collections::BTreeMap, mem, ops::Range};
+
+use call_graph::CallGraph;
+#[cfg(feature = "component-model")]
+use wasmtime_environ::component::Translator;
+use wasmtime_environ::{
+    BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionsTable,
+    CompiledFunctionsTableBuilder, CompiledModuleInfo, Compiler, DefinedFuncIndex, FilePos,
+    FinishedObject, FuncKey, FunctionBodyData, InliningCompiler, IntraModuleInlining,
+    ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap,
+    StaticModuleIndex, Tunables,
+};
+
+mod call_graph;
+mod scc;
+mod stratify;
+
+mod code_builder;
+pub use self::code_builder::{CodeBuilder, CodeHint, HashedEngineCompileEnv};
+
+#[cfg(feature = "runtime")]
+mod runtime;
+
+/// Converts an input binary-encoded WebAssembly module to compilation
+/// artifacts and type information.
+///
+/// This is where compilation actually happens of WebAssembly modules and
+/// translation/parsing/validation of the binary input occurs. The binary
+/// artifact represented in the `MmapVec` returned here is an in-memory ELF
+/// file in an owned area of virtual linear memory where permissions (such
+/// as the executable bit) can be applied.
+///
+/// Additionally compilation returns an `Option` here which is always
+/// `Some`, notably compiled metadata about the module in addition to the
+/// type information found within.
+pub(crate) fn build_artifacts<T: FinishedObject>(
+    engine: &Engine,
+    wasm: &[u8],
+    dwarf_package: Option<&[u8]>,
+    obj_state: &T::State,
+) -> Result<(
+    T,
+    Option<(CompiledModuleInfo, CompiledFunctionsTable, ModuleTypes)>,
+)> {
+    let tunables = engine.tunables();
+
+    // First a `ModuleEnvironment` is created which records type information
+    // about the wasm module. This is where the WebAssembly is parsed and
+    // validated. Afterwards `types` will have all the type information for
+    // this module.
+    let mut parser = wasmparser::Parser::new(0);
+    let mut validator = wasmparser::Validator::new_with_features(engine.features());
+    parser.set_features(*validator.features());
+    let mut types = ModuleTypesBuilder::new(&validator);
+    let mut translation = ModuleEnvironment::new(
+        tunables,
+        &mut validator,
+        &mut types,
+        StaticModuleIndex::from_u32(0),
+    )
+    .translate(parser, wasm)
+    .context("failed to parse WebAssembly module")?;
+    let functions = mem::take(&mut translation.function_body_inputs);
+
+    let compile_inputs = CompileInputs::for_module(&types, &translation, functions);
+    let unlinked_compile_outputs = compile_inputs.compile(engine)?;
+    let PreLinkOutput {
+        needs_gc_heap,
+        compiled_funcs,
+        indices,
+    } = unlinked_compile_outputs.pre_link();
+    translation.module.needs_gc_heap |= needs_gc_heap;
+
+    // Emplace all compiled functions into the object file with any other
+    // sections associated with code as well.
+    let mut object = engine.compiler().object(ObjectKind::Module)?;
+    // Insert `Engine` and type-level information into the compiled
+    // artifact so if this module is deserialized later it contains all
+    // information necessary.
+    //
+    // Note that `append_compiler_info` and `append_types` here in theory
+    // can both be skipped if this module will never get serialized.
+    // They're only used during deserialization and not during runtime for
+    // the module itself. Currently there's no need for that, however, so
+    // it's left as an exercise for later.
+    engine.append_compiler_info(&mut object);
+    engine.append_bti(&mut object);
+
+    let (mut object, compilation_artifacts) = indices.link_and_append_code(
+        object,
+        engine,
+        compiled_funcs,
+        std::iter::once(translation).collect(),
+        dwarf_package,
+    )?;
+
+    let (info, index) = compilation_artifacts.unwrap_as_module_info();
+    let types = types.finish();
+    object.serialize_info(&(&info, &index, &types));
+    let result = T::finish_object(object, obj_state)?;
+
+    Ok((result, Some((info, index, types))))
+}
+
+/// Performs the compilation phase for a component, translating and
+/// validating the provided wasm binary to machine code.
+///
+/// This method will compile all nested core wasm binaries in addition to
+/// any necessary extra functions required for operation with components.
+/// The output artifact here is the serialized object file contained within
+/// an owned mmap along with metadata about the compilation itself.
+#[cfg(feature = "component-model")]
+pub(crate) fn build_component_artifacts<T: FinishedObject>(
+    engine: &Engine,
+    binary: &[u8],
+    _dwarf_package: Option<&[u8]>,
+    obj_state: &T::State,
+) -> Result<(T, Option<wasmtime_environ::component::ComponentArtifacts>)> {
+    use wasmtime_environ::ScopeVec;
+    use wasmtime_environ::component::{
+        CompiledComponentInfo, ComponentArtifacts, ComponentTypesBuilder,
+    };
+
+    let tunables = engine.tunables();
+    let compiler = engine.compiler();
+
+    let scope = ScopeVec::new();
+    let mut validator = wasmparser::Validator::new_with_features(engine.features());
+    let mut types = ComponentTypesBuilder::new(&validator);
+    let (component, mut module_translations) =
+        Translator::new(tunables, &mut validator, &mut types, &scope)
+            .translate(binary)
+            .context("failed to parse WebAssembly module")?;
+
+    let compile_inputs = CompileInputs::for_component(
+        engine,
+        &types,
+        &component,
+        module_translations.iter_mut().map(|(i, translation)| {
+            let functions = mem::take(&mut translation.function_body_inputs);
+            (i, &*translation, functions)
+        }),
+    );
+    let unlinked_compile_outputs = compile_inputs.compile(&engine)?;
+
+    let PreLinkOutput {
+        needs_gc_heap,
+        compiled_funcs,
+        indices,
+    } = unlinked_compile_outputs.pre_link();
+    for (_, t) in &mut module_translations {
+        t.module.needs_gc_heap |= needs_gc_heap
+    }
+
+    let mut object = compiler.object(ObjectKind::Component)?;
+    engine.append_compiler_info(&mut object);
+    engine.append_bti(&mut object);
+
+    let (mut object, compilation_artifacts) = indices.link_and_append_code(
+        object,
+        engine,
+        compiled_funcs,
+        module_translations,
+        None, // TODO: Support dwarf packages for components.
+    )?;
+    let (types, ty) = types.finish(&component.component);
+
+    let info = CompiledComponentInfo {
+        component: component.component,
+    };
+    let artifacts = ComponentArtifacts {
+        info,
+        table: compilation_artifacts.table,
+        ty,
+        types,
+        static_modules: compilation_artifacts.modules,
+    };
+    object.serialize_info(&artifacts);
+
+    let result = T::finish_object(object, obj_state)?;
+    Ok((result, Some(artifacts)))
+}
+
+type CompileInput<'a> = Box<dyn FnOnce(&dyn Compiler) -> Result<CompileOutput<'a>> + Send + 'a>;
+
+struct CompileOutput<'a> {
+    key: FuncKey,
+    symbol: String,
+    function: CompiledFunctionBody,
+    start_srcloc: FilePos,
+
+    // Only present when `self.key` is a `FuncKey::DefinedWasmFunction(..)`.
+    translation: Option<&'a ModuleTranslation<'a>>,
+
+    // Only present when `self.key` is a `FuncKey::DefinedWasmFunction(..)`.
+    func_body: Option<wasmparser::FunctionBody<'a>>,
+}
+
+/// Inputs to our inlining heuristics.
+struct InlineHeuristicParams<'a> {
+    tunables: &'a Tunables,
+    caller_size: u32,
+    caller_module: StaticModuleIndex,
+    caller_def_func: DefinedFuncIndex,
+    caller_needs_gc_heap: bool,
+    callee_size: u32,
+    callee_module: StaticModuleIndex,
+    callee_def_func: DefinedFuncIndex,
+    callee_needs_gc_heap: bool,
+}
+
+/// The collection of things we need to compile for a Wasm module or component.
+#[derive(Default)]
+struct CompileInputs<'a> {
+    inputs: Vec<CompileInput<'a>>,
+}
+
+impl<'a> CompileInputs<'a> {
+    fn push_input(
+        &mut self,
+        f: impl FnOnce(&dyn Compiler) -> Result<CompileOutput<'a>> + Send + 'a,
+    ) {
+        self.inputs.push(Box::new(f));
+    }
+
+    /// Create the `CompileInputs` for a core Wasm module.
+    fn for_module(
+        types: &'a ModuleTypesBuilder,
+        translation: &'a ModuleTranslation<'a>,
+        functions: PrimaryMap<DefinedFuncIndex, FunctionBodyData<'a>>,
+    ) -> Self {
+        let mut ret = CompileInputs { inputs: vec![] };
+
+        let module_index = StaticModuleIndex::from_u32(0);
+        ret.collect_inputs_in_translations(types, [(module_index, translation, functions)]);
+
+        ret
+    }
+
+    /// Create a `CompileInputs` for a component.
+    #[cfg(feature = "component-model")]
+    fn for_component(
+        engine: &'a Engine,
+        types: &'a wasmtime_environ::component::ComponentTypesBuilder,
+        component: &'a wasmtime_environ::component::ComponentTranslation,
+        module_translations: impl IntoIterator<
+            Item = (
+                StaticModuleIndex,
+                &'a ModuleTranslation<'a>,
+                PrimaryMap<DefinedFuncIndex, FunctionBodyData<'a>>,
+            ),
+        >,
+    ) -> Self {
+        use wasmtime_environ::Abi;
+
+        let mut ret = CompileInputs { inputs: vec![] };
+
+        ret.collect_inputs_in_translations(types.module_types_builder(), module_translations);
+        let tunables = engine.tunables();
+
+        for (idx, trampoline) in component.trampolines.iter() {
+            for abi in [Abi::Wasm, Abi::Array] {
+                ret.push_input(move |compiler| {
+                    let key = FuncKey::ComponentTrampoline(abi, idx);
+                    let mut symbol = trampoline.symbol_name();
+                    symbol.push_str(match abi {
+                        Abi::Wasm => "_wasm_call",
+                        Abi::Array => "_array_call",
+                    });
+                    let function = compiler
+                        .component_compiler()
+                        .compile_trampoline(component, types, key, abi, tunables, &symbol)
+                        .with_context(|| format!("failed to compile {symbol}"))?;
+                    Ok(CompileOutput {
+                        key,
+                        function,
+                        symbol,
+                        start_srcloc: FilePos::default(),
+                        translation: None,
+                        func_body: None,
+                    })
+                });
+            }
+        }
+
+        // If there are any resources defined within this component, the
+        // signature for `resource.drop` is mentioned somewhere, and the
+        // wasm-to-native trampoline for `resource.drop` hasn't been created yet
+        // then insert that here. This is possibly required by destruction of
+        // resources from the embedder and otherwise won't be explicitly
+        // requested through initializers above or such.
+        if component.component.num_resources > 0 {
+            if let Some(sig) = types.find_resource_drop_signature() {
+                ret.push_input(move |compiler| {
+                    let key = FuncKey::ResourceDropTrampoline;
+                    let symbol = "resource_drop_trampoline".to_string();
+                    let function = compiler
+                        .compile_wasm_to_array_trampoline(types[sig].unwrap_func(), key, &symbol)
+                        .with_context(|| format!("failed to compile `{symbol}`"))?;
+                    Ok(CompileOutput {
+                        key,
+                        function,
+                        symbol,
+                        start_srcloc: FilePos::default(),
+                        translation: None,
+                        func_body: None,
+                    })
+                });
+            }
+        }
+
+        ret
+    }
+
+    fn clean_symbol(name: &str) -> Cow<'_, str> {
+        /// Maximum length of symbols generated in objects.
+        const MAX_SYMBOL_LEN: usize = 96;
+
+        // Just to be on the safe side, filter out characters that could
+        // pose issues to tools such as "perf" or "objdump".  To avoid
+        // having to update a list of allowed characters for each different
+        // language that compiles to Wasm, allows only graphic ASCII
+        // characters; replace runs of everything else with a "?".
+        let bad_char = |c: char| !c.is_ascii_graphic();
+        if name.chars().any(bad_char) {
+            let mut last_char_seen = '\u{0000}';
+            Cow::Owned(
+                name.chars()
+                    .map(|c| if bad_char(c) { '?' } else { c })
+                    .filter(|c| {
+                        let skip = last_char_seen == '?' && *c == '?';
+                        last_char_seen = *c;
+                        !skip
+                    })
+                    .take(MAX_SYMBOL_LEN)
+                    .collect::<String>(),
+            )
+        } else if name.len() <= MAX_SYMBOL_LEN {
+            Cow::Borrowed(&name[..])
+        } else {
+            Cow::Borrowed(&name[..MAX_SYMBOL_LEN])
+        }
+    }
+
+    fn collect_inputs_in_translations(
+        &mut self,
+        types: &'a ModuleTypesBuilder,
+        translations: impl IntoIterator<
+            Item = (
+                StaticModuleIndex,
+                &'a ModuleTranslation<'a>,
+                PrimaryMap<DefinedFuncIndex, FunctionBodyData<'a>>,
+            ),
+        >,
+    ) {
+        for (module, translation, functions) in translations {
+            for (def_func_index, func_body_data) in functions {
+                self.push_input(move |compiler| {
+                    let key = FuncKey::DefinedWasmFunction(module, def_func_index);
+                    let func_index = translation.module.func_index(def_func_index);
+                    let symbol = match translation
+                        .debuginfo
+                        .name_section
+                        .func_names
+                        .get(&func_index)
+                    {
+                        Some(name) => format!(
+                            "wasm[{}]::function[{}]::{}",
+                            module.as_u32(),
+                            func_index.as_u32(),
+                            Self::clean_symbol(&name)
+                        ),
+                        None => format!(
+                            "wasm[{}]::function[{}]",
+                            module.as_u32(),
+                            func_index.as_u32()
+                        ),
+                    };
+                    let func_body = func_body_data.body.clone();
+                    let data = func_body.get_binary_reader();
+                    let offset = data.original_position();
+                    let start_srcloc = FilePos::new(u32::try_from(offset).unwrap());
+                    let function = compiler
+                        .compile_function(translation, key, func_body_data, types, &symbol)
+                        .with_context(|| format!("failed to compile: {symbol}"))?;
+
+                    Ok(CompileOutput {
+                        key,
+                        symbol,
+                        function,
+                        start_srcloc,
+                        translation: Some(translation),
+                        func_body: Some(func_body),
+                    })
+                });
+
+                let func_index = translation.module.func_index(def_func_index);
+                if translation.module.functions[func_index].is_escaping() {
+                    self.push_input(move |compiler| {
+                        let key = FuncKey::ArrayToWasmTrampoline(module, def_func_index);
+                        let func_index = translation.module.func_index(def_func_index);
+                        let symbol = format!(
+                            "wasm[{}]::array_to_wasm_trampoline[{}]",
+                            module.as_u32(),
+                            func_index.as_u32()
+                        );
+                        let function = compiler
+                            .compile_array_to_wasm_trampoline(translation, types, key, &symbol)
+                            .with_context(|| format!("failed to compile: {symbol}"))?;
+                        Ok(CompileOutput {
+                            key,
+                            symbol,
+                            function,
+                            start_srcloc: FilePos::default(),
+                            translation: None,
+                            func_body: None,
+                        })
+                    });
+                }
+            }
+        }
+
+        let mut trampoline_types_seen = HashSet::new();
+        for (_func_type_index, trampoline_type_index) in types.trampoline_types() {
+            let is_new = trampoline_types_seen.insert(trampoline_type_index);
+            if !is_new {
+                continue;
+            }
+            let trampoline_func_ty = types[trampoline_type_index].unwrap_func();
+            self.push_input(move |compiler| {
+                let key = FuncKey::WasmToArrayTrampoline(trampoline_type_index);
+                let symbol = format!(
+                    "signatures[{}]::wasm_to_array_trampoline",
+                    trampoline_type_index.as_u32()
+                );
+                let function = compiler
+                    .compile_wasm_to_array_trampoline(trampoline_func_ty, key, &symbol)
+                    .with_context(|| format!("failed to compile: {symbol}"))?;
+                Ok(CompileOutput {
+                    key,
+                    function,
+                    symbol,
+                    start_srcloc: FilePos::default(),
+                    translation: None,
+                    func_body: None,
+                })
+            });
+        }
+    }
+
+    /// Compile these `CompileInput`s (maybe in parallel) and return the
+    /// resulting `UnlinkedCompileOutput`s.
+    fn compile(self, engine: &Engine) -> Result<UnlinkedCompileOutputs<'a>> {
+        let compiler = engine.compiler();
+
+        if self.inputs.len() > 0 && cfg!(miri) {
+            bail!(
+                "\
+You are attempting to compile a WebAssembly module or component that contains
+functions in Miri. Running Cranelift through Miri is known to take quite a long
+time and isn't what we want in CI at least. If this is a mistake then you should
+ignore this test in Miri with:
+
+    #[cfg_attr(miri, ignore)]
+
+If this is not a mistake then try to edit the `pulley_provenance_test` test
+which runs Cranelift outside of Miri. If you still feel this is a mistake then
+please open an issue or a topic on Zulip to talk about how best to accommodate
+the use case.
+"
+            );
+        }
+
+        let mut raw_outputs = if let Some(inlining_compiler) = compiler.inlining_compiler() {
+            if engine.tunables().inlining {
+                self.compile_with_inlining(engine, compiler, inlining_compiler)?
+            } else {
+                // Inlining compiler but inlining is disabled: compile each
+                // input and immediately finish its output in parallel, skipping
+                // call graph computation and all that.
+                engine.run_maybe_parallel::<_, _, Error, _>(self.inputs, |f| {
+                    let mut compiled = f(compiler)?;
+                    inlining_compiler.finish_compiling(
+                        &mut compiled.function,
+                        compiled.func_body.take(),
+                        &compiled.symbol,
+                    )?;
+                    Ok(compiled)
+                })?
+            }
+        } else {
+            // No inlining: just compile each individual input in parallel.
+            engine.run_maybe_parallel(self.inputs, |f| f(compiler))?
+        };
+
+        // Now that all functions have been compiled see if any
+        // wasmtime-builtin functions are necessary. If so those need to be
+        // collected and then those trampolines additionally need to be
+        // compiled.
+        compile_required_builtins(engine, &mut raw_outputs)?;
+
+        // Bucket the outputs by kind.
+        let mut outputs: BTreeMap<FuncKey, CompileOutput> = BTreeMap::new();
+        for output in raw_outputs {
+            outputs.insert(output.key, output);
+        }
+
+        Ok(UnlinkedCompileOutputs { outputs })
+    }
+
+    fn compile_with_inlining(
+        self,
+        engine: &Engine,
+        compiler: &dyn Compiler,
+        inlining_compiler: &dyn InliningCompiler,
+    ) -> Result<Vec<CompileOutput<'a>>, Error> {
+        /// The index of a function (of any kind: Wasm function, trampoline, or
+        /// etc...) in our list of unlinked outputs.
+        #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+        struct OutputIndex(u32);
+        wasmtime_environ::entity_impl!(OutputIndex);
+
+        // Our list of unlinked outputs.
+        let mut outputs = PrimaryMap::<OutputIndex, Option<CompileOutput<'_>>>::from(
+            engine.run_maybe_parallel(self.inputs, |f| f(compiler).map(Some))?,
+        );
+
+        /// Get just the output indices of the Wasm functions from our unlinked
+        /// outputs.
+        fn wasm_functions<'a>(
+            outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,
+        ) -> impl Iterator<Item = OutputIndex> + 'a {
+            outputs.iter().filter_map(|(i, o)| match o.as_ref()?.key {
+                FuncKey::DefinedWasmFunction(..) => Some(i),
+                _ => None,
+            })
+        }
+
+        // A map from a `FuncKey` to its index in our unlinked outputs.
+        //
+        // We will generally just be working with `OutputIndex`es, but
+        // occasionally we must translate from keys back to our index space, for
+        // example when we know that one module's function import is always
+        // satisfied with a particular `FuncKey::DefinedWasmFunction`. This map
+        // enables that translation.
+        let key_to_output: HashMap<FuncKey, OutputIndex> = outputs
+            .iter()
+            .filter(|(_, output)| {
+                matches!(
+                    output.as_ref().unwrap().key,
+                    FuncKey::DefinedWasmFunction(..)
+                )
+            })
+            .map(|(output_index, output)| {
+                let output = output.as_ref().unwrap();
+                (output.key, output_index)
+            })
+            .collect();
+
+        // Construct the call graph for inlining.
+        //
+        // We only inline Wasm functions, not trampolines, because we rely on
+        // trampolines being in their own stack frame when we save the entry and
+        // exit SP, FP, and PC for backtraces in trampolines.
+        let call_graph = CallGraph::<OutputIndex>::new(wasm_functions(&outputs), {
+            let mut func_keys = IndexSet::default();
+            let outputs = &outputs;
+            let key_to_output = &key_to_output;
+            move |output_index, calls| {
+                debug_assert!(calls.is_empty());
+
+                let output = outputs[output_index].as_ref().unwrap();
+                debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));
+
+                // Get this function's call graph edges as `FuncKey`s.
+                func_keys.clear();
+                inlining_compiler.calls(&output.function, &mut func_keys)?;
+
+                // Translate each of those to keys to output indices, which is
+                // what we actually need.
+                calls.extend(
+                    func_keys
+                        .iter()
+                        .copied()
+                        .filter_map(|key| key_to_output.get(&key)),
+                );
+                log::trace!(
+                    "call graph edges for {output_index:?} = {:?}: {calls:?}",
+                    output.key
+                );
+                Ok(())
+            }
+        })?;
+
+        // Stratify the call graph into a sequence of layers. We process each
+        // layer in order, but process functions within a layer in parallel
+        // (because they either do not call each other or are part of a
+        // mutual-recursion cycle; either way we won't inline members of the
+        // same layer into each other).
+        let strata = stratify::Strata::<OutputIndex>::new(wasm_functions(&outputs), &call_graph);
+        let mut layer_outputs = vec![];
+        for layer in strata.layers() {
+            // Temporarily take this layer's outputs out of our unlinked outputs
+            // list so that we can mutate these outputs (by inlining callee
+            // functions into them) while also accessing shared borrows of the
+            // unlinked outputs list (finding the callee functions we will
+            // inline).
+            debug_assert!(layer_outputs.is_empty());
+            layer_outputs.extend(layer.iter().map(|f| outputs[*f].take().unwrap()));
+
+            // Process this layer's members in parallel.
+            engine.run_maybe_parallel_mut(
+                &mut layer_outputs,
+                |output: &mut CompileOutput<'_>| {
+                    log::trace!("processing inlining for {:?}", output.key);
+                    debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));
+
+                    let caller_translation = output.translation.unwrap();
+                    let (caller_module, caller_def_func) =
+                        output.key.unwrap_defined_wasm_function();
+                    let caller_needs_gc_heap = caller_translation.module.needs_gc_heap;
+                    let caller = &mut output.function;
+
+                    let mut caller_size = inlining_compiler.size(caller);
+
+                    inlining_compiler.inline(caller, &mut |callee_key: FuncKey| {
+                        let callee_output_index: OutputIndex = key_to_output[&callee_key];
+
+                        // NB: If the callee is not inside `outputs`, then it is
+                        // in the same `Strata` layer as the caller (and
+                        // therefore is in the same strongly-connected component
+                        // as the caller, and they mutually recursive). In this
+                        // case, we do not do any inlining; communicate this
+                        // command via `?`-propagation.
+                        let callee_output = outputs[callee_output_index].as_ref()?;
+
+                        debug_assert_eq!(callee_output.key, callee_key);
+
+                        let callee = &callee_output.function;
+                        let callee_size = inlining_compiler.size(callee);
+
+                        let callee_needs_gc_heap =
+                            callee_output.translation.unwrap().module.needs_gc_heap;
+
+                        let (callee_module, callee_def_func) =
+                            callee_key.unwrap_defined_wasm_function();
+                        if Self::should_inline(InlineHeuristicParams {
+                            tunables: engine.tunables(),
+                            caller_size,
+                            caller_module,
+                            caller_def_func,
+                            caller_needs_gc_heap,
+                            callee_size,
+                            callee_module,
+                            callee_def_func,
+                            callee_needs_gc_heap,
+                        }) {
+                            caller_size = caller_size.saturating_add(callee_size);
+                            Some(callee)
+                        } else {
+                            None
+                        }
+                    })
+                },
+            )?;
+
+            for (f, func) in layer.iter().zip(layer_outputs.drain(..)) {
+                debug_assert!(outputs[*f].is_none());
+                outputs[*f] = Some(func);
+            }
+        }
+
+        // Fan out in parallel again and finish compiling each function.
+        engine.run_maybe_parallel(outputs.into(), |output| {
+            let mut output = output.unwrap();
+            inlining_compiler.finish_compiling(
+                &mut output.function,
+                output.func_body.take(),
+                &output.symbol,
+            )?;
+            Ok(output)
+        })
+    }
+
+    /// Implementation of our inlining heuristics.
+    ///
+    /// TODO: We should improve our heuristics:
+    ///
+    /// * One potentially promising hint that we don't currently make use of is
+    ///   how many times a function appears as the callee in call sites. For
+    ///   example, a function that appears in only a single call site, and does
+    ///   not otherwise escape, is often beneficial to inline regardless of its
+    ///   size (assuming we can then GC away the non-inlined version of the
+    ///   function, which we do not currently attempt to do).
+    ///
+    /// * Another potentially promising hint would be whether any of the call
+    ///   site's actual arguments are constants.
+    ///
+    /// * A general improvement would be removing the decision-tree style of
+    ///   control flow below and replacing it with (1) a pure estimated-benefit
+    ///   formula and (2) a benefit threshold. Whenever the estimated benefit
+    ///   reaches the threshold, we would inline the call. Both the formula and
+    ///   the threshold would be parameterized by tunables. This would
+    ///   effectively allow reprioritizing the relative importance of different
+    ///   hint sources, rather than being stuck with the sequence hard-coded in
+    ///   the decision tree below.
+    fn should_inline(
+        InlineHeuristicParams {
+            tunables,
+            caller_size,
+            caller_module,
+            caller_def_func,
+            caller_needs_gc_heap,
+            callee_size,
+            callee_module,
+            callee_def_func,
+            callee_needs_gc_heap,
+        }: InlineHeuristicParams,
+    ) -> bool {
+        log::trace!(
+            "considering inlining:\n\
+             \tcaller = ({caller_module:?}, {caller_def_func:?})\n\
+             \t\tsize = {caller_size}\n\
+             \t\tneeds_gc_heap = {caller_needs_gc_heap}\n\
+             \tcallee = ({callee_module:?}, {callee_def_func:?})\n\
+             \t\tsize = {callee_size}\n\
+             \t\tneeds_gc_heap = {callee_needs_gc_heap}"
+        );
+
+        debug_assert!(
+            tunables.inlining,
+            "shouldn't even call this method if we aren't configured for inlining"
+        );
+        debug_assert!(
+            caller_module != callee_module || caller_def_func != callee_def_func,
+            "we never inline recursion"
+        );
+
+        // Put a limit on how large we can make a function via inlining to cap
+        // code bloat.
+        let sum_size = caller_size.saturating_add(callee_size);
+        if sum_size > tunables.inlining_sum_size_threshold {
+            log::trace!(
+                "  --> not inlining: the sum of the caller's and callee's sizes is greater than \
+                 the inlining-sum-size threshold: {callee_size} + {caller_size} > {}",
+                tunables.inlining_sum_size_threshold
+            );
+            return false;
+        }
+
+        // Consider whether this is an intra-module call.
+        //
+        // Inlining within a single core module has most often already been done
+        // by the toolchain that produced the module, e.g. LLVM, and any extant
+        // function calls to small callees were presumably annotated with the
+        // equivalent of `#[inline(never)]` or `#[cold]` but we don't have that
+        // information anymore.
+        if caller_module == callee_module {
+            match tunables.inlining_intra_module {
+                IntraModuleInlining::Yes => {}
+
+                IntraModuleInlining::WhenUsingGc
+                    if caller_needs_gc_heap || callee_needs_gc_heap => {}
+
+                IntraModuleInlining::WhenUsingGc => {
+                    log::trace!("  --> not inlining: intra-module call that does not use GC");
+                    return false;
+                }
+
+                IntraModuleInlining::No => {
+                    log::trace!("  --> not inlining: intra-module call");
+                    return false;
+                }
+            }
+        }
+
+        // Small callees are often worth inlining regardless of the size of the
+        // caller.
+        if callee_size <= tunables.inlining_small_callee_size {
+            log::trace!(
+                "  --> inlining: callee's size is less than the small-callee size: \
+                 {callee_size} <= {}",
+                tunables.inlining_small_callee_size
+            );
+            return true;
+        }
+
+        log::trace!("  --> inlining: did not find a reason we should not");
+        true
+    }
+}
+
+fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutput>) -> Result<()> {
+    let compiler = engine.compiler();
+    let mut builtins = HashSet::new();
+    let mut new_inputs: Vec<CompileInput<'_>> = Vec::new();
+
+    let compile_builtin = |builtin: BuiltinFunctionIndex| {
+        Box::new(move |compiler: &dyn Compiler| {
+            let key = FuncKey::WasmToBuiltinTrampoline(builtin);
+            let symbol = format!("wasmtime_builtin_{}", builtin.name());
+            let mut function = compiler
+                .compile_wasm_to_builtin(key, &symbol)
+                .with_context(|| format!("failed to compile `{symbol}`"))?;
+            if let Some(compiler) = compiler.inlining_compiler() {
+                compiler.finish_compiling(&mut function, None, &symbol)?;
+            }
+            Ok(CompileOutput {
+                key,
+                function,
+                symbol,
+                start_srcloc: FilePos::default(),
+                translation: None,
+                func_body: None,
+            })
+        })
+    };
+
+    for output in raw_outputs.iter() {
+        for reloc in compiler.compiled_function_relocation_targets(&*output.function.code) {
+            if let FuncKey::WasmToBuiltinTrampoline(builtin) = reloc {
+                if builtins.insert(builtin) {
+                    new_inputs.push(compile_builtin(builtin));
+                }
+            }
+        }
+    }
+    raw_outputs.extend(engine.run_maybe_parallel(new_inputs, |c| c(compiler))?);
+    Ok(())
+}
+
+#[derive(Default)]
+struct UnlinkedCompileOutputs<'a> {
+    // A map from kind to `CompileOutput`.
+    outputs: BTreeMap<FuncKey, CompileOutput<'a>>,
+}
+
+impl UnlinkedCompileOutputs<'_> {
+    /// Flatten all our functions into a single list and remember each of their
+    /// indices within it.
+    fn pre_link(self) -> PreLinkOutput {
+        // We must ensure that `compiled_funcs` contains the function bodies
+        // sorted by their `FuncKey`, as `CompiledFunctionsTable` relies on that
+        // property.
+        //
+        // Furthermore, note that, because the order functions end up in
+        // `compiled_funcs` is the order they will ultimately be laid out inside
+        // the object file, we will group all trampolines together, all defined
+        // Wasm functions from the same module together, and etc... This is a
+        // nice property, because it means that (a) cold functions, like builtin
+        // trampolines, are not interspersed between hot Wasm functions, and (b)
+        // Wasm functions that are likely to call each other (i.e. are in the
+        // same module together) are grouped together.
+        let mut compiled_funcs = vec![];
+
+        let mut indices = FunctionIndices::default();
+        let mut needs_gc_heap = false;
+
+        // NB: Iteration over this `BTreeMap` ensures that we uphold
+        // `compiled_func`'s sorted property.
+        for output in self.outputs.into_values() {
+            needs_gc_heap |= output.function.needs_gc_heap;
+
+            let index = compiled_funcs.len();
+            compiled_funcs.push((output.symbol, output.function.code));
+
+            if output.start_srcloc != FilePos::none() {
+                indices
+                    .start_srclocs
+                    .insert(output.key, output.start_srcloc);
+            }
+
+            indices.indices.insert(output.key, index);
+        }
+
+        PreLinkOutput {
+            needs_gc_heap,
+            compiled_funcs,
+            indices,
+        }
+    }
+}
+
+/// Our pre-link functions that have been flattened into a single list.
+struct PreLinkOutput {
+    /// Whether or not any of these functions require a GC heap
+    needs_gc_heap: bool,
+    /// The flattened list of (symbol name, compiled function) pairs, as they
+    /// will be laid out in the object file.
+    compiled_funcs: Vec<(String, Box<dyn Any + Send + Sync>)>,
+    /// The `FunctionIndices` mapping our function keys to indices in that flat
+    /// list.
+    indices: FunctionIndices,
+}
+
+#[derive(Default)]
+struct FunctionIndices {
+    // A map of wasm functions and where they're located in the original file.
+    start_srclocs: HashMap<FuncKey, FilePos>,
+
+    // The index of each compiled function in `compiled_funcs`.
+    indices: BTreeMap<FuncKey, usize>,
+}
+
+impl FunctionIndices {
+    /// Link the compiled functions together, resolving relocations, and append
+    /// them to the given ELF file.
+    fn link_and_append_code<'a>(
+        self,
+        mut obj: object::write::Object<'static>,
+        engine: &'a Engine,
+        compiled_funcs: Vec<(String, Box<dyn Any + Send + Sync>)>,
+        translations: PrimaryMap<StaticModuleIndex, ModuleTranslation<'_>>,
+        dwarf_package_bytes: Option<&[u8]>,
+    ) -> Result<(wasmtime_environ::ObjectBuilder<'a>, Artifacts)> {
+        // Append all the functions to the ELF file.
+        //
+        // The result is a vector parallel to `compiled_funcs` where
+        // `symbol_ids_and_locs[i]` is the symbol ID and function location of
+        // `compiled_funcs[i]`.
+        let compiler = engine.compiler();
+        let tunables = engine.tunables();
+        let symbol_ids_and_locs = compiler.append_code(
+            &mut obj,
+            &compiled_funcs,
+            &|_caller_index: usize, callee: FuncKey| {
+                self.indices.get(&callee).copied().unwrap_or_else(|| {
+                    panic!("cannot resolve relocation! no index for callee {callee:?}")
+                })
+            },
+        )?;
+
+        // If requested, generate and add DWARF information.
+        if tunables.generate_native_debuginfo {
+            compiler.append_dwarf(
+                &mut obj,
+                &translations,
+                &|module, func| {
+                    let i = self.indices[&FuncKey::DefinedWasmFunction(module, func)];
+                    let (symbol, _) = symbol_ids_and_locs[i];
+                    let (_, compiled_func) = &compiled_funcs[i];
+                    (symbol, &**compiled_func)
+                },
+                dwarf_package_bytes,
+                tunables,
+            )?;
+        }
+
+        let mut table_builder = CompiledFunctionsTableBuilder::new();
+        for (key, compiled_func_index) in &self.indices {
+            let (_, func_loc) = symbol_ids_and_locs[*compiled_func_index];
+            let src_loc = self
+                .start_srclocs
+                .get(key)
+                .copied()
+                .unwrap_or_else(FilePos::none);
+            table_builder.push_func(*key, func_loc, src_loc);
+        }
+
+        let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables);
+        let modules = translations
+            .into_iter()
+            .map(|(_, mut translation)| {
+                // If configured attempt to use static memory initialization
+                // which can either at runtime be implemented as a single memcpy
+                // to initialize memory or otherwise enabling
+                // virtual-memory-tricks such as mmap'ing from a file to get
+                // copy-on-write.
+                if engine.tunables().memory_init_cow {
+                    let align = compiler.page_size_align();
+                    let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
+                    translation.try_static_init(align, max_always_allowed);
+                }
+
+                // Attempt to convert table initializer segments to FuncTable
+                // representation where possible, to enable table lazy init.
+                if engine.tunables().table_lazy_init {
+                    translation.try_func_table_init();
+                }
+
+                obj.append(translation)
+            })
+            .collect::<Result<PrimaryMap<_, _>>>()?;
+
+        let artifacts = Artifacts {
+            modules,
+            table: table_builder.finish(),
+        };
+
+        Ok((obj, artifacts))
+    }
+}
+
+/// The artifacts necessary for finding and calling Wasm functions at runtime,
+/// to be serialized into an ELF file.
+struct Artifacts {
+    modules: PrimaryMap<StaticModuleIndex, CompiledModuleInfo>,
+    table: CompiledFunctionsTable,
+}
+
+impl Artifacts {
+    /// Assuming this compilation was for a single core Wasm module, get the
+    /// resulting `CompiledModuleInfo`.
+    fn unwrap_as_module_info(self) -> (CompiledModuleInfo, CompiledFunctionsTable) {
+        assert_eq!(self.modules.len(), 1);
+        let info = self.modules.into_iter().next().unwrap().1;
+        let table = self.table;
+        (info, table)
+    }
+}
+
+/// Extend `dest` with `items` and return the range of indices in `dest` where
+/// they ended up.
+fn extend_with_range<T>(dest: &mut Vec<T>, items: impl IntoIterator<Item = T>) -> Range<u32> {
+    let start = dest.len();
+    let start = u32::try_from(start).unwrap();
+
+    dest.extend(items);
+
+    let end = dest.len();
+    let end = u32::try_from(end).unwrap();
+
+    start..end
+}