@swmansion/popcorn 0.3.2 → 0.4.0-next.0

package/dist/plugins/beam_tools/lib/popcorn/beam_tools/packager.ex

@@ -0,0 +1,540 @@
+defmodule Popcorn.BeamTools.Packager do
+  alias Popcorn.BeamTools.BeamPatcher
+
+  @static_nif_beams MapSet.new(["wasm.beam", "prim_tty.beam", "zstd.beam"])
+
+  # :beam_lib.significant_chunks/0 is undocumented.
+  @retained_chunks :beam_lib.significant_chunks() -- [~c"Type"]
+  # Applications that only work when the emulator was built with the matching
+  # native support. The runtime manifest declares what the build provides.
+  @app_capabilities %{
+    "asn1" => "crypto",
+    "crypto" => "crypto",
+    "public_key" => "crypto",
+    "ssl" => "crypto"
+  }
+
+  @boot_name "bin/vm.boot"
+
+  # Using `__DIR__` is safe – the plugin is compiled on user's machine
+  @patches_dir Path.expand("../../../patches", __DIR__)
+
+  # To run, Beam needs following apps:
+  # - kernel
+  # - stdlib (kernel dep)
+  #
+  # We also add elixir support out of the box and need:
+  # - compiler (elixir dep)
+  # - elixir
+  #
+  # All of them should be in elixir's transitive dependency closure
+  @base_apps ["elixir"]
+
+  @type options :: %{
+          root_dir: Path.t(),
+          entrypoint_app: String.t() | nil,
+          extra_apps: [String.t()],
+          out_dir: Path.t(),
+          runtimes_dir: Path.t(),
+          runtime_variant: String.t() | nil,
+          strip: boolean()
+        }
+
+  defp strip_tarball(path, out_dir) do
+    {:ok, entries} = :erl_tar.extract(to_charlist(path), [:memory])
+
+    stripped =
+      entries
+      |> Enum.sort_by(fn {name, _content} -> to_string(name) end)
+      |> Enum.map(&strip_beam/1)
+
+    output = Path.join(out_dir, Path.basename(path))
+    opts = [mtime: 0, atime: 0, ctime: 0, uid: 0, gid: 0]
+
+    :ok = :erl_tar.create(to_charlist(output), stripped, opts)
+  end
+
+  defp beam?(path), do: Path.extname(path) == ".beam"
+
+  # Reimplementation of :beam_lib.strip_files/2.
+  # Original also gzip compresses modules.
+  defp strip_beam({name, content}) do
+    if beam?(to_string(name)) do
+      {:ok, _module, chunks} = :beam_lib.all_chunks(content)
+      chunks_by_name = Map.new(chunks)
+
+      chunks =
+        Enum.flat_map(@retained_chunks, fn name ->
+          case Map.fetch(chunks_by_name, name) do
+            {:ok, data} -> [{name, data}]
+            :error -> []
+          end
+        end)
+
+      {:ok, stripped} = :beam_lib.build_module(chunks)
+
+      {name, stripped}
+    else
+      {name, content}
+    end
+  end
+
+  @spec run(options()) :: {:ok, map()} | {:error, map()}
+  def run(args) do
+    %{
+      root_dir: root_dir,
+      entrypoint_app: entrypoint_app,
+      extra_apps: extra_apps,
+      out_dir: out_dir,
+      runtimes_dir: runtimes_dir,
+      runtime_variant: runtime_variant,
+      strip: strip
+    } = args
+
+    toolchain = fetch_toolchain_info()
+
+    with {:ok, project_apps} <- root_dir |> project_build_dir() |> get_apps_info(),
+         {:ok, builtin_apps} <- get_builtin_apps(toolchain),
+         {:ok, apps_info} <- apps_to_pack(project_apps, builtin_apps, extra_apps, entrypoint_app),
+         variant = runtime_variant || required_runtime(apps_info),
+         {:ok, manifest} <- read_manifest(Path.join([runtimes_dir, variant, "manifest.json"])),
+         :ok <- check_otp_version(toolchain.otp, manifest.version),
+         :ok <- check_capabilities(apps_info, manifest.capabilities),
+         {:ok, boot_path} <- create_boot(out_dir, toolchain.otp_root, manifest.preloaded),
+         staged_apps = stage_apps(Path.join(out_dir, "staging"), apps_info),
+         :ok <- patch_apps(staged_apps) do
+      vm_version = manifest.version
+      toolchain = Map.take(toolchain, ~w(otp elixir)a)
+
+      File.mkdir_p!(out_dir)
+
+      packed_apps =
+        staged_apps
+        |> async_stream(fn {app, info} ->
+          version = Keyword.get(info.props, :vsn, ~c"") |> to_string()
+          tar_path = create_tarball(out_dir, app, info.ebin_dir)
+
+          {app, %{tar: tar_path, version: version}}
+        end)
+        |> Map.new()
+
+      diagnostics =
+        staged_apps
+        |> async_stream(fn {app, info} ->
+          case loaded_dynamic_nifs(app, info.ebin_dir) do
+            [] ->
+              []
+
+            beams ->
+              {:error, context} = err(:dynamic_nifs_loading, {app, beams})
+              [context]
+          end
+        end)
+        |> Enum.concat()
+
+      manifest_path = Path.join(out_dir, "manifest.json")
+
+      packed_tar_paths =
+        packed_apps
+        |> Map.values()
+        |> Enum.map(&Path.expand(Path.join(out_dir, &1.tar)))
+
+      tar_paths = maybe_strip_tarballs(packed_tar_paths, strip, out_dir)
+
+      manifest = %{
+        entrypoint: entrypoint_app,
+        apps: packed_apps,
+        notes: diagnostics,
+        toolchain: toolchain,
+        vm: %{boot: @boot_name, version: vm_version}
+      }
+
+      File.write!(manifest_path, encode_json(manifest))
+
+      result = %{
+        ok: true,
+        runtimeVariant: variant,
+        entrypoint: entrypoint_app,
+        manifestPath: Path.expand(manifest_path),
+        bootPath: Path.expand(boot_path),
+        tarPaths: tar_paths,
+        apps: packed_apps,
+        notes: diagnostics,
+        toolchain: toolchain
+      }
+
+      {:ok, result}
+    end
+  end
+
+  defp create_boot(out_dir, otp_root, runtime_preloaded) do
+    boot_path = Path.join([Path.dirname(otp_root), "bin", "no_dot_erlang.boot"])
+    {:script, id, commands} = boot_path |> File.read!() |> :erlang.binary_to_term()
+
+    preloaded =
+      Enum.flat_map(commands, fn
+        {:preLoaded, modules} -> Enum.map(modules, &to_string/1)
+        _ -> []
+      end)
+
+    case preloaded -- runtime_preloaded do
+      [] ->
+        boot = {:script, id, Enum.map(commands, &drop_app_versions/1)}
+        path = Path.join(out_dir, @boot_name)
+
+        File.mkdir_p!(Path.dirname(path))
+
+        File.write!(path, :erlang.term_to_binary(boot))
+
+        {:ok, path}
+
+      missing ->
+        err(:unsupported_boot, {boot_path, missing})
+    end
+  end
+
+  # `$ROOT/lib/kernel-10.5/ebin` -> `$ROOT/lib/kernel/ebin`
+  defp drop_app_versions({:path, dirs}) do
+    {:path, Enum.map(dirs, &drop_version_fragment/1)}
+  end
+
+  defp drop_app_versions(command), do: command
+
+  defp drop_version_fragment(dir) do
+    [root, "lib", app_version, "ebin"] = dir |> to_string() |> Path.split()
+    [app, _version] = String.split(app_version, "-", parts: 2)
+
+    to_charlist(Path.join([root, "lib", app, "ebin"]))
+  end
+
+  # Apps are copied out of the host installation so packing can modify them.
+  defp stage_apps(staging_dir, apps_info) do
+    apps_info
+    |> async_stream(fn {app, info} ->
+      ebin_dir = Path.join([staging_dir, app, "ebin"])
+
+      File.mkdir_p!(Path.dirname(ebin_dir))
+      File.cp_r!(info.ebin_dir, ebin_dir)
+
+      {app, %{info | ebin_dir: ebin_dir}}
+    end)
+    |> Map.new()
+  end
+
+  # we patch only selected modules from OTP, see patches/
+  defp patch_apps(staged_apps) do
+    [@patches_dir, "*", "*.erl"]
+    |> Path.join()
+    |> Path.wildcard()
+    |> reduce_while_ok(:ok, fn patch_path, :ok ->
+      app = patch_path |> Path.dirname() |> Path.basename()
+      name = Path.basename(patch_path, ".erl") <> ".beam"
+      ebin_dir = Map.fetch!(staged_apps, app).ebin_dir
+      beam_path = Path.join(ebin_dir, name)
+
+      BeamPatcher.patch_beam(beam_path, patch_path)
+    end)
+    |> case do
+      {:ok, _} -> :ok
+      {:error, _} = error -> error
+    end
+  end
+
+  defp async_stream(enumerable, fun) do
+    enumerable
+    |> Task.async_stream(fun, timeout: :infinity)
+    |> Enum.map(fn {:ok, result} -> result end)
+  end
+
+  defp maybe_strip_tarballs(paths, false, _out_dir), do: paths
+
+  defp maybe_strip_tarballs(paths, true, out_dir) do
+    Enum.map(paths, fn path ->
+      :ok = strip_tarball(path, out_dir)
+      Path.expand(Path.join(out_dir, Path.basename(path)))
+    end)
+  end
+
+  defp project_build_dir(root_dir) do
+    build_env = System.get_env("MIX_ENV", "dev")
+    build_lib_dir = Path.join([root_dir, "_build", build_env, "lib"])
+
+    build_lib_dir
+  end
+
+  defp get_apps_info(root_dir) do
+    extract_info = fn app_path ->
+      {:ok, [{:application, name, props}]} = :file.consult(app_path)
+      dir = Path.dirname(app_path)
+
+      {to_string(name), %{props: props, ebin_dir: dir, app_path: app_path}}
+    end
+
+    get_name = fn {name, _info} -> name end
+    get_app_path = fn {_name, info} -> info.app_path end
+    duplicated? = fn {_name, paths} -> match?([_, _ | _], paths) end
+
+    apps =
+      root_dir
+      |> Path.join("*/ebin/*.app")
+      |> Path.wildcard()
+      |> Enum.map(extract_info)
+
+    duplicates =
+      apps
+      |> Enum.group_by(get_name, get_app_path)
+      |> Enum.filter(duplicated?)
+      |> Map.new()
+
+    if Enum.empty?(duplicates) do
+      {:ok, Map.new(apps)}
+    else
+      err(:duplicated_apps, {root_dir, duplicates})
+    end
+  end
+
+  defp get_builtin_apps(toolchain) do
+    with {:ok, elixir_apps} <- get_apps_info(toolchain.elixir_root),
+         {:ok, otp_apps} <- get_apps_info(toolchain.otp_root) do
+      {:ok, Map.merge(elixir_apps, otp_apps)}
+    end
+  end
+
+  defp read_manifest(manifest_path) do
+    with {:ok, json} <- File.read(manifest_path),
+         {:ok, %{"vm" => vm}} <- decode_json(json),
+         %{"version" => version, "preloaded" => preloaded, "capabilities" => capabilities} <- vm do
+      {:ok, %{version: version, preloaded: preloaded, capabilities: capabilities}}
+    else
+      _ -> err(:bad_manifest, manifest_path)
+    end
+  end
+
+  defp apps_to_pack(project_apps, builtin_apps, extra_apps, entrypoint) do
+    all_apps_info = Map.merge(builtin_apps, project_apps)
+
+    gather_from_root = fn app, selected ->
+      gather_required_apps(all_apps_info, project_apps, app, selected)
+    end
+
+    with {:ok, roots} <- root_apps(all_apps_info, extra_apps, entrypoint),
+         {:ok, selected_apps} <- reduce_while_ok(roots, MapSet.new(), gather_from_root) do
+      all_apps_info
+      |> Map.filter(fn {app, _info} -> MapSet.member?(selected_apps, app) end)
+      |> Enum.sort()
+      |> then(&{:ok, &1})
+    end
+  end
+
+  defp required_runtime(apps_info) do
+    needs_crypto = Enum.any?(apps_info, fn {app, _info} -> @app_capabilities[app] == "crypto" end)
+    if needs_crypto, do: "crypto", else: "core"
+  end
+
+  defp check_capabilities(apps_info, capabilities) do
+    unsupported =
+      apps_info
+      |> Enum.flat_map(fn {app, _info} ->
+        case Map.fetch(@app_capabilities, app) do
+          {:ok, capability} -> [%{app: app, capability: capability}]
+          :error -> []
+        end
+      end)
+      |> Enum.reject(&Map.fetch!(capabilities, &1.capability))
+
+    if unsupported == [], do: :ok, else: err(:unsupported_apps, unsupported)
+  end
+
+  defp root_apps(all_apps_info, extra_apps, entrypoint) do
+    with {:ok, roots} <- entrypoint_roots(all_apps_info, entrypoint),
+         {:ok, extra} <- extra_roots(all_apps_info, extra_apps) do
+      {:ok, extra ++ roots}
+    end
+  end
+
+  defp entrypoint_roots(_all_apps_info, nil), do: {:ok, @base_apps}
+
+  # The entrypoint may be a builtin app, so a project with no code of its own
+  # doesn't need a stub application just to depend on one.
+  defp entrypoint_roots(all_apps_info, entrypoint)
+       when is_map_key(all_apps_info, entrypoint) do
+    {:ok, [entrypoint | @base_apps]}
+  end
+
+  defp entrypoint_roots(_all_apps_info, entrypoint),
+    do: err(:missing_entrypoint, entrypoint)
+
+  defp extra_roots(all_apps_info, extra_apps) do
+    case Enum.reject(extra_apps, &is_map_key(all_apps_info, &1)) do
+      [] -> {:ok, extra_apps}
+      missing -> err(:missing_extra_apps, Enum.sort(missing))
+    end
+  end
+
+  defp gather_required_apps(all_apps_info, project_apps, app, selected) do
+    if MapSet.member?(selected, app) do
+      {:ok, selected}
+    else
+      info = Map.fetch!(all_apps_info, app)
+      selected = MapSet.put(selected, app)
+
+      info.props
+      |> get_required_apps()
+      |> reduce_while_ok(selected, fn dep, acc ->
+        if Map.has_key?(all_apps_info, dep) do
+          gather_required_apps(all_apps_info, project_apps, dep, acc)
+        else
+          project_app_names = Enum.sort(Map.keys(project_apps))
+          err(:missing_dep, {app, dep, project_app_names})
+        end
+      end)
+    end
+  end
+
+  defp reduce_while_ok(enumerable, acc, f) do
+    Enum.reduce_while(enumerable, {:ok, acc}, fn value, {:ok, acc} ->
+      case f.(value, acc) do
+        :ok -> {:cont, {:ok, acc}}
+        {:ok, new_acc} -> {:cont, {:ok, new_acc}}
+        {:error, _} = error -> {:halt, error}
+      end
+    end)
+  end
+
+  defp create_tarball(outdir, app, ebin_dir) do
+    tar = "lib/#{app}.tar"
+    tar_path = Path.join(outdir, tar)
+    tar_path_c = to_charlist(tar_path)
+    arc_name = ~c"lib/#{app}/ebin"
+    ebin_dir_c = to_charlist(ebin_dir)
+
+    File.mkdir_p!(Path.dirname(tar_path))
+    :ok = :erl_tar.create(tar_path_c, [{arc_name, ebin_dir_c}], [])
+
+    tar
+  end
+
+  defp loaded_dynamic_nifs(app, ebin_dir) do
+    Path.join(ebin_dir, "*.beam")
+    |> Path.wildcard()
+    |> Enum.filter(&imports_load_nif?/1)
+    |> Enum.map(&Path.basename/1)
+    |> Enum.reject(&MapSet.member?(@static_nif_beams, &1))
+  end
+
+  defp imports_load_nif?(beam_path) do
+    case :beam_lib.chunks(to_charlist(beam_path), [:imports]) do
+      {:ok, {_mod, [imports: imports]}} -> {:erlang, :load_nif, 2} in imports
+      _ -> false
+    end
+  end
+
+  defp fetch_toolchain_info() do
+    %{
+      otp: host_otp_version(),
+      elixir: System.version(),
+      otp_root: Path.join(to_string(:code.root_dir()), "lib"),
+      elixir_root: :elixir |> :code.lib_dir() |> to_string() |> Path.dirname()
+    }
+  end
+
+  defp check_otp_version(host_version, runtime_version) do
+    # host: computer this runs on
+    # runtime: vm compiled to wasm
+    host = otp_version(host_version)
+    runtime = otp_version(runtime_version)
+
+    [host_major | _] = host
+    [runtime_major | _] = runtime
+
+    compatible = runtime_major - 2 <= host_major and version_lte?(host, runtime)
+
+    if compatible do
+      :ok
+    else
+      err(:unsupported_otp, {host_version, runtime_version})
+    end
+  end
+
+  defp host_otp_version do
+    path =
+      Path.join([to_string(:code.root_dir()), "releases", System.otp_release(), "OTP_VERSION"])
+
+    path
+    |> File.read!()
+    |> String.trim()
+  end
+
+  defp otp_version(version) do
+    version
+    |> to_string()
+    |> String.split("-", parts: 2)
+    |> hd()
+    |> String.split(".")
+    |> Enum.map(&String.to_integer/1)
+  end
+
+  defp version_lte?(left, right) do
+    width = max(length(left), length(right))
+    pad = fn version -> version ++ List.duplicate(0, width - length(version)) end
+    pad.(left) <= pad.(right)
+  end
+
+  defp get_required_apps(props) do
+    prop = &Keyword.get/3
+    optional = prop.(props, :optional_applications, []) |> MapSet.new()
+    applications = prop.(props, :applications, []) |> MapSet.new()
+    included = prop.(props, :included_applications, []) |> MapSet.new()
+
+    MapSet.union(applications, included)
+    |> MapSet.difference(optional)
+    |> Enum.map(&to_string/1)
+  end
+
+  defp err(:missing_entrypoint, app) do
+    {:error, %{code: "missing_entrypoint", app: app}}
+  end
+
+  defp err(:missing_extra_apps, apps) do
+    {:error, %{code: "missing_extra_apps", apps: apps}}
+  end
+
+  defp err(:bad_manifest, path) do
+    {:error, %{code: "bad_manifest", path: path}}
+  end
+
+  defp err(:unsupported_boot, {boot_path, missing_preloaded}) do
+    missing = Enum.sort(missing_preloaded)
+    {:error, %{code: "unsupported_boot", boot: boot_path, missing_preloaded: missing}}
+  end
+
+  defp err(:unsupported_otp, {host, runtime}) do
+    {:error, %{code: "unsupported_otp", host: host, runtime: runtime}}
+  end
+
+  defp err(:duplicated_apps, {root_dir, duplicates}) do
+    {:error, %{code: "duplicated_apps", root_dir: root_dir, duplicates: duplicates}}
+  end
+
+  defp err(:unsupported_apps, unsupported) do
+    {:error, %{code: "unsupported_apps", apps: Enum.sort_by(unsupported, & &1.app)}}
+  end
+
+  defp err(:missing_dep, {app, dep, project_apps}) do
+    {:error, %{code: "missing_dep", app: app, dep: dep, available_apps: project_apps}}
+  end
+
+  defp err(:dynamic_nifs_loading, {app, beams}) do
+    {:error, %{code: "dynamic_nifs_loading", app: app, beams: beams}}
+  end
+
+  defp encode_json(term) do
+    term |> :json.encode() |> IO.iodata_to_binary()
+  end
+
+  defp decode_json(json) do
+    {:ok, :json.decode(json)}
+  rescue
+    _ -> :error
+  end
+end

package/dist/plugins/beam_tools/mix.exs

@@ -0,0 +1,16 @@
+defmodule Popcorn.BeamTools.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :popcorn_beam_tools,
+      version: "0.1.0",
+      elixir: "~> 1.17",
+      deps: []
+    ]
+  end
+
+  def application do
+    []
+  end
+end

package/dist/plugins/beam_tools/patches/kernel/prim_tty.erl

@@ -0,0 +1,13 @@
+%% The emscripten target has no terminal to query, so terminal capability
+%% detection is skipped and the state is returned unchanged.
+-module(prim_tty).
+-compile([{popcorn_patch_private, [{init, 2}]}]).
+-export([init/2]).
+
+init(State, {unix, _} = OsType) ->
+    case erlang:system_info(system_architecture) of
+        "wasm32-unknown-emscripten" -> State;
+        _ -> popcorn_module:init(State, OsType)
+    end;
+init(State, OsType) ->
+    popcorn_module:init(State, OsType).

package/dist/plugins/beam_tools/patches/stdlib/beam_lib.erl

@@ -0,0 +1,27 @@
+-module(beam_lib).
+-export([checksum/1]).
+
+checksum(File0) ->
+    case catch checksum_1(File0) of
+        {ok, _} = Result -> Result;
+        Error -> Error
+    end.
+
+checksum_1(File0) ->
+    File = popcorn_module:beam_filename(File0),
+    Beam = read_beam(File),
+    {ok, Module, _} = popcorn_module:scan_beam(Beam, []),
+    case erts_internal:beamfile_module_checksum(Beam) of
+        Digest when is_binary(Digest) ->
+            {ok, {Module, {xxh3_128, Digest}}};
+        undefined ->
+            popcorn_module:error({not_a_beam_file, File})
+    end.
+
+read_beam(Beam) when is_binary(Beam) ->
+    popcorn_module:maybe_uncompress(Beam);
+read_beam(File) ->
+    case file:read_file(File) of
+        {ok, Beam} -> popcorn_module:maybe_uncompress(Beam);
+        Error -> popcorn_module:file_error(File, Error)
+    end.

package/dist/plugins/esbuild.d.ts

@@ -1,3 +1,11 @@
 import type { Plugin } from "esbuild";
-import { type PopcornPluginOptions } from "./shared";
-export declare function popcorn(options: PopcornPluginOptions): Plugin;
+import { type Options } from "./shared";
+/**
+ * Copies the worker, VM runtime, and `otp/` assets into the output directory after a successful esbuild build.
+ *
+ * Requires `format: "esm"` and either `outdir` or `outfile`.
+ * If the application bundle is elsewhere, set `PopcornOpts.workerUrl` to the copied `worker.mjs`.
+ *
+ * @see {@link Options} for application packaging and server requirements.
+ */
+export declare function popcorn(options: Options): Plugin;