ruby_everywhere 0.6.0 → 0.7.0

data/lib/everywhere/commands/platform/build.rb

@@ -21,8 +21,8 @@ module Everywhere
 
       option :root,    default: ".",       desc: "App root directory"
       option :url,     default: nil,       desc: "Platform base URL"
-      option :target,  default: nil,       desc: "Targets (comma-separated os-arch); overrides everywhere.yml"
-      option :channel, default: "direct",  desc: "Distribution channel"
+      option :target,  default: nil,       desc: "Targets (comma-separated os-arch[:channel]); overrides everywhere.yml"
+      option :channel, default: "direct",  desc: "Default distribution channel for targets without a :channel suffix"
       option :ruby,    default: nil,       desc: "Ruby version (default: build.ruby or 4.0.6)"
       option :output,  default: nil,       desc: "Where to save artifacts (default: <root>/dist)"
       option :wait,        type: :boolean, default: true,  desc: "Wait for the build and download results"
@@ -38,29 +38,26 @@ module Everywhere
 
         client = Everywhere::Platform::Client.new(base, token: creds["token"])
         framework = config.remote? ? nil : Everywhere::Framework.detect(root_path)
-        targets = resolve_targets(target, config)
+        specs = resolve_targets(target, config, channel)
         ruby ||= config.build_ruby || "4.0.6"
 
         Dir.mktmpdir("every-snapshot") do |tmp|
           signed_id = snapshot_and_upload(client, root_path, tmp)
-          manifest = Everywhere::Receipt.new(
-            root: root_path, config: config, framework: framework, ruby: ruby,
-            target: targets.first, channel: channel
-          ).manifest
+          manifests = build_manifests(root_path, config, framework, ruby, specs)
 
-          build = submit(client, targets, channel, signed_id, manifest)
-          UI.success("submitted #{UI.bold(build["id"])} for #{targets.map { |t| UI.target_label(t) }.join(", ")}")
+          build = submit(client, specs, channel, signed_id, manifests)
+          UI.success("submitted #{UI.bold(build["id"])} for #{specs.map(&:display_label).join(", ")}")
           if (build_url = build["url"] || "#{client.base_url}/builds/#{build["id"]}")
             UI.substep("watch it live → #{UI.cyan(build_url)}")
           end
 
           return unless wait
 
-          # Set expectations before the quiet part: a hosted macOS runner is not
+          # Set expectations before the quiet part: a hosted runner is not
           # instant, and knowing that up front is the difference between "slow"
           # and "broken".
-          UI.substep(UI.dim("hosted macOS runners usually start within a few minutes"))
-          final = poll_and_tail(client, build["id"])
+          UI.substep(UI.dim("#{runner_wait_subject(specs)} usually start within a few minutes"))
+          final = poll_and_tail(client, build["id"], build_url)
           report(client, final, output || File.join(root_path, "dist"), download)
         end
       rescue Everywhere::Error => e
@@ -69,6 +66,30 @@ module Everywhere
 
       private
 
+      # Which hosted runner OS each target builds on — not the target's own OS
+      # (iOS builds on macOS, Android on Linux).
+      RUNNER_OS_LABELS = {
+        "macos" => "macOS", "ios" => "macOS",
+        "android" => "Linux", "linux" => "Linux", "windows" => "Windows"
+      }.freeze
+
+      # One requested target. `token` is the literal string the user wrote
+      # ("ios-arm64:testflight") and is what keys the per-target manifests on the
+      # wire; `os_arch` is the bare form everything else (receipts, labels,
+      # filenames) is built from.
+      TargetSpec = Struct.new(:token, :os, :arch, :channel) do
+        def os_arch = "#{os}-#{arch}"
+        def label = UI.target_label(os_arch)
+        def display_label = channel == "direct" ? label : "#{label} (#{channel})"
+      end
+
+      def runner_wait_subject(specs)
+        kinds = specs.filter_map { |s| RUNNER_OS_LABELS[s.os] }.uniq
+        return "hosted build runners" if kinds.empty?
+
+        "hosted #{kinds.join("/")} runners"
+      end
+
       def snapshot_and_upload(client, root_path, tmp)
         path = File.join(tmp, "snapshot.tar.gz")
         UI.step("snapshotting source #{UI.dim("(honoring .everywhereignore)")}")
@@ -79,6 +100,7 @@ module Everywhere
         reservation = client.post("/cli/direct_uploads",
           filename: "snapshot.tar.gz", byte_size: size,
           checksum: Everywhere::Platform::Snapshot.md5_base64(path), content_type: "application/gzip")
+        die_unauthorized! if reservation.unauthorized?
         UI.die!("couldn't reserve upload (HTTP #{reservation.status})") unless reservation.ok?
 
         upload = reservation.body["direct_upload"] || {}
@@ -89,21 +111,49 @@ module Everywhere
         reservation.body["signed_id"]
       end
 
-      def submit(client, targets, channel, signed_id, manifest)
+      # `manifests` is keyed by the literal target token; `manifest` (singular)
+      # stays = the first target's, which is all an older Platform reads.
+      def submit(client, specs, channel, signed_id, manifests)
         res = client.post("/cli/builds",
-          targets: targets.join(","), channel: channel, snapshot_signed_id: signed_id, manifest: manifest)
+          targets: specs.map(&:token).join(","), channel: channel, snapshot_signed_id: signed_id,
+          manifest: manifests.values.first, manifests: manifests)
         return res.body if res.ok?
 
-        if res.status == 422 && res.body["error"] == "missing_credentials"
+        die_unauthorized! if res.unauthorized?
+        render_submit_error!(res)
+      end
+
+      def render_submit_error!(res)
+        body = res.body.is_a?(Hash) ? res.body : {}
+
+        case body["error"]
+        when "missing_credentials"
           UI.bad("can't build yet — missing signing credentials:")
-          Array(res.body["targets"]).each do |t|
-            tgt = t["target"] || {}
-            UI.step("#{UI.os_label(tgt["os"])}-#{tgt["arch"]}/#{tgt["distribution_channel"]}: add #{Array(t["missing"]).join(", ")}")
+          Array(body["targets"]).each do |t|
+            UI.step("#{server_target_label(t["target"])}: add #{Array(t["missing"]).join(", ")}")
+          end
+          UI.step("set them up → #{UI.cyan(body["manage_url"])}")
+          exit 1
+        when "invalid_targets"
+          UI.bad(body["message"] || "some targets can't be built as configured:")
+          Array(body["targets"]).each do |t|
+            UI.step("#{server_target_label(t["target"])}: #{Array(t["errors"]).join("; ")}")
           end
-          UI.step("set them up → #{UI.cyan(res.body["manage_url"])}")
           exit 1
+        when "unsupported_targets"
+          UI.step("supported: #{Array(body["supported"]).join(", ")}") if body["supported"]
         end
-        UI.die!("build submission failed (HTTP #{res.status}) #{res.body["error"]}")
+
+        UI.die!("build submission failed (HTTP #{res.status}) #{body["message"] || body["error"]}")
+      end
+
+      def server_target_label(target)
+        target ||= {}
+        "#{UI.os_label(target["os"])}-#{target["arch"]}/#{target["distribution_channel"]}"
+      end
+
+      def die_unauthorized!
+        UI.die!("token expired or revoked — run `every platform login`")
       end
 
       # How often we ask the Platform for new logs. A build spends most of its
@@ -114,14 +164,30 @@ module Everywhere
       ACTIVE_POLL  = 2
       SPINNER_TICK = 0.2
 
-      def poll_and_tail(client, build_id)
+      # The build keeps running on the Platform whether or not we can reach it,
+      # so a blip mid-tail must not end the wait. Give up only after this many
+      # polls in a row have failed.
+      MAX_POLL_ERRORS = 5
+
+      def poll_and_tail(client, build_id, build_url = nil)
         printed = Hash.new("")
         last = {}
         started = now
         status = UI::Status.new
+        errors = 0
 
         loop do
-          res = client.get("/cli/builds/#{build_id}/logs")
+          begin
+            res = client.get("/cli/builds/#{build_id}/logs")
+            errors = 0
+          rescue Everywhere::Error => e
+            errors += 1
+            status.clear
+            detach!(build_id, build_url, e) if errors >= MAX_POLL_ERRORS
+            UI.warn("lost contact with the Platform (#{e.message}) — retrying #{errors}/#{MAX_POLL_ERRORS}")
+            spin(status, [], started, IDLE_POLL)
+            next
+          end
           artifacts = res.ok? ? (res.body["artifacts"] || []) : []
 
           status.clear
@@ -140,6 +206,16 @@ module Everywhere
         status&.clear
       end
 
+      # We stopped watching; the build didn't stop. Leave the user everything
+      # they need to re-attach or grab the artifacts later.
+      def detach!(build_id, build_url, error)
+        UI.bad("gave up tailing #{UI.bold(build_id)} after #{MAX_POLL_ERRORS} failed polls — #{error.message}")
+        UI.step("the build is still running on the Platform")
+        UI.step("watch it → #{UI.cyan(build_url)}") if build_url
+        UI.die!("re-attach with `every platform build --wait` once you're back online, " \
+                "or download the artifacts from the build page")
+      end
+
       # Hold the status line alive for `interval` seconds. The spinner ticks far
       # faster than we poll so the line reads as "alive" during the multi-minute
       # pre-runner wait rather than freezing between requests.
@@ -224,11 +300,44 @@ module Everywhere
         end
       end
 
-      # everywhere.yml build.targets, or --target, or the macOS default.
-      def resolve_targets(override, config)
-        list = override ? override.split(",") : config.targets
-        list = ["macos-arm64"] if list.nil? || list.empty?
-        list.map(&:strip).reject(&:empty?)
+      # everywhere.yml build.targets, or --target, or the macOS default. Each
+      # entry is "os-arch" with an optional ":channel" suffix; --channel is the
+      # default for the ones that leave it off. No channel allowlist here — the
+      # Platform owns which channels exist and says so in its 422.
+      def resolve_targets(override, config, default_channel)
+        list = override ? override.split(",") : config.build_target_specs
+        list = Array(list).map { |t| t.to_s.strip }.reject(&:empty?)
+        list = ["macos-arm64"] if list.empty?
+        list.map { |token| parse_target_spec(token, default_channel) }
+      end
+
+      def parse_target_spec(token, default_channel)
+        os_arch, channel, extra = token.split(":", -1)
+        if !extra.nil? || os_arch.to_s.empty? || (!channel.nil? && channel.empty?)
+          UI.die!("invalid target #{token.inspect} — expected os-arch[:channel], e.g. ios-arm64:testflight")
+        end
+
+        os, arch = os_arch.split("-", 2)
+        TargetSpec.new(token, os, arch, channel || default_channel)
+      end
+
+      # One manifest per target — they differ in `target`, and may differ in
+      # bundle_id/name/version via `platforms:` overrides. The source digest is
+      # the same for all of them, so hash the tree once and thread it through.
+      def build_manifests(root_path, config, framework, ruby, specs)
+        source = nil
+        lockfile = nil
+
+        specs.each_with_object({}) do |spec, out|
+          manifest = Everywhere::Receipt.new(
+            root: root_path, config: config, framework: framework, ruby: ruby,
+            target: spec.os_arch, channel: spec.channel,
+            source_checksum: source, lockfile_checksum: lockfile
+          ).manifest
+          source ||= manifest.dig("source", "checksum")
+          lockfile ||= manifest.dig("source", "lockfile_checksum")
+          out[spec.token] = manifest
+        end
       end
 
       def label(target)

data/lib/everywhere/commands/platform/login.rb

@@ -23,6 +23,14 @@ module Everywhere
         start = client.post("/cli/device_authorizations", client_name: client_name)
         UI.die!("couldn't start login — HTTP #{start.status}") unless start.ok?
 
+        # A captive portal or a misconfigured proxy answers 200 with HTML, which
+        # parses to {} — without this we'd print an empty code and poll nothing
+        # for ten minutes.
+        unless start.body["user_code"] && start.body["device_code"]
+          UI.die!("#{base} answered without a device code — check EVERYWHERE_PLATFORM_URL " \
+                  "(a captive portal or proxy may be intercepting the request)")
+        end
+
         verify_url = start.body["verification_uri_complete"] || start.body["verification_uri"]
         UI.step("confirmation code: #{UI.bold(start.body["user_code"])}")
         UI.step("opening #{UI.cyan(verify_url)}")
@@ -48,9 +56,15 @@ module Everywhere
           sleep(interval)
           UI.die!("login timed out — run `every platform login` again") if monotonic > deadline
 
-          res = client.post("/cli/device_authorizations/token", device_code: device_code)
+          # The user is mid-approval in a browser; a dropped connection here must
+          # not kill the login. Keep polling until the deadline instead.
+          res = begin
+            client.post("/cli/device_authorizations/token", device_code: device_code)
+          rescue Everywhere::Error
+            next
+          end
           UI.die!("login failed — #{res.body["error"] || "invalid request"}") if res.status == 400
-          next unless res.ok? # transient 5xx/network: keep polling until the deadline
+          next unless res.ok? # transient 5xx: keep polling until the deadline
 
           case res.body["status"]
           when "approved" then return res.body

data/lib/everywhere/commands/platform/runner.rb

@@ -7,15 +7,16 @@ require "shellwords"
 require "base64"
 require "json"
 require "open3"
+require "securerandom"
 require_relative "../../ui"
 require_relative "../../paths"
 require_relative "../../platform/client"
 
 module Everywhere
   module Commands
-    # `every platform runner` — runs ON a build runner (a GitHub Actions macOS
-    # machine, or any Mac). It's the glue between the Runner API and the
-    # `every release` signing engine:
+    # `every platform runner` — runs ON a build runner: a macOS machine for the
+    # macos/ios targets, macOS or Linux for android. It's the glue between the
+    # Runner API and the `every release` signing engine:
     #
     #   job spec → mark running → download+extract snapshot → pull JIT creds into
     #   a throwaway keychain (+ .p8) → every release (streaming logs) → upload the
@@ -24,6 +25,28 @@ module Everywhere
     # Authenticated with the per-artifact runner token, so it can only touch the
     # one artifact it was dispatched for.
     class PlatformRunner < Dry::CLI::Command
+      # What `every release` leaves in dist/ per os, newest first when several
+      # match, and how the blob store should label it.
+      ARTIFACTS = { "macos" => %w[*.zip], "ios" => %w[*.ipa], "android" => %w[*.aab *.apk] }.freeze
+      CONTENT_TYPES = {
+        ".zip" => "application/zip",
+        ".ipa" => "application/octet-stream",
+        ".apk" => "application/vnd.android.package-archive",
+        ".aab" => "application/octet-stream"
+      }.freeze
+      RELEASE_NARRATION = {
+        "macos"   => "every release — press → sign → notarize → staple",
+        "ios"     => "every release — build → archive → export .ipa",
+        "android" => "every release — assemble → sign → bundle"
+      }.freeze
+
+      # A .mobileprovision has to be in both: Xcode reads the UserData copy,
+      # xcodebuild's own resolution still reads the MobileDevice one.
+      PROFILE_DIRS = [
+        "~/Library/MobileDevice/Provisioning Profiles",
+        "~/Library/Developer/Xcode/UserData/Provisioning Profiles"
+      ].freeze
+
       desc "Run a Platform build on this machine (used by the hosted runner)"
 
       option :artifact,  required: true, desc: "Artifact public id (art_…)"
@@ -49,10 +72,10 @@ module Everywhere
           src = fetch_snapshot(work, job)
           signing_env = pull_credentials(work)
 
-          status, release_json, zip = run_release(src, job, signing_env, shell_dir)
+          status, release_json, built = run_release(src, job, signing_env, shell_dir)
 
-          if status == "succeeded" && zip
-            complete_success(zip, release_json)
+          if status == "succeeded" && built
+            complete_success(built, release_json)
           else
             complete_failure("build failed — see log")
           end
@@ -62,6 +85,12 @@ module Everywhere
       rescue Everywhere::Error => e
         complete_failure(e.message)
         UI.die!(e.message)
+      rescue Everywhere::Fatal => e
+        # UI.die! raises Fatal (a SystemExit), which the rescue above can't see —
+        # without this an upload/signing failure would leave the artifact stuck
+        # "running" on the Platform forever.
+        complete_failure(e.message.to_s.empty? ? "build failed" : e.message)
+        raise
       end
 
       private
@@ -86,18 +115,64 @@ module Everywhere
           path = File.join(work, cred["filename"])
           File.binwrite(path, Base64.strict_decode64(cred["data_base64"]))
           File.chmod(0o600, path)
+          meta = cred["metadata"] || {}
           case cred["kind"]
           when "apple_developer_id", "apple_distribution"
-            env["APPLE_SIGNING_IDENTITY"] = install_certificate(path, cred["metadata"] || {}, work)
+            env["EVERY_IOS_TEAM_ID"] ||= meta["team_id"]
+            # An android job runs on Linux, where there's no keychain to import
+            # into. A stray apple credential there is the Platform's mistake, not
+            # a reason to lose the build.
+            if darwin?
+              env["APPLE_SIGNING_IDENTITY"] = install_certificate(path, meta, work)
+            else
+              remote_log("skipping #{cred["kind"]} — keychain import needs macOS")
+            end
           when "apple_asc_api_key"
             env["NOTARY_KEY"]    = path
-            env["NOTARY_KEY_ID"] = cred.dig("metadata", "key_id")
-            env["NOTARY_ISSUER"] = cred.dig("metadata", "issuer_id")
+            env["NOTARY_KEY_ID"] = meta["key_id"]
+            env["NOTARY_ISSUER"] = meta["issuer_id"]
+          when "apple_provisioning_profile"
+            profile = install_provisioning_profile(path)
+            env["EVERY_IOS_PROVISIONING_PROFILE"] = profile[:name]
+            env["EVERY_IOS_TEAM_ID"] ||= profile[:team]
+          when "android_keystore"
+            env["EVERY_ANDROID_KEYSTORE"]          = path
+            env["EVERY_ANDROID_KEYSTORE_PASSWORD"] = meta["store_password"]
+            env["EVERY_ANDROID_KEY_ALIAS"]         = meta["key_alias"]
+            # One-key keystores are usually created with a single password.
+            env["EVERY_ANDROID_KEY_PASSWORD"]      = meta["key_password"] || meta["store_password"]
+          end
+        end
+        env.compact
+      end
+
+      # A .mobileprovision is a CMS-signed DER blob wrapped around a plain XML
+      # plist. Scanning for the plist reads it identically on Linux, where
+      # `security cms -D` doesn't exist.
+      def install_provisioning_profile(path)
+        plist = File.binread(path)[/<\?xml.*<\/plist>/m]
+        UI.die!("the delivered provisioning profile has no embedded plist") unless plist
+
+        uuid = plist[%r{<key>UUID</key>\s*<string>([^<]+)</string>}m, 1]
+        name = plist[%r{<key>Name</key>\s*<string>([^<]+)</string>}m, 1]
+        # TeamIdentifier is an <array> of one — the first <string> after the key.
+        team = plist[%r{<key>TeamIdentifier</key>.*?<string>([^<]+)</string>}m, 1]
+        UI.die!("the delivered provisioning profile has no UUID") unless uuid
+
+        if darwin?
+          profile_dirs.each do |dir|
+            FileUtils.mkdir_p(dir)
+            FileUtils.cp(path, File.join(dir, "#{uuid}.mobileprovision"))
           end
         end
-        env
+        remote_log("provisioning profile #{UI.bold(name || uuid)}#{team ? " #{UI.dim("(team #{team})")}" : ""}")
+        { name: name, team: team, uuid: uuid }
       end
 
+      def profile_dirs = PROFILE_DIRS.map { |dir| File.expand_path(dir) }
+
+      def darwin? = RUBY_PLATFORM.include?("darwin")
+
       # Import the .p12 into a throwaway keychain and add it to the search list so
       # codesign finds the identity. Returns the identity name for notarize.sh.
       def install_certificate(p12, metadata, work)
@@ -124,11 +199,16 @@ module Everywhere
       end
 
       def run_release(src, job, signing_env, shell_dir)
-        remote_log("every release — press → sign → notarize → staple")
+        os = job.dig("target", "os").to_s
+        channel = job.dig("target", "distribution_channel").to_s
+        remote_log(RELEASE_NARRATION.fetch(os, "every release"))
         cmd = [RbConfig.ruby, "-I", lib_dir, every_exe, "release",
                "--root", src,
-               "--target", "#{job.dig("target", "os")}-#{job.dig("target", "arch")}",
-               "--channel", job.dig("target", "distribution_channel").to_s]
+               "--target", "#{os}-#{job.dig("target", "arch")}",
+               "--channel", channel]
+        # Play wants an App Bundle; every other Android channel (direct download,
+        # sideload, Amazon) wants the installable apk `every release` defaults to.
+        cmd += ["--format", "aab"] if os == "android" && channel == "play"
         cmd += ["--shell-dir", shell_dir] if shell_dir
 
         # The web UI renders ANSI, but `every release` runs here as a pipe (not a
@@ -144,37 +224,45 @@ module Everywhere
 
         dist = File.join(src, "dist")
         release_json = JSON.parse(File.read(File.join(dist, "release.json"))) rescue nil
-        zip = Dir[File.join(dist, "*.zip")].max_by { |f| File.mtime(f) }
-        [success ? "succeeded" : "failed", release_json, zip]
+        built = ARTIFACTS.fetch(os, %w[*.zip])
+                         .filter_map { |p| Dir[File.join(dist, p)].max_by { |f| File.mtime(f) } }.first
+        [success ? "succeeded" : "failed", release_json, built]
       end
 
-      def complete_success(zip, release_json)
+      def complete_success(built, release_json)
         report_phase("uploading")
-        remote_log("uploading artifact #{UI.dim("(#{(File.size(zip) / 1024.0 / 1024.0).round(1)}MB)")}")
-        signed_id = upload_artifact(zip)
+        remote_log("uploading artifact #{UI.dim("(#{(File.size(built) / 1024.0 / 1024.0).round(1)}MB)")}")
+        signed_id = upload_artifact(built)
         post!("/runner/artifacts/#{@artifact}/complete",
               status: "succeeded", file_signed_id: signed_id, release_json: release_json)
-        UI.success("build succeeded — #{File.basename(zip)}")
+        UI.success("build succeeded — #{File.basename(built)}")
       end
 
+      # Best-effort: if the thing that failed WAS connectivity, reporting the
+      # failure fails too — raising here would replace the real error and abort
+      # cleanup, so log locally and move on.
       def complete_failure(message)
         return if @completed
 
         @client.post("/runner/artifacts/#{@artifact}/complete", status: "failed", error: message)
         @completed = true
+      rescue Everywhere::Error => e
+        UI.warn("couldn't report the failure to the Platform (#{e.message})")
       end
 
-      def upload_artifact(zip)
+      def upload_artifact(built)
         require_relative "../../platform/snapshot"
         reservation = post!("/runner/artifacts/#{@artifact}/upload",
-          filename: File.basename(zip), byte_size: File.size(zip),
-          checksum: Everywhere::Platform::Snapshot.md5_base64(zip), content_type: "application/zip")
+          filename: File.basename(built), byte_size: File.size(built),
+          checksum: Everywhere::Platform::Snapshot.md5_base64(built), content_type: content_type_for(built))
         du = reservation["direct_upload"] || {}
-        res = @client.put_file(du["url"], zip, headers: du["headers"] || {})
+        res = @client.put_file(du["url"], built, headers: du["headers"] || {})
         UI.die!("artifact upload failed (HTTP #{res.code})") unless res.code.to_i.between?(200, 299)
         reservation["signed_id"]
       end
 
+      def content_type_for(path) = CONTENT_TYPES.fetch(File.extname(path), "application/octet-stream")
+
       # ---- log streaming -------------------------------------------------------
 
       # Run cmd, echoing output to this process's stdout AND forwarding it to the
@@ -241,6 +329,8 @@ module Everywhere
       end
 
       def restore_keychains
+        return unless darwin?
+
         run_security("list-keychains", "-d", "user", "-s", *@saved_keychains) if @saved_keychains
         run_security("delete-keychain", @keychain) if @keychain && File.exist?(@keychain)
       end