native-packages 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af0462ecd5b50ab3aeb8415ff1c95c7d910b858ef1f3804b18526cfd41ada930
-  data.tar.gz: 48c6d9f15ad4b8ed60617416780fa5a03f294bc4f629949a5e31af7e041c84fd
+  metadata.gz: db11bd5b39c572e4983697e34112948cf182169145c1504a5c2223bd6c01bad0
+  data.tar.gz: 379244306eeb13a7e501cd3808dbe3fd81337079e70b129ca446e243a311daaa
 SHA512:
-  metadata.gz: ed735938724f514a10b46a50b21ac3131abc1d07981544b0f6f89d5010137b9e7674e041aeed97828de39d1fc2ff243241047f051a7ae78ac75beda8a446e30b
-  data.tar.gz: 5bb0021eeab1b651c10170de7a57c1e38036714477e15a9e8d698ad68fe836d136de70107b8f180734225b502cd161ada0b01df234a382c9b582e9789a79bc54
+  metadata.gz: 4169d1495a88f410c99a59ff1de20b9d6162468e5ddd4a3082a89e843ded13f84d74401cd91f4fde3b6e4457dfa91aa9ea78e19c7c0560c7fd89c3bd18785ec8
+  data.tar.gz: 2e89f5a56a85d642e8139c1c6725c3f90db7d664580c33ac64c6a432cac1998f5fa1e7cb220e415597532a83802b068c72f3be47c36a301ce51c795e89494244

data/README.md

@@ -84,6 +84,14 @@ Release mode verifies the declared binary assets against the release checksum fi
 
 Outputs include packages, prepared recipes, checksums and `build.json`. Publication verifies their hashes, configuration identity and complete target set. Partial builds can be combined with `aggregate DIR... --output DIRECTORY`. Native filenames come from nFPM.
 
+Since 0.4.0, `build --defer-recipes` lets each platform package its
+own inputs without acquiring global recipe assets or requiring AUR tools.
+Then `aggregate --finalize-recipes` generates the downstream recipes once,
+using hashes of the completed packages. This allows a Homebrew cask to reference
+the DMG being built in the same release. Deferred builds cannot be published.
+See [deferred recipe generation](docs/configuration.md#deferred-recipe-generation)
+for the finalizer's inputs and checks.
+
 Optional `repositories` and `templates` sections replace the old separate registries. Existing `stage`, `diff`, `publish TARGET` and `status` commands support reviewed downstream updates. AUR uses PKGBUILD/`.SRCINFO`; Homebrew uses formulae/casks. Each destination has an independent ignored Git clone, with no application submodules or extra remotes. Native source repositories still apply their own validation and review.
 
 ## GitHub Actions
@@ -95,7 +103,7 @@ packaging:
   needs: release
   permissions:
     contents: write
-  uses: crmne/native-packages/.github/workflows/package.yml@v0.3.1
+  uses: crmne/native-packages/.github/workflows/package.yml@v0.4.0
   with:
     version: ${{ github.ref_name }}
     publish: true
@@ -121,7 +129,7 @@ Migration combines `packaging/project.yml`, its nFPM definition and repository r
 bundle install
 bundle exec ruby -Ilib -e 'Dir["test/*_test.rb"].sort.each { |path| require_relative path }'
 gem build native-packages.gemspec
-ruby test/gem_install.rb native-packages-0.3.1.gem
+ruby test/gem_install.rb native-packages-0.4.0.gem
 ```
 
 Tests build real packages, inspect their payloads and exercise repository publication against local Git fixtures. CI additionally runs disposable Linux install/upgrade/remove checks, SRPM rebuilds and Windows MSIX acceptance. Runtime build manifests report installation as `not-tested`: CI fixture coverage is not a substitute for testing each application's packages.

data/docs/configuration.md

@@ -65,3 +65,62 @@ native-packages publish --from dist/complete --to github
 ```
 
 Each build must use the same configuration, release version and timestamp. Aggregate rejects duplicate target/format outputs, conflicting recipe files and incomplete sets. Publishing recipes can still use the existing reviewed sequence: `stage TARGET DIRECTORY/recipes`, `diff TARGET`, then `publish TARGET --body-file FILE` where required by a submission destination.
+
+
+## Deferred recipe generation
+
+Since 0.4.0, use this opt-in sequence when downstream recipes depend
+on packages built in separate jobs. The existing build and aggregate commands
+keep generating and checking recipes as before when the new flags are absent.
+There are no new configuration keys.
+
+```sh
+# Run each target on its required host, with the same checkout and epoch.
+native-packages doctor --target linux-amd64 --defer-recipes
+native-packages build --version 1.2.3 --target linux-amd64 --defer-recipes --output dist/linux
+native-packages build --version 1.2.3 --target macos-universal --defer-recipes --output dist/macos
+native-packages build --version 1.2.3 --target windows-amd64 --defer-recipes --output dist/windows
+
+# Download those build directories and remaining recipe assets onto Linux.
+native-packages aggregate dist/linux dist/macos dist/windows --finalize-recipes --output dist/complete
+# Run the application's package installation/upgrade/removal checks here.
+native-packages publish --from dist/complete --to github,aur,homebrew
+```
+
+A deferred build still validates configuration, selected inputs, architecture,
+container contents and signing hooks. It does not acquire the global `assets`
+or generate `templates`. `doctor --defer-recipes` likewise omits recipe tooling;
+the selected package formats still need their own tools. Asset filenames and
+URLs are available as tokens, but unknown `@KEY_SHA256@` values cannot be used
+by a target's input, package definition or hooks. Such a target must use the
+normal build path with its assets already available. Release-mode target inputs
+still require their published checksums, even with `--defer-recipes`.
+
+The finalizer first verifies every input manifest and the complete target set.
+All inputs must defer recipes and agree on configuration, version, timestamp,
+tool and recipe metadata, including Git version. Use the same Git history depth
+and `SOURCE_DATE_EPOCH` on every host. Mixing ordinary and deferred builds fails.
+Neither a partial target set nor a deferred build can be published. An ordinary
+aggregate rejects deferred inputs unless `--finalize-recipes` is supplied.
+
+For each global asset, finalization first looks for a verified package whose
+release filename exactly matches the asset's rendered `file`. For example,
+`assets.MACOS.file: app-v@VERSION@-macos-universal.dmg` resolves to a native target
+with that output filename. Its hash includes the completed signing/notarization
+hook. A stale `local` file cannot override that package. Duplicate package
+filenames are rejected. The manifest records this asset by package filename
+and digest, without a temporary staging path.
+
+Other assets, such as portable or source archives, must be staged at their
+configured `local` paths on the finalization host. Finalization reads local
+inputs and does not download recipe assets from a release. This also applies
+when the deferred target builds used `--release`. Recipe tools run only here:
+AUR templates require makepkg or working Docker, and recipe archives require
+tar and xz. The finalizer renders and validates recipes, writes the recipe
+archive and complete manifest, and atomically creates a fresh output directory.
+Failure leaves the original target directories intact and creates no completed
+output. Existing destinations are never overwritten.
+
+Deferral does not enable stable downstream recipes in a prerelease configuration.
+The existing preview-format and stable-recipe restrictions still apply. Use a
+preview configuration without downstream templates when building previews.

data/docs/native-recipes.md

@@ -43,7 +43,13 @@ targets:
 
 Run each target on its own host using `build --version VERSION --target ID`.
 Use the normal `aggregate` command to combine complete target results later.
-Use the same configuration and `SOURCE_DATE_EPOCH` across jobs. No nFPM binary
+Use the same configuration and `SOURCE_DATE_EPOCH` across jobs.
+Since 0.4.0, configurations with AUR/Homebrew recipes can use
+`build --defer-recipes` on each host, then `aggregate --finalize-recipes` on
+Linux. This avoids requiring the finished DMG before its own build or running
+AUR tools on macOS/Windows. The finalizer resolves recipe assets from the
+completed package filenames and locally staged external assets. See
+[deferred recipe generation](configuration.md#deferred-recipe-generation). No nFPM binary
 is required when only native targets are selected; its version remains part of
 the configuration for compatibility with mixed Linux/native builds.
 

data/docs/releasing.md

@@ -23,7 +23,7 @@ Update the gemspec, `NativePackages::VERSION`, example/version references and ch
 Create a GitHub release for that commit:
 
 ```sh
-gh release create v0.3.1 --target main --title 'native-packages 0.3.1' --notes-file release-notes.md
+gh release create v0.4.0 --target main --title 'native-packages 0.4.0' --notes-file release-notes.md
 ```
 
 Adding `--draft` permits review before publication and does not trigger the publisher. The tag must match the exact gem version and point to a commit on `main`. Set `--prerelease` only for a prerelease gem version.

data/examples/native-packages-all-formats.yaml

@@ -1,7 +1,7 @@
 # Supply the listed build outputs before running this example.
 schema: 1
 tool:
-  version: '0.3.1'
+  version: '0.4.0'
   nfpm: '2.47.0'
 nfpm:
   name: example-app

data/examples/native-packages.yaml

@@ -1,6 +1,6 @@
 schema: 1
 tool:
-  version: '0.3.1'
+  version: '0.4.0'
   nfpm: '2.47.0'
 nfpm:
   name: example-app

data/lib/native_packages/build.rb

@@ -27,7 +27,7 @@ module NativePackages
       version
     end
 
-    def doctor(ids: [], formats: [], release: false)
+    def doctor(ids: [], formats: [], release: false, defer_recipes: false)
       configuration.validate
       selected = configuration.select(ids: ids, formats: formats)
       nfpm_version if selected.values.any? { |target| (target.fetch("formats") & Configuration::NATIVE_FORMATS).empty? }
@@ -40,13 +40,19 @@ module NativePackages
       required << "readelf" if selected.values.any? { |target| target["platform"] == "linux" && target.fetch("kind", "binary") == "binary" }
       required << "bsdtar" if selected.values.any? { |target| target.fetch("input").fetch("kind", "archive") == "archive" }
       required << "curl" if release
-      required += %w[tar xz] unless configuration.data.fetch("templates").empty?
       missing = required.uniq.reject { |tool| available?(tool) }
       raise Error, "install required tools: #{missing.join(', ')}" unless missing.empty?
+      recipe_tools unless defer_recipes
+      selected
+    end
+
+    def recipe_tools
+      return if configuration.data.fetch("templates").empty?
+      missing = %w[tar xz].reject { |tool| available?(tool) }
+      raise Error, "install required recipe tools: #{missing.join(', ')}" unless missing.empty?
       if configuration.data.fetch("templates").keys.any? { |path| path.match?(%r{\Aarch/[^/]+/PKGBUILD\z}) }
         raise Error, "AUR metadata needs makepkg or accessible Docker" unless available?("makepkg") || available?("docker")
       end
-      selected
     end
 
     def version(value, release)
@@ -73,7 +79,7 @@ module NativePackages
       configuration.tokens(version, epoch: epoch).merge("GIT_VERSION" => git_version)
     end
 
-    def run_build(value: nil, release: nil, ids: [], formats: [], output: nil, dry_run: false)
+    def run_build(value: nil, release: nil, ids: [], formats: [], output: nil, dry_run: false, defer_recipes: false)
       configuration.validate
       selected = configuration.select(ids: ids, formats: formats)
       number = version(value, release)
@@ -84,15 +90,26 @@ module NativePackages
       output = Pathname.new(output || root / "dist/packages" / number).expand_path(root)
       if dry_run
         puts JSON.pretty_generate("version" => number, "mode" => release ? "release" : "local", "output" => output.to_s,
-          "targets" => selected.transform_values { |target| target.slice("formats", "platform", "arch", "input", "before_build", "after_package", "native") })
+          "recipes" => defer_recipes ? "deferred" : "included", "targets" => selected.transform_values { |target| target.slice("formats", "platform", "arch", "input", "before_build", "after_package", "native") })
         return
       end
       raise Error, "output exists: #{output}; choose a fresh --output" if output.exist?
-      doctor(ids: ids, formats: formats, release: !!release)
+      doctor(ids: ids, formats: formats, release: !!release, defer_recipes: defer_recipes)
       info = metadata(number, release: !!release)
       sources = []
       expected = release ? release_checksums(info) : nil
-      acquire_assets(info, expected, sources)
+      if defer_recipes
+        # Asset hashes are unknown until finalization. Do not silently pass the
+        # validation placeholders into a target's input, hooks or package data.
+        configuration.data.fetch("assets").each_key { |key| info.delete("#{key}_SHA256") }
+        selected.each do |id, target|
+          tokens = configuration.target_tokens(info, id, target, "/payload").merge("PACKAGE" => "/output/package", "FORMAT" => target.fetch("formats").first)
+          render_tree(target, tokens)
+          render_tree(configuration.package(target), tokens)
+        end
+      else
+        acquire_assets(info, expected, sources)
+      end
       output.dirname.mkpath
       Dir.mktmpdir(".native-packages-", output.dirname) do |temporary|
         staging = Pathname.new(temporary) / "result"
@@ -161,17 +178,14 @@ module NativePackages
             end
           end
         end
-        recipes = staging / "recipes"
-        project.generate(recipes, info)
-        project.check(recipes, prerelease: number.include?("-"))
-        if !configuration.data.fetch("templates").empty?
-          project.recipe_archive(recipes, staging, name: configuration.name, version: number, epoch: info.fetch("SOURCE_DATE_EPOCH"))
-          (staging / "packaging-checksums.txt").delete
-        end
+        write_recipes(staging, info) unless defer_recipes
         manifest = { "schema" => 1, "name" => configuration.name, "version" => number, "configuration" => configuration.digest,
           "tool" => configuration.data.fetch("tool"), "epoch" => info.fetch("SOURCE_DATE_EPOCH"), "inputs" => sources,
           "targets" => selected.transform_values { |target| target.fetch("formats") }, "packages" => records,
           "files" => files(staging).to_h { |file| [file.relative_path_from(staging).to_s, sha256(file)] } }
+        if defer_recipes
+          manifest["recipes"] = { "state" => "deferred", "metadata" => info.reject { |key, _| key == "ROOT" } }
+        end
         write(staging / "build.json", JSON.pretty_generate(manifest) + "\n")
         write(staging / "packaging-checksums.txt", files(staging).map { |file| "#{sha256(file)}  #{file.relative_path_from(staging)}\n" }.join)
         staging.rename(output)
@@ -212,10 +226,12 @@ module NativePackages
       path
     end
 
-    def acquire_assets(info, expected, sources)
+    def acquire_assets(info, expected, sources, packages: {})
       configuration.data.fetch("assets").each do |key, definition|
         asset = render_tree(definition, info)
-        path = if expected
+        path = if packages.key?(asset.fetch("file"))
+          packages.fetch(asset.fetch("file"))
+        elsif expected
           release_input(asset.merge("release_asset" => asset.fetch("file")), info, expected, checksummed: asset.fetch("checksummed", true))
         else
           root / asset.fetch("local") { raise Error, "assets.#{key}: declare local input or use --release" }
@@ -224,7 +240,15 @@ module NativePackages
         info["#{key}_SHA256"] = digest
         info["#{key}_FILE"] = asset.fetch("file")
         info["#{key}_URL"] = asset.fetch("url", "#{info.fetch('UPSTREAM')}/releases/download/#{info.fetch('TAG')}/#{asset.fetch('file')}")
-        sources << { "asset" => key, "path" => path.to_s, "sha256" => digest }
+        source = { "asset" => key, "sha256" => digest }
+        # An aggregated package survives under its release filename; the
+        # finalizer's temporary staging path does not survive the rename.
+        if packages.key?(asset.fetch("file"))
+          source["package"] = asset.fetch("file")
+        else
+          source["path"] = path.to_s
+        end
+        sources << source
       end
     end
 
@@ -248,6 +272,48 @@ module NativePackages
       end
     end
 
+    def write_recipes(output, info)
+      recipes = output / "recipes"
+      project.generate(recipes, info)
+      project.check(recipes, prerelease: info.fetch("VERSION").include?("-"))
+      unless configuration.data.fetch("templates").empty?
+        project.recipe_archive(recipes, output, name: configuration.name, version: info.fetch("VERSION"), epoch: info.fetch("SOURCE_DATE_EPOCH"))
+        (output / "packaging-checksums.txt").delete
+      end
+    end
+
+    def verify_recipes(output, manifest, complete:)
+      if manifest.key?("recipes")
+        phase = manifest.fetch("recipes")
+        unless phase.is_a?(Hash) && phase["state"] == "deferred" && phase["metadata"].is_a?(Hash)
+          raise Error, "invalid recipe phase"
+        end
+        info = phase.fetch("metadata")
+        unless info.values_at("NAME", "VERSION", "SOURCE_DATE_EPOCH") == manifest.values_at("name", "version", "epoch")
+          raise Error, "deferred recipe metadata disagrees with build"
+        end
+        package_paths = manifest.fetch("packages").map { |entry| entry.fetch("path") }
+        raise Error, "deferred build contains recipe files" unless (manifest.fetch("files").keys - package_paths).empty?
+        raise Error, "recipes are deferred; aggregate with --finalize-recipes before publishing" if complete
+        return
+      end
+      # Require the recipe files as well as hashes, so dropping the deferred
+      # marker cannot make an unfinished build appear publishable.
+      required = ["recipes/release.json"]
+      raise Error, "missing recipe metadata" unless manifest.fetch("files").key?(required.first)
+      info = JSON.parse((output / required.first).read)
+      configuration.data.fetch("templates").each_key do |path|
+        relative = render(path, info.merge("ROOT" => root.to_s))
+        required << "recipes/#{relative_path(relative)}"
+        required << "recipes/#{File.dirname(relative)}/.SRCINFO" if relative.match?(%r{\Aarch/[^/]+/PKGBUILD\z})
+      end
+      unless configuration.data.fetch("templates").empty?
+        required << "#{configuration.name}-#{manifest.fetch('version')}-packaging.tar.xz"
+      end
+      missing = required - manifest.fetch("files").keys
+      raise Error, "missing recipe outputs: #{missing.join(', ')}" unless missing.empty?
+    end
+
     def verify(output, complete: true)
       output = Pathname.new(output).expand_path(root)
       manifest = JSON.parse((output / "build.json").read)
@@ -271,6 +337,7 @@ module NativePackages
       manifest.fetch("packages").each do |entry|
         raise Error, "package hash does not match file manifest" unless entry.fetch("sha256") == manifest.fetch("files").fetch(entry.fetch("path"))
       end
+      verify_recipes(output, manifest, complete: complete)
       manifest
     end
 
@@ -304,7 +371,7 @@ module NativePackages
       end
     end
 
-    def aggregate(inputs, output:)
+    def aggregate(inputs, output:, finalize_recipes: false)
       configuration.validate
       output = Pathname.new(output).expand_path(root)
       raise Error, "output exists: #{output}" if output.exist?
@@ -314,6 +381,14 @@ module NativePackages
       %w[version epoch tool].each do |key|
         raise Error, "builds disagree on #{key}" unless manifests.map { |entry| entry.fetch(key) }.uniq.length == 1
       end
+      phases = manifests.map { |entry| entry["recipes"] }
+      if phases.any?
+        raise Error, "do not mix deferred and completed recipe builds" if phases.any?(&:nil?)
+        raise Error, "deferred recipe metadata differs between builds" unless phases.uniq.length == 1
+        raise Error, "deferred builds require --finalize-recipes" unless finalize_recipes
+      elsif finalize_recipes
+        raise Error, "--finalize-recipes requires builds made with --defer-recipes"
+      end
       result = Marshal.load(Marshal.dump(manifests.first))
       result.merge!("targets" => {}, "packages" => [], "files" => {}, "inputs" => manifests.flat_map { |entry| entry.fetch("inputs") }.uniq)
       output.dirname.mkpath
@@ -338,6 +413,21 @@ module NativePackages
         configuration.targets.each do |id, target|
           result["targets"][id] = target.fetch("formats").select { |format| result["targets"].fetch(id, []).include?(format) }
         end
+        if finalize_recipes
+          expected = configuration.targets.transform_values { |target| target.fetch("formats") }
+          raise Error, "incomplete target set; aggregate all configured targets before finalizing recipes" unless result.fetch("targets") == expected
+          recipe_tools
+          info = result.fetch("recipes").fetch("metadata").merge("ROOT" => root.to_s)
+          packages = result.fetch("packages").each_with_object({}) do |entry, paths|
+            name = File.basename(entry.fetch("path"))
+            raise Error, "ambiguous recipe asset filename: #{name}" if paths.key?(name)
+            paths[name] = staging / entry.fetch("path")
+          end
+          acquire_assets(info, nil, result.fetch("inputs"), packages: packages)
+          write_recipes(staging, info)
+          result.delete("recipes")
+          result["files"] = files(staging).to_h { |file| [file.relative_path_from(staging).to_s, sha256(file)] }
+        end
         write(staging / "build.json", JSON.pretty_generate(result) + "\n")
         write(staging / "packaging-checksums.txt", files(staging).map { |file| "#{sha256(file)}  #{file.relative_path_from(staging)}\n" }.join)
         verify(staging)

data/lib/native_packages/cli.rb

@@ -13,7 +13,9 @@ module NativePackages
         build [--version VERSION | --release TAG]  Build all configured packages
           [--target ID] [--format FORMAT]          Select targets/formats (repeatable)
           [--output DIRECTORY] [--dry-run]         Choose output or inspect the plan
+          [--defer-recipes]                        Build targets without recipe assets/tools
         aggregate DIR... --output DIRECTORY        Combine target builds before publishing
+          [--finalize-recipes]                    Generate recipes from deferred builds
         publish --from DIRECTORY --to github,aur   Publish a complete, verified build
         migrate [--dry-run]                        Convert packaging/project.yml
         repositories                               List downstream destinations
@@ -50,11 +52,13 @@ module NativePackages
           flags.on("--target ID") { |value| options[:ids] << value }
           flags.on("--format FORMAT") { |value| options[:formats] << value }
           flags.on("--release TAG") { |value| options[:release] = value }
+          flags.on("--defer-recipes") { options[:defer_recipes] = true }
         end
         if command == "build"
           flags.on("--version VERSION") { |value| options[:value] = value }
           flags.on("--dry-run") { options[:dry_run] = true }
         end
+        flags.on("--finalize-recipes") { options[:finalize_recipes] = true } if command == "aggregate"
         if command == "init"
           flags.on("--interactive") { options[:interactive] = true }
           flags.on("--name NAME") { |value| options[:name] = value }
@@ -88,13 +92,13 @@ module NativePackages
         raise Error, "create native-packages.yaml with init, or convert existing packaging with migrate" unless configuration
         builder = Build.new(configuration)
         case command
-        when "build" then return builder.run_build(**options.slice(:value, :release, :ids, :formats, :output, :dry_run))
+        when "build" then return builder.run_build(**options.slice(:value, :release, :ids, :formats, :output, :dry_run, :defer_recipes))
         when "doctor"
-          builder.doctor(**options.slice(:ids, :formats, :release))
+          builder.doctor(**options.slice(:ids, :formats, :release, :defer_recipes))
           return puts "Configuration and packaging tools are ready. Build checks input files and package contents."
         when "aggregate"
           raise Error, "aggregate requires --output" unless options[:output]
-          return builder.aggregate(arguments, output: options.fetch(:output))
+          return builder.aggregate(arguments, output: options.fetch(:output), finalize_recipes: options.fetch(:finalize_recipes, false))
         when "publish" then return builder.publish(options.fetch(:from), options.fetch(:destinations), body_file: options[:body_file])
         end
       end

data/lib/native_packages/support.rb

@@ -12,7 +12,7 @@ require "tmpdir"
 require "yaml"
 
 module NativePackages
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
   class Error < StandardError; end
 
   module Support

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: native-packages
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Carmine Paolino