metanorma-release 0.2.24 → 0.2.26

data/lib/metanorma/release/aggregation_pipeline.rb

@@ -2,17 +2,30 @@
 
 module Metanorma
   module Release
-    FetchResult = Struct.new(:releases, :etag, :unchanged?, keyword_init: true)
+    # failed?: the release listing itself failed (rate limit, network) —
+    # distinct from an empty result. A failed listing must never be
+    # treated as "all releases deleted".
+    FetchResult = Struct.new(:releases, :etag, :unchanged?, :failed?,
+                             keyword_init: true) do
+      def initialize(releases:, etag:, **flags)
+        super(releases: releases, etag: etag,
+              unchanged?: flags.fetch(:unchanged?, false),
+              failed?: flags.fetch(:failed?, false))
+      end
+    end
     RepoReport  = Struct.new(:releases, :included, :skipped, :reason, :errors,
                              keyword_init: true)
     RepoError   = Struct.new(:tag, :message, keyword_init: true)
+    Collision   = Struct.new(:path, :claimed_by, keyword_init: true)
 
-    class AggregationPipeline # rubocop:disable Metrics/ClassLength
+    class AggregationPipeline
       Dependencies = Struct.new(
         :discoverer, :fetcher, :manifest_reader,
         :metadata_filter, :asset_processor, :delta_state,
         keyword_init: true
       ) do
+        include DependencyValidation
+
         def initialize(**kwargs)
           super
           validate_types!
@@ -27,16 +40,6 @@ module Metanorma
                               "manifest_reader")
           validate_interface!(delta_state, DeltaStateManager, "delta_state")
         end
-
-        def validate_interface!(obj, mod, name)
-          return if obj.is_a?(mod) || begin
-            obj.class.ancestors.include?(mod)
-          rescue StandardError
-            false
-          end
-
-          raise ArgumentError, "#{name} must include #{mod}, got #{obj.class}"
-        end
       end
 
       Config = Struct.new(
@@ -47,9 +50,14 @@ module Metanorma
 
       Result = Struct.new(
         :publications, :repo_count, :channels_found,
-        :report, :failed_repos,
+        :report, :failed_repos, :collisions,
         keyword_init: true
-      )
+      ) do
+        def initialize(publications:, repo_count:, channels_found:, report:,
+                       failed_repos:, collisions: [])
+          super
+        end
+      end
 
       def initialize(deps)
         @deps = deps
@@ -58,17 +66,13 @@ module Metanorma
       def run(config, output_dir)
         @deps.delta_state.load
         repos = @deps.discoverer.discover
-        publications = []
-        reports = []
-        failed_repos = []
 
-        repos.each do |repo|
-          repo_docs, report = process_repo(repo, output_dir, config)
-          publications.concat(repo_docs)
-          reports << report
-        rescue StandardError => e
-          failed_repos << RepoError.new(tag: repo.to_s, message: e.message)
-          raise if config.fail_on_error
+        if config.concurrency > 1
+          publications, reports, failed_repos = run_concurrent(repos,
+                                                               output_dir, config)
+        else
+          publications, reports, failed_repos = run_sequential(repos,
+                                                               output_dir, config)
         end
 
         @deps.delta_state.save
@@ -79,11 +83,73 @@ module Metanorma
           channels_found: publications.flat_map(&:channels).uniq.sort,
           report: reports,
           failed_repos: failed_repos,
+          collisions: detect_collisions(publications),
         )
       end
 
+      # Two releases extracting to the same path (same slug from
+      # different repos, or same-year editions under flat routing)
+      # silently overwrite each other. Surface them so callers can
+      # disambiguate in source repos or fail loudly.
+      def detect_collisions(publications)
+        claims = Hash.new { |h, k| h[k] = [] }
+        publications.each do |pub|
+          owner = pub.source ? "#{pub.source.repo_key}@#{pub.source.tag}" : pub.slug
+          pub.files.each { |file| claims[file.path] << owner }
+        end
+        claims.select { |_path, owners| owners.uniq.length > 1 }.map do |path, owners|
+          Collision.new(path: path, claimed_by: owners.uniq)
+        end
+      end
+
       private
 
+      def run_sequential(repos, output_dir, config)
+        publications = []
+        reports = []
+        failed_repos = []
+
+        repos.each do |repo|
+          repo_docs, report = process_repo(repo, output_dir, config)
+          publications.concat(repo_docs)
+          reports << report
+        rescue StandardError => e
+          failed_repos << RepoError.new(tag: repo.to_s, message: e.message)
+          raise if config.fail_on_error
+        end
+
+        [publications, reports, failed_repos]
+      end
+
+      def run_concurrent(repos, output_dir, config)
+        publications = []
+        reports = []
+        failed_repos = []
+        mutex = Mutex.new
+        threads = repos.each_slice([
+          (repos.length.to_f / config.concurrency).ceil, 1
+        ].max).map do |batch|
+          Thread.new(batch) do |slice|
+            slice.each do |repo|
+              repo_docs, report = process_repo(repo, output_dir, config)
+              mutex.synchronize do
+                publications.concat(repo_docs)
+                reports << report
+              end
+            rescue StandardError => e
+              mutex.synchronize do
+                failed_repos << RepoError.new(tag: repo.to_s,
+                                              message: e.message)
+              end
+              raise if config.fail_on_error
+            end
+          end
+        end
+        threads.each { |t| t.join(300) }
+
+        [publications, reports, failed_repos]
+      end
+
       def process_repo(repo, output_dir, config)
         repo_key = repo.to_s
 
@@ -96,6 +162,17 @@ module Metanorma
         etag = @deps.delta_state.etag(repo_key)
         fetch_result = @deps.fetcher.fetch(repo, etag: etag)
 
+        if fetch_result.failed?
+          # The listing failed (rate limit, network) — this is NOT "the
+          # releases are gone". Keep the existing extracted files and
+          # delta state untouched so a later run can reconcile.
+          return [], RepoReport.new(
+            releases: 0, included: 0, skipped: 0,
+            reason: "fetch failed — existing state kept",
+            errors: [RepoError.new(tag: nil, message: "release listing failed")]
+          )
+        end
+
         if fetch_result.unchanged?
           return [], RepoReport.new(releases: 0, included: 0, skipped: 0,
                                     reason: "etag unchanged", errors: [])
@@ -116,25 +193,27 @@ module Metanorma
           tag = release.tag_name
           current_tags << tag
 
-          content_hash = extract_content_hash(release.body)
-
-          if @deps.delta_state.processed?(repo_key, tag, content_hash)
-            files = @deps.delta_state.release_files(repo_key, tag)
-            if files.all? { |f| File.exist?(File.join(output_dir, f)) }
-              publications << build_publication(metadata, files, content_hash,
-                                                release, repo)
-              next
-            end
+          zip_asset = find_zip_asset(release)
+          fingerprint = release_fingerprint(release, zip_asset)
+          cached_files = @deps.delta_state.release_files(repo_key, tag)
+          cached_fingerprint = @deps.delta_state.release_fingerprint(repo_key, tag)
+          if fingerprint &&
+             cached_files.any? &&
+             cached_fingerprint == fingerprint &&
+             cached_files.all? { |f| File.exist?(File.join(output_dir, f)) }
+            publications << build_publication(metadata, cached_files, release,
+                                              repo)
+            next
           end
 
-          zip_asset = find_zip_asset(release)
           next unless zip_asset
 
           result = @deps.asset_processor.process(zip_asset.data, metadata_h)
-          @deps.delta_state.mark_processed(repo_key, tag, content_hash,
-                                           result.files.map(&:path))
+          @deps.delta_state.mark_processed(repo_key, tag, nil,
+                                           result.files.map(&:path),
+                                           fingerprint: fingerprint)
           publications << build_publication(metadata, result.files.map(&:path),
-                                            content_hash, release, repo)
+                                            release, repo)
         rescue StandardError => e
           errors << RepoError.new(tag: release.tag_name, message: e.message)
         end
@@ -150,22 +229,26 @@ module Metanorma
         )]
       end
 
-      def build_publication(metadata, files, _content_hash, release, repo)
+      def build_publication(metadata, files, release, repo)
         metadata.with_files_and_source(files, release, repo)
       end
 
-      def extract_content_hash(body)
-        return nil if body.nil?
-
-        match = body.match(/^content-hash:([a-f0-9]+)/)
-        match ? match[1] : nil
-      end
-
       def find_zip_asset(release)
         return nil unless release.assets
 
         release.assets.find { |a| a.name.end_with?(".zip") }
       end
+
+      # Identity of a release's CONTENT from listing data alone (no
+      # downloads): a force-recreated release gets a new timestamp and
+      # asset, so a fingerprint mismatch means the cached extraction is
+      # stale even though the tag (and the on-disk files) still exist.
+      def release_fingerprint(release, zip_asset)
+        return nil if zip_asset.nil?
+
+        size = zip_asset.respond_to?(:size) ? zip_asset.size : nil
+        [release.published_at, zip_asset.name, size].join("|")
+      end
     end
   end
 end

data/lib/metanorma/release/asset_processor.rb

@@ -12,8 +12,6 @@ module Metanorma
     class AssetProcessor
       ProcessResult = Struct.new(:files, :channels, keyword_init: true)
 
-      CANONICALIZE_PATTERN = /-ed\d+(\.\d+)?-/
-
       def initialize(output_dir:, routing:, canonicalize: true)
         @output_dir = output_dir
         @routing = routing
@@ -37,7 +35,9 @@ module Metanorma
               dest_path = File.join(@output_dir, relative_path)
 
               FileUtils.mkdir_p(File.dirname(dest_path))
-              entry.extract(dest_path) { true }
+              entry.extract(File.basename(dest_path),
+                            destination_directory:
+                              File.dirname(dest_path)) { true }
 
               ext = File.extname(file_name).delete_prefix(".")
               files << PublicationFile.new(format: ext, name: file_name,

data/lib/metanorma/release/cache_store.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "fileutils"
 require "json"
 
 module Metanorma

data/lib/metanorma/release/change_detector.rb

@@ -8,12 +8,13 @@ module Metanorma
     class ContentHashChangeDetector
       include ChangeDetector
 
-      def initialize(previous_releases:)
+      def initialize(previous_releases:, output_dir: ".")
         @previous_releases = previous_releases
+        @output_dir = output_dir
       end
 
       def detect(publication, tag, force: false)
-        current = publication.content_hash
+        current = publication.content_hash(from_directory: @output_dir)
         previous = @previous_releases[tag.to_s]
         changed = force || previous.nil? || !current.eql?(previous)
         ChangeResult.new(changed?: changed, current_hash: current,

data/lib/metanorma/release/cli.rb

@@ -11,6 +11,17 @@ module Metanorma
 
       class PipelineError < Thor::Error; end
 
+      desc "version", "Print version"
+      def version
+        puts "metanorma-release #{VERSION}"
+      end
+      map %w[--version -v] => :version
+
+      class_option :verbose, type: :boolean, default: false,
+                             desc: "Verbose output"
+      class_option :quiet, type: :boolean, default: false,
+                           desc: "Suppress warnings"
+
       desc "package", "Package compiled documents"
       option :output_dir, type: :string, default: "_site",
                           desc: "Compiled docs directory"
@@ -21,6 +32,7 @@ module Metanorma
       option :config, type: :string, desc: "Config file"
 
       def package
+        configure_logging
         config = PackageCommand::Config.new(
           output_dir: options[:output_dir],
           dest: options[:dest],
@@ -49,6 +61,7 @@ module Metanorma
       option :config, type: :string, desc: "Config file"
 
       def release
+        configure_logging
         config = ReleaseCommand::Config.new(
           output_dir: options[:output_dir],
           platform: options[:platform],
@@ -75,21 +88,24 @@ module Metanorma
       option :repos, type: :array, desc: "Explicit repo list"
       option :channels, type: :array, default: [],
                         desc: "Filter channels"
-      option :output_dir, type: :string, default: "_site/cc",
-                          desc: "Output directory"
+      option :output_dir, type: :string,
+                          desc: "Output directory (default: config file, or _site/cc)"
       option :file_routing, type: :string,
                             desc: "File routing (by-document|flat|by-format)"
       option :cache_dir, type: :string, desc: "Cache directory"
-      option :data_dir, type: :string, desc: "Write flattened documents.json for site generators"
+      option :data_dir, type: :string,
+                        desc: "Write flattened documents.json for site generators"
       option :include_drafts, type: :boolean, default: false,
                               desc: "Include draft releases"
       option :concurrency, type: :numeric, default: 4
       option :min_documents, type: :numeric, default: 0,
                              desc: "Minimum required documents"
       option :token, type: :string, desc: "Platform auth token"
-      option :config, type: :string, desc: "Config file (default: metanorma.aggregate.yml)"
+      option :config, type: :string,
+                      desc: "Config file (default: metanorma.aggregate.yml)"
 
       def aggregate
+        configure_logging
         config = AggregateCommand.build_config(
           source: options[:source],
           organizations: options[:organizations],
@@ -123,6 +139,14 @@ module Metanorma
 
       private
 
+      def configure_logging
+        if options[:verbose]
+          Metanorma::Release.logger.level = Logger::DEBUG
+        elsif options[:quiet]
+          Metanorma::Release.logger.level = Logger::ERROR
+        end
+      end
+
       def print_package_result(result, dest)
         released = result.released
         puts "Packaged #{released.length} documents → #{dest}/"

data/lib/metanorma/release/commands/aggregate.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "base64"
 require "yaml"
 
 module Metanorma
@@ -33,7 +32,6 @@ module Metanorma
         site.enrich!
         site.package! if @config.create_zip
 
-        stamp_primary_identifiers(index)
         result
       end
 
@@ -74,7 +72,7 @@ module Metanorma
           topic: cli_options[:topic] || gh["topic"],
           repos: cli_options[:repos] || file_data["repos"],
           channels: cli_options[:channels].any? ? cli_options[:channels] : Array(file_data["channels"]),
-          output_dir: cli_options[:output_dir] || file_data["output_dir"],
+          output_dir: cli_options[:output_dir] || file_data["output_dir"] || "_site/cc",
           file_routing: cli_options[:file_routing] || file_data["file_routing"] || "by-document",
           cache_dir: cli_options[:cache_dir] || file_data["cache_dir"],
           data_dir: cli_options[:data_dir] || file_data["data_dir"],
@@ -151,20 +149,6 @@ module Metanorma
           output_dir: @config.output_dir,
         )
       end
-
-      def stamp_primary_identifiers(index)
-        index.publications.each do |pub|
-          next unless pub.to_h["bibliographic"]
-
-          ids = pub.to_h.dig("bibliographic", "docidentifier")
-          next unless ids&.any?
-
-          primary = ids.find { |di| di["primary"] == true } || ids.first
-          pub.to_h.merge("primary_identifier" => primary["content"])
-        end
-      rescue LoadError
-        warn "  (relaton gem not available — bibliography skipped)"
-      end
     end
   end
 end

data/lib/metanorma/release/commands/package.rb

@@ -3,6 +3,8 @@
 module Metanorma
   module Release
     class PackageCommand
+      extend ConfigLoader
+
       Config = Struct.new(
         :output_dir, :dest, :manifest, :config_source,
         keyword_init: true
@@ -13,11 +15,15 @@ module Metanorma
       end
 
       def call
-        config = load_config
+        config = self.class.load_config(
+          config_source: @config.config_source,
+          manifest: @config.manifest,
+        )
         deps = ReleasePipeline::Dependencies.new(
-          extractor: Publication,
+          extractor: RxlExtractor,
           filters: [],
-          change_detector: ContentHashChangeDetector.new(previous_releases: {}),
+          change_detector: ContentHashChangeDetector.new(previous_releases: {},
+                                                         output_dir: @config.output_dir),
           packager: ZipPackager.new(output_dir: @config.output_dir),
           publisher: PlatformFactory.build_publisher("null", {}),
           slug_registry: SlugRegistry.from_config(config),
@@ -28,27 +34,13 @@ module Metanorma
 
         pipeline_config = ReleasePipeline::Config.new(
           output_dir: @config.output_dir,
-          manifest_path: @config.manifest,
           force: false,
           force_replace_patterns: nil,
           concurrency: 4,
-          default_visibility: "public",
         )
 
         ReleasePipeline.new(deps).run(pipeline_config)
       end
-
-      private
-
-      def load_config
-        if @config.config_source && File.exist?(@config.config_source)
-          Metanorma::Release::Config.from_file(@config.config_source)
-        elsif @config.manifest && File.exist?(@config.manifest)
-          Metanorma::Release::Config.from_file(@config.manifest)
-        else
-          Metanorma::Release::Config.defaults
-        end
-      end
     end
   end
 end

data/lib/metanorma/release/commands/release_command.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require "yaml"
-
 module Metanorma
   module Release
     class ReleaseCommand
+      extend ConfigLoader
+
       Config = Struct.new(
         :output_dir, :platform, :manifest, :force,
         :force_replace, :channels, :concurrency, :token, :config_source,
@@ -16,15 +16,20 @@ module Metanorma
       end
 
       def call
-        config = load_config
+        config = self.class.load_config(
+          config_source: @config.config_source,
+          manifest: @config.manifest,
+        )
         options = { token: @config.token }
         publisher = PlatformFactory.build_publisher(@config.platform, options)
         channel_override = Channel.parse_list(@config.channels) if @config.channels
 
+        previous_releases = build_previous_releases
         deps = ReleasePipeline::Dependencies.new(
-          extractor: Publication,
+          extractor: RxlExtractor,
           filters: [],
-          change_detector: ContentHashChangeDetector.new(previous_releases: {}),
+          change_detector: ContentHashChangeDetector.new(previous_releases: previous_releases,
+                                                         output_dir: @config.output_dir),
           packager: ZipPackager.new(output_dir: @config.output_dir),
           publisher: publisher,
           slug_registry: SlugRegistry.from_config(config),
@@ -35,11 +40,9 @@ module Metanorma
 
         pipeline_config = ReleasePipeline::Config.new(
           output_dir: @config.output_dir,
-          manifest_path: @config.manifest,
           force: @config.force,
           force_replace_patterns: @config.force_replace && !@config.force_replace.empty? ? @config.force_replace : nil,
           concurrency: @config.concurrency,
-          default_visibility: "public",
         )
 
         ReleasePipeline.new(deps).run(pipeline_config)
@@ -47,13 +50,15 @@ module Metanorma
 
       private
 
-      def load_config
-        if @config.config_source && File.exist?(@config.config_source)
-          Metanorma::Release::Config.from_file(@config.config_source)
-        elsif @config.manifest && File.exist?(@config.manifest)
-          Metanorma::Release::Config.from_file(@config.manifest)
-        else
-          Metanorma::Release::Config.defaults
+      def build_previous_releases
+        pubs = RxlExtractor.discover(@config.output_dir)
+        pubs.each_with_object({}) do |pub, map|
+          pub.slug
+          registry = SlugRegistry.default
+          strategy = registry.resolve(SlugStrategy.publisher_from_identifier(pub.identifier))
+          tag_info = strategy.compute_tag(pub)
+          content_hash = pub.content_hash(from_directory: @config.output_dir)
+          map[tag_info[:tag]] = content_hash if content_hash
         end
       end
     end

data/lib/metanorma/release/config.rb

@@ -60,13 +60,26 @@ module Metanorma
       end
     end
 
-    # Single routing entry — matches by any combination of pattern, source,
+    # Shared config resolution logic for CLI commands.
+    module ConfigLoader
+      def load_config(config_source:, manifest:)
+        if config_source && File.exist?(config_source)
+          Config.from_file(config_source)
+        elsif manifest && File.exist?(manifest)
+          Config.from_file(manifest)
+        else
+          Config.defaults
+        end
+      end
+    end
+
+    # Single routing entry — matches by any combination of pattern,
     # stage, and doctype. An entry with no criteria matches everything (catch-all).
-    DocumentEntry = Struct.new(:pattern, :source, :stages, :doctypes, :channels, keyword_init: true) do
+    DocumentEntry = Struct.new(:pattern, :stages, :doctypes,
+                               :channels, keyword_init: true) do
       def initialize(data)
         super(
           pattern: data["pattern"],
-          source: data["source"],
           stages: Array(data["stage"]).map(&:to_s),
           doctypes: Array(data["doctype"]).map(&:to_s),
           channels: Array(data["channels"]).map(&:to_s),
@@ -80,10 +93,6 @@ module Metanorma
           return false
         end
 
-        if source && !(publication.source_path&.end_with?(source) || false)
-          return false
-        end
-
         if !stages.empty? && !stages.include?(publication.stage.to_s)
           return false
         end

data/lib/metanorma/release/delta_state.rb

@@ -29,6 +29,10 @@ module Metanorma
         raise NotImplementedError, "#{self.class} must implement #release_files"
       end
 
+      def release_fingerprint(repo_key, tag)
+        raise NotImplementedError, "#{self.class} must implement #release_fingerprint"
+      end
+
       def mark_processed(repo_key, tag, content_hash, files)
         raise NotImplementedError,
               "#{self.class} must implement #mark_processed"
@@ -85,11 +89,20 @@ module Metanorma
         releases[tag]["files"] || []
       end
 
-      def mark_processed(repo_key, tag, content_hash, files)
+      def release_fingerprint(repo_key, tag)
+        releases = repo_state(repo_key)["releases"]
+        return nil unless releases.key?(tag)
+
+        releases[tag]["fingerprint"]
+      end
+
+      def mark_processed(repo_key, tag, content_hash, files,
+                         fingerprint: nil)
         repo = ensure_repo(repo_key)
         repo["releases"][tag] = {
           "content_hash" => content_hash.to_s,
           "files" => files,
+          "fingerprint" => fingerprint,
         }
       end
 
@@ -138,6 +151,7 @@ module Metanorma
       def set_etag(_repo_key, _etag) = nil
       def processed?(*_args) = false
       def release_files(*_args) = []
+      def release_fingerprint(*_args) = nil
       def mark_processed(*_args); end
       def cleanup_stale(*_args) = 0
     end

data/lib/metanorma/release/dependency_validation.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    module DependencyValidation
+      private
+
+      def validate_interface!(obj, mod, name)
+        return if obj.is_a?(mod) || begin
+          obj.class.ancestors.include?(mod)
+        rescue StandardError
+          false
+        end
+
+        raise ArgumentError, "#{name} must include #{mod}, got #{obj.class}"
+      end
+    end
+  end
+end