metanorma-release 0.2.2 → 0.2.3

data/lib/metanorma/release/aggregation_pipeline.rb

@@ -2,13 +2,42 @@
 
 module Metanorma
   module Release
-    class AggregationPipeline
+    FetchResult = Struct.new(:releases, :etag, :unchanged?, keyword_init: true)
+    RepoReport  = Struct.new(:releases, :included, :skipped, :reason, :errors,
+                             keyword_init: true)
+    RepoError   = Struct.new(:tag, :message, keyword_init: true)
+
+    class AggregationPipeline # rubocop:disable Metrics/ClassLength
       Dependencies = Struct.new(
         :discoverer, :fetcher, :manifest_reader,
-        :channel_filter, :stage_filter,
-        :asset_processor, :delta_state,
+        :metadata_filter, :asset_processor, :delta_state,
         keyword_init: true
-      )
+      ) do
+        def initialize(**kwargs)
+          super
+          validate_types!
+        end
+
+        private
+
+        def validate_types!
+          validate_interface!(discoverer, RepoDiscoverer, "discoverer")
+          validate_interface!(fetcher, ReleaseFetcher, "fetcher")
+          validate_interface!(manifest_reader, ManifestReader,
+                              "manifest_reader")
+          validate_interface!(delta_state, DeltaStateManager, "delta_state")
+        end
+
+        def validate_interface!(obj, mod, name)
+          return if obj.is_a?(mod) || begin
+            obj.class.ancestors.include?(mod)
+          rescue StandardError
+            false
+          end
+
+          raise ArgumentError, "#{name} must include #{mod}, got #{obj.class}"
+        end
+      end
 
       Config = Struct.new(
         :organizations, :channels, :topic,
@@ -17,7 +46,7 @@ module Metanorma
       )
 
       Result = Struct.new(
-        :documents, :repo_count, :channels_found,
+        :publications, :repo_count, :channels_found,
         :report, :failed_repos,
         keyword_init: true
       )
@@ -29,13 +58,13 @@ module Metanorma
       def run(config, output_dir)
         @deps.delta_state.load
         repos = @deps.discoverer.discover
-        documents = []
+        publications = []
         reports = []
         failed_repos = []
 
         repos.each do |repo|
           repo_docs, report = process_repo(repo, output_dir, config)
-          documents.concat(repo_docs)
+          publications.concat(repo_docs)
           reports << report
         rescue StandardError => e
           failed_repos << RepoError.new(tag: repo.to_s, message: e.message)
@@ -45,11 +74,11 @@ module Metanorma
         @deps.delta_state.save
 
         Result.new(
-          documents: documents,
+          publications: publications,
           repo_count: repos.length,
-          channels_found: documents.flat_map { |d| d.channels || [] }.uniq.sort,
+          channels_found: publications.flat_map(&:channels).uniq.sort,
           report: reports,
-          failed_repos: failed_repos
+          failed_repos: failed_repos,
         )
       end
 
@@ -59,9 +88,9 @@ module Metanorma
         repo_key = repo.to_s
 
         manifest_channels = @deps.manifest_reader.read(repo)
-        if manifest_channels && !@deps.channel_filter.overlaps?(manifest_channels)
+        if manifest_channels && !@deps.metadata_filter.overlaps?(manifest_channels)
           return [], RepoReport.new(releases: 0, included: 0, skipped: 0,
-                                    reason: 'channel manifest', errors: [])
+                                    reason: "channel manifest", errors: [])
         end
 
         etag = @deps.delta_state.etag(repo_key)
@@ -69,19 +98,19 @@ module Metanorma
 
         if fetch_result.unchanged?
           return [], RepoReport.new(releases: 0, included: 0, skipped: 0,
-                                    reason: 'etag unchanged', errors: [])
+                                    reason: "etag unchanged", errors: [])
         end
 
         current_tags = []
-        documents = []
+        publications = []
         errors = []
 
         fetch_result.releases.each do |release|
-          metadata = ReleaseMetadata.from_release_body(release.body)
+          metadata = Publication.from_release_body(release.body)
           next if metadata.nil?
 
-          next unless @deps.channel_filter.matches?(metadata.to_h)
-          next unless @deps.stage_filter.matches?(metadata.to_h)
+          metadata_h = metadata.to_h
+          next unless @deps.metadata_filter.matches?(metadata_h)
           next if release.prerelease && !config.include_drafts
 
           tag = release.tag_name
@@ -91,16 +120,19 @@ module Metanorma
 
           if @deps.delta_state.processed?(repo_key, tag, content_hash)
             files = @deps.delta_state.release_files(repo_key, tag)
-            documents << build_document(metadata, files, content_hash, release, repo)
+            publications << build_publication(metadata, files, content_hash,
+                                              release, repo)
             next
           end
 
           zip_asset = find_zip_asset(release)
           next unless zip_asset
 
-          result = @deps.asset_processor.process(zip_asset.data, metadata.to_h)
-          @deps.delta_state.mark_processed(repo_key, tag, content_hash, result.files.map(&:path))
-          documents << build_document(metadata, result.files.map(&:path), content_hash, release, repo)
+          result = @deps.asset_processor.process(zip_asset.data, metadata_h)
+          @deps.delta_state.mark_processed(repo_key, tag, content_hash,
+                                           result.files.map(&:path))
+          publications << build_publication(metadata, result.files.map(&:path),
+                                            content_hash, release, repo)
         rescue StandardError => e
           errors << RepoError.new(tag: release.tag_name, message: e.message)
         end
@@ -108,34 +140,16 @@ module Metanorma
         @deps.delta_state.cleanup_stale(repo_key, current_tags)
         @deps.delta_state.set_etag(repo_key, fetch_result.etag)
 
-        [documents, RepoReport.new(
+        [publications, RepoReport.new(
           releases: fetch_result.releases.length,
-          included: documents.length,
-          skipped: fetch_result.releases.length - documents.length,
+          included: publications.length,
+          skipped: fetch_result.releases.length - publications.length,
           reason: nil, errors: errors
         )]
       end
 
-      def build_document(metadata, files, content_hash, release, repo)
-        source = DocumentSource.new(
-          owner: repo.owner, repo: repo.repo,
-          tag: release.tag_name,
-          release_url: release.html_url,
-          release_date: release.published_at
-        )
-
-        file_structs = files.map { |f| DocumentFile.new(name: File.basename(f), path: f) }
-
-        AggregatedDocument.new(
-          id: metadata.id, title: metadata.title,
-          edition: metadata.edition, stage: metadata.stage,
-          doctype: metadata.doctype,
-          channels: metadata.channels,
-          formats: metadata.formats,
-          flavor: metadata.flavor,
-          content_hash: content_hash.to_s,
-          source: source, files: file_structs
-        )
+      def build_publication(metadata, files, _content_hash, release, repo)
+        metadata.with_files_and_source(files, release, repo)
       end
 
       def extract_content_hash(body)
@@ -148,7 +162,7 @@ module Metanorma
       def find_zip_asset(release)
         return nil unless release.assets
 
-        release.assets.find { |a| a.name.end_with?('.zip') }
+        release.assets.find { |a| a.name.end_with?(".zip") }
       end
     end
   end

data/lib/metanorma/release/asset_processor.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 begin
-  require 'zip'
+  require "zip"
 rescue LoadError
-  raise LoadError, "The rubyzip gem is required for AssetProcessor. Add `gem 'rubyzip'` to your Gemfile."
+  raise LoadError,
+        "The rubyzip gem is required for AssetProcessor. Add `gem 'rubyzip'` to your Gemfile."
 end
 
 module Metanorma
@@ -23,7 +24,7 @@ module Metanorma
         files = []
 
         Dir.mktmpdir do |tmp_dir|
-          zip_path = File.join(tmp_dir, 'archive.zip')
+          zip_path = File.join(tmp_dir, "archive.zip")
           File.binwrite(zip_path, zip_data)
 
           Zip::File.open(zip_path) do |zip_file|
@@ -38,20 +39,21 @@ module Metanorma
               FileUtils.mkdir_p(File.dirname(dest_path))
               entry.extract(dest_path) { true }
 
-              files << DocumentFile.new(name: file_name, path: relative_path)
+              ext = File.extname(file_name).delete_prefix(".")
+              files << PublicationFile.new(format: ext, name: file_name,
+                                           path: relative_path)
             end
           end
         end
 
-        ProcessResult.new(files: files, channels: metadata['channels'])
+        ProcessResult.new(files: files, channels: metadata["channels"])
       end
 
       private
 
       def canonicalize_name(name)
-        # Strip edition suffix: -ed1. → ., -ed1-wd. → -wd.
-        name.sub(/-ed\d+(\.\d+)?-(?=[a-z0-9])/, '-')
-            .sub(/-ed\d+(\.\d+)?\./, '.')
+        name.sub(/-ed\d+(\.\d+)?-(?=[a-z0-9])/, "-")
+          .sub(/-ed\d+(\.\d+)?\./, ".")
       end
     end
   end

data/lib/metanorma/release/cache_store.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
 module Metanorma
   module Release
@@ -47,13 +47,13 @@ module Metanorma
 
       def delete(key)
         path = file_path(key)
-        File.delete(path) if File.exist?(path)
+        FileUtils.rm_f(path)
       end
 
       def clear
         return unless Dir.exist?(@directory)
 
-        Dir.glob(File.join(@directory, '*')).each do |f|
+        Dir.glob(File.join(@directory, "*")).each do |f|
           File.delete(f) if File.file?(f)
         end
       end
@@ -61,14 +61,14 @@ module Metanorma
       def keys
         return [] unless Dir.exist?(@directory)
 
-        Dir.glob(File.join(@directory, '*')).select { |f| File.file?(f) }
-                                            .map { |f| File.basename(f) }
+        Dir.glob(File.join(@directory, "*")).select { |f| File.file?(f) }
+          .map { |f| File.basename(f) }
       end
 
       private
 
       def file_path(key)
-        sanitized = key.gsub(/[^a-zA-Z0-9._-]/, '_')
+        sanitized = key.gsub(/[^a-zA-Z0-9._-]/, "_")
         File.join(@directory, sanitized)
       end
     end

data/lib/metanorma/release/change_detector.rb

@@ -2,6 +2,9 @@
 
 module Metanorma
   module Release
+    ChangeResult = Struct.new(:changed?, :current_hash, :previous_hash,
+                              keyword_init: true)
+
     class ContentHashChangeDetector
       include ChangeDetector
 
@@ -9,11 +12,12 @@ module Metanorma
         @previous_releases = previous_releases
       end
 
-      def detect(metadata, tag, force: false)
-        current = ContentHash.of_directory(metadata.output_dir, base: metadata.file_base_name)
+      def detect(publication, tag, force: false)
+        current = publication.content_hash
         previous = @previous_releases[tag.to_s]
         changed = force || previous.nil? || !current.eql?(previous)
-        ChangeResult.new(changed?: changed, current_hash: current, previous_hash: previous)
+        ChangeResult.new(changed?: changed, current_hash: current,
+                         previous_hash: previous)
       end
     end
   end

data/lib/metanorma/release/channel.rb

@@ -3,61 +3,35 @@
 module Metanorma
   module Release
     class Channel
-      def self.parse(channel_string)
-        parts = channel_string.to_s.strip.split('/', 2)
-        if ChannelAudience.values.include?(parts[0])
-          new(audience: parts[0], category: parts[1] || 'default')
-        else
-          new(audience: ChannelAudience::PUBLIC, category: parts[0])
-        end
-      end
-
-      def self.parse_list(strings)
-        (strings || []).map { |s| parse(s) }
-      end
-
-      def self.public(category)
-        new(audience: ChannelAudience::PUBLIC, category: category)
-      end
-
-      def self.members(category)
-        new(audience: ChannelAudience::MEMBERS, category: category)
-      end
+      attr_reader :name
 
-      def self.internal(category)
-        new(audience: ChannelAudience::INTERNAL, category: category)
-      end
-
-      attr_reader :audience, :category
-
-      def initialize(audience:, category:)
-        @audience = audience
-        @category = category
+      def initialize(name)
+        @name = name.to_s.strip
         freeze
       end
 
       def to_s
-        "#{audience}/#{category}"
+        @name
       end
 
-      def public?
-        audience == ChannelAudience::PUBLIC
+      def eql?(other)
+        other.is_a?(self.class) && @name == other.name
       end
 
-      def members?
-        audience == ChannelAudience::MEMBERS
+      def hash
+        @name.hash
       end
 
       def matches?(filter_channels)
-        filter_channels.any? { |c| eql?(Channel.parse(c)) }
+        filter_channels.any? { |c| eql?(Channel.new(c)) }
       end
 
-      def eql?(other)
-        other.is_a?(self.class) && audience == other.audience && category == other.category
+      def self.parse(channel_string)
+        new(channel_string.to_s.strip)
       end
 
-      def hash
-        [audience, category].hash
+      def self.parse_list(strings)
+        (strings || []).map { |s| parse(s) }
       end
     end
   end

data/lib/metanorma/release/channel_filter.rb

@@ -2,25 +2,41 @@
 
 module Metanorma
   module Release
-    class ChannelFilter
-      def initialize(channels)
-        @channels = channels.map { |c| Channel.parse(c) }
-        @all = @channels.empty?
+    class MetadataFilter
+      def initialize(channels: [], stages: [])
+        @channels = channels.map { |c| Channel.new(c) }
+        @stages = Set.new(stages.map(&:downcase))
+        @all_channels = @channels.empty?
+        @all_stages = @stages.empty?
       end
 
       def matches?(release_metadata)
-        return true if @all
-
-        release_channels = (release_metadata['channels'] || []).map { |c| Channel.parse(c) }
-        release_channels.any? { |rc| @channels.any? { |fc| fc.eql?(rc) } }
+        channel_match?(release_metadata) && stage_match?(release_metadata)
       end
 
       def overlaps?(manifest_channels)
-        return true if @all
+        return true if @all_channels
 
-        parsed = manifest_channels.map { |c| Channel.parse(c) }
+        parsed = manifest_channels.map { |c| Channel.new(c) }
         parsed.any? { |mc| @channels.any? { |fc| fc.eql?(mc) } }
       end
+
+      private
+
+      def channel_match?(release_metadata)
+        return true if @all_channels
+
+        release_channels = (release_metadata["channels"] || []).map do |c|
+          Channel.new(c)
+        end
+        release_channels.any? { |rc| @channels.any? { |fc| fc.eql?(rc) } }
+      end
+
+      def stage_match?(release_metadata)
+        return true if @all_stages
+
+        @stages.include?(release_metadata["stage"].to_s.downcase)
+      end
     end
   end
 end