metanorma-release 0.2.2 → 0.2.3

data/lib/metanorma/release/cli.rb

@@ -1,129 +1,158 @@
 # frozen_string_literal: true
 
-require 'optparse'
+require "thor"
 
 module Metanorma
   module Release
-    module CLI
-      module_function
-
-      def run(argv)
-        command = argv.shift
-        case command
-        when 'package'   then run_package(argv)
-        when 'publish'   then run_publish(argv)
-        when 'aggregate' then run_aggregate(argv)
-        when nil
-          warn 'Usage: metanorma-release <package|publish|aggregate> [options]'
-          exit 2
-        else
-          warn "Unknown command: #{command}"
-          exit 2
-        end
+    class CLI < Thor
+      def self.exit_on_failure?
+        true
       end
 
-      def run_package(argv)
-        config = parse_package_args(argv)
+      class PipelineError < Thor::Error; end
+
+      desc "package", "Package compiled documents"
+      option :output_dir, type: :string, default: "_site",
+                          desc: "Compiled docs directory"
+      option :dest, type: :string, default: "dist",
+                    desc: "Destination for packages"
+      option :manifest, type: :string, default: "metanorma.release.yml",
+                        desc: "Release manifest file"
+      option :config, type: :string, desc: "Config file"
+
+      def package
+        config = PackageCommand::Config.new(
+          output_dir: options[:output_dir],
+          dest: options[:dest],
+          manifest: options[:manifest],
+          config_source: options[:config],
+        )
         result = PackageCommand.new(config).call
         print_package_result(result, config.dest)
-        exit(result.failed.empty? ? 0 : 1)
+        raise PipelineError, format_failures(result) unless result.failed.empty?
       end
 
-      def run_publish(argv)
-        config = parse_publish_args(argv)
-        result = PublishCommand.new(config).call
+      desc "release", "Package and release documents"
+      option :platform, type: :string, default: "github",
+                        desc: "Publishing platform (github|local)"
+      option :output_dir, type: :string, default: "_site",
+                          desc: "Compiled docs directory"
+      option :manifest, type: :string, default: "metanorma.release.yml",
+                        desc: "Release manifest file"
+      option :force, type: :boolean, default: false,
+                     desc: "Force release even if unchanged"
+      option :force_replace, type: :array, default: [],
+                             desc: "Glob patterns for force-replace"
+      option :channels, type: :array, desc: "Override channels"
+      option :concurrency, type: :numeric, default: 4
+      option :token, type: :string, desc: "Platform auth token"
+      option :config, type: :string, desc: "Config file"
+
+      def release
+        config = ReleaseCommand::Config.new(
+          output_dir: options[:output_dir],
+          platform: options[:platform],
+          manifest: options[:manifest],
+          force: options[:force],
+          force_replace: options[:force_replace],
+          channels: options[:channels],
+          concurrency: options[:concurrency],
+          token: options[:token],
+          config_source: options[:config],
+        )
+        result = ReleaseCommand.new(config).call
         print_publish_result(result)
-        exit(result.failed.empty? ? 0 : 1)
+        raise PipelineError, format_failures(result) unless result.failed.empty?
       end
 
-      def run_aggregate(argv)
-        config = parse_aggregate_args(argv)
+      desc "aggregate", "Aggregate released documents"
+      option :source, type: :string, default: "github",
+                      desc: "Source (github|local:PATH)"
+      option :organizations, type: :array, default: [],
+                             desc: "Organizations to discover"
+      option :topic, type: :string, default: "metanorma-release",
+                     desc: "Repository topic"
+      option :repos, type: :array, desc: "Explicit repo list"
+      option :channels, type: :array, default: [],
+                        desc: "Filter channels"
+      option :stages, type: :array, default: [],
+                      desc: "Filter stages"
+      option :output_dir, type: :string, default: "_site/cc",
+                          desc: "Output directory"
+      option :file_routing, type: :string, default: "by-document",
+                            desc: "File routing (by-document|flat|by-format)"
+      option :cache_dir, type: :string, desc: "Cache directory"
+      option :include_drafts, type: :boolean, default: false,
+                              desc: "Include draft releases"
+      option :concurrency, type: :numeric, default: 4
+      option :min_documents, type: :numeric, default: 0,
+                             desc: "Minimum required documents"
+      option :token, type: :string, desc: "Platform auth token"
+
+      def aggregate
+        config = AggregateCommand::Config.new(
+          source: options[:source],
+          organizations: options[:organizations],
+          topic: options[:topic],
+          repos: options[:repos],
+          channels: options[:channels],
+          stages: options[:stages],
+          output_dir: options[:output_dir],
+          file_routing: options[:file_routing],
+          cache_dir: options[:cache_dir],
+          include_drafts: options[:include_drafts],
+          concurrency: options[:concurrency],
+          min_documents: options[:min_documents],
+          token: options[:token],
+          create_zip: nil,
+        )
         result = AggregateCommand.new(config).call
         print_aggregate_result(result)
-        if config.min_documents.positive? && result.documents.length < config.min_documents
-          warn "Error: Found #{result.documents.length} documents, minimum is #{config.min_documents}"
-          exit 1
-        end
-        exit(result.failed_repos.empty? ? 0 : 1)
-      end
 
-      class << self
-        private
-
-        def parse_package_args(argv)
-          options = { output_dir: '_site', dest: 'dist', manifest: 'metanorma.release.yml',
-                      config_source: nil }
-          OptionParser.new do |opts|
-            opts.banner = 'Usage: metanorma-release package [options]'
-            opts.on('--output-dir DIR', 'Compiled docs directory') { |v| options[:output_dir] = v }
-            opts.on('--dest DIR', 'Destination for packages') { |v| options[:dest] = v }
-            opts.on('--manifest FILE', 'Release manifest file') { |v| options[:manifest] = v }
-            opts.on('--config SOURCE', 'Channel config (file path or platform ref)') { |v| options[:config_source] = v }
-          end.parse!(argv)
-          PackageCommand::Config.new(**options)
+        if options[:min_documents].positive? && result.publications.length < options[:min_documents]
+          raise PipelineError,
+                "Found #{result.publications.length} documents, minimum is #{options[:min_documents]}"
         end
 
-        def parse_publish_args(argv)
-          options = { output_dir: '_site', platform: 'github', manifest: 'metanorma.release.yml',
-                      force: false, force_replace: [], channels: nil, concurrency: 4, token: nil,
-                      config_source: nil }
-          OptionParser.new do |opts|
-            opts.banner = 'Usage: metanorma-release publish [options]'
-            opts.on('--platform NAME', 'github|local') { |v| options[:platform] = v }
-            opts.on('--output-dir DIR', 'Compiled docs directory') { |v| options[:output_dir] = v }
-            opts.on('--manifest FILE', 'Release manifest file') { |v| options[:manifest] = v }
-            opts.on('--force', 'Force release even if unchanged') { |v| options[:force] = v }
-            opts.on('--force-replace PAT', 'Glob patterns for force-replace') { |v| options[:force_replace] << v }
-            opts.on('--channels CHANS', 'Override channels (comma-separated)') { |v| options[:channels] = v.split(',') }
-            opts.on('--concurrency N', Integer) { |v| options[:concurrency] = v }
-            opts.on('--token TOKEN', 'Platform auth token') { |v| options[:token] = v }
-            opts.on('--config SOURCE', 'Channel config (file path or platform ref)') { |v| options[:config_source] = v }
-          end.parse!(argv)
-          PublishCommand::Config.new(**options)
+        unless result.failed_repos.empty?
+          raise PipelineError,
+                format_repo_failures(result)
         end
+      end
 
-        def parse_aggregate_args(argv)
-          options = { source: 'github', organizations: [], topic: 'metanorma-release',
-                      repos: nil, channels: [], stages: [], output_dir: '_site/cc',
-                      file_routing: 'by-document', cache_dir: nil,
-                      include_drafts: false, concurrency: 4, min_documents: 0, token: nil }
-          OptionParser.new do |opts|
-            opts.banner = 'Usage: metanorma-release aggregate [options]'
-            opts.on('--source SOURCE', 'github|local:PATH') { |v| options[:source] = v }
-            opts.on('--organizations ORGS', 'Comma-separated org list') { |v| options[:organizations] = v.split(',') }
-            opts.on('--topic TOPIC', 'Repository topic') { |v| options[:topic] = v }
-            opts.on('--repos REPOS', 'Explicit repo list (comma-separated)') { |v| options[:repos] = v.split(',') }
-            opts.on('--channels CHANS', 'Filter channels (comma-separated)') { |v| options[:channels] = v.split(',') }
-            opts.on('--stages STAGES', 'Filter stages (comma-separated)') { |v| options[:stages] = v.split(',') }
-            opts.on('--output-dir DIR', 'Output directory') { |v| options[:output_dir] = v }
-            opts.on('--file-routing MODE', 'by-document|flat|by-format') { |v| options[:file_routing] = v }
-            opts.on('--cache-dir DIR', 'Cache directory') { |v| options[:cache_dir] = v }
-            opts.on('--[no-]include-drafts', 'Include draft releases') { |v| options[:include_drafts] = v }
-            opts.on('--concurrency N', Integer) { |v| options[:concurrency] = v }
-            opts.on('--min-documents N', Integer) { |v| options[:min_documents] = v }
-            opts.on('--token TOKEN', 'Platform auth token') { |v| options[:token] = v }
-          end.parse!(argv)
-          AggregateCommand::Config.new(**options)
-        end
+      private
 
-        def print_package_result(result, dest)
-          released = result.released
-          puts "Packaged #{released.length} documents → #{dest}/"
-          released.each { |doc| puts "  #{doc.id}" }
-        end
+      def print_package_result(result, dest)
+        released = result.released
+        puts "Packaged #{released.length} documents → #{dest}/"
+        released.each { |pub| puts "  #{pub.slug}" }
+      end
 
-        def print_publish_result(result)
-          puts "Released #{result.released.length}, skipped #{result.skipped.length}, failed #{result.failed.length}"
-          result.released.each { |doc| puts "  RELEASED: #{doc.id} (#{doc.version.tag_component})" }
-          result.skipped.each { |doc| puts "  SKIPPED: #{doc.id} (unchanged)" }
-          result.failed.each { |f| puts "  FAILED: #{f[:document].id} - #{f[:error]}" }
+      def print_publish_result(result)
+        puts "Released #{result.released.length}, skipped #{result.skipped.length}, failed #{result.failed.length}"
+        result.released.each do |pub|
+          puts "  RELEASED: #{pub.slug} (ed#{pub.edition})"
         end
-
-        def print_aggregate_result(result)
-          puts "Aggregated #{result.documents.length} documents from #{result.repo_count} repos"
-          puts "Index: #{result.channels_found.join(', ')}" unless result.channels_found.empty?
+        result.skipped.each { |pub| puts "  SKIPPED: #{pub.slug} (unchanged)" }
+        result.failed.each do |f|
+          puts "  FAILED: #{f[:document].slug} - #{f[:error]}"
         end
       end
+
+      def print_aggregate_result(result)
+        puts "Aggregated #{result.publications.length} documents from #{result.repo_count} repos"
+        puts "Channels: #{result.channels_found.join(', ')}" unless result.channels_found.empty?
+      end
+
+      def format_failures(result)
+        result.failed.map do |f|
+          "#{f[:document].slug}: #{f[:error]}"
+        end.join("\n")
+      end
+
+      def format_repo_failures(result)
+        result.failed_repos.map { |r| "#{r.tag}: #{r.message}" }.join("\n")
+      end
     end
   end
 end

data/lib/metanorma/release/commands/aggregate.rb

@@ -6,7 +6,7 @@ module Metanorma
       Config = Struct.new(
         :source, :organizations, :topic, :repos, :repo_pattern, :local_path,
         :channels, :stages, :output_dir, :file_routing, :cache_dir,
-        :include_drafts, :concurrency, :min_documents, :token, :zip,
+        :include_drafts, :concurrency, :min_documents, :token, :create_zip,
         keyword_init: true
       )
 
@@ -16,8 +16,15 @@ module Metanorma
 
       def call
         result = run_aggregation
-        enrich(result) if result.documents.any?
-        zip_output if @config.zip
+        return result unless result.publications.any?
+
+        index = build_index(result)
+        site = Site.new(index: index, output_dir: @config.output_dir)
+        site.write!
+        site.enrich!
+        site.package! if @config.create_zip
+
+        stamp_primary_identifiers(index)
         result
       end
 
@@ -29,18 +36,18 @@ module Metanorma
           organizations: @config.organizations,
           topic: @config.topic,
           repos: @config.repos,
-          token: @config.token
+          token: @config.token,
         )
 
-        channel_filter = ChannelFilter.new(
-          channels: Channel.parse_list(@config.channels)
+        metadata_filter = MetadataFilter.new(
+          channels: Channel.parse_list(@config.channels),
+          stages: @config.stages || [],
         )
-        stage_filter = StageFilter.new(@config.stages || [])
         routing = FileRoutingFactory.from_name(@config.file_routing)
         asset_processor = AssetProcessor.new(
           output_dir: @config.output_dir,
           routing: routing,
-          canonicalize: true
+          canonicalize: true,
         )
         delta_state = build_delta_state
 
@@ -48,10 +55,9 @@ module Metanorma
           discoverer: adapters[:discoverer],
           fetcher: adapters[:fetcher],
           manifest_reader: adapters[:manifest_reader],
-          channel_filter: channel_filter,
-          stage_filter: stage_filter,
+          metadata_filter: metadata_filter,
           asset_processor: asset_processor,
-          delta_state: delta_state
+          delta_state: delta_state,
         )
 
         config = AggregationPipeline::Config.new(
@@ -60,66 +66,45 @@ module Metanorma
           topic: @config.topic,
           concurrency: @config.concurrency,
           include_drafts: @config.include_drafts,
-          fail_on_error: false
+          fail_on_error: false,
         )
 
         AggregationPipeline.new(deps).run(config, @config.output_dir)
       end
 
-      def build_delta_state
-        return NullDeltaState.new unless @config.cache_dir
-
-        DeltaState.new(
-          cache_store: FileCacheStore.new(@config.cache_dir),
-          output_dir: @config.output_dir
-        )
-      end
-
-      def enrich(result)
-        index = DocumentIndex.from_documents(
-          result.documents,
-          parameters: IndexParameters.new(
+      def build_index(result)
+        Index.from_documents(
+          result.publications,
+          parameters: {
             organizations: @config.organizations,
             channels: @config.channels || [],
             topic: @config.topic,
-            repo_count: result.repo_count
-          )
+            repo_count: result.repo_count,
+          },
         )
-        enricher = RelatonEnricher.new
-        enrich_result = enricher.enrich(index, @config.output_dir)
-        return unless enrich_result
-
-        stamp_primary_identifiers(enrich_result.documents)
-      rescue LoadError
-        warn '  (relaton gem not available — bibliography skipped)'
       end
 
-      def stamp_primary_identifiers(documents)
-        documents.map do |doc|
-          next doc unless doc['bibliographic']
-
-          ids = doc['bibliographic']['docidentifier']
-          next doc unless ids&.any?
+      def build_delta_state
+        return NullDeltaState.new unless @config.cache_dir
 
-          primary = ids.find { |di| di['primary'] == true } || ids.first
-          doc.merge('primary_identifier' => primary['content'])
-        end
+        DeltaState.new(
+          cache_store: FileCacheStore.new(@config.cache_dir),
+          output_dir: @config.output_dir,
+        )
       end
 
-      def zip_output
-        require 'zip'
+      def stamp_primary_identifiers(index)
+        index.publications.each do |pub|
+          next unless pub.to_h["bibliographic"]
 
-        dir = @config.output_dir
-        zip_path = "#{dir}.zip"
+          ids = pub.to_h.dig("bibliographic", "docidentifier")
+          next unless ids&.any?
 
-        Zip::File.open(zip_path, Zip::File::CREATE) do |zipfile|
-          Dir.glob("#{dir}/**/*").each do |file|
-            next if File.directory?(file)
-
-            entry_name = file.sub("#{File.dirname(dir)}/", '')
-            zipfile.add(entry_name, file)
-          end
+          primary = ids.find { |di| di["primary"] == true } || ids.first
+          pub.to_h.merge("primary_identifier" => primary["content"])
         end
+      rescue LoadError
+        warn "  (relaton gem not available — bibliography skipped)"
       end
     end
   end

data/lib/metanorma/release/commands/package.rb

@@ -8,26 +8,22 @@ module Metanorma
         keyword_init: true
       )
 
-      include ConfigResolver
-
       def initialize(config)
         @config = config
       end
 
       def call
-        manifest = load_manifest(@config.manifest)
-        channel_config = resolve_channel_config(@config.config_source, manifest)
-
+        config = load_config
         deps = ReleasePipeline::Dependencies.new(
-          extractor: RxlExtractor.new,
+          extractor: Publication,
           filters: [],
           change_detector: ContentHashChangeDetector.new(previous_releases: {}),
-          packager: ZipPackager.new,
-          publisher: PlatformFactory.build_publisher('null', {}),
-          naming_registry: NamingRegistry.default_registry,
-          manifest: manifest,
+          packager: ZipPackager.new(output_dir: @config.output_dir),
+          publisher: PlatformFactory.build_publisher("null", {}),
+          slug_registry: SlugRegistry.from_config(config),
+          manifest: nil,
           channel_override: nil,
-          channel_config: channel_config
+          config: config,
         )
 
         pipeline_config = ReleasePipeline::Config.new(
@@ -36,11 +32,23 @@ module Metanorma
           force: false,
           force_replace_patterns: nil,
           concurrency: 4,
-          default_visibility: 'public'
+          default_visibility: "public",
         )
 
         ReleasePipeline.new(deps).run(pipeline_config)
       end
+
+      private
+
+      def load_config
+        if @config.config_source && File.exist?(@config.config_source)
+          Metanorma::Release::Config.from_file(@config.config_source)
+        elsif @config.manifest && File.exist?(@config.manifest)
+          Metanorma::Release::Config.from_file(@config.manifest)
+        else
+          Metanorma::Release::Config.defaults
+        end
+      end
     end
   end
 end

data/lib/metanorma/release/commands/{publish.rb → release_command.rb}

@@ -2,37 +2,33 @@
 
 module Metanorma
   module Release
-    class PublishCommand
+    class ReleaseCommand
       Config = Struct.new(
         :output_dir, :platform, :manifest, :force,
         :force_replace, :channels, :concurrency, :token, :config_source,
         keyword_init: true
       )
 
-      include ConfigResolver
-
       def initialize(config)
         @config = config
       end
 
       def call
-        manifest = load_manifest(@config.manifest)
-        channel_config = resolve_channel_config(@config.config_source, manifest)
-
+        config = load_config
         options = { token: @config.token }
         publisher = PlatformFactory.build_publisher(@config.platform, options)
         channel_override = Channel.parse_list(@config.channels) if @config.channels
 
         deps = ReleasePipeline::Dependencies.new(
-          extractor: RxlExtractor.new,
+          extractor: Publication,
           filters: [],
           change_detector: ContentHashChangeDetector.new(previous_releases: {}),
-          packager: ZipPackager.new,
+          packager: ZipPackager.new(output_dir: @config.output_dir),
           publisher: publisher,
-          naming_registry: NamingRegistry.default_registry,
-          manifest: manifest,
+          slug_registry: SlugRegistry.from_config(config),
+          manifest: nil,
           channel_override: channel_override,
-          channel_config: channel_config
+          config: config,
         )
 
         pipeline_config = ReleasePipeline::Config.new(
@@ -41,11 +37,23 @@ module Metanorma
           force: @config.force,
           force_replace_patterns: @config.force_replace && !@config.force_replace.empty? ? @config.force_replace : nil,
           concurrency: @config.concurrency,
-          default_visibility: 'public'
+          default_visibility: "public",
         )
 
         ReleasePipeline.new(deps).run(pipeline_config)
       end
+
+      private
+
+      def load_config
+        if @config.config_source && File.exist?(@config.config_source)
+          Metanorma::Release::Config.from_file(@config.config_source)
+        elsif @config.manifest && File.exist?(@config.manifest)
+          Metanorma::Release::Config.from_file(@config.manifest)
+        else
+          Metanorma::Release::Config.defaults
+        end
+      end
     end
   end
 end

data/lib/metanorma/release/config.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Metanorma
+  module Release
+    class Config
+      def self.from_yaml(yaml_string)
+        data = YAML.safe_load(yaml_string, permitted_classes: [Symbol])
+        new(data || {})
+      end
+
+      def self.from_file(path)
+        unless File.exist?(path)
+          raise ArgumentError,
+                "Config file not found: #{path}"
+        end
+
+        from_yaml(File.read(path))
+      end
+
+      def self.defaults
+        new({})
+      end
+
+      def initialize(data)
+        @data = data
+      end
+
+      def channels
+        @data.fetch("channels", [])
+      end
+
+      def routing
+        @data.fetch("routing", {})
+      end
+
+      def routing_default
+        routing.fetch("default", ["public"])
+      end
+
+      def routing_rules
+        routing.fetch("rules", [])
+      end
+
+      def slug_config
+        @data.fetch("slug", {})
+      end
+
+      def slug_default_strategy
+        slug_config.fetch("default", "edition")
+      end
+
+      def slug_strategies
+        slug_config.fetch("strategies", {})
+      end
+
+      def documents
+        @data.fetch("documents", [])
+      end
+
+      def defaults
+        @data.fetch("defaults", {})
+      end
+
+      def default_channels
+        list = defaults.fetch("channels", nil)
+        return ["public"] unless list
+
+        list
+      end
+
+      def resolve_channels(publication)
+        manifest_channels = resolve_manifest_channels(publication)
+        return manifest_channels if manifest_channels
+
+        rule_channels = resolve_routing_rules(publication)
+        return rule_channels if rule_channels
+
+        default_channels
+      end
+
+      private
+
+      def resolve_manifest_channels(publication)
+        documents.each do |entry|
+          next unless entry["source"] && publication.source_path&.end_with?(entry["source"])
+          return entry["channels"] if entry["channels"]
+        end
+        nil
+      end
+
+      def resolve_routing_rules(publication)
+        routing_rules.each do |rule|
+          match = true
+          match &&= Array(rule["stage"]).map(&:to_s).include?(publication.stage.to_s) if rule["stage"]
+          match &&= Array(rule["doctype"]).map(&:to_s).include?(publication.doctype.to_s) if rule["doctype"]
+          return rule["channels"] if match && rule["channels"]
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/metanorma/release/content_hash.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'digest'
+require "digest"
 
 module Metanorma
   module Release
@@ -25,8 +25,16 @@ module Metanorma
       end
 
       def self.of_directory(directory, base: nil)
-        pattern = base ? File.join(directory, "#{base}.*") : File.join(directory, '**', '*')
-        files = Dir.glob(pattern).reject { |f| File.directory?(f) || f.end_with?('.zip') }
+        pattern = if base
+                    File.join(directory,
+                              "#{base}.*")
+                  else
+                    File.join(directory, "**",
+                              "*")
+                  end
+        files = Dir.glob(pattern).reject do |f|
+          File.directory?(f) || f.end_with?(".zip")
+        end
         of_files(files)
       end