metanorma-release 0.2.1

data/lib/metanorma/release/commands/aggregate.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    class AggregateCommand
+      Config = Struct.new(
+        :source, :organizations, :topic, :repos, :repo_pattern, :local_path,
+        :channels, :stages, :output_dir, :file_routing, :cache_dir,
+        :include_drafts, :concurrency, :min_documents, :token, :zip,
+        keyword_init: true
+      )
+
+      def initialize(config)
+        @config = config
+      end
+
+      def call
+        result = run_aggregation
+        enrich(result) if result.documents.any?
+        zip_output if @config.zip
+        result
+      end
+
+      private
+
+      def run_aggregation
+        adapters = PlatformFactory.build_aggregation_adapters(
+          source: @config.local_path ? "local:#{@config.local_path}" : @config.source,
+          organizations: @config.organizations,
+          topic: @config.topic,
+          repos: @config.repos,
+          token: @config.token
+        )
+
+        channel_filter = ChannelFilter.new(
+          channels: Channel.parse_list(@config.channels)
+        )
+        stage_filter = StageFilter.new(@config.stages || [])
+        routing = FileRoutingFactory.from_name(@config.file_routing)
+        asset_processor = AssetProcessor.new(
+          output_dir: @config.output_dir,
+          routing: routing,
+          canonicalize: true
+        )
+        delta_state = build_delta_state
+
+        deps = AggregationPipeline::Dependencies.new(
+          discoverer: adapters[:discoverer],
+          fetcher: adapters[:fetcher],
+          manifest_reader: adapters[:manifest_reader],
+          channel_filter: channel_filter,
+          stage_filter: stage_filter,
+          asset_processor: asset_processor,
+          delta_state: delta_state
+        )
+
+        config = AggregationPipeline::Config.new(
+          organizations: @config.organizations,
+          channels: @config.channels,
+          topic: @config.topic,
+          concurrency: @config.concurrency,
+          include_drafts: @config.include_drafts,
+          fail_on_error: false
+        )
+
+        AggregationPipeline.new(deps).run(config, @config.output_dir)
+      end
+
+      def build_delta_state
+        return NullDeltaState.new unless @config.cache_dir
+
+        DeltaState.new(
+          cache_store: FileCacheStore.new(@config.cache_dir),
+          output_dir: @config.output_dir
+        )
+      end
+
+      def enrich(result)
+        index = DocumentIndex.from_documents(
+          result.documents,
+          parameters: IndexParameters.new(
+            organizations: @config.organizations,
+            channels: @config.channels || [],
+            topic: @config.topic,
+            repo_count: result.repo_count
+          )
+        )
+        enricher = RelatonEnricher.new
+        enrich_result = enricher.enrich(index, @config.output_dir)
+        return unless enrich_result
+
+        stamp_primary_identifiers(enrich_result.documents)
+      rescue LoadError
+        warn '  (relaton gem not available — bibliography skipped)'
+      end
+
+      def stamp_primary_identifiers(documents)
+        documents.map do |doc|
+          next doc unless doc['bibliographic']
+
+          ids = doc['bibliographic']['docidentifier']
+          next doc unless ids&.any?
+
+          primary = ids.find { |di| di['primary'] == true } || ids.first
+          doc.merge('primary_identifier' => primary['content'])
+        end
+      end
+
+      def zip_output
+        require 'zip'
+
+        dir = @config.output_dir
+        zip_path = "#{dir}.zip"
+
+        Zip::File.open(zip_path, Zip::File::CREATE) do |zipfile|
+          Dir.glob("#{dir}/**/*").each do |file|
+            next if File.directory?(file)
+
+            entry_name = file.sub("#{File.dirname(dir)}/", '')
+            zipfile.add(entry_name, file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/release/commands/package.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    class PackageCommand
+      Config = Struct.new(
+        :output_dir, :dest, :manifest, :config_source,
+        keyword_init: true
+      )
+
+      include ConfigResolver
+
+      def initialize(config)
+        @config = config
+      end
+
+      def call
+        manifest = load_manifest(@config.manifest)
+        channel_config = resolve_channel_config(@config.config_source, manifest)
+
+        deps = ReleasePipeline::Dependencies.new(
+          extractor: RxlExtractor.new,
+          filters: [],
+          change_detector: ContentHashChangeDetector.new(previous_releases: {}),
+          packager: ZipPackager.new,
+          publisher: PlatformFactory.build_publisher('null', {}),
+          naming_registry: NamingRegistry.default_registry,
+          manifest: manifest,
+          channel_override: nil,
+          channel_config: channel_config
+        )
+
+        pipeline_config = ReleasePipeline::Config.new(
+          output_dir: @config.output_dir,
+          manifest_path: @config.manifest,
+          force: false,
+          force_replace_patterns: nil,
+          concurrency: 4,
+          default_visibility: 'public'
+        )
+
+        ReleasePipeline.new(deps).run(pipeline_config)
+      end
+    end
+  end
+end

data/lib/metanorma/release/commands/publish.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    class PublishCommand
+      Config = Struct.new(
+        :output_dir, :platform, :manifest, :force,
+        :force_replace, :channels, :concurrency, :token, :config_source,
+        keyword_init: true
+      )
+
+      include ConfigResolver
+
+      def initialize(config)
+        @config = config
+      end
+
+      def call
+        manifest = load_manifest(@config.manifest)
+        channel_config = resolve_channel_config(@config.config_source, manifest)
+
+        options = { token: @config.token }
+        publisher = PlatformFactory.build_publisher(@config.platform, options)
+        channel_override = Channel.parse_list(@config.channels) if @config.channels
+
+        deps = ReleasePipeline::Dependencies.new(
+          extractor: RxlExtractor.new,
+          filters: [],
+          change_detector: ContentHashChangeDetector.new(previous_releases: {}),
+          packager: ZipPackager.new,
+          publisher: publisher,
+          naming_registry: NamingRegistry.default_registry,
+          manifest: manifest,
+          channel_override: channel_override,
+          channel_config: channel_config
+        )
+
+        pipeline_config = ReleasePipeline::Config.new(
+          output_dir: @config.output_dir,
+          manifest_path: @config.manifest,
+          force: @config.force,
+          force_replace_patterns: @config.force_replace && !@config.force_replace.empty? ? @config.force_replace : nil,
+          concurrency: @config.concurrency,
+          default_visibility: 'public'
+        )
+
+        ReleasePipeline.new(deps).run(pipeline_config)
+      end
+    end
+  end
+end

data/lib/metanorma/release/config_fetcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    module ConfigFetcher
+      def fetch(source)
+        raise NotImplementedError, "#{self.class} must implement #fetch"
+      end
+    end
+  end
+end

data/lib/metanorma/release/config_locator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    class ConfigLocator
+      CONFIG_FILES = ['.metanorma.yml', '.metanorma.yaml'].freeze
+      CONFIG_DIRS = ['.metanorma'].freeze
+
+      def self.find(start_dir = Dir.pwd)
+        new.find(start_dir)
+      end
+
+      def find(start_dir)
+        dir = File.expand_path(start_dir)
+        loop do
+          CONFIG_FILES.each do |name|
+            path = File.join(dir, name)
+            return ChannelConfig.from_file(path) if File.exist?(path)
+          end
+
+          CONFIG_DIRS.each do |name|
+            path = File.join(dir, name)
+            next unless File.directory?(path)
+
+            channels = File.join(path, 'channels.yml')
+            return ChannelConfig.from_file(channels) if File.exist?(channels)
+          end
+
+          parent = File.dirname(dir)
+          return nil if parent == dir
+
+          dir = parent
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/release/config_resolver.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    module ConfigResolver
+      def resolve_channel_config(cli_source, manifest)
+        return fetch_config(cli_source) if cli_source
+        return fetch_config(manifest.config_source) if manifest&.config_source
+
+        found = ConfigLocator.find
+        return found if found
+
+        ChannelConfig.empty
+      end
+
+      def load_manifest(path)
+        return nil unless path && File.exist?(path)
+
+        ChannelManifest.from_file(path)
+      end
+
+      private
+
+      def fetch_config(source)
+        if source.start_with?('local:')
+          Platform::Local::ConfigFetcher.new.fetch(source)
+        elsif source.include?('/')
+          Platform::Local::ConfigFetcher.new.fetch("local:#{source}")
+        else
+          require 'octokit'
+          client = PlatformFactory.build_github_client(nil)
+          Platform::GitHub::ConfigFetcher.new(client: client).fetch(source)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/release/content_hash.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Metanorma
+  module Release
+    class ContentHash
+      def self.from_hex(hex_string)
+        new(hex_string.to_s)
+      end
+
+      def self.of_content(data)
+        new(Digest::SHA256.hexdigest(data))
+      end
+
+      def self.of_file(path)
+        new(Digest::SHA256.file(path).hexdigest)
+      end
+
+      def self.of_files(paths)
+        sorted = paths.sort
+        digest = Digest::SHA256.new
+        sorted.each { |p| digest << File.binread(p) }
+        new(digest.hexdigest)
+      end
+
+      def self.of_directory(directory, base: nil)
+        pattern = base ? File.join(directory, "#{base}.*") : File.join(directory, '**', '*')
+        files = Dir.glob(pattern).reject { |f| File.directory?(f) || f.end_with?('.zip') }
+        of_files(files)
+      end
+
+      def initialize(hex)
+        @hex = hex
+        freeze
+      end
+
+      def to_s
+        @hex
+      end
+
+      def eql?(other)
+        other.is_a?(self.class) && @hex == other.to_s
+      end
+
+      def hash
+        @hex.hash
+      end
+    end
+  end
+end

data/lib/metanorma/release/delta_state.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Metanorma
+  module Release
+    class DeltaState
+      def initialize(cache_store:, output_dir:)
+        @cache = cache_store
+        @output_dir = output_dir
+        @state = empty_state
+      end
+
+      def load
+        raw = @cache.get('delta_state')
+        return if raw.nil?
+
+        @state = JSON.parse(raw)
+      rescue JSON::ParserError
+        @state = empty_state
+      end
+
+      def save
+        @cache.set('delta_state', JSON.generate(@state))
+      end
+
+      def etag(repo_key)
+        repo_state(repo_key)['etag']
+      end
+
+      def set_etag(repo_key, etag_value)
+        repo = ensure_repo(repo_key)
+        repo['etag'] = etag_value
+      end
+
+      def processed?(repo_key, tag, content_hash)
+        releases = repo_state(repo_key)['releases']
+        return false unless releases.key?(tag)
+        return false if content_hash.nil?
+
+        releases[tag]['content_hash'] == content_hash.to_s
+      end
+
+      def release_files(repo_key, tag)
+        releases = repo_state(repo_key)['releases']
+        return [] unless releases.key?(tag)
+
+        releases[tag]['files'] || []
+      end
+
+      def mark_processed(repo_key, tag, content_hash, files)
+        repo = ensure_repo(repo_key)
+        repo['releases'][tag] = {
+          'content_hash' => content_hash.to_s,
+          'files' => files
+        }
+      end
+
+      def cleanup_stale(repo_key, current_tags)
+        repo = repo_state(repo_key)
+        releases = repo['releases']
+        removed = 0
+
+        releases.each do |tag, entry|
+          next if current_tags.include?(tag)
+
+          (entry['files'] || []).each do |file|
+            path = File.join(@output_dir, file)
+            if File.exist?(path)
+              File.delete(path)
+              removed += 1
+            end
+          end
+          releases.delete(tag)
+        end
+
+        removed
+      end
+
+      private
+
+      def empty_state
+        { 'last_run' => Time.now.utc.iso8601, 'repos' => {} }
+      end
+
+      def repo_state(repo_key)
+        @state['repos'][repo_key] || { 'etag' => nil, 'releases' => {} }
+      end
+
+      def ensure_repo(repo_key)
+        @state['repos'][repo_key] ||= { 'etag' => nil, 'releases' => {} }
+      end
+    end
+
+    class NullDeltaState
+      def initialize; end
+
+      def load; end
+      def save; end
+      def etag(_repo_key) = nil
+      def set_etag(_repo_key, _etag) = nil
+      def processed?(*_args) = false
+      def release_files(*_args) = []
+      def mark_processed(*_args); end
+      def cleanup_stale(*_args) = 0
+    end
+  end
+end

data/lib/metanorma/release/document_id.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    class DocumentId
+      def self.from_raw(raw_identifier)
+        normalized = raw_identifier.to_s.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-|-$/, '')
+        raise ArgumentError, 'Document ID cannot be empty' if normalized.empty?
+
+        new(normalized)
+      end
+
+      def self.from_normalized(value)
+        raise ArgumentError, 'Document ID cannot be empty' if value.nil? || value.strip.empty?
+
+        new(value.to_s.strip)
+      end
+
+      def initialize(value)
+        @value = value
+        freeze
+      end
+
+      def to_s
+        @value
+      end
+
+      def tag_prefix
+        @value
+      end
+
+      def file_name
+        @value
+      end
+
+      def eql?(other)
+        other.is_a?(self.class) && @value == other.to_s
+      end
+
+      def hash
+        @value.hash
+      end
+    end
+  end
+end

data/lib/metanorma/release/document_index.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Metanorma
+  module Release
+    DocumentFile = Struct.new(:name, :path, keyword_init: true) do
+      def extension
+        File.extname(name).delete_prefix('.')
+      end
+    end
+
+    DocumentSource = Struct.new(
+      :owner, :repo, :tag, :release_url, :release_date,
+      keyword_init: true
+    ) do
+      def repo_key
+        "#{owner}/#{repo}"
+      end
+    end
+
+    IndexParameters = Struct.new(
+      :organizations, :channels, :topic, :repo_count,
+      keyword_init: true
+    )
+
+    IndexSummary = Struct.new(
+      :repo_count, :document_count, :channels_found,
+      keyword_init: true
+    )
+
+    class AggregatedDocument
+      def self.from_h(hash)
+        files = (hash['files'] || []).map do |f|
+          DocumentFile.new(name: f['name'], path: f['path'])
+        end
+        source_data = hash['source'] || {}
+        source = DocumentSource.new(
+          owner: source_data['owner'], repo: source_data['repo'],
+          tag: source_data['tag'], release_url: source_data['releaseUrl'],
+          release_date: source_data['releaseDate']
+        )
+        new(
+          id: hash['id'], title: hash['title'], edition: hash['edition'],
+          stage: hash['stage'], doctype: hash.fetch('doctype', ''),
+          channels: hash['channels'] || [], formats: hash['formats'] || [],
+          flavor: hash['flavor'], content_hash: hash['contentHash'],
+          source: source, files: files
+        )
+      end
+
+      attr_reader :id, :title, :edition, :stage, :doctype, :channels,
+                  :formats, :flavor, :content_hash, :source, :files
+
+      def initialize(id:, title:, edition:, stage:, doctype:, channels:,
+                     formats:, flavor:, content_hash:, source:, files:)
+        @id = id
+        @title = title
+        @edition = edition
+        @stage = stage
+        @doctype = doctype
+        @channels = channels.freeze
+        @formats = formats.freeze
+        @flavor = flavor
+        @content_hash = content_hash
+        @source = source
+        @files = files.freeze
+        freeze
+      end
+
+      def to_h
+        {
+          'id' => id, 'title' => title, 'edition' => edition,
+          'stage' => stage, 'doctype' => doctype,
+          'channels' => channels, 'formats' => formats,
+          'flavor' => flavor, 'contentHash' => content_hash,
+          'source' => {
+            'owner' => source.owner, 'repo' => source.repo,
+            'tag' => source.tag, 'releaseUrl' => source.release_url,
+            'releaseDate' => source.release_date
+          },
+          'files' => files.map { |f| { 'name' => f.name, 'path' => f.path } }
+        }
+      end
+    end
+
+    class DocumentIndex
+      SCHEMA_VERSION = 1
+
+      SchemaError = Class.new(StandardError)
+
+      def self.from_json(json_string)
+        data = JSON.parse(json_string)
+        validate!(data)
+        new(
+          documents: (data['documents'] || []).map { |d| AggregatedDocument.from_h(d) },
+          parameters: IndexParameters.new(
+            organizations: data.dig('parameters', 'organizations') || [],
+            channels: data.dig('parameters', 'channels') || [],
+            topic: data.dig('parameters', 'topic'),
+            repo_count: data.dig('parameters', 'repoCount') || 0
+          ),
+          generated_at: data['generatedAt']
+        )
+      end
+
+      def self.from_documents(documents, parameters:)
+        new(documents: documents, parameters: parameters)
+      end
+
+      def initialize(documents:, parameters:, generated_at: nil)
+        @documents = documents.freeze
+        @parameters = parameters
+        @generated_at = generated_at || Time.now.utc.iso8601
+        freeze
+      end
+
+      attr_reader :documents, :parameters
+
+      def summary
+        IndexSummary.new(
+          repo_count: @parameters.repo_count,
+          document_count: @documents.length,
+          channels_found: channels
+        )
+      end
+
+      def channels
+        @documents.flat_map(&:channels).uniq.sort
+      end
+
+      def document_count
+        @documents.length
+      end
+
+      def empty?
+        @documents.empty?
+      end
+
+      def to_h
+        {
+          'version' => SCHEMA_VERSION,
+          'generatedAt' => @generated_at,
+          'parameters' => {
+            'organizations' => @parameters.organizations,
+            'channels' => @parameters.channels,
+            'topic' => @parameters.topic,
+            'repoCount' => @parameters.repo_count
+          },
+          'summary' => {
+            'repoCount' => summary.repo_count,
+            'documentCount' => summary.document_count,
+            'channelsFound' => summary.channels_found
+          },
+          'documents' => @documents.map(&:to_h)
+        }
+      end
+
+      def to_json(*_args)
+        JSON.generate(to_h)
+      end
+
+      def write(path)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, to_json)
+      end
+
+      def self.validate!(data)
+        raise SchemaError, "Missing 'version' field" unless data.key?('version')
+        unless data['version'] == SCHEMA_VERSION
+          raise SchemaError,
+                "Unsupported schema version: #{data['version']}. Expected #{SCHEMA_VERSION}"
+        end
+        raise SchemaError, "Missing 'documents' field" unless data.key?('documents')
+
+        data['documents'].each do |doc|
+          raise SchemaError, "Document missing required field 'id'" unless doc.key?('id')
+          raise SchemaError, "Document missing required field 'title'" unless doc.key?('title')
+        end
+      end
+    end
+  end
+end