metanorma-release 0.2.2 → 0.2.3

data/lib/metanorma/release/delta_state.rb

@@ -1,10 +1,47 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
 module Metanorma
   module Release
+    module DeltaStateManager
+      def load
+        raise NotImplementedError, "#{self.class} must implement #load"
+      end
+
+      def save
+        raise NotImplementedError, "#{self.class} must implement #save"
+      end
+
+      def etag(repo_key)
+        raise NotImplementedError, "#{self.class} must implement #etag"
+      end
+
+      def set_etag(repo_key, etag_value)
+        raise NotImplementedError, "#{self.class} must implement #set_etag"
+      end
+
+      def processed?(repo_key, tag, content_hash)
+        raise NotImplementedError, "#{self.class} must implement #processed?"
+      end
+
+      def release_files(repo_key, tag)
+        raise NotImplementedError, "#{self.class} must implement #release_files"
+      end
+
+      def mark_processed(repo_key, tag, content_hash, files)
+        raise NotImplementedError,
+              "#{self.class} must implement #mark_processed"
+      end
+
+      def cleanup_stale(repo_key, current_tags)
+        raise NotImplementedError, "#{self.class} must implement #cleanup_stale"
+      end
+    end
+
     class DeltaState
+      include DeltaStateManager
+
       def initialize(cache_store:, output_dir:)
         @cache = cache_store
         @output_dir = output_dir
@@ -12,7 +49,7 @@ module Metanorma
       end
 
       def load
-        raw = @cache.get('delta_state')
+        raw = @cache.get("delta_state")
         return if raw.nil?
 
         @state = JSON.parse(raw)
@@ -21,50 +58,50 @@ module Metanorma
       end
 
       def save
-        @cache.set('delta_state', JSON.generate(@state))
+        @cache.set("delta_state", JSON.generate(@state))
       end
 
       def etag(repo_key)
-        repo_state(repo_key)['etag']
+        repo_state(repo_key)["etag"]
       end
 
       def set_etag(repo_key, etag_value)
         repo = ensure_repo(repo_key)
-        repo['etag'] = etag_value
+        repo["etag"] = etag_value
       end
 
       def processed?(repo_key, tag, content_hash)
-        releases = repo_state(repo_key)['releases']
+        releases = repo_state(repo_key)["releases"]
         return false unless releases.key?(tag)
         return false if content_hash.nil?
 
-        releases[tag]['content_hash'] == content_hash.to_s
+        releases[tag]["content_hash"] == content_hash.to_s
       end
 
       def release_files(repo_key, tag)
-        releases = repo_state(repo_key)['releases']
+        releases = repo_state(repo_key)["releases"]
         return [] unless releases.key?(tag)
 
-        releases[tag]['files'] || []
+        releases[tag]["files"] || []
       end
 
       def mark_processed(repo_key, tag, content_hash, files)
         repo = ensure_repo(repo_key)
-        repo['releases'][tag] = {
-          'content_hash' => content_hash.to_s,
-          'files' => files
+        repo["releases"][tag] = {
+          "content_hash" => content_hash.to_s,
+          "files" => files,
         }
       end
 
       def cleanup_stale(repo_key, current_tags)
         repo = repo_state(repo_key)
-        releases = repo['releases']
+        releases = repo["releases"]
         removed = 0
 
         releases.each do |tag, entry|
           next if current_tags.include?(tag)
 
-          (entry['files'] || []).each do |file|
+          (entry["files"] || []).each do |file|
             path = File.join(@output_dir, file)
             if File.exist?(path)
               File.delete(path)
@@ -80,20 +117,20 @@ module Metanorma
       private
 
       def empty_state
-        { 'last_run' => Time.now.utc.iso8601, 'repos' => {} }
+        { "last_run" => Time.now.utc.iso8601, "repos" => {} }
       end
 
       def repo_state(repo_key)
-        @state['repos'][repo_key] || { 'etag' => nil, 'releases' => {} }
+        @state["repos"][repo_key] || { "etag" => nil, "releases" => {} }
       end
 
       def ensure_repo(repo_key)
-        @state['repos'][repo_key] ||= { 'etag' => nil, 'releases' => {} }
+        @state["repos"][repo_key] ||= { "etag" => nil, "releases" => {} }
       end
     end
 
     class NullDeltaState
-      def initialize; end
+      include DeltaStateManager
 
       def load; end
       def save; end

data/lib/metanorma/release/file_routing.rb

@@ -28,21 +28,24 @@ module Metanorma
       include FileRouting
 
       def compute_path(file_name, _metadata)
-        ext = File.extname(file_name).delete_prefix('.')
+        ext = File.extname(file_name).delete_prefix(".")
         "#{ext}/#{file_name}"
       end
     end
 
     module FileRoutingFactory
       ROUTING_MAP = {
-        'by-document' => ByDocument,
-        'flat' => Flat,
-        'by-format' => ByFormat
+        "by-document" => ByDocument,
+        "flat" => Flat,
+        "by-format" => ByFormat,
       }.freeze
 
       def self.from_name(name)
         klass = ROUTING_MAP[name]
-        raise ArgumentError, "Unknown routing mode: #{name}. Available: #{ROUTING_MAP.keys.join(', ')}" unless klass
+        unless klass
+          raise ArgumentError,
+                "Unknown routing mode: #{name}. Available: #{ROUTING_MAP.keys.join(', ')}"
+        end
 
         klass.new
       end

data/lib/metanorma/release/index.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Metanorma
+  module Release
+    class Index
+      SCHEMA_VERSION = 1
+
+      class SchemaError < StandardError
+      end
+
+      attr_reader :publications, :parameters, :generated_at
+
+      def initialize(publications:, parameters:, generated_at: nil)
+        @publications = publications.freeze
+        @parameters = parameters
+        @generated_at = generated_at || Time.now.utc.iso8601
+        freeze
+      end
+
+      def channels
+        @publications.flat_map(&:channels).uniq.sort
+      end
+
+      def publication_count
+        @publications.length
+      end
+
+      def empty?
+        @publications.empty?
+      end
+
+      def to_h
+        {
+          "version" => SCHEMA_VERSION,
+          "generatedAt" => @generated_at,
+          "parameters" => {
+            "organizations" => @parameters[:organizations] || [],
+            "channels" => @parameters[:channels] || [],
+            "topic" => @parameters[:topic],
+            "repoCount" => @parameters[:repo_count] || 0,
+          },
+          "summary" => {
+            "repoCount" => @parameters[:repo_count] || 0,
+            "documentCount" => publication_count,
+            "channelsFound" => channels,
+          },
+          "documents" => @publications.map(&:to_h),
+        }
+      end
+
+      def to_json(*_args)
+        JSON.generate(to_h)
+      end
+
+      def write(path)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, to_json)
+      end
+
+      def self.from_documents(publications, parameters:)
+        new(publications: publications, parameters: parameters)
+      end
+
+      def self.from_json(json_string)
+        data = JSON.parse(json_string)
+        validate!(data)
+        publications = (data["documents"] || []).map do |d|
+          publication_from_h(d)
+        end
+        new(
+          publications: publications,
+          parameters: {
+            organizations: data.dig("parameters", "organizations") || [],
+            channels: data.dig("parameters", "channels") || [],
+            topic: data.dig("parameters", "topic"),
+            repo_count: data.dig("parameters", "repoCount") || 0,
+          },
+          generated_at: data["generatedAt"],
+        )
+      end
+
+      def self.validate!(data)
+        raise SchemaError, "Missing 'version' field" unless data.key?("version")
+        unless data["version"] == SCHEMA_VERSION
+          raise SchemaError,
+                "Unsupported schema version: #{data['version']}. Expected #{SCHEMA_VERSION}"
+        end
+        unless data.key?("documents")
+          raise SchemaError,
+                "Missing 'documents' field"
+        end
+
+        data["documents"].each do |doc|
+          unless doc.key?("id")
+            raise SchemaError,
+                  "Document missing required field 'id'"
+          end
+        end
+      end
+
+      def self.publication_from_h(hash)
+        files = (hash["files"] || []).map do |f|
+          PublicationFile.new(format: f["format"], name: f["name"],
+                              path: f["path"])
+        end
+        source = if hash["source"]
+                   PublicationSource.new(
+                     owner: hash["source"]["owner"],
+                     repo: hash["source"]["repo"],
+                     tag: hash["source"]["tag"],
+                     url: hash["source"]["releaseUrl"],
+                     date: hash["source"]["releaseDate"],
+                   )
+                 end
+        Publication.new(
+          identifier: hash["identifier"] || hash["id"],
+          slug: hash["id"],
+          title: hash["title"],
+          edition: hash["edition"],
+          stage: hash["stage"],
+          doctype: hash.fetch("doctype", ""),
+          revdate: hash["revdate"],
+          channels: hash["channels"] || [],
+          files: files,
+          source: source,
+        )
+      end
+    end
+  end
+end

data/lib/metanorma/release/interfaces.rb

@@ -2,16 +2,6 @@
 
 module Metanorma
   module Release
-    module Extractor
-      def discover(output_dir)
-        raise NotImplementedError, "#{self.class} must implement #discover"
-      end
-
-      def extract(rxl_path)
-        raise NotImplementedError, "#{self.class} must implement #extract"
-      end
-    end
-
     module Filter
       def apply(documents)
         raise NotImplementedError, "#{self.class} must implement #apply"
@@ -36,12 +26,22 @@ module Metanorma
       end
     end
 
-    ChangeResult = Struct.new(:changed?, :current_hash, :previous_hash, keyword_init: true)
-    Artifact = Struct.new(:zip_path, :asset_name, :size, keyword_init: true)
-    PublishResult = Struct.new(:tag, :url, :created?, keyword_init: true)
+    module RepoDiscoverer
+      def discover
+        raise NotImplementedError, "#{self.class} must implement #discover"
+      end
+    end
 
-    ReleasedArtifact = Struct.new(:id, :tag, :url, :channels, keyword_init: true)
+    module ReleaseFetcher
+      def fetch(repo, etag: nil)
+        raise NotImplementedError, "#{self.class} must implement #fetch"
+      end
+    end
 
-    ReleaseResult = Struct.new(:released, :skipped, :failed, :released_artifacts, keyword_init: true)
+    module ManifestReader
+      def read(repo)
+        raise NotImplementedError, "#{self.class} must implement #read"
+      end
+    end
   end
 end

data/lib/metanorma/release/platform/github/manifest_reader.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'yaml'
+require "yaml"
 
 module Metanorma
   module Release
@@ -14,14 +14,14 @@ module Metanorma
           end
 
           def read(repo)
-            content = @client.contents(repo.to_s, path: 'metanorma.release.yml')
+            content = @client.contents(repo.to_s, path: "metanorma.release.yml")
             return nil unless content
 
-            yaml = content['content'].unpack1('m0')
+            yaml = content["content"].unpack1("m0")
             parsed = YAML.safe_load(yaml, permitted_classes: [Symbol])
             return nil unless parsed.is_a?(Hash)
 
-            (parsed['channels'] || []).map(&:to_s)
+            (parsed["channels"] || []).map(&:to_s)
           rescue StandardError
             nil
           end

data/lib/metanorma/release/platform/github/publisher.rb

@@ -17,7 +17,8 @@ module Metanorma
 
             if force_replace
               delete_existing_release(tag_name)
-              return create_release(tag_name, metadata, artifact)
+              return create_release(tag_name, metadata, artifact,
+                                    prerelease: channels_pre_release?(channels))
             end
 
             existing = find_release(tag_name)
@@ -25,41 +26,52 @@ module Metanorma
             if existing
               update_release(existing, metadata)
             else
-              create_release(tag_name, metadata, artifact)
+              create_release(tag_name, metadata, artifact,
+                             prerelease: channels_pre_release?(channels))
             end
           end
 
           private
 
+          def channels_pre_release?(channels)
+            channels.any? do |c|
+              c.to_s.include?("draft") || c.to_s.include?("internal")
+            end
+          end
+
           def find_release(tag_name)
-            @client.releases(@repo).find { |r| r['tag_name'] == tag_name }
+            @client.releases(@repo).find { |r| r["tag_name"] == tag_name }
           rescue StandardError
             nil
           end
 
-          def create_release(tag_name, metadata, artifact)
+          def create_release(tag_name, metadata, artifact, prerelease: false)
             release = @client.create_release(
               @repo, tag_name,
               name: tag_name,
               body: metadata.to_release_body,
-              prerelease: tag_name.match?(/-(wd|cd|ds|fd|proposal)$/)
+              prerelease: prerelease
             )
-            upload_asset(release['id'], artifact)
-            PublishResult.new(tag: tag_name, url: release['html_url'], created?: true)
+            upload_asset(release["id"], artifact)
+            PublishResult.new(tag: tag_name, url: release["html_url"],
+                              created?: true)
           end
 
           def update_release(release, metadata)
-            @client.update_release(release['url'], body: metadata.to_release_body)
-            PublishResult.new(tag: release['tag_name'], url: release['html_url'], created?: false)
+            @client.update_release(release["url"],
+                                   body: metadata.to_release_body)
+            PublishResult.new(tag: release["tag_name"],
+                              url: release["html_url"], created?: false)
           end
 
           def upload_asset(release_id, artifact)
-            @client.upload_asset(release_id, artifact.zip_path, content_type: 'application/zip')
+            @client.upload_asset(release_id, artifact.zip_path,
+                                 content_type: "application/zip")
           end
 
           def delete_existing_release(tag_name)
             release = find_release(tag_name)
-            @client.delete_release(release['url']) if release
+            @client.delete_release(release["url"]) if release
             begin
               @client.delete_ref(@repo, "tags/#{tag_name}")
             rescue StandardError

data/lib/metanorma/release/platform/github/release_fetcher.rb

@@ -20,18 +20,20 @@ module Metanorma
           def fetch(repo, etag: nil)
             releases = @client.releases(repo.to_s)
             parsed = releases.map { |r| parse_release(r) }
-            FetchResult.new(releases: parsed, etag: "etag-#{repo}", unchanged?: false)
+            FetchResult.new(releases: parsed, etag: "etag-#{repo}",
+                            unchanged?: false)
           end
 
           private
 
           def parse_release(r)
             assets = (r[:assets] || []).map do |a|
+              data = download_asset(a[:url]) if a[:name].end_with?(".zip")
               GitHubAsset.new(
                 name: a[:name],
                 browser_download_url: a[:browser_download_url],
                 size: a[:size],
-                data: nil
+                data: data,
               )
             end
             GitHubRelease.new(
@@ -42,9 +44,16 @@ module Metanorma
               html_url: r[:html_url],
               published_at: r[:published_at],
               created_at: r[:created_at],
-              assets: assets
+              assets: assets,
             )
           end
+
+          def download_asset(url)
+            @client.get(url, accept: "application/octet-stream")
+          rescue StandardError => e
+            warn "Warning: Failed to download asset #{url}: #{e.message}"
+            nil
+          end
         end
       end
     end

data/lib/metanorma/release/platform/github.rb

@@ -1,20 +1,23 @@
 # frozen_string_literal: true
 
 begin
-  require 'octokit'
+  require "octokit"
 rescue LoadError
-  raise LoadError, "The octokit gem is required for GitHub adapters. Add `gem 'octokit'` to your Gemfile."
+  raise LoadError,
+        "The octokit gem is required for GitHub adapters. Add `gem 'octokit'` to your Gemfile."
 end
 
 module Metanorma
   module Release
     module Platform
       module GitHub
-        autoload :Publisher, 'metanorma/release/platform/github/publisher'
-        autoload :TopicDiscoverer, 'metanorma/release/platform/github/topic_discoverer'
-        autoload :ReleaseFetcher, 'metanorma/release/platform/github/release_fetcher'
-        autoload :ManifestReader, 'metanorma/release/platform/github/manifest_reader'
-        autoload :ConfigFetcher,  'metanorma/release/platform/github/config_fetcher'
+        autoload :Publisher, "metanorma/release/platform/github/publisher"
+        autoload :TopicDiscoverer,
+                 "metanorma/release/platform/github/topic_discoverer"
+        autoload :ReleaseFetcher,
+                 "metanorma/release/platform/github/release_fetcher"
+        autoload :ManifestReader,
+                 "metanorma/release/platform/github/manifest_reader"
 
         def self.cache_store(cache_dir:)
           FileCacheStore.new(cache_dir)

data/lib/metanorma/release/platform/local/directory_discoverer.rb

@@ -16,7 +16,7 @@ module Metanorma
 
             Dir.children(@base_path).filter_map do |entry|
               full = File.join(@base_path, entry)
-              RepoRef.new(owner: 'local', repo: entry) if File.directory?(full)
+              RepoRef.new(owner: "local", repo: entry) if File.directory?(full)
             end
           end
         end

data/lib/metanorma/release/platform/local/fetcher.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
 module Metanorma
   module Release
@@ -21,9 +21,13 @@ module Metanorma
 
           def fetch(repo, etag: nil)
             dir = File.join(@base_path, repo.repo)
-            return FetchResult.new(releases: [], etag: nil, unchanged?: false) unless Dir.exist?(dir)
+            unless Dir.exist?(dir)
+              return FetchResult.new(releases: [], etag: nil,
+                                     unchanged?: false)
+            end
 
-            releases = Dir.glob(File.join(dir, '*.meta.json')).filter_map do |meta_path|
+            releases = Dir.glob(File.join(dir,
+                                          "*.meta.json")).filter_map do |meta_path|
               build_release(dir, meta_path)
             end
 
@@ -34,7 +38,7 @@ module Metanorma
 
           def build_release(dir, meta_path)
             data = JSON.parse(File.read(meta_path))
-            base = File.basename(meta_path, '.meta.json')
+            base = File.basename(meta_path, ".meta.json")
             zip_path = File.join(dir, "#{base}.zip")
 
             unless File.exist?(zip_path)
@@ -42,32 +46,33 @@ module Metanorma
               return nil
             end
 
-            metadata = ReleaseMetadata.new(data)
+            metadata = Publication.from_metadata_hash(data)
             asset = LocalAsset.new(
               name: "#{base}.zip",
               browser_download_url: "file://#{File.expand_path(zip_path)}",
               size: File.size(zip_path),
-              data: File.binread(zip_path)
+              data: File.binread(zip_path),
             )
 
             LocalRelease.new(
-              tag_name: "#{data['id']}/#{data.fetch('edition', '1')}",
+              tag_name: "#{metadata.slug}/#{metadata.edition || '1'}",
               body: metadata.to_release_body,
-              prerelease: prerelease?(data),
+              prerelease: prerelease?(metadata),
               draft: false,
               html_url: "file://#{File.expand_path(dir)}",
               published_at: File.mtime(zip_path).iso8601,
               created_at: File.mtime(zip_path).iso8601,
-              assets: [asset]
+              assets: [asset],
             )
           rescue JSON::ParserError
             warn "Warning: Invalid metadata JSON in #{meta_path}, skipping"
             nil
           end
 
-          def prerelease?(data)
-            stage = data['stage'].to_s
-            %w[working-draft committee-draft draft-standard final-draft].include?(stage)
+          def prerelease?(metadata)
+            stage = metadata.stage.to_s
+            %w[working-draft committee-draft draft-standard
+               final-draft].include?(stage)
           end
         end
       end

data/lib/metanorma/release/platform/local/publisher.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'fileutils'
+require "json"
+require "fileutils"
 
 module Metanorma
   module Release
@@ -18,23 +18,25 @@ module Metanorma
             FileUtils.mkdir_p(@output_dir)
 
             zip_dest = File.join(@output_dir, artifact.asset_name)
-            meta_dest = File.join(@output_dir, meta_file_name(artifact.asset_name))
+            meta_dest = File.join(@output_dir,
+                                  meta_file_name(artifact.asset_name))
 
             if force_replace
-              File.delete(zip_dest) if File.exist?(zip_dest)
-              File.delete(meta_dest) if File.exist?(meta_dest)
+              FileUtils.rm_f(zip_dest)
+              FileUtils.rm_f(meta_dest)
             end
 
             FileUtils.cp(artifact.zip_path, zip_dest)
             File.write(meta_dest, metadata.to_json)
 
-            PublishResult.new(tag: tag.to_s, url: "file://#{File.expand_path(zip_dest)}", created?: true)
+            PublishResult.new(tag: tag.to_s,
+                              url: "file://#{File.expand_path(zip_dest)}", created?: true)
           end
 
           private
 
           def meta_file_name(asset_name)
-            base = asset_name.sub(/\.zip$/, '')
+            base = asset_name.sub(/\.zip$/, "")
             "#{base}.meta.json"
           end
         end

data/lib/metanorma/release/platform/local.rb

@@ -4,10 +4,10 @@ module Metanorma
   module Release
     module Platform
       module Local
-        autoload :Publisher, 'metanorma/release/platform/local/publisher'
-        autoload :DirectoryDiscoverer, 'metanorma/release/platform/local/directory_discoverer'
-        autoload :Fetcher, 'metanorma/release/platform/local/fetcher'
-        autoload :ConfigFetcher, 'metanorma/release/platform/local/config_fetcher'
+        autoload :Publisher, "metanorma/release/platform/local/publisher"
+        autoload :DirectoryDiscoverer,
+                 "metanorma/release/platform/local/directory_discoverer"
+        autoload :Fetcher, "metanorma/release/platform/local/fetcher"
       end
     end
   end