metanorma-release 0.2.24 → 0.2.26

data/lib/metanorma/release/document_flattener.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    class DocumentFlattener
+      def initialize(display_categories: [])
+        @display_categories = display_categories || []
+      end
+
+      def flatten(doc)
+        bib = doc["bibliographic"] || {}
+        doctype = extract_doctype(bib) || doc.fetch("doctype", "")
+        formats = doc["formats"] || []
+        base = {
+          "slug" => doc["id"],
+          "id" => resolve_doc_id(bib, doc),
+          "title" => doc["title"].to_s,
+          "abstract" => extract_abstract(bib),
+          "stage" => (doc["stage"] || "published").to_s.downcase,
+          "doctype" => doctype,
+          "edition" => doc["edition"],
+          "date" => extract_date(doc),
+          "channels" => doc["channels"] || [],
+          "formats" => formats,
+          "files" => doc["files"] || [],
+          "bibliographic" => bib,
+        }
+        add_format_flags(base, formats)
+        add_display_category(base, doctype)
+        add_contributors(base, bib)
+        base
+      end
+
+      private
+
+      def add_format_flags(hash, formats)
+        hash["has_html"] = formats.include?("html")
+        hash["has_pdf"] = formats.include?("pdf")
+        hash["has_xml"] = formats.include?("xml")
+        hash["has_rxl"] = formats.include?("rxl")
+
+        files = hash["files"]
+        %w[html pdf xml rxl].each do |fmt|
+          file = files.find { |f| f["format"] == fmt }
+          hash["#{fmt}_path"] = file["path"] if file
+        end
+      end
+
+      def add_display_category(hash, doctype)
+        cat = resolve_display_category(doctype)
+        hash["stage_css"] = hash["stage"].gsub(/\s+/, "-")
+        hash["doctype_class"] = "type-#{doctype.downcase}"
+        hash["display_category"] = cat&.fetch("name", nil)
+        hash["display_category_slug"] = cat&.fetch("slug", nil)
+      end
+
+      def resolve_display_category(doctype)
+        return nil if doctype.nil? || doctype.empty?
+
+        @display_categories.each do |cat|
+          doctypes = cat["doctypes"] || []
+          if doctypes.include?(doctype)
+            return { "name" => cat["name"],
+                     "slug" => cat["slug"] }
+          end
+        end
+        nil
+      end
+
+      def resolve_doc_id(bib, doc)
+        extract_primary_id(bib) || doc["identifier"] || doc["id"]
+      end
+
+      def extract_primary_id(bib)
+        ids = bib["docidentifier"]
+        return nil unless ids&.any?
+
+        primary = ids.find { |di| di["primary"] == true } || ids.first
+        primary["content"]
+      end
+
+      def extract_doctype(bib)
+        bib.dig("ext", "doctype", "content")
+      end
+
+      def extract_abstract(bib)
+        abstracts = bib["abstract"]
+        return nil unless abstracts&.any?
+
+        abstracts.first["content"]
+      end
+
+      def extract_date(doc)
+        bib_date = extract_bib_date(doc["bibliographic"])
+        return bib_date if bib_date
+
+        release_date = doc.dig("source", "releaseDate")
+        return nil unless release_date
+
+        release_date.to_s.split(/[T ]/).first
+      end
+
+      def extract_bib_date(bib)
+        return nil unless bib
+
+        dates = bib["date"]
+        return nil if dates.nil? || dates.empty?
+
+        published = dates.find { |d| d["type"] == "published" }
+        entry = published || dates.first
+        at = entry["at"]
+        at ? at.to_s.split(/[T ]/).first : nil
+      end
+
+      def add_contributors(hash, bib)
+        contribs = bib["contributor"] || []
+        persons, committees = partition_contributors(contribs)
+        hash["authors"] = persons
+        hash["committee"] = committees.first
+      end
+
+      def partition_contributors(contribs)
+        persons = contribs.filter_map { |c| parse_person(c) }
+        committees = contribs.filter_map { |c| parse_committee(c) }
+        [persons, committees]
+      end
+
+      def parse_person(contrib)
+        return nil unless contrib["person"]
+
+        name = extract_person_name(contrib["person"])
+        return nil unless name
+
+        role = (contrib["role"] || []).first&.fetch("type", nil)
+        { "name" => name, "role" => role }
+      end
+
+      def parse_committee(contrib)
+        return nil unless contrib["organization"]
+
+        extract_org_subdivision(contrib["organization"])
+      end
+
+      def extract_person_name(person)
+        n = person["name"] || {}
+        complete = n["completename"]
+        return complete["content"] if complete.is_a?(Hash) && complete["content"]
+        return complete if complete.is_a?(String)
+
+        surname = n["surname"]
+        given = n["given"]
+        given_str = given.is_a?(Hash) ? given["content"].to_s : given.to_s
+        parts = [given_str, surname].compact
+        parts.empty? ? nil : parts.join(" ")
+      end
+
+      def extract_org_subdivision(org)
+        subs = org["subdivision"]
+        return nil unless subs&.any?
+
+        sd = subs.first
+        sd_name = sd["name"]
+        if sd_name.is_a?(Array)
+          sd_name.first&.dig("content")
+        elsif sd_name.is_a?(Hash)
+          sd_name["content"]
+        else
+          sd_name.to_s
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/release/index.rb

@@ -51,7 +51,7 @@ module Metanorma
       end
 
       def to_json(*_args)
-        JSON.generate(to_h)
+        JSON.pretty_generate(to_h)
       end
 
       def write(path)

data/lib/metanorma/release/interfaces.rb

@@ -2,6 +2,18 @@
 
 module Metanorma
   module Release
+    module Extractor
+      def discover(output_dir)
+        raise NotImplementedError, "#{self} must implement .discover"
+      end
+    end
+
+    Release = Struct.new(:tag_name, :body, :prerelease, :draft,
+                         :html_url, :published_at, :created_at,
+                         :assets, keyword_init: true)
+    Asset = Struct.new(:name, :browser_download_url, :size, :data,
+                       keyword_init: true)
+
     module Filter
       def apply(documents)
         raise NotImplementedError, "#{self.class} must implement #apply"

data/lib/metanorma/release/platform/github/manifest_reader.rb

@@ -22,7 +22,9 @@ module Metanorma
             return nil unless parsed.is_a?(Hash)
 
             channels = Array(parsed["channels"])
-            Array(parsed["documents"]).each { |doc| channels.concat(Array(doc["channels"])) }
+            Array(parsed["documents"]).each do |doc|
+              channels.concat(Array(doc["channels"]))
+            end
             channels.map(&:to_s).uniq
           rescue StandardError
             nil

data/lib/metanorma/release/platform/github/release_fetcher.rb

@@ -6,12 +6,6 @@ module Metanorma
   module Release
     module Platform
       module GitHub
-        GitHubRelease = Struct.new(:tag_name, :body, :prerelease, :draft,
-                                   :html_url, :published_at, :created_at,
-                                   :assets, keyword_init: true)
-        GitHubAsset = Struct.new(:name, :browser_download_url, :size, :data,
-                                 keyword_init: true)
-
         class ReleaseFetcher
           include Metanorma::Release::ReleaseFetcher
 
@@ -21,14 +15,16 @@ module Metanorma
           end
 
           def fetch(repo, etag: nil)
-            releases = paginate_releases(repo.to_s)
+            releases, failed = paginate_releases(repo.to_s)
             parsed = releases.map { |r| parse_release(r) }
             FetchResult.new(releases: parsed, etag: "etag-#{repo}",
-                            unchanged?: false)
+                            unchanged?: false, failed?: failed)
           end
 
           private
 
+          # Returns [releases, failed] — failed is true when the listing
+          # errored, which callers must not treat as an empty release set.
           def paginate_releases(repo_slug)
             all = []
             page = 1
@@ -41,23 +37,23 @@ module Metanorma
 
               page += 1
             end
-            all
+            [all, false]
           rescue StandardError => e
-            warn "Warning: Failed to fetch releases for #{repo_slug}: #{e.message}"
-            []
+            Metanorma::Release.logger.warn "Failed to fetch releases for #{repo_slug}: #{e.message}"
+            [[], true]
           end
 
           def parse_release(r)
             assets = (r[:assets] || []).map do |a|
               data = download_asset(a[:url]) if a[:name].end_with?(".zip")
-              GitHubAsset.new(
+              Asset.new(
                 name: a[:name],
                 browser_download_url: a[:browser_download_url],
                 size: a[:size],
                 data: data,
               )
             end
-            GitHubRelease.new(
+            Release.new(
               tag_name: r[:tag_name],
               body: r[:body],
               prerelease: r[:prerelease],
@@ -82,7 +78,7 @@ module Metanorma
             end
             data
           rescue StandardError => e
-            warn "Warning: Failed to download asset #{url}: #{e.message}"
+            Metanorma::Release.logger.warn "Failed to download asset #{url}: #{e.message}"
             nil
           end
 

data/lib/metanorma/release/platform/github/topic_discoverer.rb

@@ -20,7 +20,7 @@ module Metanorma
               page = 1
               loop do
                 results = @client.search_repositories(query, per_page: 100,
-                                                      page: page)
+                                                             page: page)
                 items = results[:items]
                 break if items.nil? || items.empty?
 

data/lib/metanorma/release/platform/github.rb

@@ -18,10 +18,6 @@ module Metanorma
                  "metanorma/release/platform/github/release_fetcher"
         autoload :ManifestReader,
                  "metanorma/release/platform/github/manifest_reader"
-
-        def self.cache_store(cache_dir:)
-          FileCacheStore.new(cache_dir)
-        end
       end
     end
   end

data/lib/metanorma/release/platform/local/fetcher.rb

@@ -6,12 +6,6 @@ module Metanorma
   module Release
     module Platform
       module Local
-        LocalRelease = Struct.new(:tag_name, :body, :prerelease, :draft,
-                                  :html_url, :published_at, :created_at,
-                                  :assets, keyword_init: true)
-        LocalAsset = Struct.new(:name, :browser_download_url, :size, :data,
-                                keyword_init: true)
-
         class Fetcher
           include Metanorma::Release::ReleaseFetcher
 
@@ -42,22 +36,22 @@ module Metanorma
             zip_path = File.join(dir, "#{base}.zip")
 
             unless File.exist?(zip_path)
-              warn "Warning: Missing zip for #{meta_path}, skipping"
+              Metanorma::Release.logger.warn "Missing zip for #{meta_path}, skipping"
               return nil
             end
 
             metadata = Publication.from_metadata_hash(data)
-            asset = LocalAsset.new(
+            asset = Asset.new(
               name: "#{base}.zip",
               browser_download_url: "file://#{File.expand_path(zip_path)}",
               size: File.size(zip_path),
               data: File.binread(zip_path),
             )
 
-            LocalRelease.new(
+            Release.new(
               tag_name: "#{metadata.slug}/#{metadata.edition || '1'}",
               body: metadata.to_release_body,
-              prerelease: prerelease?(metadata),
+              prerelease: metadata.draft?,
               draft: false,
               html_url: "file://#{File.expand_path(dir)}",
               published_at: File.mtime(zip_path).iso8601,
@@ -65,15 +59,9 @@ module Metanorma
               assets: [asset],
             )
           rescue JSON::ParserError
-            warn "Warning: Invalid metadata JSON in #{meta_path}, skipping"
+            Metanorma::Release.logger.warn "Invalid metadata JSON in #{meta_path}, skipping"
             nil
           end
-
-          def prerelease?(metadata)
-            stage = metadata.stage.to_s
-            %w[working-draft committee-draft draft-standard
-               final-draft].include?(stage)
-          end
         end
       end
     end

data/lib/metanorma/release/platform/null/manifest_reader.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    module Platform
+      module Null
+        class ManifestReader
+          include Metanorma::Release::ManifestReader
+
+          def read(_repo) = nil
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/release/platform/null/publisher.rb

@@ -8,7 +8,7 @@ module Metanorma
           include Metanorma::Release::Publisher
 
           def publish(tag, _artifact, _metadata, channels:,
-force_replace: false)
+                      force_replace: false)
             PublishResult.new(tag: tag.to_s, url: "null://", created?: true)
           end
         end

data/lib/metanorma/release/platform/null.rb

@@ -5,6 +5,8 @@ module Metanorma
     module Platform
       module Null
         autoload :Publisher, "metanorma/release/platform/null/publisher"
+        autoload :ManifestReader,
+                 "metanorma/release/platform/null/manifest_reader"
       end
     end
   end

data/lib/metanorma/release/platform/static_discoverer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Release
+    module Platform
+      class StaticDiscoverer
+        include RepoDiscoverer
+
+        def initialize(repos:)
+          @repos = repos
+        end
+
+        def discover
+          @repos
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/release/platform.rb

@@ -6,6 +6,7 @@ module Metanorma
       autoload :GitHub, "metanorma/release/platform/github"
       autoload :Local, "metanorma/release/platform/local"
       autoload :Null, "metanorma/release/platform/null"
+      autoload :StaticDiscoverer, "metanorma/release/platform/static_discoverer"
     end
   end
 end

data/lib/metanorma/release/platform_factory.rb

@@ -24,14 +24,17 @@ module Metanorma
 
           discoverer = if opts[:repos]
                          repos = opts[:repos].map { |r| RepoRef.from_string(r) }
-                         StaticDiscoverer.new(repos: repos)
+                         Platform::StaticDiscoverer.new(repos: repos)
                        else
                          Platform::GitHub::TopicDiscoverer.new(
                            client: client, organizations: opts[:organizations], topic: opts[:topic],
                          )
                        end
 
-          download_cache = opts[:cache_dir] ? File.join(opts[:cache_dir], "downloads") : nil
+          download_cache = if opts[:cache_dir]
+                             File.join(opts[:cache_dir],
+                                       "downloads")
+                           end
 
           {
             discoverer: discoverer,
@@ -58,7 +61,7 @@ module Metanorma
         if source.start_with?("local:")
           adapters = AGGREGATION_REGISTRY["local"].call(options,
                                                         options[:token])
-          adapters[:manifest_reader] = NullManifestReader.new
+          adapters[:manifest_reader] = Platform::Null::ManifestReader.new
           return adapters
         end
 
@@ -78,24 +81,6 @@ module Metanorma
       def self.register_aggregation(name, factory)
         AGGREGATION_REGISTRY[name] = factory
       end
-
-      class StaticDiscoverer
-        include RepoDiscoverer
-
-        def initialize(repos:)
-          @repos = repos
-        end
-
-        def discover
-          @repos
-        end
-      end
-
-      class NullManifestReader
-        include Metanorma::Release::ManifestReader
-
-        def read(_repo) = nil
-      end
     end
   end
 end