metanorma-release 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83c0192b0bb05996d578d29961debb99d4600e0e1a178f259b1f4be4b8bbf065
-  data.tar.gz: fe1669b07a72cceb289350d042e9a5eec6eea71ad77b3f5ae6699f30977a3ad3
+  metadata.gz: fd5a3df2d9c64ed9d7d70d8f4c3fd3646d8c932a4cbffc39c07490e0276e37d9
+  data.tar.gz: f688bc55c1dff3751cdc1903e420e64c7503f7716710655c123e034dd0a3a3fb
 SHA512:
-  metadata.gz: 4bf35ae312524e91297d084901835d75255a9af7dd5bff78f4fcd45eea0813ba3ac574eca6529ebc31b21624f5fd28eeefbc4c328fe42b4121a1211d538a0856
-  data.tar.gz: c430938830a22af1df1a904253fd26a6710108b0bb544010f903ac7b5df141d2f6780961d4924579782a77fc6b19aa8dfc23fedebc8ccbad001fd6d2acf1c8e7
+  metadata.gz: 8127051648875b3ea36b988542c9ae82954bf68caf4ad91b785543d8ff6518ef95e1caa7d55334a777c22567806fbfa6cd32014812e6b2885388f42997f881d1
+  data.tar.gz: 77c9e3f867de101f3e962d674c8ddcabc630b9ba45557afe4c6338af0669a572dee37432b8231887cde7964f143817ea4ee1b50c9549bf2875dcdc76542152fe

data/README.adoc

@@ -133,6 +133,8 @@ metanorma-release release [options]
 === `metanorma-release aggregate`
 
 Aggregate published releases from multiple repositories into a unified file tree.
+Reads config from `metanorma.aggregate.yml` if present (auto-detected).
+Idempotent: uses cached delta state (`.cache/aggregate/`) to skip unchanged repos.
 
 [source,sh]
 ----
@@ -142,21 +144,40 @@ metanorma-release aggregate [options]
 [cols="1m,3",options="header"]
 |===
 |Option |Description
+|`--config FILE` |Config file (default: `metanorma.aggregate.yml`)
 |`--source SOURCE` |Discovery source: `github`, `local:PATH` (default: `github`)
-|`--organizations ORGS` |Organization list
+|`--organizations ORGS` |Organization list (overrides config)
 |`--topic TOPIC` |Repository topic filter (default: `metanorma-release`)
 |`--repos REPOS` |Explicit repo list
 |`--channels CHANS` |Filter channels
 |`--stages STAGES` |Filter stages
 |`--output-dir DIR` |Output directory (default: `_site/cc`)
 |`--file-routing MODE` |File layout: `by-document`, `flat`, `by-format` (default: `by-document`)
-|`--cache-dir DIR` |Cache directory for delta state
 |`--[no-]include-drafts` |Include draft releases
 |`--concurrency N` |Parallel repos (default: 4)
 |`--min-documents N` |Fail if fewer documents found (default: 0)
-|`--token TOKEN` |Platform auth token
+|`--token TOKEN` |Platform auth token (falls back to `GITHUB_TOKEN` env)
 |===
 
+==== Aggregate config file
+
+Create `metanorma.aggregate.yml` in your project root:
+
+[source,yaml]
+----
+source: github
+output_dir: _site/cc
+file_routing: flat
+cache_dir: .cache/aggregate
+
+github:
+  organizations:
+    - MyOrg
+  topic: metanorma-release
+----
+
+CLI flags override config file values. Cache is always enabled (defaults to `.cache/aggregate/`).
+
 == Concepts
 
 === Publication

data/lib/metanorma/release/cli.rb

@@ -82,15 +82,17 @@ module Metanorma
       option :file_routing, type: :string, default: "by-document",
                             desc: "File routing (by-document|flat|by-format)"
       option :cache_dir, type: :string, desc: "Cache directory"
+      option :data_dir, type: :string, desc: "Write flattened documents.json for site generators"
       option :include_drafts, type: :boolean, default: false,
                               desc: "Include draft releases"
       option :concurrency, type: :numeric, default: 4
       option :min_documents, type: :numeric, default: 0,
                              desc: "Minimum required documents"
       option :token, type: :string, desc: "Platform auth token"
+      option :config, type: :string, desc: "Config file (default: metanorma.aggregate.yml)"
 
       def aggregate
-        config = AggregateCommand::Config.new(
+        config = AggregateCommand.build_config(
           source: options[:source],
           organizations: options[:organizations],
           topic: options[:topic],
@@ -100,18 +102,20 @@ module Metanorma
           output_dir: options[:output_dir],
           file_routing: options[:file_routing],
           cache_dir: options[:cache_dir],
+          data_dir: options[:data_dir],
           include_drafts: options[:include_drafts],
           concurrency: options[:concurrency],
           min_documents: options[:min_documents],
           token: options[:token],
           create_zip: nil,
+          config: options[:config],
         )
         result = AggregateCommand.new(config).call
         print_aggregate_result(result)
 
-        if options[:min_documents].positive? && result.publications.length < options[:min_documents]
+        if config.min_documents.positive? && result.publications.length < config.min_documents
           raise PipelineError,
-                "Found #{result.publications.length} documents, minimum is #{options[:min_documents]}"
+                "Found #{result.publications.length} documents, minimum is #{config.min_documents}"
         end
 
         unless result.failed_repos.empty?

data/lib/metanorma/release/commands/aggregate.rb

@@ -1,15 +1,21 @@
 # frozen_string_literal: true
 
+require "yaml"
+
 module Metanorma
   module Release
     class AggregateCommand
       Config = Struct.new(
         :source, :organizations, :topic, :repos, :repo_pattern, :local_path,
         :channels, :stages, :output_dir, :file_routing, :cache_dir,
-        :include_drafts, :concurrency, :min_documents, :token, :create_zip,
+        :data_dir, :include_drafts, :concurrency, :min_documents, :token,
+        :create_zip,
         keyword_init: true
       )
 
+      DEFAULT_CONFIG_FILE = "metanorma.aggregate.yml"
+      DEFAULT_CACHE_DIR = ".cache/aggregate"
+
       def initialize(config)
         @config = config
       end
@@ -19,7 +25,8 @@ module Metanorma
         return result unless result.publications.any?
 
         index = build_index(result)
-        site = Site.new(index: index, output_dir: @config.output_dir)
+        site = Site.new(index: index, output_dir: @config.output_dir,
+                        data_dir: @config.data_dir)
         site.write!
         site.enrich!
         site.package! if @config.create_zip
@@ -28,6 +35,56 @@ module Metanorma
         result
       end
 
+      def self.build_config(cli_options)
+        file_data = load_config_file(cli_options[:config])
+        merged = merge_config(file_data, cli_options)
+        Config.new(
+          source: merged[:source],
+          organizations: merged[:organizations],
+          topic: merged[:topic],
+          repos: merged[:repos],
+          channels: merged[:channels],
+          stages: merged[:stages],
+          output_dir: merged[:output_dir],
+          file_routing: merged[:file_routing],
+          cache_dir: merged[:cache_dir] || DEFAULT_CACHE_DIR,
+          data_dir: merged[:data_dir],
+          include_drafts: merged[:include_drafts],
+          concurrency: merged[:concurrency],
+          min_documents: merged[:min_documents],
+          token: merged[:token],
+          create_zip: merged[:create_zip],
+        )
+      end
+
+      def self.load_config_file(path)
+        path ||= DEFAULT_CONFIG_FILE
+        return {} unless File.exist?(path)
+
+        YAML.safe_load_file(path, permitted_classes: [Symbol]) || {}
+      end
+
+      def self.merge_config(file_data, cli_options)
+        gh = file_data["github"] || {}
+        {
+          source: cli_options[:source] || file_data["source"],
+          organizations: cli_options[:organizations].any? ? cli_options[:organizations] : Array(gh["organizations"]),
+          topic: cli_options[:topic] || gh["topic"],
+          repos: cli_options[:repos] || file_data["repos"],
+          channels: cli_options[:channels].any? ? cli_options[:channels] : Array(file_data["channels"]),
+          stages: cli_options[:stages].any? ? cli_options[:stages] : Array(file_data["stages"]),
+          output_dir: cli_options[:output_dir] || file_data["output_dir"],
+          file_routing: cli_options[:file_routing] || file_data["file_routing"],
+          cache_dir: cli_options[:cache_dir] || file_data["cache_dir"],
+          data_dir: cli_options[:data_dir] || file_data["data_dir"],
+          include_drafts: cli_options[:include_drafts] || file_data["include_drafts"],
+          concurrency: cli_options[:concurrency] || file_data["concurrency"],
+          min_documents: cli_options[:min_documents] || file_data["min_documents"],
+          token: cli_options[:token],
+          create_zip: cli_options[:create_zip],
+        }
+      end
+
       private
 
       def run_aggregation

data/lib/metanorma/release/platform_factory.rb

@@ -64,7 +64,8 @@ module Metanorma
 
       def self.build_github_client(token)
         require "octokit"
-        token ? Octokit::Client.new(access_token: token) : Octokit::Client.new
+        access_token = token || ENV.fetch("GITHUB_TOKEN", nil)
+        access_token ? Octokit::Client.new(access_token: access_token) : Octokit::Client.new
       end
 
       def self.register_publisher(name, factory)

data/lib/metanorma/release/site.rb

@@ -9,9 +9,10 @@ module Metanorma
     class Site
       attr_reader :index, :output_dir
 
-      def initialize(index:, output_dir:)
+      def initialize(index:, output_dir:, data_dir: nil)
         @index = index
         @output_dir = output_dir
+        @data_dir = data_dir
       end
 
       def write!
@@ -22,22 +23,49 @@ module Metanorma
       def enrich!
         return if index.empty?
 
-        documents = index.publications.map do |pub|
-          rxl_file = pub.files.find { |f| f.format == "rxl" }
-          next pub.to_h unless rxl_file
+        documents = enrich_documents
+        write_relaton_index(documents)
+        write_data_file(documents) if @data_dir
+      end
+
+      def package!(zip_path: nil)
+        require "zip"
 
-          rxl_path = File.join(output_dir, rxl_file.path)
-          next pub.to_h unless File.exist?(rxl_path)
+        path = zip_path || "#{output_dir}.zip"
+        Zip::File.open(path, Zip::File::CREATE) do |zipfile|
+          Dir.glob("#{output_dir}/**/*").each do |file|
+            next if File.directory?(file)
 
-          bib = Relaton::Bib::Item.from_xml(File.read(rxl_path))
-          enriched = pub.to_h
-          enriched["bibliographic"] = bib.to_h
-          enriched
+            entry_name = file.sub("#{File.dirname(output_dir)}/", "")
+            zipfile.add(entry_name, file)
+          end
+        end
+        path
+      end
+
+      private
+
+      def enrich_documents
+        index.publications.map do |pub|
+          enrich_publication(pub)
         rescue StandardError => e
           warn "  Skip #{pub.identifier}: #{e.message}"
           pub.to_h
         end
+      end
+
+      def enrich_publication(pub)
+        rxl_file = pub.files.find { |f| f.format == "rxl" }
+        return pub.to_h unless rxl_file
 
+        rxl_path = File.join(output_dir, rxl_file.path)
+        return pub.to_h unless File.exist?(rxl_path)
+
+        bib = Relaton::Bib::Item.from_xml(File.read(rxl_path))
+        pub.to_h.merge("bibliographic" => bib.to_h)
+      end
+
+      def write_relaton_index(documents)
         dest = File.join(output_dir, "relaton")
         FileUtils.mkdir_p(dest)
         index_data = { "root" => { "title" => "Document Registry",
@@ -47,19 +75,59 @@ module Metanorma
         File.write(File.join(dest, "index.yaml"), YAML.dump(index_data))
       end
 
-      def package!(zip_path: nil)
-        require "zip"
+      def write_data_file(documents)
+        FileUtils.mkdir_p(@data_dir)
+        items = documents.compact.map { |doc| flatten_for_site(doc) }
+        File.write(File.join(@data_dir, "documents.json"),
+                   JSON.pretty_generate({ "items" => items }))
+      end
 
-        path = zip_path || "#{output_dir}.zip"
-        Zip::File.open(path, Zip::File::CREATE) do |zipfile|
-          Dir.glob("#{output_dir}/**/*").each do |file|
-            next if File.directory?(file)
+      def flatten_for_site(doc)
+        bib = doc["bibliographic"] || {}
+        doc_id = resolve_doc_id(bib, doc)
+        {
+          "slug" => doc["id"],
+          "id" => doc_id,
+          "title" => doc["title"].to_s,
+          "abstract" => extract_abstract(bib),
+          "stage" => (doc["stage"] || "published").to_s.downcase,
+          "doctype" => extract_doctype(bib) || doc.fetch("doctype", ""),
+          "edition" => doc["edition"],
+          "date" => extract_date(doc),
+          "channels" => doc["channels"] || [],
+          "formats" => doc["formats"] || [],
+          "files" => doc["files"] || [],
+        }
+      end
 
-            entry_name = file.sub("#{File.dirname(output_dir)}/", "")
-            zipfile.add(entry_name, file)
-          end
-        end
-        path
+      def resolve_doc_id(bib, doc)
+        extract_primary_id(bib) || doc["identifier"] || doc["id"]
+      end
+
+      def extract_primary_id(bib)
+        ids = bib["docidentifier"]
+        return nil unless ids&.any?
+
+        primary = ids.find { |di| di["primary"] == true } || ids.first
+        primary["content"]
+      end
+
+      def extract_doctype(bib)
+        bib.dig("ext", "doctype", "content")
+      end
+
+      def extract_abstract(bib)
+        abstracts = bib["abstract"]
+        return nil unless abstracts&.any?
+
+        abstracts.first["content"]
+      end
+
+      def extract_date(doc)
+        release_date = doc.dig("source", "releaseDate")
+        return nil unless release_date
+
+        release_date.to_s.split(/[T ]/).first
       end
     end
   end

data/lib/metanorma/release/version.rb

@@ -2,6 +2,6 @@
 
 module Metanorma
   module Release
-    VERSION = "0.2.4"
+    VERSION = "0.2.6"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-release
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-17 00:00:00.000000000 Z
+date: 2026-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: relaton-bib