relaton 3.0.0.pre.alpha.18 → 3.0.0.pre.alpha.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e4cd375143aef3a8e9dacfec4c86c06937243afe6a28a50d2960926e9c3396c
-  data.tar.gz: 8f96b7587af7145d05843de68b465263fdd524ad25adfd84fd53970ec97fd4a7
+  metadata.gz: eee3114e443e307b96ca18a96dff2a48dcb4d5b5dae424fd2ecb3ca6e57202be
+  data.tar.gz: 7208a6c3f058fca1874370a24bfe89d03516d5c4d66477d8c4160c4517717c44
 SHA512:
-  metadata.gz: f74dc77c912c9894872cfae0f04883badc152bf4c0f4e4abf4e2871a8352a1445b89cfb8ef0cac7f9270fd297327823a4c703013fb559bb268a39b0319fc120b
-  data.tar.gz: 9ffb0a15a5cb561f8699ef307c966a470ca96ac2cc97a968161cb7095dedfc5c2f6a5a2db8f59d8e53722776d61f72643ec2b32c8e6ccc50b8e9dd837fdfdbed
+  metadata.gz: f8c13c3bbdbe96caa0874d0b0c4671c05fa4c6652070372b0fc55992ac3c87834a6e03bda4a7fe17945d31db33b5f1a5a99d927d8d1d203d8238a7e837c29d77
+  data.tar.gz: 3e5e49f9fdd82e5abe32bc4afb35b2baf075bc1b10aee9aad0e663f4361a0486069ff94cf3d4c066d17ccf4f32dacfac5eccad6eb1534008978d39734577dc84

data/data/equivalence/adoption-chains.yaml

@@ -27,6 +27,18 @@ chains:
   - 'EN \1'
   - '\1'
 
+# Adoption ladders: the reverse edges. A code matching `pattern` is a base
+# document; `forms` are its national/regional adoptions, ordered
+# outermost-last, so the resolver can walk a chain from either end
+# (TODO.sota: adoption chains resolvable end-to-end, bidirectional).
+adoptions:
+- pattern: '^(ISO[ /].+)$'
+  forms:
+  - 'EN \1'
+  - 'BS EN \1'
+  - 'DIN EN \1'
+  - 'NF EN \1'
+
 # Explicit equivalences: every form in `equivalents` resolves to
 # `canonical` (and the canonical resolves to itself). Used where no pattern
 # expresses the relationship (joint publications, historical adoptions).
@@ -36,3 +48,12 @@ equivalences:
   - EN ISO/IEC 17025
   - BS EN ISO/IEC 17025
   - DIN EN ISO/IEC 17025
+# Joint publication: the same document under a different publisher
+# ordering (Law 3.1) — the direction each corpus prints is the data.
+- canonical: ISO/IEC 14515-1
+  equivalents:
+  - IEC/ISO 14515-1
+# Variant spelling: the prefixed form of a bare reference (Law 3.3).
+- canonical: RFC 7200
+  equivalents:
+  - IETF RFC 7200

data/lib/relaton/bib/model/item_shared.rb

@@ -23,7 +23,11 @@ module Relaton
         attribute :docnumber, :string
         attribute :date, Date, collection: true, initialize_empty: true
         attribute :contributor, Contributor, collection: true, initialize_empty: true
-        attribute :edition, Edition
+        # Lenient ingest: some upstream corpus documents carry more than one
+        # <edition> in a bibitem (multi-edition bibdata, relaton-db#149); keep
+        # the first instead of failing the whole round-trip with
+        # CollectionTrueMissingError (lutaml-model#971, singular_tolerance).
+        attribute :edition, Edition, singular_tolerance: :first
         attribute :version, Version, collection: true, initialize_empty: true
         attribute :note, Note, collection: true, initialize_empty: true
         attribute :language, :string, collection: true, initialize_empty: true

data/lib/relaton/core/data_fetcher.rb

@@ -36,6 +36,73 @@ module Relaton
       # @return [Boolean] whether this run prunes the existing output tree
       attr_accessor :prune
 
+      # @return [Boolean] whether this run consults and maintains the
+      #   content-digest ledger (see #unchanged?). Off by default: a flavor
+      #   turns it on only where its enumeration entry determines the record
+      #   (the TODO.sota crawl-audit rule).
+      attr_accessor :digest_ledger
+
+      # -- Content-digest ledger -----------------------------------------
+      #
+      # A per-output sidecar (`.digests.yaml`: key -> digest + file) lets a
+      # run skip the fetch+parse+write of a record whose enumeration entry is
+      # unchanged, where the entry determines the record. The sidecar is
+      # tracked in the data repo so a fresh CI clone still skips. Skipped
+      # files are recorded as written, so #prune_unwritten keeps them; the
+      # saved ledger keeps only files this run owned, so deleted records
+      # drop out with their file.
+      #
+      # Sound only where the entry bounds the record: a per-item detail
+      # fetch whose content the entry does not determine, or relation output
+      # discovered across records, must stay outside the skip (TODO.sota,
+      # crawl audit 2026-10-23).
+
+      LEDGER_FILE = ".digests.yaml"
+
+      # True when the ledger holds `key` with the entry's digest - the
+      # record is unchanged and the caller may skip it. Records the
+      # ledger's file as written, so prune keeps a skipped record.
+      #
+      # @param key [String] the record's stable identity (its docid)
+      # @param entry [Object] the enumeration entry; digested as JSON
+      # @return [Boolean]
+      def unchanged?(key, entry)
+        return false unless @digest_ledger
+
+        row = ledger[key]
+        return false unless row.is_a?(Hash) && row["digest"] == entry_digest(entry)
+
+        @written << row["file"] if row["file"]
+        warn "Skipping unchanged `#{key}`"
+        true
+      end
+
+      # Record the entry's digest for `key`, with the file the record was
+      # written to (nil when this run wrote no file for it - the entry
+      # drops out of the ledger at save time).
+      #
+      # @param key [String]
+      # @param entry [Object]
+      # @param file [String, nil]
+      # @return [void]
+      def note_digest(key, entry, file: nil)
+        return unless @digest_ledger
+
+        ledger_new[key] = { "digest" => entry_digest(entry), "file" => file }
+      end
+
+      # Persist the ledger: this run's noted entries over the loaded ones,
+      # keeping only entries whose file this run recorded as written.
+      #
+      # @return [void]
+      def save_ledger
+        kept = ledger.merge(ledger_new).select do |_, row|
+          row.is_a?(Hash) && row["file"] && @written.include?(row["file"])
+        end
+        File.write ledger_path, YAML.dump(kept), encoding: "UTF-8"
+        @ledger = kept
+      end
+
       # API method for external service
       #
       # @param prune [Boolean] delete output files this run did not write. The
@@ -43,7 +110,8 @@ module Relaton
       #   ratio guard stops a partial crawl from wiping the dataset.
       # @return the value returned by the instance `#fetch`, so callers can act
       #   on the outcome (e.g. relaton-iso returns whether it rebuilt).
-      def self.fetch(source = nil, output: "data", format: "yaml", prune: false)
+      def self.fetch(source = nil, output: "data", format: "yaml", prune: false,
+                     digest_ledger: false)
         t1 = Time.now
         puts "Started at: #{t1}"
         FileUtils.mkdir_p output
@@ -51,8 +119,10 @@ module Relaton
         # Only touched in prune mode: the default path stays exactly what it
         # was, and existing ::fetch callers (and their specs) see no new call.
         fetcher.prune = true if prune
+        fetcher.digest_ledger = digest_ledger
         result = fetcher.fetch(source)
         fetcher.prune_unwritten if prune
+        fetcher.save_ledger if digest_ledger
         t2 = Time.now
         puts "Stopped at: #{t2}"
         puts "Done in: #{(t2 - t1).round} sec."
@@ -251,6 +321,28 @@ module Relaton
 
       private
 
+      def ledger_path
+        File.join @output, LEDGER_FILE
+      end
+
+      def ledger
+        @ledger ||= if File.exist?(ledger_path)
+                      YAML.safe_load_file(ledger_path) || {}
+                    else
+                      {}
+                    end
+      end
+
+      def ledger_new
+        @ledger_new ||= {}
+      end
+
+      def entry_digest(entry)
+        require "digest"
+        require "json"
+        Digest::SHA256.hexdigest JSON.generate(entry)
+      end
+
       # Write, clobbering whatever is there. The caller has already established
       # that the path is ours to take. Identical content is left untouched:
       # an incremental crawl (one that keeps the data tree) must not churn

data/lib/relaton/db.rb

@@ -158,13 +158,15 @@ module Relaton
     end
 
     # Cross-SDO equivalence as data: on a total miss, walk the equivalence
-    # tables from relaton/equivalence.rb — a national adoption falls back to
-    # the EN, the EN to the ISO. Each candidate goes through the full routed
-    # fetch (cache, co-publishers); an unrouteable candidate is skipped.
+    # tables from relaton/equivalence.rb in both directions — a national
+    # adoption falls back to the EN, the EN to the ISO, and a base document
+    # falls forward to its declared adoptions (.related_for). Each candidate
+    # goes through the full routed fetch (cache, co-publishers); an
+    # unrouteable candidate is skipped.
     #
     # @return [Relaton::Bib::ItemData, nil]
     def fetch_equivalent(code, year, opts)
-      Equivalence.equivalents_for(code).each do |candidate|
+      Equivalence.related_for(code).each do |candidate|
         Util.info "Not found; trying equivalent `#{candidate}`", key: code
         std, pubid = @registry.route(candidate)
         bib = fetch_routed(candidate, year, opts, std, pubid) and return bib

data/lib/relaton/equivalence.rb

@@ -84,6 +84,34 @@ module Relaton
         end
         out
       end
+
+      # Every edge of `code`'s equivalence set: the canonical walk
+      # (.equivalents_for — a national adoption falls back to the EN, the EN
+      # to the ISO) PLUS the reverse edges, generated from the tables'
+      # `adoptions` ladders (the ISO resolves to its EN, BS EN, DIN EN and
+      # NF EN adoptions). The code itself is excluded; the walk order is
+      # canonical-first, ladder outermost-last.
+      #
+      # @param [String] code
+      # @return [Array<String>]
+      def related_for(code)
+        out = equivalents_for(code)
+        load_seed
+        ([code] + out.uniq).each do |candidate|
+          tables.each do |table|
+            (table["adoptions"] || []).each do |ladder|
+              m = Regexp.new(ladder["pattern"]).match(candidate)
+              next unless m
+
+              Array(ladder["forms"]).each do |template|
+                form = template.gsub(/\\(\d+)/) { m[Regexp.last_match(1).to_i] }
+                out << form unless form == code || out.include?(form)
+              end
+            end
+          end
+        end
+        out
+      end
     end
   end
 end

data/lib/relaton/ogc/data_fetcher.rb

@@ -46,9 +46,13 @@ module Relaton
 
       def fetch_doc(hit)
         return if hit["type"] == "CC"
+        # The list entry carries the whole record payload, so an unchanged
+        # entry skips the parse and the write (the digest ledger).
+        return if unchanged?(hit["identifier"], hit)
 
         bib = Scraper.parse_page hit, @errors
-        write_document bib
+        file = write_document bib
+        note_digest(hit["identifier"], hit, file: file) if file
         true
       rescue StandardError => e
         Util.error "Fetching document: #{hit['identifier']}\n" \
@@ -56,17 +60,19 @@ module Relaton
         false
       end
 
+      # @return [String, nil] the file written, or nil for a duplicate
       def write_document(bib) # rubocop:disable Metrics/AbcSize
         docid = bib.docidentifier[0].content
         if @docids.include?(docid)
           @dupids << docid
-          return
+          return nil
         end
 
         @docids << docid
         file = file_name bib
         index.add_or_update index_id(docid), file
-        File.write file, serialize(bib), encoding: "UTF-8"
+        force_write file, serialize(bib)
+        file
       end
 
       #

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.0.0.pre.alpha.18".freeze
+  VERSION = "3.0.0.pre.alpha.19".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre.alpha.18
+  version: 3.0.0.pre.alpha.19
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-24 00:00:00.000000000 Z
+date: 2026-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -442,14 +442,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre.alpha.28
+        version: 2.0.0.pre.alpha.59
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.pre.alpha.28
+        version: 2.0.0.pre.alpha.59
 - !ruby/object:Gem::Dependency
   name: yeptris
   requirement: !ruby/object:Gem::Requirement