relaton 3.0.7 → 3.1.0

data/lib/relaton/render/iso690/template.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Render
+    module Iso690
+      # A citation template as declared in a style instance: literal text
+      # interleaved with {{slot}} placeholders resolved against the ISO 690
+      # data element inventory. A leading literal attaches to the first
+      # slot, and a literal between two slots to the preceding one, so an
+      # absent field never orphans punctuation around it. The last rendered
+      # slot gives up its trailing literal in favour of the template
+      # terminator, appended once when any field rendered and the output
+      # does not already end with it.
+      class Template
+        SLOT = /\{\{\s*([\w-]+)\s*\}\}/.freeze
+
+        Field = Struct.new(:present?, :text)
+
+        def initialize(source)
+          @chunks = []
+          @terminator = ""
+          scan(source.to_s)
+        end
+
+        # The short cite renders the reference with the first-biblio
+        # marker appended to its first component: isodoc's styled
+        # references (eref2linkshort) split there (the 1.x
+        # fmt-first-biblio-delim)
+        def evaluate_short(fields, delim)
+          present = present_indices(fields)
+          return "" if present.empty?
+
+          # The split marker follows the leading run of present slots
+          # (the 1.x first SEGMENT: "{{creator}}{{role}}" splits after
+          # the role), before the first literal that follows it
+          lead_slots = []
+          @chunks.each_with_index do |(kind, _value), i|
+            break if kind != :slot || !present.include?(i)
+
+            lead_slots << i
+          end
+          split_at = lead_slots.last ? lead_slots.last + 1 : nil
+
+          out = +""
+          @chunks.each_with_index do |(kind, value), i|
+            text =
+              case kind
+              when :slot
+                field = fields[normalise(value)]
+                next unless field&.present?
+
+                field.text
+              else
+                next unless bridged?(i, present)
+
+                value
+              end
+            if split_at == i
+              out << fieldless(lead_slots, fields) if false
+              out << delim.to_s
+              split_at = nil
+            end
+            out << text unless text.nil?
+          end
+          out << delim.to_s if split_at == @chunks.length
+          terminate(out).rstrip
+        end
+
+        # 1.x segment-join semantics: present elements joined with the
+        # template's separators — a literal renders only when it
+        # separates (a run of absent elements between) two present
+        # elements; leading and trailing literals drop with the
+        # elements they would have attached to
+        def evaluate(fields)
+          present = present_indices(fields)
+          return "" if present.empty?
+
+          out = +"".dup
+          # Parenthetical groups balance on the template's literal
+          # parens alone: data text (a series run "(N.S.)", a host
+          # creator's closing paren) carries parens of its own that
+          # are no part of the template's group structure
+          paren_balance = 0
+          @chunks.each_with_index do |(kind, value), i|
+            if kind == :slot
+              field = fields[normalise(value)]
+              out << field.text if field&.present?
+            elsif value.to_s.match?(/\A[）)]/)
+              # Symmetric close: with no pending open the close-paren
+              # drops, and any trailing punctuation bridges normally
+              closes = value.to_s[/\A[）)]+/]
+              emitted = [paren_balance, closes.size].min
+              if emitted.positive?
+                out << closes[0, emitted] +
+                  value.to_s[closes.size..].to_s
+                paren_balance -= emitted
+              else
+                rest = value.to_s.sub(/\A[）)]+/, "")
+                # the trailing separator of a dropped group bridges as
+                # though the group were never there
+                if !rest.empty? && present.any? { |j| j > i } &&
+                   group_gap?(i, present)
+                  out << rest
+                end
+              end
+            elsif value.to_s.match?(/[（(]\z/)
+              # A literal opening a parenthetical group ("({{series}})",
+              # "。（{{series}}）") attaches its paren to the slot it
+              # opens — the pair drops with an absent element, while
+              # any leading punctuation bridges normally
+              lead, paren = value.to_s.match(/\A(.*)([（(])\z/).captures
+              out << lead if bridged_text?(i, lead, present)
+              if present.include?(i + 1)
+                out << paren
+                paren_balance += 1
+              end
+            elsif bridged?(i, present) ||
+                  (value.include?("<") && present.any? { |j| j < i })
+              # markup attaches to the element it closes
+              out << value
+            elsif close_pending_paren?(out, value)
+              out << value
+            end
+          end
+          terminate(out).rstrip
+        end
+
+        private
+
+        def scan(source)
+          rest = source
+          while (slot = SLOT.match(rest))
+            @chunks << [:text, rest[0...slot.begin(0)]] if
+              slot.begin(0).positive?
+            @chunks << [:slot, slot[1]]
+            rest = rest[slot.end(0)..]
+          end
+          @terminator = rest
+        end
+
+        def present_indices(fields)
+          @chunks.each_index.select do |i|
+            kind, value = @chunks[i]
+            next false unless kind == :slot
+
+            field = fields[normalise(value)]
+            !field.nil? && field.present?
+          end
+        end
+
+        # The texts between this close and the last present slot are
+        # all parenthetical-group parts (each dropped or partial)
+        def group_gap?(index, present)
+          left = present.select { |i| i < index }.max
+          @chunks[(left + 1)...index].all? do |(_, text)|
+            text.to_s.match?(/\A[）)]|[（(]\z/)
+          end
+        end
+
+        # The leading punctuation of a paren-opening literal bridges
+        # per the segment-join rule (a bare paren has no lead)
+        def bridged_text?(index, lead, present)
+          return true if lead.empty?
+
+          first_slot = @chunks.index { |(kind, _)| kind == :slot }
+          return present.include?(first_slot) if index < first_slot
+
+          left = present.select { |i| i < index }.max
+          left && present.any? { |i| i > index } &&
+            !@chunks[(left + 1)...index].any? do |(kind, _)|
+              kind == :text
+            end
+        end
+
+        # A literal is the trailing separator of the element that
+        # precedes it: it renders when that element renders and a
+        # later element renders, and it is the FIRST literal of its
+        # gap (the 1.x segment join — absent elements between two
+        # present ones leave the single separator that follows the
+        # earlier present element; an absent slot directly before a
+        # present one, as "{{date}}{{disambiguator}}, {{access}}",
+        # does not swallow it). A leading literal (no slot precedes
+        # it) attaches to the first slot, emitting only when that
+        # slot is present.
+        def bridged?(index, present)
+          first_slot = @chunks.index { |(kind, _)| kind == :slot }
+          return present.include?(first_slot) if index < first_slot
+
+          left = present.select { |i| i < index }.max
+          left && present.any? { |i| i > index } &&
+            !@chunks[(left + 1)...index].any? do |(kind, _)|
+              kind == :text
+            end
+        end
+
+        # A template's paired literal ("({{production}})") closes even
+        # when nothing follows the present element it wraps. Fullwidth
+        # parens (CJK) pair the same way
+        def close_pending_paren?(out, value)
+          return false unless value.start_with?(")", "\uFF09")
+
+          opens = out.count("(") + out.count("\uFF08")
+          closes = out.count(")") + out.count("\uFF09")
+          opens > closes
+        end
+
+        def terminate(body)
+          # 1.x punctuation cleanup: a sentence period before a spaced
+          # comma yields to the comma (", 2013" over ". , 2013"; an
+          # initials period before a comma ("P.,") stands, being
+          # unspaced), and stacked separators around absent elements
+          # collapse into one (". : ." into ". "); an empty pair left
+          # by an absent element drops
+          body = body.gsub(/:\s*\.\s*/, ": ")
+          body = body.gsub(/\.\s+,/, ",")
+          body = body.gsub(/(?:[.,:;]\s+)+[.,:;]/, ". ")
+          body = body.gsub(/\.\s*\./, ". ")
+          body = body.gsub(/:\s*\.\s*/, ": ")
+          body = body.gsub(/\(\s*\)/, "")
+          body = body.gsub(/ +/, " ").gsub(" ,", ",")
+          terminator = unbalanced_close(body, @terminator)
+          return body if terminator.strip.empty?
+          if terminator !~ /[<\w]/ && body.rstrip.end_with?(terminator.strip)
+            return body
+          end
+
+          "#{body}#{terminator}"
+        end
+
+        # The terminator's closing parens stand only when the body
+        # opened them (their "(" died with an absent element)
+        def unbalanced_close(body, terminator)
+          return terminator unless terminator.start_with?(")")
+
+          open = body.count("(") - body.count(")")
+          return terminator.sub(/\A\)+/) { |closes| closes[0, open] } if
+            open.positive?
+
+          terminator.sub(/\A\)+/, "")
+        end
+
+        def normalise(slot)
+          slot.to_s.delete("_").downcase
+        end
+      end
+    end
+  end
+end

data/lib/relaton/render/iso690.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Render
+    # Citation rendering per the ISO 690 architecture: a style instance
+    # places the inventory's data elements (clause 7) in each resource
+    # kind's order (clause 8); the engine is a template evaluator over a
+    # field resolver and holds no styles, no vocabularies, no i18n.
+    # The v1 compatibility facade (relaton-render#90): isodoc's
+    # render-isodoc subclasses Relaton::Render::General.
+    autoload :General, "relaton/render/general"
+
+    module Iso690
+      autoload :Element, "relaton/render/iso690/element"
+      autoload :Elements, "relaton/render/iso690/elements"
+      autoload :Fields, "relaton/render/iso690/fields"
+      autoload :I18n, "relaton/render/iso690/i18n"
+      autoload :IndexRenderer, "relaton/render/iso690/index_renderer"
+      autoload :IdentifierKinds, "relaton/render/iso690/identifier_kinds"
+      autoload :Kinds, "relaton/render/iso690/kinds"
+      autoload :Renderer, "relaton/render/iso690/renderer"
+        autoload :Rules, "relaton/render/iso690/rules"
+      autoload :Style, "relaton/render/iso690/style"
+      autoload :CslImport, "relaton/render/iso690/csl_import"
+      autoload :Template, "relaton/render/iso690/template"
+    end
+  end
+end

data/lib/relaton/sae/CLAUDE.md

@@ -0,0 +1,89 @@
+# CLAUDE.md — SAE flavor
+
+`Relaton::Sae` fetches SAE International standards (relaton/relaton-sae#1).
+
+## Development
+
+- `bundle exec rake spec:sae` — this flavor's suite (from the repo root).
+- No VCR: the specs stub HTTP with WebMock. `spec/sae/fixtures/crossref/`
+  holds three Crossref cursor pages built from real records (11 works:
+  six J3016/J3194 editions, three AMS4121 revisions, a part and a
+  letter-pair number). `spec/sae/fixtures/index-v2.yaml` and `fixtures/data/` are the
+  **output of `DataFetcher` over those pages**: regenerate both together by
+  running the crawler over the page fixtures, never by hand.
+
+## Where the data comes from
+
+**sae.org answers every script request with a CAPTCHA** (product pages and
+search alike), so no scraper is possible. SAE deposits every standard edition
+with Crossref instead: prefix `10.4271`, type `standard`, ~47,700 records. The
+DOI suffix is SAE's own edition code in lowercase:
+
+| DOI | SAE code | scheme |
+|---|---|---|
+| `10.4271/j3016_202104` | `SAE J3016_202104` | J-series: `_YYYYMM` edition date |
+| `10.4271/ams4121j` | `SAE AMS4121J` | aerospace: revision letter, no date |
+| `10.4271/j2337/2_201702` | `SAE J2337/2_201702` | a part |
+
+Crossref's text search does **not** find a standard by its id (it matches
+titles), so an undated reference cannot be answered live. The flavor is
+therefore index-based, the ECMA/IALA shape:
+
+- **Producer** — `DataFetcher` (`relaton fetch-data sae-crossref`) walks
+  `api.crossref.org/prefixes/10.4271/works?filter=type:standard` with the
+  deep-paging cursor (1000 a request, one request a second through
+  `Core::Pacer`; 429/5xx and transport errors retried with back-off, five
+  attempts, then `RequestError`, because
+  a partial crawl would publish a partial index). Each work goes through
+  `Relaton::Doi::Parser`, then gets SAE's code as its **primary**
+  docidentifier (type `SAE`), the DOI beside it, and `ext` doctype
+  `standard`, flavor `sae`.
+- **Consumer** — `Bibliography.get` reads the `relaton-data-sae` Pages index
+  (`PAGES_URL`) and the document from `ENDPOINT` (raw GitHub).
+
+## Three identifier forms
+
+`DoiCode` gives each record three strings:
+
+- `#native` — the code in the DOI's spelling, `SAE J3016_202104`,
+  `SAE AMSQQA250/4B`. The output file is named after it
+  (`data/sae-amsqqa250-4b.yaml`), so a file name never depends on pubid.
+- `#printed` — the code as SAE prints it (`SAE AMS-QQ-A-250/4B`): pubid's
+  render of the native code, or the native code when pubid cannot read it.
+  It is the primary docidentifier.
+- `#reference` — what the index row is parsed from: the native code when the
+  DOI has a date, else the code with the year of the record's `approved` date
+  (`SAE AMS4121J:2017`), so a dated reference also reaches an aerospace
+  revision.
+
+`Pubid::Sae` (pubid >= 2.0.9, pubid#665) reads all three, the month
+included, so the `Db` keys the query and the fetched document with the
+default `cache_pubid`. A reference pubid cannot read (`SAE J3016_2021`,
+`SAE CPGM2016`) answers nil with a log line (relaton#235); it does not raise.
+
+## Selection
+
+`index.search(pubid)` with no block, so the subset match `query === row`:
+a revision, a year or a month the query omits matches any value; the type
+(`J`, `AMS`, …) is compared, and a part and an amendment are `subset_strict`
+(`SAE J2337` does not answer with `SAE J2337/2`). Of the matches the highest
+revision wins (`[length, letters]`, so `AA` follows `Z`), then the latest
+year and month, then the file name (the index sort is not stable). A year
+argument goes on a **copy** of a pubid handed in by `Db`.
+
+The index key carries the month, so two editions of one year are two rows.
+Two DOIs that spell one code (`ams-4121j`, `ams4121j`) share a key, and
+`DataFetcher#add_to_index` keeps the later one (DOI date, then approval
+date).
+
+## What pubid still cannot read
+
+About 960 of the ~47,700 deposits: 791 `CP*` collected-paper products (not
+standards) and ~170 one-off shapes (`SUP1` supplements, language tails such
+as `_PTBR`, `_4Q2012`, `_MMYYYY` dates). See pubid's `lib/pubid/sae/CLAUDE.md`.
+Such a record is written, reported in `@errors` (the crawl's GitHub issue),
+and not indexed: one row that fails to deserialize makes `Relaton::Index`
+reject the whole index.
+
+A multi-letter tail stays in `number` (`J1939DA`, and also `AMS4121AA`), so
+a two-letter revision is its own bucket: `SAE AMS4121` does not reach it.

data/lib/relaton/sae/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Sae
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/sae/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Sae
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/sae/bibliography.rb

@@ -0,0 +1,128 @@
+require "net/http"
+
+module Relaton
+  module Sae
+    module Bibliography
+      # Transport failures of the document fetch. They become
+      # `Relaton::RequestError`, the one class `Relaton::Db#net_retry` retries.
+      NET_ERRORS = [
+        SocketError, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED,
+        Errno::ETIMEDOUT, EOFError, Net::HTTPBadResponse,
+        Net::HTTPHeaderSyntaxError, Net::ProtocolError, Net::OpenTimeout,
+        Net::ReadTimeout, OpenSSL::SSL::SSLError
+      ].freeze
+
+      class << self
+        #
+        # Fetch an SAE standard.
+        #
+        # @param ref [String, Pubid::Sae::Identifier] `SAE J3016`,
+        #   `SAE J3016:2021`, or SAE's native edition code `SAE J3016_202104`
+        # @param year [String, nil]
+        # @param _opts [Hash]
+        # @return [Relaton::Sae::ItemData, nil]
+        #
+        def get(ref, year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: ref.to_s
+          pubid = parse_ref ref, year
+          return not_found(ref) unless pubid
+
+          row = best_match pubid
+          item = row && fetch_doc(row[:file])
+          return not_found(ref) unless item
+
+          Util.info "Found: `#{primary_id item}`", key: ref.to_s
+          item
+        end
+
+        private
+
+        def not_found(ref)
+          Util.info "Not found.", key: ref.to_s
+          nil
+        end
+
+        #
+        # The query as a `Pubid::Sae::Identifier`.
+        #
+        # A parsed pubid comes from `Relaton::Db` (relaton#205): it is not
+        # mutated, so a year argument goes on a copy. A reference pubid cannot
+        # read answers nil, so `get` answers not-found with a log line instead
+        # of raising (relaton#235): `Db` passes such a reference as a String.
+        #
+        # @return [Pubid::Sae::Identifier, nil]
+        #
+        def parse_ref(ref, year)
+          ref = parse_text(ref.to_s.strip) unless ref.is_a?(::Pubid::Sae::Identifier)
+          return ref unless ref && year
+
+          copy = ::Pubid::Sae::Identifier.from_hash ref.to_hash
+          copy.date = ::Pubid::Sae::Components::Date.new(year: year.to_i)
+          copy
+        end
+
+        def parse_text(text)
+          ::Pubid::Sae::Identifier.parse text
+        rescue ::Pubid::Errors::Error, ArgumentError => e
+          Util.info "Not an SAE identifier pubid reads: #{e.message}", key: text
+          nil
+        end
+
+        #
+        # The index row for a query: the newest edition among its matches.
+        #
+        # Passing the pubid (not a String) is what makes `Index::Type#search`
+        # narrow to the `number` bucket, and with no block it selects with
+        # pubid's subset match `query === row`: a revision, a year or a month
+        # the query omits matches any value, and the type (`J`, `AMS`, …), a
+        # part and an amendment must agree.
+        #
+        # Of the matches, the highest revision wins (aerospace standards), then
+        # the latest date (J-series editions). `r[:file]` breaks a tie, because
+        # the index sort is not stable.
+        #
+        # @param pubid [Pubid::Sae::Identifier]
+        # @return [Hash, nil]
+        #
+        def best_match(pubid)
+          index.search(pubid).max_by do |r|
+            id = r[:id]
+            [revision_rank(id.revision), id.year.to_i, id.date&.month.to_i,
+             r[:file]]
+          end
+        end
+
+        # `B` follows `A`, and a two-letter revision (`AA`) follows `Z`.
+        def revision_rank(revision)
+          [revision.to_s.size, revision.to_s]
+        end
+
+        # @param file [String] the row's path in the data repo
+        # @return [Relaton::Sae::ItemData, nil] nil when the file is missing
+        def fetch_doc(file)
+          uri = URI("#{ENDPOINT}#{file}")
+          resp = Net::HTTP.get_response uri
+          return if resp.code == "404"
+          unless resp.code == "200"
+            raise Relaton::RequestError,
+                  "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          Item.from_yaml(resp.body).tap { |i| i.fetched = Date.today.to_s }
+        rescue *NET_ERRORS => e
+          raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
+        end
+
+        def primary_id(item)
+          (item.docidentifier.detect(&:primary) || item.docidentifier.first)&.content
+        end
+
+        def index
+          Relaton::Index.find_or_create(
+            :sae, pages_url: PAGES_URL, pubid_class: ::Pubid::Sae::Identifier
+          )
+        end
+      end
+    end
+  end
+end