relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/jcgm.rb

@@ -0,0 +1,24 @@
+require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+require_relative "jcgm/util"
+require_relative "jcgm/docidentifier"
+require_relative "jcgm/item"
+require_relative "jcgm/bibitem"
+require_relative "jcgm/bibdata"
+require_relative "jcgm/bibliography"
+
+module Relaton
+  module Jcgm
+    INDEXFILE = "index-v1".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/jis/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jis
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/jis/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jis
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/jis/bibliography.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    module Bibliography
+      extend self
+
+      #
+      # Search JIS by reference, returning the full candidate collection.
+      #
+      # The reference is parsed into a {Pubid::Jis::Identifier} and matched
+      # against the pubid-based `index-v2` via {HitCollection}. Candidates share
+      # the reference's series and number (a supplement is filed under its base
+      # number), so an edition and its amendments are returned together.
+      #
+      # @param [String] code JIS document code
+      # @param [String, nil] year JIS document year
+      #
+      # @return [Relaton::Jis::HitCollection] search result
+      #
+      # @raise [Pubid::Errors::ParseError] when the reference is not a JIS
+      #   identifier, so a malformed reference is not reported as "not found"
+      #
+      def search(code, year = nil)
+        pubid = ::Pubid::Jis::Identifier.parse code
+        pubid.year ||= year.to_i if year
+        HitCollection.new pubid
+      end
+
+      #
+      # Get JIS document by reference
+      #
+      # @param [String] ref JIS document reference
+      # @param [String, nil] year JIS document year
+      # @param [Hash] opts options
+      # @option opts [Boolean] :all_parts return all parts of document
+      #
+      # @return [Relaton::Jis::Item, nil] JIS document
+      #
+      # @raise [Pubid::Errors::ParseError] when the reference is not a JIS
+      #   identifier
+      #
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize
+        code = ref.sub(/\s\((all parts|規格群)\)/, "")
+        opts[:all_parts] ||= !$1.nil?
+        Util.info "Fetching from webdesk.jsa.or.jp ...", key: ref
+        hits = search(code, year)
+        result = opts[:all_parts] ? hits.find_all_parts : hits.find
+        if result.is_a? Bib::ItemData
+          Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+          return result
+        end
+        hint result, ref, year
+      end
+
+      #
+      # Log hint message
+      #
+      # @param [Array] result search result (missed edition years)
+      # @param [String] ref reference to search
+      # @param [String, nil] year year to search
+      #
+      def hint(result, ref, year)
+        Util.info "Not found.", key: ref
+        if result&.any?
+          Util.info "TIP: No match for edition year `#{year}`, but " \
+                    "matches exist for `#{result.uniq.join('`, `')}`.", key: ref
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/jis/data_fetcher.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require_relative "../jis"
+require_relative "scraper"
+
+module Relaton
+  module Jis
+    class DataFetcher < Core::DataFetcher
+      URL = "https://webdesk.jsa.or.jp/books/"
+
+      def initialize(output, format)
+        super
+        @queue = SizedQueue.new 10
+        @threads = create_thread_pool 5
+        @mutex = Mutex.new
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      # The pubid index-v2, the only index a crawl writes. relaton-data-jis
+      # rebuilds the legacy string index-v1 from data/ in its own crawler.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :jis, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Jis::Identifier
+        )
+      end
+
+      # Parse a primary docidentifier string into a pubid identifier; nil (with
+      # a warning) if pubid can't parse it, so a single bad id never aborts the
+      # crawl or corrupts the index.
+      def pubid(id)
+        ::Pubid::Jis::Identifier.parse id
+      rescue StandardError => e
+        Util.warn "Failed to parse `#{id}` with pubid: #{e.message}"
+        nil
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml bib
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml bib
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+
+      def create_thread_pool(size)
+        Array.new(size) do
+          Thread.new do
+            until (url = @queue.shift) == :END
+              fetch_doc url
+            end
+          end
+        end
+      end
+
+      def fetch_doc(url) # rubocop:disable Metrics/MethodLength
+        attempts = 0
+        begin
+          bib = Scraper.new(url, @errors).fetch
+        rescue StandardError => e
+          attempts += 1
+          if attempts < 5
+            sleep 2
+            retry
+          else
+            Util.warn "URL: #{url}"
+            Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+          end
+        else
+          save_doc bib, url
+        end
+      end
+
+      def fetch(_source = nil)
+        return unless initial_post
+
+        resp = agent.get "#{URL}W11M0070/index"
+        parse_page resp
+        index.save
+        report_errors
+      end
+
+      def initial_post
+        return true if @initial_time && Time.now - @initial_time < 600
+
+        body = { record: 0, dantai: "JIS", searchtype2: 1,
+                 status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
+        resp = agent.post "#{URL}W11M0270/index", body
+        disp = JSON.parse resp.body
+        @initial_time = Time.now
+        disp["status"] || Util.warn("No results found for JIS")
+      end
+
+      def agent
+        @agent ||= Mechanize.new
+      end
+
+      def parse_page(resp)
+        while resp
+          xpath = '//div[@class="blockGenaral"]/a'
+          resp.xpath(xpath).each { |a| @queue << a[:href] }
+          offset = parse_offset resp
+          break if offset >= count
+
+          resp = get_next_page(offset)
+        end
+        end_threads_and_wait
+      end
+
+      def parse_offset(resp) # rubocop:disable Metrics/AbcSize
+        if resp.at('//*[@id="btnPaging"]') # first page
+          xpath = '//script[contains(.,"var count =")]'
+          @count = resp.at(xpath).text.match(/var count = (\d+);/)[1]
+          resp.at("//*[@id='offset']")[:value].to_i
+        else
+          script = resp.at("//script").text
+          script.match(/\("offset"\)\.value = '(\d+)'/)[1].to_i
+        end
+      end
+
+      def end_threads_and_wait
+        @threads.size.times { @queue << :END }
+        @queue.close
+        @threads.each(&:join)
+      end
+
+      def count
+        @count.to_i
+      end
+
+      def get_next_page(offset) # rubocop:disable Metrics/MethodLength
+        attempts = 0
+        begin
+          if initial_post
+            url = "#{URL}W11M0070/getAddList"
+            agent.post url, search_type: "JIS", offset: offset
+          end
+        rescue StandardError => e
+          attempts += 1
+          if attempts < 5
+            sleep 2
+            retry
+          else
+            Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+          end
+        end
+      end
+
+      def save_doc(bib, url) # rubocop:disable Metrics/MethodLength
+        return unless bib
+
+        id = bib.docidentifier.find(&:primary).content
+        @mutex.synchronize do
+          # Two distinct ids can sanitize to one filename, in which case the
+          # second document used to be dropped. Give it a path of its own; a
+          # genuine duplicate (same id) still resolves to one path and is still
+          # skipped. Inside the lock: unique_output_file reserves state.
+          file = unique_output_file id
+          # A reserved path only ever belongs to one id, so a hit here is the
+          # same document again. Checked FIRST: a disambiguated path stays !=
+          # output_file forever, so gating this on that comparison would make a
+          # repeat of a disambiguated id skip the skip and overwrite itself.
+          if @files.include?(file)
+            Util.warn "File #{file} already exists. Duplication URL: #{url}"
+            next
+          end
+
+          Util.warn "File #{output_file id} already exists; writing #{file} instead" if file != output_file(id)
+          @files << file
+          File.write file, serialize(bib), encoding: "UTF-8"
+          pid = pubid id
+          index.add_or_update pid, file if pid
+        end
+      end
+    end
+  end
+end

data/lib/relaton/jis/docidentifier.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    # A JIS document identifier, backed by `Pubid::Jis`.
+    #
+    # `content` stays the source string for serialization. `#pubid` parses it
+    # into a `Pubid::Jis::Identifier`, and the three mutators change that pubid
+    # and render it back into `content`. There is no string surgery.
+    #
+    # The parse is lazy and soft, as in `Relaton::Itu::Docidentifier`: it runs
+    # on first use, and content that does not parse gives a nil `#pubid`, with
+    # no log. The all-parts form "JIS C 0364 (all parts)" does not parse, and
+    # a cached all-parts item is deserialized again, so a log would be noise.
+    # With a nil `#pubid` the mutators do nothing.
+    #
+    # ## Rendering rules
+    #
+    # - The `JIS ` publisher is rendered only when the source has it. pubid
+    #   always adds it, but the source spells a technical report
+    #   `TR Z 0010:2008`.
+    # - Everything else is the pubid canonical spelling. pubid does not keep the
+    #   source spelling `AMENDMENT 1` / `EXPLANATION 1`, so a changed supplement
+    #   renders as `AMD 1` / `EXPL 1`, the form that the index uses.
+    # - The all-parts marker is ` (all parts)`, as in
+    #   `Relaton::Jis::StructuredIdentifier`, not the pubid `（規格群）`.
+    #   `Bibliography.get` accepts both as a reference.
+    class Docidentifier < Bib::Docidentifier
+      # Capture the inherited content setter before the override, so that a
+      # mutation can write the rendered string back and keep the changed pubid.
+      # `content=` would clear it, and a new parse of "… (all parts)" fails.
+      alias_method :store_content, :content=
+
+      def content=(value)
+        super
+        @pubid = nil
+        @pubid_parsed = false
+      end
+
+      # @return [Pubid::Jis::Identifier, nil] nil if the content does not parse
+      def pubid
+        return @pubid if @pubid_parsed
+
+        @pubid_parsed = true
+        @pubid = parse
+      end
+
+      # Removes every part level: `JIS C 0364-2-21:1999` -> `JIS C 0364:1999`.
+      # For a supplement, the part is on its base.
+      def remove_part!
+        return unless pubid
+
+        change_document { |id| id.exclude(:parts) }
+        refresh_content!
+      end
+
+      # Removes the year and the reaffirmation marker:
+      # `JIS C 9901:2019R` -> `JIS C 9901`.
+      #
+      # For a supplement, only the base loses its date, and the supplement keeps
+      # its own year: `JIS B 3700-101:1996/CORRIGENDUM 1:2002` ->
+      # `JIS B 3700-101/CORRIGENDUM 1:2002`. pubid renders a supplement without
+      # a year as `…/CORRIGENDUM 1:`, with a trailing colon.
+      def remove_date!
+        return unless pubid
+
+        change_document { |id| id.exclude(:year, :reaffirmed) }
+        refresh_content!
+      end
+
+      # `JIS C 0364-2-21:1999` -> `JIS C 0364 (all parts)`, with `all_parts`
+      # set on `#pubid`.
+      def to_all_parts!
+        return if !pubid || pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy;
+        # `change_document` itself never mutates in place (it only reassigns
+        # `@pubid`), but this stays correct even if that changes.
+        original = @pubid.exclude
+        change_document { |id| id.exclude(:parts, :year, :reaffirmed) }
+        document = render(@pubid)
+        @pubid = original.to_all_parts
+        store_content "#{document} (all parts)"
+      end
+
+      private
+
+      def parse
+        return unless content
+
+        require "pubid"
+        ::Pubid::Jis::Identifier.parse content.to_s
+      rescue LoadError, StandardError
+        nil
+      end
+
+      # Applies the change to the document identifier: the pubid itself, or the
+      # base of a supplement. Copy on write, as `Pubid::Identifier#exclude` is:
+      # an argument-less `exclude` copies the supplement, so a pubid that a
+      # caller got before the mutation does not change.
+      def change_document(&)
+        @pubid = document_changed(@pubid, &)
+      end
+
+      def document_changed(id, &)
+        return yield(id) unless id.is_a?(::Pubid::Jis::SupplementIdentifier)
+
+        id.exclude.tap { |copy| copy.base &&= document_changed(copy.base, &) }
+      end
+
+      def refresh_content!
+        store_content render(@pubid)
+      end
+
+      def render(id)
+        id.to_s(with_publisher: content.to_s.start_with?("JIS"))
+      end
+    end
+  end
+end

data/lib/relaton/jis/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Jis
+    class Doctype < Bib::Doctype
+      TYPES = %w[japanese-industrial-standard technical-report technical-specification amendment].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/jis/ext.rb

@@ -0,0 +1,13 @@
+require_relative "doctype"
+require_relative "structured_identifier"
+
+module Relaton
+  module Jis
+    class Ext < Iso::Ext
+      attribute :doctype, Doctype
+      attribute :structuredidentifier, StructuredIdentifier
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-jis"]
+    end
+  end
+end

data/lib/relaton/jis/hit.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class Hit < Core::Hit
+      #
+      # Create new hit
+      #
+      # @param [Hash] hit found hit
+      # @param [Relaton::Jis::HitCollection] collection hit collection
+      #
+      # @return [Relaton::Jis::Hit] new hit
+      #
+      def self.create(hit, collection)
+        new hit, collection
+      end
+
+      # The edition aspects. {HitCollection} picks the edition, so a match
+      # ignores them. The reaffirmation mark `R` goes with the year:
+      # `exclude(:year)` keeps it, and `JIS L 4107` must match
+      # `JIS L 4107:2000R`.
+      EDITION = %i[year reaffirmed].freeze
+
+      #
+      # Check if the hit matches the collection's reference.
+      #
+      # The candidate must be the same document type, so a plain standard
+      # query never matches its amendments. The class check is explicit
+      # because pubid's JIS `==` does not compare the class
+      # (`JIS TR X 0014:1999 == JIS X 0014:1999`). The other aspects are
+      # compared, except the edition, and the parts when `all_parts` is on.
+      # This includes the base document of a supplement. The `SYMBOL` is
+      # compared on a standard, but not on a supplement: pubid's supplement
+      # `==` ignores it. {HitCollection#find_by_year} puts the exact printed id
+      # first for that reason.
+      #
+      # @param [Boolean] all_parts match any part of the document
+      #
+      # @return [Boolean] true if the hit matches
+      #
+      def matches?(all_parts: false)
+        return false unless pubid.instance_of?(reference.class)
+
+        ignore = all_parts ? EDITION + [:parts] : EDITION
+        reference.matches? pubid, ignore: ignore
+      end
+
+      #
+      # The hit's pubid identifier. `index-v2` rows are already deserialized to
+      # {Pubid::Jis::Identifier} via `pubid_class`; a Hash or String id is
+      # converted for robustness.
+      #
+      # @return [Pubid::Jis::Identifier, nil] identifier, or nil when it cannot
+      #   be built
+      #
+      def pubid
+        return @pubid if defined? @pubid
+
+        id = hit[:id]
+        @pubid = case id
+                 when Hash then ::Pubid::Jis::Identifier.from_hash id
+                 when String then ::Pubid::Jis::Identifier.parse id
+                 else id
+                 end
+      rescue StandardError
+        Util.warn "Unable to create an identifier from `#{hit[:id]}`"
+        @pubid = nil
+      end
+
+      # @return [Relaton::Jis::Item]
+      def item
+        @item ||= begin
+          url = "#{HitCollection::GH_URL}#{hit[:file]}"
+          resp = Net::HTTP.get_response URI(url)
+          item = Item.from_yaml resp.body
+          item.fetched = Date.today.to_s
+          item
+        end
+      end
+
+      private
+
+      def reference
+        hit_collection.pubid
+      end
+    end
+  end
+end

data/lib/relaton/jis/hit_collection.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require_relative "hit"
+
+module Relaton
+  module Jis
+    class HitCollection < Core::HitCollection
+      GH_URL = "https://raw.githubusercontent.com/relaton/relaton-data-jis/v2/"
+
+      #
+      # Initialize hit collection.
+      #
+      # Searches the pubid-based `index-v2` for every entry sharing the
+      # reference's series and number (a supplement is filed under its base
+      # number, so editions and amendments come back together). Narrowing to a
+      # specific type/year/part happens later in {#find}.
+      #
+      # @param [Pubid::Jis::Identifier] pubid parsed reference
+      #
+      def initialize(pubid)
+        super(pubid, pubid.year&.to_s)
+        @array = index.search(pubid) { |row| same_base? row[:id] }
+          .map { |row| Hit.new row, self }
+          .sort_by { |hit| hit.pubid.to_s }
+      end
+
+      # @return [Pubid::Jis::Identifier] parsed reference
+      def pubid
+        ref
+      end
+
+      #
+      # Find the best hit for the reference.
+      #
+      # @return [Relaton::Bib::ItemData, Array<Integer>] the matching item, or
+      #   the list of available edition years when none match the requested year
+      #
+      def find
+        pubid.year ? find_by_year : find_all_years
+      end
+
+      # The item of the requested edition. When no edition has the requested
+      # year, the years of the matching editions are returned.
+      def find_by_year
+        editions = @array.select(&:matches?)
+        hit = edition_hit editions
+        hit ? hit.item : editions.map { |h| h.pubid.year }
+      end
+
+      # The main item is the latest edition of the requested type; every other
+      # candidate sharing the series and number (older editions, amendments,
+      # explanations) is attached as an `instanceOf` relation.
+      def find_all_years
+        editions = @array.select(&:matches?)
+        return [] if editions.empty?
+
+        item = editions.max_by { |hit| hit.pubid.year.to_i }.item
+        attach_relations item.to_most_recent_reference,
+                         item.docidentifier.first.content
+      end
+
+      # The lowest-numbered part becomes the all-parts umbrella; every candidate
+      # sharing the series and number is attached as an `instanceOf` relation.
+      # A part-less document counts as part 0. Its index row deserializes with
+      # `parts` nil, not [].
+      def find_all_parts
+        parts = @array.select { |hit| hit.matches? all_parts: true }
+        lowest = parts.min_by { |hit| Array(hit.pubid.parts).first.to_i }
+        item = lowest.item.to_all_parts
+        attach_relations item, item.docidentifier.first.content
+      end
+
+      # Attach every candidate except `skip_id` to `umbrella` as an `instanceOf`
+      # relation and return the umbrella.
+      def attach_relations(umbrella, skip_id)
+        @array.each do |hit|
+          next if hit.pubid.to_s == skip_id
+
+          umbrella.relation << create_relation(hit)
+        end
+        umbrella
+      end
+
+      def create_relation(hit)
+        id = hit.pubid.to_s
+        docid = Docidentifier.new content: id, type: "JIS", primary: true
+        bibitem = ItemData.new(
+          formattedref: Bib::Formattedref.new(content: id),
+          docidentifier: [docid],
+        )
+        Relation.new type: "instanceOf", bibitem: bibitem
+      end
+
+      # Index of pubid identifiers (`index-v2`), deserialized via `pubid_class`.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :jis,
+          url: "#{GH_URL}#{INDEXFILE}.zip",
+          file: "#{INDEXFILE}.yaml",
+          pubid_class: ::Pubid::Jis::Identifier,
+        )
+      end
+
+      private
+
+      # The exact printed id comes first, then the same identifier without the
+      # reaffirmation mark (`JIS L 4107:2000` finds `JIS L 4107:2000R`).
+      def edition_hit(editions)
+        editions.find { |h| h.pubid.to_s == pubid.to_s } ||
+          editions.find { |h| h.pubid.matches? pubid, ignore: [:reaffirmed] }
+      end
+
+      # Broad candidate filter: same series and number as the reference. For a
+      # supplement (amendment/corrigendum/explanation) the document series and
+      # number live on `base`, so compare against that.
+      def same_base?(candidate)
+        cand = base_of candidate
+        ours = base_of pubid
+        cand.series == ours.series && cand.number.to_s == ours.number.to_s
+      end
+
+      def base_of(identifier)
+        supplement = identifier.respond_to?(:base) && identifier.base
+        supplement || identifier
+      end
+    end
+  end
+end

data/lib/relaton/jis/item.rb

@@ -0,0 +1,22 @@
+require_relative "docidentifier"
+require_relative "ext"
+require_relative "item_data"
+
+module Relaton
+  module Jis
+    class Relation < Bib::Relation
+    end
+
+    class Item < Iso::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier,
+                collection: true, initialize_empty: true
+      attribute :relation, Relation,
+                collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end
+
+require_relative "relation"