relaton 2.1.0 → 3.0.0.pre.alpha.1

data/lib/relaton/itu/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Itu
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-itu".freeze
+    end
+  end
+end

data/lib/relaton/itu.rb

@@ -0,0 +1,33 @@
+require "mechanize"
+require "parslet"
+require "digest/md5"
+require "relaton/index"
+require "relaton/bib"
+require "relaton/core"
+require_relative "version"
+require_relative "itu/util"
+require_relative "itu/item_data"
+require_relative "itu/model/item"
+require_relative "itu/model/bibitem"
+require_relative "itu/model/bibdata"
+require_relative "itu/pubid"
+require_relative "itu/scraper"
+require_relative "itu/hit_collection"
+require_relative "itu/bibliography"
+
+module Relaton
+  module Itu
+    INDEXFILE = "index-v1"
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/jis/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jis
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/jis/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jis
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/jis/bibliography.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    module Bibliography
+      extend self
+
+      #
+      # Search JIS by reference, returning the full candidate collection.
+      #
+      # The reference is parsed into a {Pubid::Jis::Identifier} and matched
+      # against the pubid-based `index-v2` via {HitCollection}. Candidates share
+      # the reference's series and number (a supplement is filed under its base
+      # number), so an edition and its amendments are returned together.
+      #
+      # @param [String] code JIS document code
+      # @param [String, nil] year JIS document year
+      #
+      # @return [Relaton::Jis::HitCollection, nil] search result, or nil when
+      #   the reference cannot be parsed
+      #
+      def search(code, year = nil)
+        pubid = ::Pubid::Jis::Identifier.parse code
+        pubid.year ||= year.to_i if year
+        HitCollection.new pubid
+      rescue StandardError => e
+        Util.warn "Unable to parse `#{code}` with pubid: #{e.message}"
+        nil
+      end
+
+      #
+      # Get JIS document by reference
+      #
+      # @param [String] ref JIS document reference
+      # @param [String, nil] year JIS document year
+      # @param [Hash] opts options
+      # @option opts [Boolean] :all_parts return all parts of document
+      #
+      # @return [Relaton::Jis::Item, nil] JIS document
+      #
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        code = ref.sub(/\s\((all parts|規格群)\)/, "")
+        opts[:all_parts] ||= !$1.nil?
+        Util.info "Fetching from webdesk.jsa.or.jp ...", key: ref
+        hits = search(code, year)
+        unless hits
+          hint [], ref, year
+          return
+        end
+        result = opts[:all_parts] ? hits.find_all_parts : hits.find
+        if result.is_a? Bib::ItemData
+          Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+          return result
+        end
+        hint result, ref, year
+      end
+
+      #
+      # Log hint message
+      #
+      # @param [Array] result search result (missed edition years)
+      # @param [String] ref reference to search
+      # @param [String, nil] year year to search
+      #
+      def hint(result, ref, year)
+        Util.info "Not found.", key: ref
+        if result&.any?
+          Util.info "TIP: No match for edition year `#{year}`, but " \
+                    "matches exist for `#{result.uniq.join('`, `')}`.", key: ref
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/jis/data_fetcher.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require_relative "../jis"
+require_relative "scraper"
+
+module Relaton
+  module Jis
+    class DataFetcher < Core::DataFetcher
+      URL = "https://webdesk.jsa.or.jp/books/"
+
+      def initialize(output, format)
+        super
+        @queue = SizedQueue.new 10
+        @threads = create_thread_pool 5
+        @mutex = Mutex.new
+      end
+
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def index
+        @index ||= Relaton::Index.find_or_create :jis, file: "#{INDEXFILE}.yaml"
+      end
+
+      # Pubid-based index built in parallel with the legacy string index. The
+      # pool keys by type, so requesting a second :jis index with a different
+      # file evicts the v1 Type from the pool, but we keep our own reference in
+      # @index, so both stay live for the duration of the crawl.
+      def index_v2
+        @index_v2 ||= Relaton::Index.find_or_create(
+          :jis, file: "#{INDEXFILE_V2}.yaml", pubid_class: ::Pubid::Jis::Identifier
+        )
+      end
+
+      # Parse a primary docidentifier string into a pubid identifier; nil (with
+      # a warning) if pubid can't parse it, so a single bad id never aborts the
+      # crawl or corrupts index-v2.
+      def pubid(id)
+        ::Pubid::Jis::Identifier.parse id
+      rescue StandardError => e
+        Util.warn "Failed to parse `#{id}` with pubid: #{e.message}"
+        nil
+      end
+
+      def to_yaml(bib)
+        Item.to_yaml bib
+      end
+
+      def to_xml(bib)
+        Bibdata.to_xml bib
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+
+      def create_thread_pool(size)
+        Array.new(size) do
+          Thread.new do
+            until (url = @queue.shift) == :END
+              fetch_doc url
+            end
+          end
+        end
+      end
+
+      def fetch_doc(url) # rubocop:disable Metrics/MethodLength
+        attempts = 0
+        begin
+          bib = Scraper.new(url, @errors).fetch
+        rescue StandardError => e
+          attempts += 1
+          if attempts < 5
+            sleep 2
+            retry
+          else
+            Util.warn "URL: #{url}"
+            Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+          end
+        else
+          save_doc bib, url
+        end
+      end
+
+      def fetch(_source = nil)
+        return unless initial_post
+
+        resp = agent.get "#{URL}W11M0070/index"
+        parse_page resp
+        index.save
+        index_v2.save
+        report_errors
+      end
+
+      def initial_post
+        return true if @initial_time && Time.now - @initial_time < 600
+
+        body = { record: 0, dantai: "JIS", searchtype2: 1,
+                 status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
+        resp = agent.post "#{URL}W11M0270/index", body
+        disp = JSON.parse resp.body
+        @initial_time = Time.now
+        disp["status"] || Util.warn("No results found for JIS")
+      end
+
+      def agent
+        @agent ||= Mechanize.new
+      end
+
+      def parse_page(resp)
+        while resp
+          xpath = '//div[@class="blockGenaral"]/a'
+          resp.xpath(xpath).each { |a| @queue << a[:href] }
+          offset = parse_offset resp
+          break if offset >= count
+
+          resp = get_next_page(offset)
+        end
+        end_threads_and_wait
+      end
+
+      def parse_offset(resp) # rubocop:disable Metrics/AbcSize
+        if resp.at('//*[@id="btnPaging"]') # first page
+          xpath = '//script[contains(.,"var count =")]'
+          @count = resp.at(xpath).text.match(/var count = (\d+);/)[1]
+          resp.at("//*[@id='offset']")[:value].to_i
+        else
+          script = resp.at("//script").text
+          script.match(/\("offset"\)\.value = '(\d+)'/)[1].to_i
+        end
+      end
+
+      def end_threads_and_wait
+        @threads.size.times { @queue << :END }
+        @queue.close
+        @threads.each(&:join)
+      end
+
+      def count
+        @count.to_i
+      end
+
+      def get_next_page(offset) # rubocop:disable Metrics/MethodLength
+        attempts = 0
+        begin
+          if initial_post
+            url = "#{URL}W11M0070/getAddList"
+            agent.post url, search_type: "JIS", offset: offset
+          end
+        rescue StandardError => e
+          attempts += 1
+          if attempts < 5
+            sleep 2
+            retry
+          else
+            Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+          end
+        end
+      end
+
+      def save_doc(bib, url) # rubocop:disable Metrics/MethodLength
+        return unless bib
+
+        id = bib.docidentifier.find(&:primary).content
+        file = output_file id
+        @mutex.synchronize do
+          if @files.include?(file)
+            Util.warn "File #{file} already exists. Duplication URL: #{url}"
+          else
+            @files << file
+            File.write file, serialize(bib), encoding: "UTF-8"
+            index.add_or_update id, file
+            pid = pubid id
+            index_v2.add_or_update pid, file if pid
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/jis/docidentifier.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class Docidentifier < Bib::Docidentifier
+      def remove_part!
+        content&.sub!(/-\d+/, "")
+      end
+
+      def remove_date!
+        content&.sub!(/:\d{4}/, "")
+      end
+
+      def to_all_parts!
+        remove_part!
+        remove_date!
+        self.content = "#{content} (all parts)" if content
+      end
+    end
+  end
+end

data/lib/relaton/jis/doctype.rb

@@ -0,0 +1,9 @@
+module Relaton
+  module Jis
+    class Doctype < Bib::Doctype
+      TYPES = %w[japanese-industrial-standard technical-report technical-specification amendment].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/jis/ext.rb

@@ -0,0 +1,13 @@
+require_relative "doctype"
+require_relative "structured_identifier"
+
+module Relaton
+  module Jis
+    class Ext < Iso::Ext
+      attribute :doctype, Doctype
+      attribute :structuredidentifier, StructuredIdentifier
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-jis"]
+    end
+  end
+end

data/lib/relaton/jis/hit.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class Hit < Core::Hit
+      #
+      # Create new hit
+      #
+      # @param [Hash] hit found hit
+      # @param [Relaton::Jis::HitCollection] collection hit collection
+      #
+      # @return [Relaton::Jis::Hit] new hit
+      #
+      def self.create(hit, collection)
+        new hit, collection
+      end
+
+      #
+      # Check if the hit matches the collection's reference.
+      #
+      # The candidate must be the same document type (so a plain standard query
+      # never matches its amendments) and share series and number. Part is
+      # compared only when the reference names a specific part and `all_parts`
+      # is off. Year is filtered separately by {HitCollection}.
+      #
+      # @param [Boolean] all_parts match any part of the document
+      #
+      # @return [Boolean] true if the hit matches
+      #
+      def matches?(all_parts: false)
+        cand = pubid
+        return false unless cand && same_document?(cand)
+        return true if all_parts || reference_partless?
+
+        Array(cand.parts).map(&:to_s) == reference_parts
+      end
+
+      #
+      # The hit's pubid identifier. `index-v2` rows are already deserialized to
+      # {Pubid::Jis::Identifier} via `pubid_class`; a Hash or String id is
+      # converted for robustness.
+      #
+      # @return [Pubid::Jis::Identifier, nil] identifier, or nil when it cannot
+      #   be built
+      #
+      def pubid
+        return @pubid if defined? @pubid
+
+        id = hit[:id]
+        @pubid = case id
+                 when Hash then ::Pubid::Jis::Identifier.from_hash id
+                 when String then ::Pubid::Jis::Identifier.parse id
+                 else id
+                 end
+      rescue StandardError
+        Util.warn "Unable to create an identifier from `#{hit[:id]}`"
+        @pubid = nil
+      end
+
+      # @return [Relaton::Jis::Item]
+      def item
+        @item ||= begin
+          url = "#{HitCollection::GH_URL}#{hit[:file]}"
+          resp = Net::HTTP.get_response URI(url)
+          item = Item.from_yaml resp.body
+          item.fetched = Date.today.to_s
+          item
+        end
+      end
+
+      private
+
+      def reference
+        hit_collection.pubid
+      end
+
+      # Same document type, series and number as the reference.
+      def same_document?(cand)
+        cand.instance_of?(reference.class) &&
+          cand.series == reference.series &&
+          cand.number.to_s == reference.number.to_s
+      end
+
+      def reference_partless?
+        reference.parts.nil? || reference.parts.empty?
+      end
+
+      def reference_parts
+        Array(reference.parts).map(&:to_s)
+      end
+    end
+  end
+end

data/lib/relaton/jis/hit_collection.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "hit"
+
+module Relaton
+  module Jis
+    class HitCollection < Core::HitCollection
+      GH_URL = "https://raw.githubusercontent.com/relaton/relaton-data-jis/v2/"
+
+      #
+      # Initialize hit collection.
+      #
+      # Searches the pubid-based `index-v2` for every entry sharing the
+      # reference's series and number (a supplement is filed under its base
+      # number, so editions and amendments come back together). Narrowing to a
+      # specific type/year/part happens later in {#find}.
+      #
+      # @param [Pubid::Jis::Identifier] pubid parsed reference
+      #
+      def initialize(pubid)
+        super(pubid, pubid.year&.to_s)
+        @array = index.search(pubid) { |row| same_base? row[:id] }
+          .map { |row| Hit.new row, self }
+          .sort_by { |hit| hit.pubid.to_s }
+      end
+
+      # @return [Pubid::Jis::Identifier] parsed reference
+      def pubid
+        ref
+      end
+
+      #
+      # Find the best hit for the reference.
+      #
+      # @return [Relaton::Bib::ItemData, Array<Integer>] the matching item, or
+      #   the list of available edition years when none match the requested year
+      #
+      def find
+        if pubid.year
+          find_by_year pubid.year
+        else
+          find_all_years
+        end
+      end
+
+      def find_by_year(ref_year)
+        missed_years = []
+        @array.each do |hit|
+          next unless hit.matches?
+
+          return hit.item if hit.pubid.year.to_s == ref_year.to_s
+
+          missed_years << hit.pubid.year
+        end
+        missed_years
+      end
+
+      # The main item is the latest edition of the requested type; every other
+      # candidate sharing the series and number (older editions, amendments,
+      # explanations) is attached as an `instanceOf` relation.
+      def find_all_years
+        editions = @array.select(&:matches?)
+        return [] if editions.empty?
+
+        item = editions.max_by { |hit| hit.pubid.year.to_i }.item
+        attach_relations item.to_most_recent_reference,
+                         item.docidentifier.first.content
+      end
+
+      # The lowest-numbered part becomes the all-parts umbrella; every candidate
+      # sharing the series and number is attached as an `instanceOf` relation.
+      def find_all_parts
+        parts = @array.select { |hit| hit.matches? all_parts: true }
+        lowest = parts.min_by { |hit| hit.pubid.parts.first.to_i }
+        item = lowest.item.to_all_parts
+        attach_relations item, item.docidentifier.first.content
+      end
+
+      # Attach every candidate except `skip_id` to `umbrella` as an `instanceOf`
+      # relation and return the umbrella.
+      def attach_relations(umbrella, skip_id)
+        @array.each do |hit|
+          next if hit.pubid.to_s == skip_id
+
+          umbrella.relation << create_relation(hit)
+        end
+        umbrella
+      end
+
+      def create_relation(hit)
+        id = hit.pubid.to_s
+        docid = Docidentifier.new content: id, type: "JIS", primary: true
+        bibitem = ItemData.new(
+          formattedref: Bib::Formattedref.new(content: id),
+          docidentifier: [docid],
+        )
+        Relation.new type: "instanceOf", bibitem: bibitem
+      end
+
+      # Index of pubid identifiers (`index-v2`), deserialized via `pubid_class`.
+      def index
+        @index ||= Relaton::Index.find_or_create(
+          :jis,
+          url: "#{GH_URL}#{INDEXFILE_V2}.zip",
+          file: "#{INDEXFILE_V2}.yaml",
+          pubid_class: ::Pubid::Jis::Identifier,
+        )
+      end
+
+      private
+
+      # Broad candidate filter: same series and number as the reference. For a
+      # supplement (amendment/corrigendum/explanation) the document series and
+      # number live on `base`, so compare against that.
+      def same_base?(candidate)
+        cand = base_of candidate
+        ours = base_of pubid
+        cand.series == ours.series && cand.number.to_s == ours.number.to_s
+      end
+
+      def base_of(identifier)
+        supplement = identifier.respond_to?(:base) && identifier.base
+        supplement || identifier
+      end
+    end
+  end
+end

data/lib/relaton/jis/item.rb

@@ -0,0 +1,22 @@
+require_relative "docidentifier"
+require_relative "ext"
+require_relative "item_data"
+
+module Relaton
+  module Jis
+    class Relation < Bib::Relation
+    end
+
+    class Item < Iso::Item
+      model ItemData
+
+      attribute :docidentifier, Docidentifier,
+                collection: true, initialize_empty: true
+      attribute :relation, Relation,
+                collection: true, initialize_empty: true
+      attribute :ext, Ext
+    end
+  end
+end
+
+require_relative "relation"

data/lib/relaton/jis/item_base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "docidentifier"
+
+module Relaton
+  module Jis
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+      attribute :docidentifier, Docidentifier,
+                collection: true, initialize_empty: true
+      attribute :relation, Relation,
+                collection: true, initialize_empty: true
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/jis/item_data.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/jis/processor.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "relaton/core/processor"
+
+module Relaton
+  module Jis
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_jis
+        @prefix = "JIS"
+        @defaultprefix = %r{^(JIS|TR)\s}
+        @idtype = "JIS"
+        @datasets = %w[jis-webdesk]
+      end
+
+      def get(code, date, opts)
+        require_relative "../jis"
+        Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../jis"
+        Item.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../jis"
+        Item.from_yaml yaml
+      end
+
+      def grammar_hash
+        require_relative "../jis"
+        Digest::MD5.hexdigest Relaton::VERSION
+      end
+
+      def threads
+        3
+      end
+
+      def remove_index_file
+        require_relative "../jis"
+        Relaton::Index.find_or_create(
+          :jis, url: true, file: "#{INDEXFILE_V2}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end