relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/jis/item_base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "docidentifier"
+
+module Relaton
+  module Jis
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+      attribute :docidentifier, Docidentifier,
+                collection: true, initialize_empty: true
+      attribute :relation, Relation,
+                collection: true, initialize_empty: true
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/jis/item_data.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/jis/processor.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "relaton/core/processor"
+
+module Relaton
+  module Jis
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_jis
+        @prefix = "JIS"
+        @defaultprefix = %r{^(JIS|TR)\s}
+        @idtype = "JIS"
+        @datasets = %w[jis-webdesk]
+      end
+
+      def get(code, date, opts)
+        require_relative "../jis"
+        Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../jis"
+        Item.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../jis"
+        Item.from_yaml yaml
+      end
+
+      def grammar_hash
+        require_relative "../jis"
+        Digest::MD5.hexdigest Relaton::VERSION
+      end
+
+      def threads
+        3
+      end
+
+      def remove_index_file
+        require_relative "../jis"
+        Relaton::Index.find_or_create(
+          :jis, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/jis/relation.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "item_base"
+
+module Relaton
+  module Jis
+    class Relation
+      attribute :bibitem, ItemBase
+    end
+  end
+end

data/lib/relaton/jis/scraper.rb

@@ -0,0 +1,232 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class Scraper
+      ATTRS = %i[
+        title source abstract docidentifier docnumber date type language script
+        status contributor structuredidentifier ext
+      ].freeze
+
+      LANGS = { "和文" => { lang: "ja", script: "Jpan" },
+                "英訳" => { lang: "en", script: "Latn" } }.freeze
+
+      DATETYPES = { "発行年月日" => "issued", "確認年月日" => "confirmed" }.freeze
+      STATUSES = { "有効" => "valid", "廃止" => "withdrawn" }.freeze
+
+      def initialize(url, errors = {})
+        @url = url
+        @agent = Mechanize.new
+        @errors = errors
+      end
+
+      def fetch # rubocop:disable Metrics/MethodLength
+        @doc = @agent.get(@url).at "//div[@id='main']/section"
+        contributors = fetch_contributor
+        eg_contributor = fetch_editorialgroup_contributor
+        contributors << eg_contributor if eg_contributor
+        attrs = ATTRS.to_h { |attr| [attr, send("fetch_#{attr}")] }
+        attrs[:contributor] = contributors
+        ItemData.new(**attrs)
+      end
+
+      def fetch_title
+        result = { "ja" => "Jpan", "en" => "Latn" }.map.with_index do |(lang, script), i|
+          content = @doc.at("./h2/text()[#{i + 2}]").text.strip
+          Bib::Title.new content: content, language: lang, script: script
+        end
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def fetch_source # rubocop:disable Metrics/MethodLength
+        src = Bib::Uri.new content: @url, type: "src"
+        uri = URI @url
+        domain = "#{uri.scheme}://#{uri.host}"
+        xpath = "./dl/dt[.='プレビュー']/following-sibling::dd[1]/a"
+        result = @doc.xpath(xpath).reduce([src]) do |mem, node|
+          href = "#{domain}#{node[:href]}"
+          mem << Bib::Uri.new(content: href, type: "pdf")
+        end
+        @errors[:source] &&= result.empty?
+        result
+      end
+
+      def fetch_abstract
+        result = @doc.xpath("//div[@id='honbun']").map do |node|
+          Bib::Abstract.new(
+            content: node.text.strip,
+            language: "ja", script: "Jpan"
+          )
+        end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      def fetch_docidentifier
+        docid = document_id
+        @errors[:docidentifier] &&= docid.nil? || docid.empty?
+        return [] if docid.nil? || docid.empty?
+
+        [Docidentifier.new(
+          content: docid, type: "JIS", primary: true,
+        )]
+      end
+
+      def fetch_docnumber
+        docid = document_id
+        match = docid&.match(/^\w+\s(\w)\s?(\d+)/)
+        @errors[:docnumber] &&= match.nil?
+        return unless match
+
+        "#{match[1]}#{match[2]}"
+      end
+
+      def document_id
+        @document_id ||= @doc.at("./h2/text()[1]")&.text&.strip
+      end
+
+      def fetch_date
+        result = DATETYPES.each_with_object([]) do |(key, type), a|
+          node = @doc.at("./div/div/div/p/text()[contains(.,'#{key}')]")
+          next unless node
+
+          at = node.text.match(/\d{4}-\d{2}-\d{2}/).to_s
+          next if at.empty?
+
+          a << Bib::Date.new(type: type, at: at)
+        end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      def fetch_type
+        "standard"
+      end
+
+      def fetch_language
+        langs_scripts.map { |l| l[:lang] }
+      end
+
+      def fetch_script
+        langs_scripts.map { |l| l[:script] }
+      end
+
+      def langs_scripts # rubocop:disable Metrics/MethodLength
+        @langs_scripts ||= begin
+          result = LANGS.each_with_object([]) do |(key, lang), a|
+            l = @doc.at(
+              "./div/div/div[@class='blockContentFile']/div/div/p[1]" \
+              "/span[contains(.,'#{key}')]/following-sibling::span",
+            )
+            next if l.nil? || l.text.strip == "-"
+
+            a << lang
+          end
+          @errors[:language] &&= result.empty?
+          result
+        end
+      end
+
+      def fetch_status
+        xpath = "./div/div/div/p/text()[contains(.,'状態')]" \
+                "/following-sibling::span"
+        st = @doc.at(xpath)
+        status_val = STATUSES[st&.text&.strip]
+        @errors[:status] &&= status_val.nil?
+        return unless status_val
+
+        stage = Bib::Status::Stage.new(content: status_val)
+        Bib::Status.new(stage: stage)
+      end
+
+      def fetch_doctype # rubocop:disable Metrics/CyclomaticComplexity
+        type = case document_id
+               when /JIS\s[A-Z]\s[\w-]+:\d{4}\/AMENDMENT/ then "amendment"
+               when /JIS\s[A-Z]\s[\w-]+/ then "japanese-industrial-standard"
+               when /TR[\s\/][\w-]+/ then "technical-report"
+               when /TS[\s\/][\w-]+/ then "technical-specification"
+               end
+        @errors[:doctype] &&= type.nil?
+        return unless type
+
+        Doctype.new content: type
+      end
+
+      def fetch_ics
+        td = @doc.at("./table/tr[th[.='ICS']]/td")
+        @errors[:ics] &&= td.nil?
+        return [] unless td
+
+        td.text.strip.split.map { |code| Bib::ICS.new code: code }
+      end
+
+      def fetch_contributor
+        authorizer = create_contrib(
+          "一般財団法人　日本規格協会", "authorizer"
+        )
+        xpath = "./table/tr[th[.='原案作成団体']]/td"
+        result = @doc.xpath(xpath).reduce([authorizer]) do |a, node|
+          a << create_contrib(node.text.strip, "author")
+          a << create_contrib(node.text.strip, "publisher")
+        end
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      def create_contrib(name, role)
+        org = Bib::Organization.new name: create_orgname(name)
+        role_obj = Bib::Contributor::Role.new(type: role)
+        Bib::Contributor.new organization: org, role: [role_obj]
+      end
+
+      def create_orgname(name)
+        tls = Bib::TypedLocalizedString
+        orgname = [tls.new(content: name, language: "ja", script: "Jpan")]
+        if name.include?("日本規格協会")
+          orgname << tls.new(
+            content: "Japanese Industrial Standards",
+            language: "en", script: "Latn"
+          )
+        end
+        orgname
+      end
+
+      def fetch_editorialgroup_contributor # rubocop:disable Metrics/MethodLength
+        node = @doc.at("./table/tr[th[.='原案作成団体']]/td")
+        @errors[:editorialgroup] &&= node.nil?
+        return unless node
+
+        subdivision = Bib::Subdivision.new(
+          type: "technical-committee",
+          name: [Bib::TypedLocalizedString.new(content: node.text.strip)],
+        )
+        desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Bib::Contributor::Role.new(
+          type: "author", description: [desc],
+        )
+        org = Bib::Organization.new(
+          name: [], subdivision: [subdivision],
+        )
+        Bib::Contributor.new(role: [role], organization: org)
+      end
+
+      def fetch_structuredidentifier
+        StructuredIdentifier.new(
+          project_number: Iso::ProjectNumber.new(content: fetch_docnumber),
+          type: "JIS",
+        )
+      end
+
+      def fetch_ext
+        Ext.new(
+          doctype: fetch_doctype,
+          flavor: "jis",
+          ics: fetch_ics,
+          structuredidentifier: fetch_structuredidentifier,
+        )
+      end
+    end
+  end
+end

data/lib/relaton/jis/structured_identifier.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class StructuredIdentifier < Iso::StructuredIdentifier
+      def to_all_parts!
+        super
+        pn = project_number
+        pn.content = "#{pn.content} (all parts)" if pn&.content
+      end
+    end
+  end
+end

data/lib/relaton/jis/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jis
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-jis".freeze
+    end
+  end
+end

data/lib/relaton/jis.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "mechanize"
+require "pubid"
+require "relaton/iso"
+require "relaton/index"
+require_relative "version"
+require_relative "jis/util"
+require_relative "jis/item"
+require_relative "jis/bibitem"
+require_relative "jis/bibdata"
+require_relative "jis/hit_collection"
+require_relative "jis/bibliography"
+
+module Relaton
+  module Jis
+    # The pubid index. relaton-data-jis rebuilds the legacy index-v1 from data/.
+    INDEXFILE = "index-v2"
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/logger/channels/gh_issue.rb

@@ -0,0 +1,143 @@
+require "net/http"
+require "json"
+
+module Relaton
+  module Logger
+    module Channels
+      #
+      # This class is used to report the log content as a GitHub issue in the
+      # repository specified in the initializer.
+      # The log content is stored in the issue body. Only unique log messages
+      # are stored. When an open issue with the same title already exists, the
+      # log is added to it as a comment rather than opening a duplicate.
+      # Token is required to create an issue. It should be stored in the
+      # environment variable GITHUB_TOKEN.
+      # To create an issue, call the create_issue method after all log messages
+      # are written.
+      #
+      class GhIssue
+        #
+        # Create a new instance of the class.
+        #
+        # @param [String] repo owner/repo name
+        # @param [String] title title of the issue
+        #
+        def initialize(repo, title)
+          @repo = repo
+          @title = title
+          @log = Set.new
+          puts "GITHUB_TOKEN is not set!" if ENV["GITHUB_TOKEN"].nil?
+        end
+
+        def write(string)
+          @log << string
+        end
+
+        def close
+        end
+
+        # Report the collected log. A crawler runs on a schedule, so creating a
+        # fresh issue every time would pile up duplicates of the same standing
+        # report; when an open issue with this title already exists, add the run's
+        # log as a comment on it instead.
+        def create_issue
+          return if @log.empty? || ENV["GITHUB_TOKEN"].nil?
+
+          if (number = open_issue_number)
+            report_response post_comment(number), "Issue #{number} commented!", 201
+          else
+            report_response post_issue, "Issue created!", 201
+          end
+        end
+
+        private
+
+        def report_response(responce, success_message, success_code)
+          if responce.code.to_i == success_code
+            puts success_message
+          else
+            puts "Failed to report issue: #{responce.code} #{responce.message}\n#{responce.body}"
+          end
+        end
+
+        # Number of the open issue with exactly this title, or nil. Any failure
+        # (network, rate limit, missing search scope) degrades to nil, so the
+        # worst case is the previous behaviour — a new issue.
+        def open_issue_number
+          query = "repo:#{@repo} is:issue is:open in:title \"#{@title}\""
+          uri = URI("https://api.github.com/search/issues")
+          uri.query = URI.encode_www_form(q: query, per_page: 100)
+          responce = get(uri)
+          return nil unless responce.code.to_i == 200
+
+          items = JSON.parse(responce.body)["items"] || []
+          items.find { |i| i["title"] == @title }&.fetch("number", nil)
+        rescue StandardError => e
+          puts "Failed to search for an open `#{@title}` issue: #{e.message}"
+          nil
+        end
+
+        def get(uri)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.request(Net::HTTP::Get.new(uri.request_uri, headers))
+        end
+
+        def post(uri, body)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+
+          request = Net::HTTP::Post.new(uri.request_uri, headers)
+          request.body = body.to_json
+
+          http.request(request)
+        end
+
+        def post_issue
+          post URI("https://api.github.com/repos/#{@repo}/issues"), issue_body
+        end
+
+        def post_comment(number)
+          post URI("https://api.github.com/repos/#{@repo}/issues/#{number}/comments"),
+               { body: body }
+        end
+
+        def issue_body
+          { title: @title, body: body }
+        end
+
+        # GitHub rejects an issue or comment body longer than this with a 422,
+        # which loses the whole report — and these reports get long (a crawl can
+        # log hundreds of ids it could not index). Keep as many whole lines as
+        # fit and say how many were left out.
+        MAX_BODY = 65_536
+        TRUNCATION_ROOM = 200
+
+        def body
+          lines = @log.to_a
+          text = lines.join("\n")
+          return text if text.length <= MAX_BODY
+
+          kept = []
+          size = 0
+          lines.each do |line|
+            break if size + line.length + 1 > MAX_BODY - TRUNCATION_ROOM
+
+            kept << line
+            size += line.length + 1
+          end
+          "#{kept.join("\n")}\n\n…and #{lines.size - kept.size} more (see the run log)"
+        end
+
+        def headers
+          {
+            "Content-Type" => "application/json",
+            "Accept" => "application/vnd.github+json",
+            "Authorization" => "Bearer #{ENV['GITHUB_TOKEN']}",
+            "X-GitHub-Api-Version" => "2022-11-28",
+          }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/logger/config.rb

@@ -0,0 +1,34 @@
+module Relaton::Logger
+  module Config
+    def configure
+      yield configuration if block_given?
+    end
+
+    def configuration
+      @configuration ||= self::Configuration.new
+    end
+  end
+
+  class Configuration
+    # @return [Array<Relaton::Logger::Log>] List of loggers
+    attr_reader :logger_pool
+
+    def initialize
+      @logger_pool ||= Pool.new
+      @logger_pool[:default] = Log.new($stderr, levels: %i[info warn error fatal])
+    end
+
+    #
+    # Replace the current list of loggers with the given list of loggers.
+    #
+    # @param [Array<Relaton::Logger::Log>] loggers list of loggers
+    #
+    # @return [void]
+    #
+    def logger_pool=(loggers)
+      @logger_pool.loggers = loggers
+    end
+  end
+
+  extend Config
+end

data/lib/relaton/logger/formatter_json.rb

@@ -0,0 +1,8 @@
+module Relaton::Logger
+  class FormatterJSON < ::Logger::Formatter
+    def call(severity, _datetime, progname, msg, **args)
+      hash = { prog: progname, message: msg, severity: severity }.merge(args)
+      "#{hash.to_json}\n"
+    end
+  end
+end

data/lib/relaton/logger/formatter_string.rb

@@ -0,0 +1,12 @@
+module Relaton::Logger
+  class FormatterString < ::Logger::Formatter
+    def call(severity, _datetime, progname, msg, **args)
+      output = []
+      output << "[#{progname}]" if progname
+      output << "#{severity}:"
+      output << "(#{args[:key]})" if args[:key]
+      output << "#{msg}\n"
+      output.join " "
+    end
+  end
+end

data/lib/relaton/logger/log.rb

@@ -0,0 +1,64 @@
+module Relaton::Logger
+  class Log < ::Logger
+    attr_reader :levels
+
+    def initialize(logdev, shift_age = 0, shift_size = 1_048_576, **args) # rubocop:disable Lint/MissingSuper
+      self.levels = args.delete(:levels) || [UNKNOWN, FATAL, ERROR, WARN, INFO]
+      self.level = @levels.min
+      self.progname = args.delete(:progname)
+      @default_formatter = FormatterString.new
+      self.datetime_format = args.delete(:datetime_format)
+      self.formatter = args.delete(:formatter)
+      self.formatter = self.formatter.new if self.formatter.is_a? Class
+      @logdev = nil
+      @level_override = {}
+      if logdev && logdev != File::NULL
+        @logdev = LogDevice.new(logdev, shift_age: shift_age, shift_size: shift_size, **args)
+      end
+    end
+
+    def levels=(levels)
+      @levels = Set.new levels.map { |l| Severity.coerce l }
+      self.level = @levels.min
+    end
+
+    def add_level(level)
+      @levels << Severity.coerce(level)
+      self.level = @levels.min
+    end
+
+    def remove_level(level)
+      @levels.delete Severity.coerce(level)
+      self.level = @levels.min
+    end
+
+    def unknown(message = nil, progname = nil, **args, &block)
+      level = Object.const_get "Logger::#{__callee__.to_s.upcase}"
+      add(level, message, progname, **args, &block)
+    end
+
+    %i[fatal error warn info debug].each { |m| alias_method m, :unknown }
+
+    def add(severity, message = nil, progname = nil, **args) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+      severity ||= UNKNOWN
+      return true if @logdev.nil? || !@levels.include?(severity)
+
+      if block_given?
+        progname ||= message || @progname
+        message = yield
+      else
+        progname ||= @progname
+      end
+      @logdev.write format_message(format_severity(severity), Time.now, progname, message, **args)
+      true
+    end
+
+    def format_message(severity, datetime, progname, msg, **args)
+      (@formatter || @default_formatter).call(severity, datetime, progname, msg, **args)
+    end
+
+    def truncate
+      @logdev.truncate
+    end
+  end
+end

data/lib/relaton/logger/log_device.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Relaton::Logger
+  class LogDevice < ::Logger::LogDevice
+    #
+    # @param header [Boolean, nil] whether to add header to log file
+    #
+    def initialize(logdev, **args)
+      # TODO: the header is not used yet, maybe it will be used in the future for not JSON formatters
+      @header = args.delete :header
+      super
+    end
+
+    def add_log_header(file)
+      return unless @header
+
+      super
+    end
+
+    def truncate
+      return unless @dev.respond_to? :truncate
+
+      @dev.truncate 0
+      @dev.rewind
+    end
+  end
+end