relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.1

data/lib/relaton/jis/relation.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "item_base"
+
+module Relaton
+  module Jis
+    class Relation
+      attribute :bibitem, ItemBase
+    end
+  end
+end

data/lib/relaton/jis/scraper.rb

@@ -0,0 +1,232 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class Scraper
+      ATTRS = %i[
+        title source abstract docidentifier docnumber date type language script
+        status contributor structuredidentifier ext
+      ].freeze
+
+      LANGS = { "和文" => { lang: "ja", script: "Jpan" },
+                "英訳" => { lang: "en", script: "Latn" } }.freeze
+
+      DATETYPES = { "発行年月日" => "issued", "確認年月日" => "confirmed" }.freeze
+      STATUSES = { "有効" => "valid", "廃止" => "withdrawn" }.freeze
+
+      def initialize(url, errors = {})
+        @url = url
+        @agent = Mechanize.new
+        @errors = errors
+      end
+
+      def fetch # rubocop:disable Metrics/MethodLength
+        @doc = @agent.get(@url).at "//div[@id='main']/section"
+        contributors = fetch_contributor
+        eg_contributor = fetch_editorialgroup_contributor
+        contributors << eg_contributor if eg_contributor
+        attrs = ATTRS.to_h { |attr| [attr, send("fetch_#{attr}")] }
+        attrs[:contributor] = contributors
+        ItemData.new(**attrs)
+      end
+
+      def fetch_title
+        result = { "ja" => "Jpan", "en" => "Latn" }.map.with_index do |(lang, script), i|
+          content = @doc.at("./h2/text()[#{i + 2}]").text.strip
+          Bib::Title.new content: content, language: lang, script: script
+        end
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def fetch_source # rubocop:disable Metrics/MethodLength
+        src = Bib::Uri.new content: @url, type: "src"
+        uri = URI @url
+        domain = "#{uri.scheme}://#{uri.host}"
+        xpath = "./dl/dt[.='プレビュー']/following-sibling::dd[1]/a"
+        result = @doc.xpath(xpath).reduce([src]) do |mem, node|
+          href = "#{domain}#{node[:href]}"
+          mem << Bib::Uri.new(content: href, type: "pdf")
+        end
+        @errors[:source] &&= result.empty?
+        result
+      end
+
+      def fetch_abstract
+        result = @doc.xpath("//div[@id='honbun']").map do |node|
+          Bib::Abstract.new(
+            content: node.text.strip,
+            language: "ja", script: "Jpan"
+          )
+        end
+        @errors[:abstract] &&= result.empty?
+        result
+      end
+
+      def fetch_docidentifier
+        docid = document_id
+        @errors[:docidentifier] &&= docid.nil? || docid.empty?
+        return [] if docid.nil? || docid.empty?
+
+        [Docidentifier.new(
+          content: docid, type: "JIS", primary: true,
+        )]
+      end
+
+      def fetch_docnumber
+        docid = document_id
+        match = docid&.match(/^\w+\s(\w)\s?(\d+)/)
+        @errors[:docnumber] &&= match.nil?
+        return unless match
+
+        "#{match[1]}#{match[2]}"
+      end
+
+      def document_id
+        @document_id ||= @doc.at("./h2/text()[1]")&.text&.strip
+      end
+
+      def fetch_date
+        result = DATETYPES.each_with_object([]) do |(key, type), a|
+          node = @doc.at("./div/div/div/p/text()[contains(.,'#{key}')]")
+          next unless node
+
+          at = node.text.match(/\d{4}-\d{2}-\d{2}/).to_s
+          next if at.empty?
+
+          a << Bib::Date.new(type: type, at: at)
+        end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      def fetch_type
+        "standard"
+      end
+
+      def fetch_language
+        langs_scripts.map { |l| l[:lang] }
+      end
+
+      def fetch_script
+        langs_scripts.map { |l| l[:script] }
+      end
+
+      def langs_scripts # rubocop:disable Metrics/MethodLength
+        @langs_scripts ||= begin
+          result = LANGS.each_with_object([]) do |(key, lang), a|
+            l = @doc.at(
+              "./div/div/div[@class='blockContentFile']/div/div/p[1]" \
+              "/span[contains(.,'#{key}')]/following-sibling::span",
+            )
+            next if l.nil? || l.text.strip == "-"
+
+            a << lang
+          end
+          @errors[:language] &&= result.empty?
+          result
+        end
+      end
+
+      def fetch_status
+        xpath = "./div/div/div/p/text()[contains(.,'状態')]" \
+                "/following-sibling::span"
+        st = @doc.at(xpath)
+        status_val = STATUSES[st&.text&.strip]
+        @errors[:status] &&= status_val.nil?
+        return unless status_val
+
+        stage = Bib::Status::Stage.new(content: status_val)
+        Bib::Status.new(stage: stage)
+      end
+
+      def fetch_doctype # rubocop:disable Metrics/CyclomaticComplexity
+        type = case document_id
+               when /JIS\s[A-Z]\s[\w-]+:\d{4}\/AMENDMENT/ then "amendment"
+               when /JIS\s[A-Z]\s[\w-]+/ then "japanese-industrial-standard"
+               when /TR[\s\/][\w-]+/ then "technical-report"
+               when /TS[\s\/][\w-]+/ then "technical-specification"
+               end
+        @errors[:doctype] &&= type.nil?
+        return unless type
+
+        Doctype.new content: type
+      end
+
+      def fetch_ics
+        td = @doc.at("./table/tr[th[.='ICS']]/td")
+        @errors[:ics] &&= td.nil?
+        return [] unless td
+
+        td.text.strip.split.map { |code| Bib::ICS.new code: code }
+      end
+
+      def fetch_contributor
+        authorizer = create_contrib(
+          "一般財団法人　日本規格協会", "authorizer"
+        )
+        xpath = "./table/tr[th[.='原案作成団体']]/td"
+        result = @doc.xpath(xpath).reduce([authorizer]) do |a, node|
+          a << create_contrib(node.text.strip, "author")
+          a << create_contrib(node.text.strip, "publisher")
+        end
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      def create_contrib(name, role)
+        org = Bib::Organization.new name: create_orgname(name)
+        role_obj = Bib::Contributor::Role.new(type: role)
+        Bib::Contributor.new organization: org, role: [role_obj]
+      end
+
+      def create_orgname(name)
+        tls = Bib::TypedLocalizedString
+        orgname = [tls.new(content: name, language: "ja", script: "Jpan")]
+        if name.include?("日本規格協会")
+          orgname << tls.new(
+            content: "Japanese Industrial Standards",
+            language: "en", script: "Latn"
+          )
+        end
+        orgname
+      end
+
+      def fetch_editorialgroup_contributor # rubocop:disable Metrics/MethodLength
+        node = @doc.at("./table/tr[th[.='原案作成団体']]/td")
+        @errors[:editorialgroup] &&= node.nil?
+        return unless node
+
+        subdivision = Bib::Subdivision.new(
+          type: "technical-committee",
+          name: [Bib::TypedLocalizedString.new(content: node.text.strip)],
+        )
+        desc = Bib::LocalizedMarkedUpString.new(content: "committee")
+        role = Bib::Contributor::Role.new(
+          type: "author", description: [desc],
+        )
+        org = Bib::Organization.new(
+          name: [], subdivision: [subdivision],
+        )
+        Bib::Contributor.new(role: [role], organization: org)
+      end
+
+      def fetch_structuredidentifier
+        StructuredIdentifier.new(
+          project_number: Iso::ProjectNumber.new(content: fetch_docnumber),
+          type: "JIS",
+        )
+      end
+
+      def fetch_ext
+        Ext.new(
+          doctype: fetch_doctype,
+          flavor: "jis",
+          ics: fetch_ics,
+          structuredidentifier: fetch_structuredidentifier,
+        )
+      end
+    end
+  end
+end

data/lib/relaton/jis/structured_identifier.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Jis
+    class StructuredIdentifier < Iso::StructuredIdentifier
+      def to_all_parts!
+        super
+        pn = project_number
+        pn.content = "#{pn.content} (all parts)" if pn&.content
+      end
+    end
+  end
+end

data/lib/relaton/jis/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jis
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-jis".freeze
+    end
+  end
+end

data/lib/relaton/jis.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "mechanize"
+require "pubid"
+require "relaton/iso"
+require "relaton/index"
+require_relative "version"
+require_relative "jis/util"
+require_relative "jis/item"
+require_relative "jis/bibitem"
+require_relative "jis/bibdata"
+require_relative "jis/hit_collection"
+require_relative "jis/bibliography"
+
+module Relaton
+  module Jis
+    INDEXFILE = "index-v1"
+    # Pubid-based index produced alongside index-v1 during the pubid migration.
+    INDEXFILE_V2 = "index-v2"
+
+    class Error < StandardError; end
+
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/logger/channels/gh_issue.rb

@@ -0,0 +1,78 @@
+require "net/http"
+
+module Relaton
+  module Logger
+    module Channels
+      #
+      # This class is used to create a GitHub issue with the log content.
+      # The issue will be created in the repository specified in the
+      # initializer.
+      # The log content is stored in the issue body. Only unique log messages
+      # are stored.
+      # Token is required to create an issue. It should be stored in the
+      # environment variable GITHUB_TOKEN.
+      # To create an issue, call the create_issue method after all log messages
+      # are written.
+      #
+      class GhIssue
+        #
+        # Create a new instance of the class.
+        #
+        # @param [String] repo owner/repo name
+        # @param [String] title title of the issue
+        #
+        def initialize(repo, title)
+          @repo = repo
+          @title = title
+          @log = Set.new
+          puts "GITHUB_TOKEN is not set!" if ENV["GITHUB_TOKEN"].nil?
+        end
+
+        def write(string)
+          @log << string
+        end
+
+        def close
+        end
+
+        def create_issue
+          return if @log.empty? || ENV["GITHUB_TOKEN"].nil?
+
+          responce = post_issue
+
+          if responce.code.to_i == 201
+            puts "Issue created!"
+          else
+            puts "Failed to create issue: #{responce.code} #{responce.message}\n#{responce.body}"
+          end
+        end
+
+        private
+
+        def post_issue
+          uri = URI("https://api.github.com/repos/#{@repo}/issues")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+
+          request = Net::HTTP::Post.new(uri.request_uri, headers)
+          request.body = issue_body.to_json
+
+          http.request(request)
+        end
+
+        def issue_body
+          { title: @title, body: @log.join("\n") }
+        end
+
+        def headers
+          {
+            "Content-Type" => "application/json",
+            "Accept" => "application/vnd.github+json",
+            "Authorization" => "Bearer #{ENV['GITHUB_TOKEN']}",
+            "X-GitHub-Api-Version" => "2022-11-28",
+          }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/logger/config.rb

@@ -0,0 +1,34 @@
+module Relaton::Logger
+  module Config
+    def configure
+      yield configuration if block_given?
+    end
+
+    def configuration
+      @configuration ||= self::Configuration.new
+    end
+  end
+
+  class Configuration
+    # @return [Array<Relaton::Logger::Log>] List of loggers
+    attr_reader :logger_pool
+
+    def initialize
+      @logger_pool ||= Pool.new
+      @logger_pool[:default] = Log.new($stderr, levels: %i[info warn error fatal])
+    end
+
+    #
+    # Replace the current list of loggers with the given list of loggers.
+    #
+    # @param [Array<Relaton::Logger::Log>] loggers list of loggers
+    #
+    # @return [void]
+    #
+    def logger_pool=(loggers)
+      @logger_pool.loggers = loggers
+    end
+  end
+
+  extend Config
+end

data/lib/relaton/logger/formatter_json.rb

@@ -0,0 +1,8 @@
+module Relaton::Logger
+  class FormatterJSON < ::Logger::Formatter
+    def call(severity, _datetime, progname, msg, **args)
+      hash = { prog: progname, message: msg, severity: severity }.merge(args)
+      "#{hash.to_json}\n"
+    end
+  end
+end

data/lib/relaton/logger/formatter_string.rb

@@ -0,0 +1,12 @@
+module Relaton::Logger
+  class FormatterString < ::Logger::Formatter
+    def call(severity, _datetime, progname, msg, **args)
+      output = []
+      output << "[#{progname}]" if progname
+      output << "#{severity}:"
+      output << "(#{args[:key]})" if args[:key]
+      output << "#{msg}\n"
+      output.join " "
+    end
+  end
+end

data/lib/relaton/logger/log.rb

@@ -0,0 +1,64 @@
+module Relaton::Logger
+  class Log < ::Logger
+    attr_reader :levels
+
+    def initialize(logdev, shift_age = 0, shift_size = 1_048_576, **args) # rubocop:disable Lint/MissingSuper
+      self.levels = args.delete(:levels) || [UNKNOWN, FATAL, ERROR, WARN, INFO]
+      self.level = @levels.min
+      self.progname = args.delete(:progname)
+      @default_formatter = FormatterString.new
+      self.datetime_format = args.delete(:datetime_format)
+      self.formatter = args.delete(:formatter)
+      self.formatter = self.formatter.new if self.formatter.is_a? Class
+      @logdev = nil
+      @level_override = {}
+      if logdev && logdev != File::NULL
+        @logdev = LogDevice.new(logdev, shift_age: shift_age, shift_size: shift_size, **args)
+      end
+    end
+
+    def levels=(levels)
+      @levels = Set.new levels.map { |l| Severity.coerce l }
+      self.level = @levels.min
+    end
+
+    def add_level(level)
+      @levels << Severity.coerce(level)
+      self.level = @levels.min
+    end
+
+    def remove_level(level)
+      @levels.delete Severity.coerce(level)
+      self.level = @levels.min
+    end
+
+    def unknown(message = nil, progname = nil, **args, &block)
+      level = Object.const_get "Logger::#{__callee__.to_s.upcase}"
+      add(level, message, progname, **args, &block)
+    end
+
+    %i[fatal error warn info debug].each { |m| alias_method m, :unknown }
+
+    def add(severity, message = nil, progname = nil, **args) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+      severity ||= UNKNOWN
+      return true if @logdev.nil? || !@levels.include?(severity)
+
+      if block_given?
+        progname ||= message || @progname
+        message = yield
+      else
+        progname ||= @progname
+      end
+      @logdev.write format_message(format_severity(severity), Time.now, progname, message, **args)
+      true
+    end
+
+    def format_message(severity, datetime, progname, msg, **args)
+      (@formatter || @default_formatter).call(severity, datetime, progname, msg, **args)
+    end
+
+    def truncate
+      @logdev.truncate
+    end
+  end
+end

data/lib/relaton/logger/log_device.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Relaton::Logger
+  class LogDevice < ::Logger::LogDevice
+    #
+    # @param header [Boolean, nil] whether to add header to log file
+    #
+    def initialize(logdev, **args)
+      # TODO: the header is not used yet, maybe it will be used in the future for not JSON formatters
+      @header = args.delete :header
+      super
+    end
+
+    def add_log_header(file)
+      return unless @header
+
+      super
+    end
+
+    def truncate
+      return unless @dev.respond_to? :truncate
+
+      @dev.truncate 0
+      @dev.rewind
+    end
+  end
+end

data/lib/relaton/logger/pool.rb

@@ -0,0 +1,24 @@
+module Relaton::Logger
+  class Pool
+    extend Forwardable
+    def_delegators :@loggers, :[], :[]=, :size, :each
+
+    attr_accessor :loggers
+
+    def initialize
+      @loggers = {}
+    end
+
+    def unknown(message = nil, progname = nil, **args, &block)
+      @loggers.each { |_, logger| logger.send(__callee__, message, progname, **args, &block) }
+      nil
+    end
+
+    %i[fatal error warn info debug].each { |m| alias_method m, :unknown }
+
+    def truncate
+      @loggers.each { |_, logger| logger.truncate }
+      nil
+    end
+  end
+end

data/lib/relaton/logger.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "logger"
+require_relative "version"
+require_relative "logger/log"
+require_relative "logger/log_device"
+require_relative "logger/pool"
+require_relative "logger/formatter_string"
+require_relative "logger/formatter_json"
+require_relative "logger/config"
+require_relative "logger/channels/gh_issue"
+
+module Relaton
+  def self.logger_pool
+    Logger.configuration.logger_pool
+  end
+end

data/lib/relaton/nist/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Nist
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/nist/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Nist
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end