relaton-jis 1.19.1 → 1.19.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c223759d15403f399f92edd2272b156b0e3c7f6c7dc274babb6b24cf02f99326
-  data.tar.gz: 387192eb0a06f3f0cca389decf2e0aa9d13193aed0f52d0e8949c7089ce77b99
+  metadata.gz: 8b816ec16d7873fe9c8379b12dbe5f48f2874a50a5e113fdb439abaa6eb0a081
+  data.tar.gz: 94d2b6a1be560ec226f95da9eb6c8973af93fa983eb86ceb5cbaba8b72c3fbc3
 SHA512:
-  metadata.gz: a1ceb478a730fba7ce00809420161cc996b1ade2e3d590fab98c75668ee1450df0be2daebf1f5e3cedd8fe099121bd5e2ed01d73b56b5b755afee12e2a009302
-  data.tar.gz: 1c7fe1151443babf282595293d6c41f5945fe75d56481f06edd60a5c00f61b15f9ff65a428de6827494f08df3a64f5eb620a6fdb2f1832f826d4de346dee28f7
+  metadata.gz: f255822a0bc60c34d5b6d1074dd4b280b7626d64084710fe78a19495b8d1f6a9fe55ba2afa396ad7227024d6207c4de63ee904b8e395276e3eadbd626d9e1e93
+  data.tar.gz: f8b91c43a6f9d2a8a84da5f4812a135c841e0aa909938c30a6c6d1e885b338dfb83e95303d42fbd189918606d93154393f2d521ed5eea7d1cbb2adaf79f7b395

data/README.adoc

@@ -149,6 +149,24 @@ item.link
   @type="pdf">]
 ----
 
+=== Fetch data
+
+This gem scrapes the https://webdesk.jsa.or.jp/books/W11M0270 pages to fetch the JIS Standards metadata. By default the data is saved in the `./data` folder in YAML format.
+
+The method `RelatonJis::DataFetcher.fetch(output: "data", format: "yaml")` fetches all the documents from the dataset and saves them to the `./data` folder in YAML format.
+Arguments:
+
+- `output` - folder to save documents (default './data').
+- `format` - the format in which the documents are saved. Possible formats are: `yaml`, `xml`, `bibxml` (default `yaml`).
+
+[source,ruby]
+----
+RelatonJis::DataFetcher.fetch
+Start fetching JIS data at 2024-09-27 17:49:40 -0400
+Fetching JIS data finished at 2024-09-27 18:40:11 -0400. It took 3031.0 seconds.
+=> nil
+----
+
 === Logging
 
 RelatonJis uses the relaton-logger gem for logging. By default, it logs to STDOUT. To change the log levels and add other loggers, read the https://github.com/relaton/relaton-logger#usage[relaton-logger] documentation.

data/lib/relaton_jis/bibliography.rb

@@ -2,7 +2,8 @@ module RelatonJis
   module Bibliography
     extend self
 
-    SOURCE = "https://webdesk.jsa.or.jp/books/W11M".freeze
+    # SOURCE = "https://webdesk.jsa.or.jp/books/W11M".freeze
+    GH_URL = "https://raw.githubusercontent.com/relaton/relaton-data-jis/refs/heads/main/".freeze
 
     #
     # Search JIS by keyword
@@ -13,14 +14,9 @@ module RelatonJis
     # @return [RelatonJis::HitCollection] search result
     #
     def search(code, year = nil)
-      agent = Mechanize.new
-      resp = agent.post "#{SOURCE}0270/index", dantai: "JIS", bunsyo_id: code, searchtype2: "1", status_1: "1", status_2: "1"
-      disp = JSON.parse resp.body
-      # raise RelatonBib::RequestError, "No results found for #{code}" if disp["disp_screen"].nil?
-      return unless disp["status"]
-
-      result = agent.get "#{SOURCE}0070/index"
-      HitCollection.new code, year, result: result.xpath("//div[@class='blockGenaral']")
+      index = Relaton::Index.find_or_create(:jis, url: "#{GH_URL}index-v1.zip", file: DataFetcher::INDEX_FILE)
+      result = index.search(code).sort_by { |h| h[:id] }
+      HitCollection.new code, year, result: result # .xpath("//div[@class='blockGenaral']")
     end
 
     #

data/lib/relaton_jis/data_fetcher.rb

@@ -0,0 +1,156 @@
+module RelatonJis
+  class DataFetcher
+    URL = "https://webdesk.jsa.or.jp/books/".freeze
+    INDEX_FILE = "index-v1.yaml".freeze
+
+    def initialize(output, format)
+      @output = output
+      @format = format
+      @ext = format.sub("bibxml", "xml")
+      @files = Set.new
+      @queue = SizedQueue.new 10
+      @threads = create_thread_pool 5
+      @mutex = Mutex.new
+    end
+
+    def self.fetch(output: "data", format: "yaml")
+      start_time = Time.now
+      puts "Start fetching JIS data at #{start_time}"
+      FileUtils.mkdir_p output
+      new(output, format).fetch
+      stop_time = Time.now
+      puts "Fetching JIS data finished at #{stop_time}. It took #{stop_time - start_time} seconds."
+    end
+
+    def create_thread_pool(size)
+      Array.new(size) do
+        Thread.new do
+          until (url = @queue.shift) == :END
+            fetch_doc url
+          end
+        end
+      end
+    end
+
+    def fetch_doc(url) # rubocop:disable Metrics/MethodLength
+      attempts = 0
+      begin
+        bib = Scraper.new(url).fetch
+      rescue StandardError => e
+        attempts += 1
+        if attempts < 5
+          sleep 2
+          retry
+        else
+          Util.warn "URL: #{url}"
+          Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+        end
+      else
+        save_doc bib, url
+      end
+    end
+
+    def fetch
+      return unless initial_post
+
+      resp = agent.get "#{URL}W11M0070/index"
+      parse_page resp
+      index.save
+    end
+
+    def initial_post
+      return true if @initial_time && Time.now - @initial_time < 600
+
+      body = { record: 0, dantai: "JIS", searchtype2: 1, status_1: 1, status_2: 2 }
+      resp = agent.post "#{URL}W11M0270/index", body
+      disp = JSON.parse resp.body
+      @initial_time = Time.now
+      disp["status"] || Util.warn("No results found for JIS")
+    end
+
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    def parse_page(resp)
+      while resp
+        resp.xpath('//div[@class="blockGenaral"]/a').each { |a| @queue << a[:href] }
+        offset = parse_offset resp
+        break if offset >= count # no more pages
+
+        resp = get_next_page(offset)
+      end
+      end_threads_and_wait
+    end
+
+    def parse_offset(resp) # rubocop:disable Metrics/AbcSize
+      if resp.at('//*[@id="btnPaging"]') # first page
+        @count = resp.at('//script[contains(.,"var count =")]').text.match(/var count = (\d+);/)[1]
+        resp.at("//*[@id='offset']")[:value].to_i
+      else
+        script = resp.at("//script").text
+        script.match(/\("offset"\)\.value = '(\d+)'/)[1].to_i
+      end
+    end
+
+    def end_threads_and_wait
+      @threads.size.times { @queue << :END }
+      @queue.close
+      @threads.each(&:join)
+    end
+
+    def count
+      @count.to_i
+    end
+
+    def get_next_page(offset) # rubocop:disable Metrics/MethodLength
+      attempts = 0
+      begin
+        if initial_post
+          agent.post "#{URL}W11M0070/getAddList", search_type: "JIS", offset: offset
+        end
+      rescue StandardError => e
+        attempts += 1
+        if attempts < 5
+          sleep 2
+          retry
+        else
+          Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+        end
+      end
+    end
+
+    def save_doc(bib, url) # rubocop:disable Metrics/MethodLength
+      return unless bib
+
+      id = bib.docidentifier.find(&:primary).id
+      file = file id
+      @mutex.synchronize do
+        if @files.include?(file)
+          Util.warn "File #{file} already exists. Duplication URL: #{url}"
+        else
+          @files << file
+          File.write file, serialize(bib), encoding: "UTF-8"
+          index.add_or_update id, file
+        end
+      end
+    end
+
+    def index
+      @index ||= Relaton::Index.find_or_create :jis, file: INDEX_FILE
+    end
+
+    def file(id)
+      name = id.gsub(/[:\/\s]/, "_")
+      File.join @output, "#{name}.#{@ext}"
+    end
+
+    def serialize(bib)
+      case @format
+      when "yaml" then bib.to_hash.to_yaml
+      when "xml" then bib.to_xml bibdata: true
+      else bib.send "to_#{@format}"
+      end
+    end
+  end
+end

data/lib/relaton_jis/hash_converter.rb

@@ -1,6 +1,6 @@
 module RelatonJis
   module HashConverter
-    include RelatonBib::HashConverter
+    include RelatonIsoBib::HashConverter
     extend self
 
     # @param item_hash [Hash]

data/lib/relaton_jis/hit.rb

@@ -8,9 +8,9 @@ module RelatonJis
     #
     # @return [RelatonJis::Hit] new hit
     #
-    def self.create(node, collection)
-      a = node.at("./a")
-      hit = { id: a.at("./text()").text.strip, url: a["href"] }
+    def self.create(hit, collection)
+      # a = node.at("./a")
+      # hit = { id: a.at("./text()").text.strip, url: a["href"] }
       new hit, collection
     end
 
@@ -44,7 +44,14 @@ module RelatonJis
     end
 
     def fetch
-      @fetch ||= Scraper.new(hit[:url]).fetch
+      return @fetch if defined? @fetch
+
+      # @fetch = Scraper.new(hit[:url]).fetch
+      yaml = Mechanize.new.get("#{Bibliography::GH_URL}#{hit[:file]}").body
+      hash = YAML.safe_load yaml
+      @fetch = RelatonJis::BibliographicItem.from_hash hash
+      @fetch.fetched = Date.today.to_s
+      @fetch
     end
   end
 end

data/lib/relaton_jis/processor.rb

@@ -9,6 +9,7 @@ module RelatonJis
       @prefix = "JIS"
       @defaultprefix = %r{^(JIS|TR)\s}
       @idtype = "JIS"
+      @datasets = %w[jis-webdesk]
     end
 
     # @param code [String]
@@ -19,6 +20,18 @@ module RelatonJis
       ::RelatonJis::Bibliography.get(code, date, opts)
     end
 
+    #
+    # Fetch all the docukents from a source
+    #
+    # @param [String] _source source name
+    # @param [Hash] opts
+    # @option opts [String] :output directory to output documents
+    # @option opts [String] :format
+    #
+    def fetch_data(_source, opts)
+      DataFetcher.fetch(**opts)
+    end
+
     # @param xml [String]
     # @return [RelatonJis::BibliographicItem]
     def from_xml(xml)
@@ -37,5 +50,12 @@ module RelatonJis
     def grammar_hash
       @grammar_hash ||= ::RelatonJis.grammar_hash
     end
+
+    #
+    # Remove index file
+    #
+    def remove_index_file
+      Relaton::Index.find_or_create(:jis, url: true, file: DataFetcher::INDEX_FILE).remove_file
+    end
   end
 end

data/lib/relaton_jis/scraper.rb

@@ -1,7 +1,9 @@
+# encoding: UTF-8
+
 module RelatonJis
   class Scraper
     ATTRS = %i[
-      fetched title link abstract docid docnumber date type language script
+      title link abstract docid docnumber date type language script
       docstatus doctype ics contributor editorialgroup structuredidentifier
     ].freeze
 
@@ -24,9 +26,9 @@ module RelatonJis
       BibliographicItem.new(**attrs)
     end
 
-    def fetch_fetched
-      Date.today.to_s
-    end
+    # def fetch_fetched
+    #   Date.today.to_s
+    # end
 
     def fetch_title
       { "ja" => "Jpan", "en" => "Lant" }.map.with_index do |(lang, script), i|

data/lib/relaton_jis/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RelatonJis
-  VERSION = "1.19.1"
+  VERSION = "1.19.2"
 end

data/lib/relaton_jis.rb

@@ -2,6 +2,7 @@
 
 require "mechanize"
 require "relaton_iso_bib"
+require "relaton/index"
 require_relative "relaton_jis/version"
 require_relative "relaton_jis/util"
 require_relative "relaton_jis/document_type"
@@ -12,6 +13,7 @@ require_relative "relaton_jis/scraper"
 require_relative "relaton_jis/bibliography"
 require_relative "relaton_jis/hit_collection"
 require_relative "relaton_jis/hit"
+require_relative "relaton_jis/data_fetcher"
 
 module RelatonJis
   class Error < StandardError; end

data/relaton_jis.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   spec.add_dependency "mechanize", "~> 2.10"
+  spec.add_dependency "relaton-index", "~> 0.2.0"
   spec.add_dependency "relaton-iso-bib", "~> 1.19.0"
 
   # For more information and examples about making a new gem, check out our

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-jis
 version: !ruby/object:Gem::Version
-  version: 1.19.1
+  version: 1.19.2
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-07-26 00:00:00.000000000 Z
+date: 2024-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: relaton-index
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: relaton-iso-bib
   requirement: !ruby/object:Gem::Requirement
@@ -60,6 +74,7 @@ files:
 - lib/relaton_jis.rb
 - lib/relaton_jis/bibliographic_item.rb
 - lib/relaton_jis/bibliography.rb
+- lib/relaton_jis/data_fetcher.rb
 - lib/relaton_jis/document_type.rb
 - lib/relaton_jis/hash_converter.rb
 - lib/relaton_jis/hit.rb