relaton-core 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22d26ddd3ce683c8ec7a757cc3f164863454f9051f35960708cc5490005cda3b
-  data.tar.gz: f19cec52c8ebaa1bc7ff526f024d43dd59e3dfb6ffd864a2b386c885a796af8e
+  metadata.gz: c165688976f3768ae55ababdba7bfdccd0e3fb3114c4793fe4497d74e88970da
+  data.tar.gz: 12f685b833874257bf04d87bf2a74375e2c72681a8861712b37c7cc52f513659
 SHA512:
-  metadata.gz: 5d568bd089a08098dae0a461f4d3e53c26048755d60a40348438851285d11f0b28966e27003a019816db68453294dffe9609d3bb6b84ee1115cc1e2f86b4f15a
-  data.tar.gz: 15063d385e121622a436d1cfab3b0918ff8b0c75259467249c2ecc34ed53cd1200fc9666d0f50a5e2beb31d766c4b27d0568419942e02e9041716cf64e09a368
+  metadata.gz: 3cceb7dc3f6dbd33cb08851b63b4d78239d61775e1e91fbd14330549693305b575b966d7b8bf4a66397317308638dde39b61e4eaa5ed9567ce3b0466bb618bbb
+  data.tar.gz: 6b5d18e51d50c2e1a5d457de051c32b5dcbb7768d809fb796d67efb8c811a0a5b2eed96874ae8f8d12961cbf347a90ca54e695a53c410444f013fda0f231b9a9

data/lib/relaton/core/data_fetcher.rb

@@ -1,6 +1,6 @@
 module Relaton::Core
   class DataFetcher
-    attr_accessor :docs
+    # attr_accessor :docs
     #
     # Initialize fetcher
     #
@@ -11,14 +11,9 @@ module Relaton::Core
       @output = output
       @format = format
       @ext = format.sub "bibxml", "xml"
-      @files = []
-      @docs = []
-    end
-
-    def index
-      @index ||= Relaton::Index.find_or_create self.class::INDEX_TYPE,
-                                               file: self.class::INDEX_FILE,
-                                               pubid_class: self.class.get_identifier_class
+      @files = Set.new
+      # @docs = []
+      @errors = Hash.new(true)
     end
 
     # API method for external service
@@ -32,58 +27,35 @@ module Relaton::Core
       puts "Done in: #{(t2 - t1).round} sec."
     end
 
-    def self.get_identifier_class
-      raise NotImplementedError, "#{self.class}#get_identifier_class method must be implemented"
-    end
-
     def fetch
-      fetch_docs ACTIVE_PUBS_URL
-      fetch_docs OBSOLETE_PUBS_URL, retired: true
-      index.save
+      raise NotImplementedError, "#{self.class}#fetch method must be implemented"
     end
 
-    # Parse hash and return RelatonBib
-    # @param [Hash] doc document data
-    # @return [RelatonBib]
-    def parse(doc)
-      raise NotImplementedError, "#{self.class}#parse method must be implemented"
+    def gh_issue
+      return @gh_issue if defined? @gh_issue
+
+      @gh_issue = Relaton::Logger::Channels::GhIssue.new(*gh_issue_channel)
+      Relaton.logger_pool[:gh_issue] = Relaton::Logger::Log.new(@gh_issue, levels: [:error])
+      @gh_issue
     end
 
-    # @param [RelatonBib::BibliographicItem] bib
-    # @return [String] filename based on PubID identifier
-    def get_output_file(bib)
-      File.join @output, "#{bib.docidentifier.first.id.gsub(/[.\s-]+/, '-')}.#{@ext}"
+    def gh_issue_channel
+      raise NotImplementedError, "#{self.class}#gh_issue_channel method must be implemented"
     end
 
-    #
-    # Parse document and save to file
-    #
-    # @param [Hash] doc document data
-    # @param [Boolean] retired if true then document is retired
-    #
-    # @return [void]
-    #
-    def parse_and_save(doc)
-      bibitem = parse(doc)
-      save_bib(bibitem)
-      index_add_or_update(bibitem)
+    def repot_errors
+      @errors.select { |_, v| v }.each_key { |k| log_error "Failed to fetch #{k}" }
+      gh_issue.create_issue
     end
 
-    #
-    # Save bibitem to file
-    #
-    # @param [RelatonBib::BibliographicItem] bib bibitem
-    #
-    # @return [void]
-    #
-    def save_bib(bib)
-      file = get_output_file(bib)
-      File.write file, serialize(bib), encoding: "UTF-8"
+    def log_error(_msg)
+      raise NoMatchingPatternError, "#{self.class}#log_error method must be implemented"
     end
 
-    def index_add_or_update(bib)
-      index.add_or_update self.class.get_identifier_class.parse(bib.docidentifier.first.id),
-                          get_output_file(bib)
+    # @param [String] document ID
+    # @return [String] filename based on PubID identifier
+    def output_file(docid)
+      File.join @output, "#{docid.downcase.gsub(/[.\s\/-]+/, '-')}.#{@ext}"
     end
 
     #
@@ -94,11 +66,19 @@ module Relaton::Core
     # @return [String] serialized bibliographic item
     #
     def serialize(bib)
-      case @format
-      when "yaml" then bib.to_hash.to_yaml
-      when "xml" then bib.to_xml(bibdata: true)
-      else bib.send "to_#{@format}"
-      end
+      send "to_#{@format}", bib
+    end
+
+    def to_yaml(bib)
+      raise NotImplementedError, "#{self.class}#to_yaml method must be implemented"
+    end
+
+    def to_xml(bib)
+      raise NotImplementedError, "#{self.class}#to_xml method must be implemented"
+    end
+
+    def to_bibxml(bib)
+      raise NotImplementedError, "#{self.class}#to_bibxml method must be implemented"
     end
   end
 end

data/lib/relaton/core/hit.rb

@@ -0,0 +1,55 @@
+require "weakref"
+
+module Relaton
+  module Core
+    class Hit
+      # @return [RelatonBib::HitCollection]
+      attr_accessor :hit_collection
+
+      # @return [Array<Hash>]
+      attr_reader :hit
+
+      # @param hit [Hash]
+      # @param hit_collection [RelatonBib::HitCollection]
+      def initialize(hit, hit_collection = nil)
+        @hit            = hit
+        @hit_collection = WeakRef.new hit_collection if hit_collection
+      end
+
+      # @return [String]
+      def to_s
+        inspect
+      end
+
+      # @return [String]
+      def inspect
+        "<#{self.class}:#{format('%<id>#.14x', id: object_id << 1)} " \
+          "@text=\"#{@hit_collection&.text}\" " \
+          "@fetched=\"#{!@fetch.nil?}\" " \
+          "@fullIdentifier=\"#{@fetch&.shortref(nil)}\" " \
+          "@title=\"#{@hit[:code]}\">"
+      end
+
+      # @return [RelatonBib::ItemData]
+      def item
+        raise "Not implemented"
+      end
+
+      # @param opts [Hash]
+      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Boolean] :bibdata
+      # @option opts [String, Symbol] :lang language
+      # @return [String] XML
+      # def to_xml(**opts)
+      #   if opts[:builder]
+      #     fetch.to_xml(**opts)
+      #   else
+      #     builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
+      #       fetch.to_xml(**opts.merge(builder: xml))
+      #     end
+      #     builder.doc.root.to_xml
+      #   end
+      # end
+    end
+  end
+end

data/lib/relaton/core/hit_collection.rb

@@ -0,0 +1,106 @@
+require "forwardable"
+require_relative "hit"
+
+module Relaton
+  module Core
+    class HitCollection
+      extend Forwardable
+
+      def_delegators :@array, :<<, :[], :first, :empty?, :any?, :size, :each, :each_slice, :reduce, :map
+
+      # @return [TrueClass, FalseClass]
+      attr_reader :fetched
+
+      # @return [String]
+      attr_reader :ref
+
+      # @return [String]
+      attr_reader :year
+
+      #
+      # @param ref [String, Pubid] reference to search
+      # @param year [String, nil] year of publication
+      #
+      def initialize(ref, year = nil)
+        @array = []
+        @ref = ref
+        @year = year
+        @fetched = false
+      end
+
+      #
+      # Fetches hits from the data source
+      #
+      # @return [self] self object
+      #
+      def fetch
+        workers = WorkersPool.new 4
+        workers.worker(&:item)
+        each do |hit|
+          workers << hit
+        end
+        workers.end
+        workers.result
+        @fetched = true
+        self
+      end
+
+      #
+      # Renders the collection as XML
+      #
+      # @param opts [Hash] options
+      # @option opts [Nokogiri::XML::Builder] :builder XML builder
+      # @option opts [Boolean] :bibdata render bibdata if true
+      # @option opts [String, Symbol] :lang language
+      #
+      # @return [String] XML representation of the collection
+      #
+      # def to_xml(**opts)
+      #   builder = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |xml|
+      #     xml.documents do
+      #       @array.each do |hit|
+      #         hit.fetch
+      #         hit.to_xml(**opts.merge(builder: xml))
+      #       end
+      #     end
+      #   end
+      #   builder.to_xml
+      # end
+
+      #
+      # Selects matching hits and returns a new collection
+      #
+      # @param [Proc] &block proc to select hits
+      #
+      # @return [RelatonBib::HitCollection] new hit collection
+      #
+      def select!(&block)
+        @array.select!(&block)
+        self
+      end
+
+      def reduce!(sum, &block)
+        @array = @array.reduce sum, &block
+        self
+      end
+
+      #
+      # Returns String representation of the collection
+      #
+      # @return [String] String representation of the collection
+      #
+      def to_s
+        inspect
+      end
+
+      #
+      # Returns String representation of the collection
+      #
+      # @return [String] String representation of the collection
+      #
+      def inspect
+        "<#{self.class}:#{format('%#.14x', object_id << 1)} @ref=#{@ref} @fetched=#{@fetched}>"
+      end
+    end
+  end
+end

data/lib/relaton/core/processor.rb

@@ -0,0 +1,48 @@
+module Relaton
+  module Core
+    class Processor
+      # @rerurn [Symbol]
+      attr_reader :short
+
+      # @return [String]
+      attr_reader :prefix, :idtype
+
+      # @return [Regexp]
+      attr_reader :defaultprefix
+
+      # @return [Array<String>]
+      attr_reader :datasets
+
+      def initialize
+        raise "This is an abstract class!"
+      end
+
+      def get(_code, _date, _opts)
+        raise "This is an abstract class!"
+      end
+
+      def fetch_data(_source, _opts)
+        raise "This is an abstract class!"
+      end
+
+      def from_xml(_xml)
+        raise "This is an abstract class!"
+      end
+
+      def from_yaml(_hash)
+        raise "This is an abstract class!"
+      end
+
+      def grammar_hash
+        raise "This is an abstract class!"
+      end
+
+      # Retuns default number of workers. Should be overraded by childred classes if need.
+      #
+      # @return [Integer] nuber of wokrers
+      def threads
+        10
+      end
+    end
+  end
+end

data/lib/relaton/core/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module Core
-    VERSION = "0.0.3".freeze
+    VERSION = "0.0.4".freeze
   end
 end

data/lib/relaton/core/workers_pool.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Core
+    # Workers poll.
+    class WorkersPool
+      attr_accessor :nb_hits
+
+      def initialize(num_workers = 2)
+        @num_workers = num_workers < 2 ? 2 : num_workers
+        @queue = SizedQueue.new(num_workers * 2)
+        @result = []
+        @nb_hits = 0
+      end
+
+      def worker(&block)
+        @threads = Array.new @num_workers do
+          Thread.new do
+            until (item = @queue.pop) == :END
+              @result << yield(item) if block
+            end
+          end
+        end
+      end
+
+      def result
+        @threads.each(&:join)
+        @result
+      end
+
+      def <<(item)
+        @queue << item
+        self
+      end
+
+      def end
+        @num_workers.times { @queue << :END }
+      end
+
+      def size
+        @result.size
+      end
+    end
+  end
+end

data/lib/relaton/core.rb

@@ -1,3 +1,6 @@
-require "relaton_bib"
-require "relaton/index"
+require "relaton/logger"
+require_relative "core/version"
+require_relative "core/processor"
 require_relative "core/data_fetcher"
+require_relative "core/hit_collection"
+require_relative "core/workers_pool"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton-core
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-01-17 00:00:00.000000000 Z
+date: 2025-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -39,47 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: relaton-bib
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.20.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.20.0
-- !ruby/object:Gem::Dependency
-  name: relaton-index
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.2.16
-  type: :runtime
+        version: 0.21.2
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.16
+        version: 0.21.2
 - !ruby/object:Gem::Dependency
-  name: pubid-core
+  name: relaton-logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.12.10
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.12.10
+        version: 0.2.0
 description: Library for importing and caching bibliographic references to technical
   standards
 email:
@@ -95,7 +81,11 @@ files:
 - lib/relaton-core.rb
 - lib/relaton/core.rb
 - lib/relaton/core/data_fetcher.rb
+- lib/relaton/core/hit.rb
+- lib/relaton/core/hit_collection.rb
+- lib/relaton/core/processor.rb
 - lib/relaton/core/version.rb
+- lib/relaton/core/workers_pool.rb
 homepage: https://github.com/relaton/relaton-core
 licenses:
 - BSD-2-Clause
@@ -108,14 +98,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.27
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Library for importing and caching bibliographic references to technical standards