html_entry 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7d159801477390ab4b8934b0fa0d8db6238ef931a4e3090cbae0e5d735199966
+  data.tar.gz: a1a7f1209bbd12744f5ebe2fd54e3bd3d3fd43c556d309fde9a5607c5566cdaf
+SHA512:
+  metadata.gz: da0a2352cc43bf9bcb5339cb399303575f85eb9e9bda15aad1db69d20563c903e8bd2f11204741ee65472555e62db160f178c5bcf5280e6decbe5bd2d229ee36
+  data.tar.gz: ec7290fbf672fb50459e03de7b0156ab271092a0f194ad5b87fb51f75eef1ccd25e0d630d7f977f5e28d78e661a4fb94a837daebe0ff0268d49b2a9586dcf989

data/lib/html_entry/error.rb

@@ -0,0 +1,4 @@
+module HtmlEntry
+  class Error < StandardError
+  end
+end

data/lib/html_entry/page/entity_fetcher.rb

@@ -0,0 +1,198 @@
+require 'nokogiri'
+require_relative '../error'
+require_relative 'values_collector'
+require_relative '../page'
+
+module HtmlEntry
+  module Page
+    ##
+    # This entity-html_entry class designed for reading data from HTML/XML block
+    # according to instructions
+    #
+    # @see tests/html_entry/page/test_entity_fetcher.rb
+    #
+    class EntityFetcher
+      ##
+      # Cache fetched XML elements
+      # @type [Hash]
+
+      @selector_cache = {}
+
+      # Get instructions
+      #
+      # @return [Array]
+      #
+      attr_reader :instructions
+
+      ##
+      # Init
+      #
+      def initialize
+        @selector_cache ||= {}
+      end
+
+      ##
+      # Set instructions
+      #
+      # Example for reading simple text by CSS selector:
+      # {
+      #   :name1 => {
+      #     :type     => :instruction,
+      #     :selector => '.test-block a.deep-in',
+      #   }
+      # }
+      # There are filters allowed for type :instruction :
+      # - :node_text, returns XML of found node
+      # - :node, returns object Nokogiri::XML::Element of found node
+      # - :no_strip, returns non-stripped text
+      # - by default it use .strip for found text
+      # Example for calculating instruction according to fetch fields:
+      # {
+      #   :vote_up   => {
+      #     :type     => :instruction,
+      #     :selector => '.vote-up',
+      #   },
+      #     :vote_down => {
+      #     :type     => :instruction,
+      #     :selector => '.vote-down',
+      #   },
+      #     :vote_diff => {
+      #     :type     => :function,
+      #     :function => Proc.new { |info, name, document, instruction|
+      #       info[:vote_up].to_i - info[:vote_down].to_i
+      #     },
+      #   }
+      # }
+      #
+      # @param [Array] instructions
+      # @return [self]
+
+      def instructions=(instructions)
+        instructions = [instructions] unless instructions.instance_of? Array
+
+        @instructions = instructions
+      end
+
+      ##
+      # Fetch data from document
+      #
+      # @param [Nokogiri::HTML::Document, Nokogiri::XML::Element] document
+      # @param [TrueClass, FalseClass] plenty Get plenty of elements
+      #                                       or the only one
+      # @return [Hash, Array]
+
+      def fetch(document:, plenty: false)
+        if plenty
+          fetch_plenty(document)
+        else
+          fetch_single(document)
+        end
+      end
+
+      ##
+      # Fetch single data from document
+      #
+      # @param [Nokogiri::HTML::Document, Nokogiri::XML::Element] document
+      # @return [Hash]
+
+      def fetch_single(document)
+        collector = get_values_collector(document)
+
+        instructions.each do |instruction|
+          node = Page.fetch_node(document, instruction)
+
+          next unless instruction[:data]
+          instruction[:data].each do |name, data_instruction|
+            collector.fetch name, data_instruction, node
+          end
+        end
+
+        collector.data
+      end
+
+      ##
+      # Get value collector
+      #
+      # @param [Nokogiri::HTML::Document, Nokogiri::XML::Element] document
+      # @return [Page::ValuesCollector]
+
+      def get_values_collector(document)
+        Page::ValuesCollector.new document:     document,
+                                  instructions: instructions
+      end
+
+      ##
+      # Fetch collection data from document
+      #
+      # @param [Nokogiri::HTML::Document, Nokogiri::XML::Element] document
+      # @return [Hash]
+      def fetch_plenty(document)
+        unless instructions.instance_of? Array
+          raise 'Instructions must be an array.'
+        end
+
+        collectors, data = process_instructions(document)
+
+        collectors.each do |_i, collector|
+          # @type [HtmlEntry::Page::ValuesCollector] collector
+          data.push collector.data
+        end
+
+        data
+      end
+
+      protected
+
+      def process_instructions(document)
+        data = []
+        # @type [HtmlEntry::Page::ValuesCollector[]] collectors
+        collectors = {}
+        instructions.each do |instruction|
+          raise 'Instruction must be Hash.' unless instruction.instance_of? Hash
+          nodes = retrieve_nodes(document, instruction)
+          nodes.each_with_index do |node, i|
+            process_node(document, node, instruction, collectors, i)
+          end
+        end
+        [collectors, data]
+      end
+
+      # @param [Array, Nil] instruction
+      def retrieve_nodes(document, instruction)
+        nodes = Page.fetch_nodes(document, instruction)
+        if nodes.nil? || instruction[:allow_empty] && nodes.count.zero?
+          nodes = [nil]
+        end
+        nodes
+      end
+
+      def process_node(document, node, instruction, collectors, index)
+        if instruction[:merge]
+          # gather items under the same collector
+          index = 0
+        end
+
+        unless collectors.key? index
+          collectors[index] = get_values_collector(document)
+        end
+
+        return unless instruction[:data]
+
+        instruction[:data].each do |name, data_instruction|
+          collectors[index].fetch name, data_instruction, node
+        end
+      end
+
+      ##
+      # Check if merge nodes data must disabled
+      #
+      def data_has_option?(instruction, option:, value:)
+        return false if instruction.key :merge
+
+        !instruction[:data].select! do |_k, el|
+          (el.is_a?(Hash) && (el[option] == value))
+        end.nil?
+      end
+    end
+  end
+end

data/lib/html_entry/page/values_collector.rb

@@ -0,0 +1,172 @@
+require 'nokogiri'
+require_relative '../error'
+
+module HtmlEntry
+  module Page
+    ##
+    # This class responsible for getting values according to an instruction
+    #
+    # @see tests/html_entry/page/test_entity_fetcher.rb
+    #
+    class ValuesCollector
+      ##
+      # Extra options
+      #
+      # @type [Hash]
+      #
+      @options = {}
+
+      ##
+      # Collected data
+      #
+      # @type [Hash]
+      #
+      @data = {}
+
+      attr_reader :data
+
+      def initialize(options = {})
+        @options = options
+        @data    = {}
+      end
+
+      ##
+      # Fetch value of element
+      #
+      # @param [Symbol] name
+      # @param [Hash] instruction
+      # @param [Nokogiri::XML::Element] node
+      # @return [String, Nokogiri::XML::Element]
+
+      def fetch(name, instruction, node)
+        if node && (instruction[:type] == :attribute)
+          value = get_node_attribute(
+            node,
+            instruction
+          )
+        elsif instruction[:type] == :function
+          value = call_function(name, instruction)
+        elsif instruction[:type] == :boolean || instruction[:type] == :bool
+          value = !!node
+        elsif node && instruction[:type] == :children
+          value = children(
+            name:        name,
+            instruction: instruction,
+            node:        node,
+            plenty:      if instruction[:children_plenty].nil?
+                           true
+                         else
+                           instruction[:children_plenty]
+                         end
+          )
+        elsif node && (instruction[:type] == :value || instruction[:type].nil?)
+          # empty type should be determined as :value
+          value = node
+        elsif instruction.is_a?(Hash) && !instruction[:default].nil?
+          value = instruction[:default]
+        elsif node.nil?
+          value = nil
+        else
+          raise HtmlEntry::Error, 'Unknown instruction type or XML/HTML value not found.'
+        end
+
+        value = filter_value(value, instruction)
+        if data[name].instance_of?(Array) && value.instance_of?(Array)
+          data[name] = [data[name], value].flatten
+        else
+          unless data[name].nil? && (!instruction[:overwrite])
+            raise "Value already set for data key name '#{name}'."
+          end
+          data[name] = value
+        end
+
+        data[name]
+      end
+
+      protected
+
+      ##
+      # Fetch value of element
+      #
+      # @param [Symbol] name
+      # @param [Hash] instruction
+      # @param [Nokogiri::XML::Element] node
+      # @return [Hash, Array]
+      #
+      def children(instruction:, node:, plenty: nil, name: nil)
+        instruction = if instruction[:instructions] == :the_same
+                        @options[:instructions]
+                      else
+                        instruction[:instructions]
+                      end
+
+        fetcher              = Page::EntityFetcher.new
+        fetcher.instructions = instruction
+        fetcher.fetch document: node,
+                      plenty:   plenty.nil? ? true : plenty
+      end
+
+      ##
+      # Filter value
+      #
+      # @param [Nokogiri::XML::Element] value
+      # @param [Hash] instruction
+      # @return [String, Nokogiri::XML::Element]
+      #
+      def filter_value(value, instruction)
+        filter(value, instruction[:filter])
+      end
+
+      ##
+      # Filter fetched value
+      #
+      # @param [Nokogiri::XML::Element] value
+      # @param [Symbol] filter
+      # @return [String, Nokogiri::XML::Element]
+      #
+      def filter(value, filter = nil)
+        # return as is, :filter can be omitted in instruction
+        return value if filter == :element
+
+        # return text with tags
+        return value.to_s.strip if filter == :node_text
+
+        # return text without tags by default
+        value = value.text if value.instance_of?(Nokogiri::XML::Element)
+
+        # return integer
+        return value.to_i if filter == :to_i
+
+        # return non-stripped text
+        return value.to_s if filter == :no_strip
+
+        return value.strip if value.is_a? String
+
+        value
+      end
+
+      ##
+      # @param [Nokogiri::XML::Element] node
+      # @param [Hash] instruction
+      # @return [String]
+
+      def get_node_attribute(node, instruction)
+        node[instruction[:attribute]]
+      end
+
+      ##
+      # Call custom function
+      #
+      # @param [Hash] instruction
+      # @return [*]
+
+      def call_function(name, instruction)
+        if instruction[:function].instance_of? Proc
+          instruction[:function].call name, instruction, data, @options
+        else
+          HtmlEntry::Error.new ':function is not instance of Proc'
+        end
+      end
+    end
+  end
+end

data/lib/html_entry/page.rb

@@ -0,0 +1,44 @@
+module HtmlEntry
+  ##
+  # Page module
+  #
+  module Page
+    ##
+    # Get node by XPath or CSS selector
+    #
+    # @param [Nokogiri::HTML::Document] document
+    # @param [Hash] instruction
+    # @return [Nokogiri::XML::Element]
+
+    def fetch_node(document, instruction)
+      nodes = fetch_nodes(document, instruction)
+      nodes.first if nodes
+    end
+
+    ##
+    # Get nodes by XPath or CSS selector
+    #
+    # @param [Nokogiri::HTML::Document|Nokogiri::XML::Element] document
+    # @param [Hash] instruction
+    # @return [Nokogiri::XML::NodeSet]
+
+    def fetch_nodes(document, instruction)
+      unless document.instance_of?(Nokogiri::HTML::Document) || document.instance_of?(Nokogiri::XML::Element)
+        raise '"document" must be an instance of Nokogiri::HTML::Document.'
+      end
+      if instruction[:selector]
+        document.css(instruction[:selector])
+      elsif instruction[:css]
+        document.css(instruction[:css])
+      elsif instruction[:xpath]
+        if defined? document.xpath
+          document.xpath(instruction[:xpath])
+        else
+          raise 'Cannot use this document.'
+        end
+      end
+    end
+
+    module_function :fetch_nodes, :fetch_node
+  end
+end

data/lib/html_entry/page_fetcher.rb

@@ -0,0 +1,109 @@
+require_relative 'page/entity_fetcher'
+
+module HtmlEntry
+  ##
+  # Page fetcher
+  #
+  class PageFetcher
+    ##
+    # Set instructions
+    #
+    # @param [Hash] instructions
+    # @return [self]
+
+    attr_writer :instructions
+
+    # Get instructions
+    #
+    # @return [Hash]
+
+    attr_reader :instructions
+
+    # Fetch entities from document
+    #
+    # @param [Nokogiri::HTML::Document] document
+    # @return [Hash]
+
+    def fetch(document)
+      items = []
+      if instructions[:block].nil?
+        # "block" instructions is not defined
+        block_document = if document.instance_of?(Nokogiri::HTML::Document)
+                           fetch_block_document(
+                               document,
+                               type:     :selector,
+                               selector: 'body'
+                           ).first
+                         else
+                           document
+                         end
+
+        fetch_data(block_document, instructions[:entity]).each do |element|
+          items.push element
+        end
+      else
+        # fetch each "block" and process entities
+        fetch_block_document(document, instructions[:block]).each do |block_document|
+          fetch_data(block_document, instructions[:entity]).each do |element|
+            items.push element
+          end
+        end
+      end
+      items
+    end
+
+    ##
+    # Check if it's a last page
+    #
+    # @param [Nokogiri::HTML::Document] document
+    # @return [TrueClass, FalseClass]
+
+    def last_page?(document)
+      if instructions[:last_page][:type] == :function
+        !!call_function(document, instructions[:last_page])
+      else
+        Page.fetch_nodes(document, instructions[:last_page]).count > 0
+      end
+    end
+
+    protected
+
+    ##
+    # Fetch entity data
+    #
+    # @param [Nokogiri::XML::Element] entity_document
+    # @param [Hash] instructions
+    # @return [Hash]
+
+    def fetch_data(entity_document, instructions)
+      fetcher              = Page::EntityFetcher.new
+      fetcher.instructions = instructions
+      fetcher.fetch(document: entity_document, plenty: true)
+    end
+
+    ##
+    # Fetch entities on a page
+    #
+    # @param [Nokogiri::HTML::Document] document
+    # @return [Nokogiri::XML::NodeSet]
+
+    def fetch_block_document(document, instructions)
+      raise 'Instructions are not set.' if instructions.nil?
+
+      return call_function(document, instructions) if instructions[:type] == :function
+
+      Page.fetch_nodes(document, instructions)
+    end
+
+    ##
+    # Call custom function
+    #
+    # @param [Nokogiri::HTML::Document] document
+    # @param [Hash] instruction
+    # @return [*]
+
+    def call_function(document, instruction)
+      instruction[:function].call document, instruction
+    end
+  end
+end

data/lib/html_entry/version.rb

@@ -0,0 +1,3 @@
+module HtmlEntry
+  VERSION = '0.1.0'.freeze
+end

data/lib/html_entry.rb

@@ -0,0 +1,6 @@
+require 'html_entry/version'
+
+##
+# Html entry top module
+module HtmlEntry
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: html_entry
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kirby Rs
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-08-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: |-
+  A simple gem which allows to organize fetching entries \
+      from plain HTML.
+email: bizkirby@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/html_entry.rb
+- lib/html_entry/error.rb
+- lib/html_entry/page.rb
+- lib/html_entry/page/entity_fetcher.rb
+- lib/html_entry/page/values_collector.rb
+- lib/html_entry/page_fetcher.rb
+- lib/html_entry/version.rb
+homepage: https://github.com/rikby/html-entries
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: HTML entries fetcher
+test_files: []