aranha-parsers 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5685f5cd07ae6f7ff6adb9f03c7ebd286a0279f508157706460617feb377937e
+  data.tar.gz: 3e02eb80a205bd5b26397728d6a680a3ac4bf65afdb9cd45a3e831e75b5193e2
+SHA512:
+  metadata.gz: 468f57861a6182901b9aca6361115ae0e9bbdb192e91f9c10d34b99e4af11338b3f727a17734a70c85493495ddde7f633078560d3f52e0b4677a013fc58cd234
+  data.tar.gz: 56fafe12e79122cc78dafc92d8bde7721b3f1daad69ca959fc36c58e7c314af326f8cf017516ce102fb3109a1c652c86102a2d37859192849f39b104ba3037e9

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/lib/aranha/parsers.rb

@@ -0,0 +1,9 @@
+module Aranha
+  module Parsers
+    require 'aranha/parsers/base'
+    require 'aranha/parsers/html'
+    require 'aranha/parsers/invalid_state_exception'
+    require 'aranha/parsers/source_address'
+    require 'aranha/parsers/version'
+  end
+end

data/lib/aranha/parsers/base.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'fileutils'
+require 'aranha/parsers/source_address'
+
+module Aranha
+  module Parsers
+    class Base
+      LOG_DIR_ENVVAR = 'ARANHA_PARSERS_LOG_DIR'
+
+      attr_reader :source_address
+
+      def initialize(url)
+        @source_address = ::Aranha::Parsers::SourceAddress.new(url)
+        log_content(source_address.serialize, '-source-address')
+      end
+
+      delegate :url, to: :source_address
+
+      def content
+        s = source_address.content
+        log_content(s)
+        s
+      end
+
+      private
+
+      def log_content(content, suffix = '')
+        path = log_file(suffix)
+
+        return unless path
+        File.open(path, 'wb') { |file| file.write(content) }
+      end
+
+      def log_file(suffix)
+        dir = log_parsers_dir
+        return nil unless dir
+        f = ::File.join(dir, "#{self.class.name.parameterize}#{suffix}.log")
+        FileUtils.mkdir_p(File.dirname(f))
+        f
+      end
+
+      def log_parsers_dir
+        return ENV[LOG_DIR_ENVVAR] if ENV[LOG_DIR_ENVVAR]
+        return ::Rails.root.join('log', 'parsers') if rails_root_exist?
+        nil
+      end
+
+      def rails_root_exist?
+        ::Rails.root
+        true
+      rescue NameError
+        return false
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+module Aranha
+  module Parsers
+    module Html
+      require 'aranha/parsers/html/base'
+      require 'aranha/parsers/html/item'
+      require 'aranha/parsers/html/item_list'
+      require 'aranha/parsers/html/node'
+    end
+  end
+end

data/lib/aranha/parsers/html/base.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'aranha/parsers/base'
+require 'aranha/parsers/html/node/default'
+
+module Aranha
+  module Parsers
+    module Html
+      class Base < ::Aranha::Parsers::Base
+        class << self
+          def fields
+            @fields ||= []
+            @fields.dup
+          end
+
+          def field(name, type, xpath)
+            @fields ||= []
+            @fields << Field.new(name, type, xpath)
+          end
+
+          Field = Struct.new(:name, :type, :xpath)
+        end
+
+        def nokogiri
+          @nokogiri ||= Nokogiri::HTML(content, &:noblanks)
+        end
+
+        protected
+
+        def node_parser_class
+          ::Aranha::Parsers::Html::Node::Default
+        end
+
+        private
+
+        def node_parser
+          @node_parser ||= node_parser_class.new(fields)
+        end
+
+        def fields
+          self.class.fields.map { |f| [f.name, f.type, f.xpath] }
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/item.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/base'
+
+module Aranha
+  module Parsers
+    module Html
+      class Item < Base
+        def data
+          @data ||= node_parser.parse(item_node)
+        end
+
+        def item_node
+          @item_node ||= begin
+            r = item_xpath ? nokogiri.at_xpath(item_xpath) : nokogiri
+            raise "Item node not found (Item xpath: #{item_xpath})" unless r
+            r
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/item_list.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/base'
+
+module Aranha
+  module Parsers
+    module Html
+      class ItemList < Base
+        def data
+          count = 0
+          @data ||= nokogiri.xpath(items_xpath).map do |m|
+            count += 1
+            node_parser.parse(m)
+          end
+        rescue StandardError => e
+          raise StandardError, "#{e.message} (Count: #{count})"
+        end
+
+        def items_xpath
+          raise "Class #{self.class} has no method \"item_xpath\". Implement it"
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        require 'aranha/parsers/html/node/base'
+        require 'aranha/parsers/html/node/default'
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/base.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Base
+          attr_reader :fields
+
+          def initialize(fields)
+            @fields = fields
+          end
+
+          def parse(node)
+            Hash[fields.map { |f| [f[0], parse_field(node, f[2], f[1])] }]
+          end
+
+          private
+
+          def parse_field(node, xpath, parser_method)
+            value_method = "#{parser_method}_value"
+            return send(value_method, node, xpath) if respond_to?(value_method)
+
+            raise "Method \"#{value_method}\" not found in #{self.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/html/node/default.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/html/node/base'
+
+module Aranha
+  module Parsers
+    module Html
+      module Node
+        class Default < ::Aranha::Parsers::Html::Node::Base
+          def string_value(node, xpath)
+            if node.at_xpath(xpath)
+              node.at_xpath(xpath).text.to_s.tr("\u00A0", ' ').strip
+            else
+              ''
+            end
+          end
+
+          def quoted_value(node, xpath)
+            s = string_value(node, xpath)
+            return '' unless s
+
+            m = /\"([^\"]+)\"/.match(s)
+            return m[1] if m
+
+            ''
+          end
+
+          def integer_value(node, xpath)
+            r = string_value(node, xpath)
+            return nil if r.blank?
+
+            m = /\d+/.match(r)
+            raise "Integer not found in \"#{r}\"" unless m
+
+            m[0].to_i
+          end
+
+          def integer_optional_value(node, xpath)
+            r = string_value(node, xpath)
+            m = /\d+/.match(r)
+            m ? m[0].to_i : nil
+          end
+
+          def float_value(node, xpath)
+            parse_float(node, xpath, true)
+          end
+
+          def float_optional_value(node, xpath)
+            parse_float(node, xpath, false)
+          end
+
+          def array_value(node, xpath)
+            r = node.xpath(xpath).map { |n| n.text.strip }
+            r.join('|')
+          end
+
+          def join_value(node, xpath)
+            m = ''
+            node.xpath(xpath).each do |n|
+              m << n.text.strip
+            end
+            m
+          end
+
+          def duration_value(node, xpath)
+            m = /(\d+) m/.match(join_value(node, xpath))
+            m ? m[1].to_i : nil
+          end
+
+          def regxep(node, xpath, pattern)
+            s = string_value(node, xpath)
+            m = pattern.match(s)
+            return m if m
+
+            raise "Pattern \"#{pattern}\" not found in string \"#{s}\""
+          end
+
+          private
+
+          def parse_float(node, xpath, required)
+            s = string_value(node, xpath)
+            m = /\d+(?:[\.\,](\d+))?/.match(s)
+            if m
+              m[0].sub(',', '.').to_f
+            elsif required
+              raise "Float value not found in \"#{s}\""
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/invalid_state_exception.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Parsers
+    class InvalidStateException < StandardError
+    end
+  end
+end

data/lib/aranha/parsers/source_address.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'active_support/core_ext/module/delegation'
+require 'aranha/parsers/source_address/hash_http_get'
+require 'aranha/parsers/source_address/hash_http_post'
+require 'aranha/parsers/source_address/http_get'
+require 'aranha/parsers/source_address/file'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class << self
+        SUBS = [
+          ::Aranha::Parsers::SourceAddress::HashHttpGet,
+          ::Aranha::Parsers::SourceAddress::HashHttpPost,
+          ::Aranha::Parsers::SourceAddress::HttpGet,
+          ::Aranha::Parsers::SourceAddress::File
+        ].freeze
+
+        def detect_sub(source)
+          return source.sub if source.is_a?(self)
+          SUBS.each do |sub|
+            return sub.new(source) if sub.valid_source?(source)
+          end
+          raise "No content fetcher found for source \"#{source}\""
+        end
+
+        def deserialize(string)
+          new(string =~ %r{\A[a-z]+://} ? string.strip : ::YAML.load(string))
+        end
+
+        def from_file(path)
+          deserialize(::File.read(path))
+        end
+      end
+
+      attr_reader :sub
+
+      def initialize(source)
+        @sub = self.class.detect_sub(source)
+      end
+
+      delegate :content, :url, to: :sub
+
+      def to_s
+        sub.url
+      end
+
+      def serialize
+        sub.serialize.strip + "\n"
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/file.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/source_address/http_get'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class File < ::Aranha::Parsers::SourceAddress::HttpGet
+        SCHEME = 'file://'
+
+        class << self
+          def valid_source?(source)
+            source.to_s.start_with?(SCHEME + '/', '/')
+          end
+        end
+
+        def initialize(source)
+          super source.to_s.gsub(/\A#{Regexp.quote(SCHEME)}/, '')
+        end
+
+        def url
+          "#{SCHEME}#{source}"
+        end
+
+        def content
+          ::File.open(source, &:read)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/hash_http_get.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/source_address/hash_http_post'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HashHttpGet < ::Aranha::Parsers::SourceAddress::HashHttpPost
+        class << self
+          def valid_source?(source)
+            source.is_a?(::Hash) &&
+              source.with_indifferent_access[:method].to_s.downcase.strip == 'get'
+          end
+        end
+
+        def content
+          HTTPClient.new.get_content(
+            source[:url],
+            source[:params]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/hash_http_post.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'httpclient'
+require 'yaml'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HashHttpPost
+        class << self
+          def valid_source?(source)
+            source.is_a?(::Hash) &&
+              source.with_indifferent_access[:method].to_s.downcase.strip == 'post'
+          end
+        end
+
+        attr_reader :source
+
+        def initialize(source)
+          @source = source.with_indifferent_access
+        end
+
+        def ==(other)
+          self.class == other.class && source == other.source
+        end
+
+        def url
+          source.fetch(:url)
+        end
+
+        def serialize
+          source.to_yaml
+        end
+
+        def content
+          HTTPClient.new.post_content(
+            source[:url],
+            source[:params].merge(follow_redirect: true)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'addressable'
+require 'net/http'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HttpGet
+        class << self
+          def location_uri(source_uri, location)
+            ::Addressable::URI.join(source_uri, location).to_s
+          end
+
+          def valid_source?(source)
+            source.to_s =~ %r{\Ahttps?://}
+          end
+        end
+
+        attr_reader :source
+
+        def initialize(source)
+          @source = source.to_s
+        end
+
+        def ==(other)
+          self.class == other.class && source == other.source
+        end
+
+        def url
+          source
+        end
+
+        def content
+          content_fetch(url)
+        end
+
+        def serialize
+          url
+        end
+
+        private
+
+        def content_fetch(uri, limit = 10)
+          raise 'too many HTTP redirects' if limit.zero?
+
+          response = Net::HTTP.get_response(URI(uri))
+
+          case response
+          when Net::HTTPSuccess then
+            response.body
+          when Net::HTTPRedirection then
+            content_fetch(self.class.location_uri(uri, response['location']), limit - 1)
+          else
+            response.value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/spec/source_target_fixtures.rb

@@ -0,0 +1,67 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Aranha
+  module Spec
+    # Lists pairs of source/target files in a directory.
+    class SourceTargetFixtures
+      class << self
+        def source_target_basename(file)
+          m = /^(.+)\.(?:source|target)(?:\..+)?$/.match(File.basename(file))
+          m ? m[1] : nil
+        end
+      end
+
+      attr_reader :fixtures_directory
+
+      def initialize(fixtures_directory)
+        @fixtures_directory = fixtures_directory
+      end
+
+      def source_target_files
+        sources_targets_basenames.map do |basename|
+          OpenStruct.new(source: source_file(basename), target: target_file(basename))
+        end
+      end
+
+      def source_files
+        r = []
+        source_target_files.each do |st|
+          r << st.source if st.source
+        end
+        r
+      end
+
+      def target_file(basename)
+        fixture_file(basename, 'target')
+      end
+
+      def source_file(basename)
+        fixture_file(basename, 'source')
+      end
+
+      private
+
+      def fixture_file(basename, suffix)
+        prefix = "#{basename}.#{suffix}"
+        Dir.foreach(fixtures_directory) do |item|
+          next if item == '.' || item == '..'
+          return File.expand_path(item, fixtures_directory) if item.starts_with?(prefix)
+        end
+        nil
+      end
+
+      def sources_targets_basenames
+        basenames = Set.new
+        Dir.foreach(fixtures_directory) do |item|
+          next if item == '.' || item == '..'
+          b = self.class.source_target_basename(item)
+          basenames << b if b.present?
+        end
+        basenames
+      end
+    end
+  end
+end

data/lib/aranha/parsers/spec/source_target_fixtures_example.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative 'source_target_fixtures'
+
+RSpec.shared_examples 'source_target_fixtures' do |spec_file| # rubocop:disable Metrics/BlockLength
+  let(:spec_file) { spec_file }
+
+  it 'fixtures directory should exist' do
+    expect(::File.directory?(fixtures_dir)).to be true
+  end
+
+  context 'in fixtures directory' do
+    it 'should have at least one file' do
+      expect(source_target_fixtures.source_target_files.count).to be > 0
+    end
+
+    if ENV['WRITE_TARGET_FIXTURES']
+      it 'should write target data for all files' do
+        source_target_fixtures.source_files.each do |source_file|
+          sd = sort_results(source_data(source_file))
+          basename = ::Aranha::Spec::SourceTargetFixtures.source_target_basename(source_file)
+          target_file = File.expand_path("../#{basename}.target.yaml", source_file)
+          File.write(target_file, sd.to_yaml)
+        end
+      end
+    else
+      it 'should parse data for all files' do
+        source_target_fixtures.source_target_files.each do |st|
+          assert_source_target_complete(st)
+          sd = source_data(st.source)
+          td = YAML.load_file(st.target)
+          expect(sort_results(sd)).to eq(sort_results(td))
+        end
+      end
+    end
+  end
+
+  def source_target_fixtures
+    @source_target_fixtures ||= ::Aranha::Spec::SourceTargetFixtures.new(fixtures_dir)
+  end
+
+  def assert_source_target_complete(st)
+    expect(st.source).to(be_truthy, "Source not found (Target: #{st.target})")
+    expect(st.target).to(be_truthy, "Target not found (Source: #{st.source})")
+  end
+
+  def source_data(source_file)
+    described_class.new(source_file).data
+  end
+
+  def fixtures_dir
+    ::File.join(
+      ::File.dirname(spec_file),
+      ::File.basename(spec_file, '.*') + '_files'
+    )
+  end
+
+  def sort_results(r)
+    r
+  end
+end

data/lib/aranha/parsers/version.rb

@@ -0,0 +1,5 @@
+module Aranha
+  module Parsers
+    VERSION = '0.1.0'.freeze
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: aranha-parsers
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Esquilo Azul Company
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- lib/aranha/parsers.rb
+- lib/aranha/parsers/base.rb
+- lib/aranha/parsers/html.rb
+- lib/aranha/parsers/html/base.rb
+- lib/aranha/parsers/html/item.rb
+- lib/aranha/parsers/html/item_list.rb
+- lib/aranha/parsers/html/node.rb
+- lib/aranha/parsers/html/node/base.rb
+- lib/aranha/parsers/html/node/default.rb
+- lib/aranha/parsers/invalid_state_exception.rb
+- lib/aranha/parsers/source_address.rb
+- lib/aranha/parsers/source_address/file.rb
+- lib/aranha/parsers/source_address/hash_http_get.rb
+- lib/aranha/parsers/source_address/hash_http_post.rb
+- lib/aranha/parsers/source_address/http_get.rb
+- lib/aranha/parsers/spec/source_target_fixtures.rb
+- lib/aranha/parsers/spec/source_target_fixtures_example.rb
+- lib/aranha/parsers/version.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Parsers' utilities for Ruby.
+test_files: []