aranha 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d65a2ccecd09ab619dea2d76306dfc48c7aad83b0b111b4544afb8492a97ef04
-  data.tar.gz: fd1d57c4d7ec4a22f8bd5829b8ad9a76838b729d54440b68c559cdf8ba5f9482
+  metadata.gz: d8acb62bae7e2cede5a9d658d705bfbc28229cc9a595207d9f6f7796a7681e1f
+  data.tar.gz: debd7afd142d1de8d2ff38f6f7cf35cf89b82174ce278f7f2247324cfeb53b41
 SHA512:
-  metadata.gz: 3b34ee01cfc1f6be364017da680c64b4abf7a5bc303b40edfdedf7532cdfd69cbe17d57349fac4fbc8881dc8db336f8a5738aa1976e9e5a3f60a09b8c4808f77
-  data.tar.gz: c5129c06f6ec81bfd02da79ccfa64b1187abf644e2271ac84ea30d24fc02b1608c13b74fbf3f962ecbe4a3d69955c08c51fd0ad1f692395e82ac15a78978f638
+  metadata.gz: 7ce9a11b1c49756f67dae806d065cdd0fbdbe8f9b2e132ec4cc2efad55f198fd2c8fbea1c21a27cef9e22398a0632d8efef03a02c590a57a9abc2d7024c7ef23
+  data.tar.gz: 3dc367711f35207d2f693f01fe4ae2f485eb79f7052fbccd75876e4a90c8a2e98624947980135dc764b95b7f30b791227002f7fee609638ce5a961ad83c04e7a

data/Rakefile

@@ -16,7 +16,7 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-APP_RAKEFILE = File.expand_path('test/dummy/Rakefile', __dir__)
+APP_RAKEFILE = File.expand_path('spec/support/rails_app/Rakefile', __dir__)
 load 'rails/tasks/engine.rake'
 
 load 'rails/tasks/statistics.rake'
@@ -25,11 +25,12 @@ Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
 
-Rake::TestTask.new(:test) do |t|
-  t.libs << 'lib'
-  t.libs << 'test'
-  t.pattern = 'test/**/*_test.rb'
-  t.verbose = false
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec)
+  task test: :spec
+rescue LoadError
+  # no rspec available
 end
 
 task default: :test

data/lib/aranha/fixtures/download.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'aranha/parsers/source_address'
+
 module Aranha
   module Fixtures
     class Download
@@ -44,7 +46,7 @@ module Aranha
       end
 
       def url(file)
-        File.read(file).strip
+        ::Aranha::Parsers::SourceAddress.from_file(file)
       end
 
       def target(file)

data/lib/aranha/parsers/base.rb

@@ -2,82 +2,41 @@
 
 require 'open-uri'
 require 'fileutils'
+require 'aranha/parsers/source_address'
 
 module Aranha
   module Parsers
     class Base
       LOG_DIR_ENVVAR = 'ARANHA_PARSERS_LOG_DIR'
 
+      attr_reader :source_address
+
       def initialize(url)
-        @url = url
+        @source_address = ::Aranha::Parsers::SourceAddress.new(url)
+        log_content(source_address.serialize, '-source-address')
       end
 
-      def url
-        r = (@url.is_a?(Hash) ? @url.fetch(:url) : @url)
-        r.to_s.gsub(%r{/+$}, '')
-      end
+      delegate :url, to: :source_address
 
       def content
-        s = content_by_url_type
+        s = source_address.content
         log_content(s)
         s
       end
 
       private
 
-      def content_by_url_type
-        if @url.is_a?(Hash)
-          content_hash
-        elsif /^http/ =~ @url
-          content_get
-        else
-          content_file
-        end
-      end
-
-      def content_file
-        ::File.open(@url.to_s.gsub(%r{\Afile://}, ''), &:read)
-      end
-
-      def content_get
-        content_get_fetch(@url)
-      end
-
-      def content_get_fetch(uri, limit = 10)
-        raise 'too many HTTP redirects' if limit.zero?
-
-        response = Net::HTTP.get_response(URI(uri))
-
-        case response
-        when Net::HTTPSuccess then
-          response.body
-        when Net::HTTPRedirection then
-          content_get_fetch(response['location'], limit - 1)
-        else
-          response.value
-        end
-      end
-
-      def content_hash
-        return content_post if @url[:method] == :post
-
-        raise "Unknown URL format: #{@url}"
-      end
-
-      def content_post
-        HTTPClient.new.post_content(@url[:url], @url[:params].merge(follow_redirect: true))
-      end
+      def log_content(content, suffix = '')
+        path = log_file(suffix)
 
-      def log_content(content)
-        path = log_file
         return unless path
         File.open(path, 'wb') { |file| file.write(content) }
       end
 
-      def log_file
+      def log_file(suffix)
         dir = log_parsers_dir
         return nil unless dir
-        f = ::File.join(dir, "#{self.class.name.parameterize}.log")
+        f = ::File.join(dir, "#{self.class.name.parameterize}#{suffix}.log")
         FileUtils.mkdir_p(File.dirname(f))
         f
       end
@@ -89,9 +48,8 @@ module Aranha
       end
 
       def rails_root_exist?
-        klass = Module.const_get('Rails')
-        return false unless klass.is_a?(Class)
-        klass.respond_to?(:root)
+        ::Rails.root
+        true
       rescue NameError
         return false
       end

data/lib/aranha/parsers/source_address.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'active_support/core_ext/module/delegation'
+require 'aranha/parsers/source_address/hash_http_post'
+require 'aranha/parsers/source_address/http_get'
+require 'aranha/parsers/source_address/file'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class << self
+        SUBS = [
+          ::Aranha::Parsers::SourceAddress::HashHttpPost,
+          ::Aranha::Parsers::SourceAddress::HttpGet,
+          ::Aranha::Parsers::SourceAddress::File
+        ].freeze
+
+        def detect_sub(source)
+          return source.sub if source.is_a?(self)
+          SUBS.each do |sub|
+            return sub.new(source) if sub.valid_source?(source)
+          end
+          raise "No content fetcher found for source \"#{source}\""
+        end
+
+        def deserialize(string)
+          new(string =~ %r{\A[a-z]+://} ? string.strip : ::YAML.load(string))
+        end
+
+        def from_file(path)
+          deserialize(::File.read(path))
+        end
+      end
+
+      attr_reader :sub
+
+      def initialize(source)
+        @sub = self.class.detect_sub(source)
+      end
+
+      delegate :content, :url, to: :sub
+
+      def serialize
+        sub.serialize.strip + "\n"
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/file.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/source_address/http_get'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class File < ::Aranha::Parsers::SourceAddress::HttpGet
+        SCHEME = 'file://'
+
+        class << self
+          def valid_source?(source)
+            source.to_s.start_with?(SCHEME + '/', '/')
+          end
+        end
+
+        def initialize(source)
+          super source.to_s.gsub(/\A#{Regexp.quote(SCHEME)}/, '')
+        end
+
+        def url
+          "#{SCHEME}#{source}"
+        end
+
+        def content
+          ::File.open(source, &:read)
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/hash_http_post.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/indifferent_access'
+require 'httpclient'
+require 'yaml'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HashHttpPost
+        class << self
+          def valid_source?(source)
+            source.is_a?(::Hash) && source.with_indifferent_access.key?(:url)
+          end
+        end
+
+        attr_reader :source
+
+        def initialize(source)
+          @source = source.with_indifferent_access
+        end
+
+        def ==(other)
+          self.class == other.class && source == other.source
+        end
+
+        def url
+          source.fetch(:url)
+        end
+
+        def serialize
+          source.to_yaml
+        end
+
+        def content
+          HTTPClient.new.post_content(
+            source[:url],
+            source[:params].merge(follow_redirect: true)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aranha/parsers/source_address/http_get.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'net/http'
+
+module Aranha
+  module Parsers
+    class SourceAddress
+      class HttpGet
+        class << self
+          def valid_source?(source)
+            source.to_s.start_with?('http://')
+          end
+        end
+
+        attr_reader :source
+
+        def initialize(source)
+          @source = source.to_s
+        end
+
+        def ==(other)
+          self.class == other.class && source == other.source
+        end
+
+        def url
+          source
+        end
+
+        def content
+          content_fetch(url)
+        end
+
+        def serialize
+          url
+        end
+
+        private
+
+        def content_fetch(uri, limit = 10)
+          raise 'too many HTTP redirects' if limit.zero?
+
+          response = Net::HTTP.get_response(URI(uri))
+
+          case response
+          when Net::HTTPSuccess then
+            response.body
+          when Net::HTTPRedirection then
+            content_fetch(response['location'], limit - 1)
+          else
+            response.value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/test/test_helper.rb

@@ -3,9 +3,9 @@
 # Configure Rails Environment
 ENV['RAILS_ENV'] = 'test'
 
-require File.expand_path('../test/dummy/config/environment.rb', __dir__)
+require File.expand_path('../spec/support/rails_apps/config/environment.rb', __dir__)
 ActiveRecord::Migrator.migrations_paths = [
-  File.expand_path('../test/dummy/db/migrate', __dir__)
+  File.expand_path('../spec/support/rails_app/db/migrate', __dir__)
 ]
 require 'rails/test_help'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-04 00:00:00.000000000 Z
+date: 2019-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -86,6 +86,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.142.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -135,6 +149,10 @@ files:
 - lib/aranha/parsers/html/node/base.rb
 - lib/aranha/parsers/html/node/default.rb
 - lib/aranha/parsers/invalid_state_exception.rb
+- lib/aranha/parsers/source_address.rb
+- lib/aranha/parsers/source_address/file.rb
+- lib/aranha/parsers/source_address/hash_http_post.rb
+- lib/aranha/parsers/source_address/http_get.rb
 - lib/aranha/parsers/spec/source_target_fixtures.rb
 - lib/aranha/parsers/spec/source_target_fixtures_example.rb
 - lib/aranha/processor.rb
@@ -144,42 +162,6 @@ files:
 - lib/aranha/version.rb
 - lib/tasks/aranha_tasks.rake
 - test/aranha_test.rb
-- test/dummy/README.rdoc
-- test/dummy/Rakefile
-- test/dummy/app/assets/javascripts/application.js
-- test/dummy/app/assets/stylesheets/application.css
-- test/dummy/app/controllers/application_controller.rb
-- test/dummy/app/helpers/application_helper.rb
-- test/dummy/app/views/layouts/application.html.erb
-- test/dummy/bin/bundle
-- test/dummy/bin/rails
-- test/dummy/bin/rake
-- test/dummy/bin/setup
-- test/dummy/config.ru
-- test/dummy/config/application.rb
-- test/dummy/config/boot.rb
-- test/dummy/config/database.yml
-- test/dummy/config/environment.rb
-- test/dummy/config/environments/development.rb
-- test/dummy/config/environments/production.rb
-- test/dummy/config/environments/test.rb
-- test/dummy/config/initializers/assets.rb
-- test/dummy/config/initializers/backtrace_silencers.rb
-- test/dummy/config/initializers/cookies_serializer.rb
-- test/dummy/config/initializers/filter_parameter_logging.rb
-- test/dummy/config/initializers/inflections.rb
-- test/dummy/config/initializers/mime_types.rb
-- test/dummy/config/initializers/session_store.rb
-- test/dummy/config/initializers/to_time_preserves_timezone.rb
-- test/dummy/config/initializers/wrap_parameters.rb
-- test/dummy/config/locales/en.yml
-- test/dummy/config/routes.rb
-- test/dummy/config/secrets.yml
-- test/dummy/db/schema.rb
-- test/dummy/public/404.html
-- test/dummy/public/422.html
-- test/dummy/public/500.html
-- test/dummy/public/favicon.ico
 - test/integration/navigation_test.rb
 - test/test_helper.rb
 homepage: 
@@ -207,42 +189,6 @@ signing_key:
 specification_version: 4
 summary: Rails utilities for web crawling.
 test_files:
-- test/dummy/Rakefile
-- test/dummy/README.rdoc
-- test/dummy/config.ru
-- test/dummy/config/boot.rb
-- test/dummy/config/database.yml
-- test/dummy/config/secrets.yml
-- test/dummy/config/locales/en.yml
-- test/dummy/config/application.rb
-- test/dummy/config/environments/development.rb
-- test/dummy/config/environments/test.rb
-- test/dummy/config/environments/production.rb
-- test/dummy/config/environment.rb
-- test/dummy/config/routes.rb
-- test/dummy/config/initializers/assets.rb
-- test/dummy/config/initializers/cookies_serializer.rb
-- test/dummy/config/initializers/inflections.rb
-- test/dummy/config/initializers/session_store.rb
-- test/dummy/config/initializers/wrap_parameters.rb
-- test/dummy/config/initializers/to_time_preserves_timezone.rb
-- test/dummy/config/initializers/filter_parameter_logging.rb
-- test/dummy/config/initializers/backtrace_silencers.rb
-- test/dummy/config/initializers/mime_types.rb
-- test/dummy/db/schema.rb
-- test/dummy/app/views/layouts/application.html.erb
-- test/dummy/app/controllers/application_controller.rb
-- test/dummy/app/helpers/application_helper.rb
-- test/dummy/app/assets/stylesheets/application.css
-- test/dummy/app/assets/javascripts/application.js
-- test/dummy/public/422.html
-- test/dummy/public/404.html
-- test/dummy/public/favicon.ico
-- test/dummy/public/500.html
-- test/dummy/bin/bundle
-- test/dummy/bin/setup
-- test/dummy/bin/rails
-- test/dummy/bin/rake
 - test/aranha_test.rb
 - test/test_helper.rb
 - test/integration/navigation_test.rb