aranha 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b770597c6db06b5641e943c6b758e4bf03da504dddde3b53873042397ebd2d0f
-  data.tar.gz: 97e2aa65591ba5ef13af0d3e121b5e81f6d30ab78d10c5d6b8654b96e40e55fc
+  metadata.gz: 92786506ca137fde90c898b24a9018453468cd72a5f500d50125969d4c19ce71
+  data.tar.gz: 5cda7e665e1ec8b09ca758de1652c94cd42ab3e49fc27c447ba7d975cfe572be
 SHA512:
-  metadata.gz: c18119d14b4b101ed22289c4da45faa78949c2c7871a08c62a182722f7fbe42807f3101526f970f463657dcf6341e596f54eb32d8f040f6f529d3fe961b77db2
-  data.tar.gz: 83f31a41541d7a700e448894b665109b43c2ee8f7a8adfe561292dc675ace9fa118cb59e6891a03d916ac03043c9eb36879eb1fa08e14a2c78ee693de91de3e4
+  metadata.gz: 2aca9aff975e8b027b17ea8e8afc2ef2edc815c8c4ae4a10eec6f7e2ec3181024c1b73702dcc4ff0ec0b40464157e9233a07b2bcbf396c1649dbf53a0dbd006d
+  data.tar.gz: b019d925812e0c8209ba4b5d31c9737f9c72dedbff8e9b018ef0246e127afdbb0d6a1bd91ba997fcb1f473edaf76b383b3735432184f8ffd9543d3b209e71bac

data/app/models/aranha/address.rb

@@ -12,13 +12,14 @@ module Aranha
       end
 
       def add_start_points
+        ::Rails.logger.info("Start points: #{start_points.count}")
         start_points.each do |url, processor|
           add(url, processor)
         end
       end
 
       def add(url, processor, extra_data = nil)
-        a = find_or_initialize_by(url: url)
+        a = find_or_initialize_by(url: sanitize_url(url))
         a.processor = processor
         a.extra_data = extra_data.to_yaml
         a.save!
@@ -32,6 +33,14 @@ module Aranha
 
       private
 
+      def sanitize_url(url)
+        if url.is_a?(Hash)
+          url.to_yaml
+        else
+          url.to_s
+        end
+      end
+
       def start_points
         @start_points ||= {}
       end
@@ -60,14 +69,18 @@ module Aranha
 
     def instanciate_processor
       if processor_instancier_arity == 2 || processor_instancier_arity < 0
-        processor_instancier.call(url, YAML.load(extra_data))
+        processor_instancier.call(url_to_process, YAML.load(extra_data))
       elsif processor_instancier_arity == 1
-        processor_instancier.call(url)
+        processor_instancier.call(url_to_process)
       else
         raise("#{processor}.initialize should has 1 or 2 or * arguments")
       end
     end
 
+    def url_to_process
+      ::YAML.load(url)
+    end
+
     def processor_instancier
       processor.constantize.method(:new)
     end

data/lib/aranha.rb

@@ -9,6 +9,7 @@ module Aranha
 end
 
 require_dependency 'aranha/default_processor'
+require_dependency 'aranha/fixtures/download'
 require_dependency 'aranha/processor'
 require_dependency 'aranha/parsers/base'
 require_dependency 'aranha/parsers/html/base'

data/lib/aranha/default_processor.rb

@@ -4,11 +4,18 @@ module Aranha
   class DefaultProcessor
     attr_reader :source_uri, :extra_data
 
-    def initialize(source_uri, extra_data)
-      unless source_uri.is_a?(Addressable::URI)
-        source_uri = source_uri.to_s.gsub(%r{\A/}, 'file:///')
+    class << self
+      def sanitize_uri(uri)
+        return uri if uri.is_a?(Hash)
+        unless uri.is_a?(Addressable::URI)
+          uri = uri.to_s.gsub(%r{\A/}, 'file:///')
+        end
+        Addressable::URI.parse(uri)
       end
-      @source_uri = Addressable::URI.parse(source_uri)
+    end
+
+    def initialize(source_uri, extra_data)
+      @source_uri = self.class.sanitize_uri(source_uri)
       @extra_data = extra_data
     end
 

data/lib/aranha/fixtures/download.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Fixtures
+    class Download
+      def initialize(prefix, download)
+        @prefix = prefix
+        @prefix = '' if @prefix.blank?
+        @download = download
+      end
+
+      def run
+        url_files.each do |f|
+          Rails.logger.info(relative_path(f))
+          download(url(f), target(f)) if @download
+        end
+      end
+
+      private
+
+      def url_files
+        files = []
+        Dir["#{fixtures_root}/**/*.url"].map do |path|
+          files << path if match_pattern(path)
+        end
+        files
+      end
+
+      def match_pattern(path)
+        relative_path(path).start_with?(@prefix)
+      end
+
+      def fixtures_root
+        Rails.root.to_s
+      end
+
+      def download(url, target)
+        Rails.logger.info "Baixando \"#{url}\"..."
+        File.open(target, 'wb') { |file| file.write(::Aranha::Parsers::Base.new(url).content) }
+      end
+
+      def url(file)
+        File.read(file).strip
+      end
+
+      def target(file)
+        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
+      end
+
+      def relative_path(path)
+        path.sub(%r{^#{Regexp.quote(fixtures_root)}/}, '')
+      end
+    end
+  end
+end

data/lib/aranha/parsers/base.rb

@@ -11,7 +11,8 @@ module Aranha
       end
 
       def url
-        @url.gsub(%r{/+$}, '')
+        r = (@url.is_a?(Hash) ? @url.fetch(:url) : @url)
+        r.to_s.gsub(%r{/+$}, '')
       end
 
       def content

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/tasks/aranha_tasks.rake

@@ -8,4 +8,11 @@ namespace(:aranha) do
   task clear: :environment do
     Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
   end
+
+  namespace :fixtures do
+    desc 'Download remote content for fixtures.'
+    task download: :environment do
+      ::Aranha::Fixtures::Download.new(ENV['PREFIX'], ENV['DOWNLOAD'].present?).run
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-02 00:00:00.000000000 Z
+date: 2019-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -107,6 +107,7 @@ files:
 - lib/aranha/dom_elements_traverser/cursor.rb
 - lib/aranha/dom_elements_traverser/data.rb
 - lib/aranha/engine.rb
+- lib/aranha/fixtures/download.rb
 - lib/aranha/parsers/base.rb
 - lib/aranha/parsers/html/base.rb
 - lib/aranha/parsers/html/item_list.rb
@@ -180,42 +181,42 @@ signing_key:
 specification_version: 4
 summary: Rails utilities for web crawling.
 test_files:
-- test/aranha_test.rb
+- test/dummy/Rakefile
+- test/dummy/README.rdoc
+- test/dummy/config.ru
+- test/dummy/config/boot.rb
+- test/dummy/config/database.yml
+- test/dummy/config/secrets.yml
+- test/dummy/config/locales/en.yml
+- test/dummy/config/application.rb
+- test/dummy/config/environments/development.rb
+- test/dummy/config/environments/test.rb
+- test/dummy/config/environments/production.rb
+- test/dummy/config/environment.rb
 - test/dummy/config/routes.rb
 - test/dummy/config/initializers/assets.rb
 - test/dummy/config/initializers/cookies_serializer.rb
-- test/dummy/config/initializers/to_time_preserves_timezone.rb
+- test/dummy/config/initializers/inflections.rb
 - test/dummy/config/initializers/session_store.rb
-- test/dummy/config/initializers/backtrace_silencers.rb
 - test/dummy/config/initializers/wrap_parameters.rb
+- test/dummy/config/initializers/to_time_preserves_timezone.rb
 - test/dummy/config/initializers/filter_parameter_logging.rb
-- test/dummy/config/initializers/inflections.rb
+- test/dummy/config/initializers/backtrace_silencers.rb
 - test/dummy/config/initializers/mime_types.rb
-- test/dummy/config/database.yml
-- test/dummy/config/secrets.yml
-- test/dummy/config/locales/en.yml
-- test/dummy/config/environment.rb
-- test/dummy/config/boot.rb
-- test/dummy/config/application.rb
-- test/dummy/config/environments/production.rb
-- test/dummy/config/environments/test.rb
-- test/dummy/config/environments/development.rb
-- test/dummy/Rakefile
-- test/dummy/public/favicon.ico
-- test/dummy/public/404.html
-- test/dummy/public/500.html
-- test/dummy/public/422.html
-- test/dummy/config.ru
-- test/dummy/app/assets/stylesheets/application.css
-- test/dummy/app/assets/javascripts/application.js
-- test/dummy/app/helpers/application_helper.rb
+- test/dummy/db/schema.rb
 - test/dummy/app/views/layouts/application.html.erb
 - test/dummy/app/controllers/application_controller.rb
-- test/dummy/bin/setup
+- test/dummy/app/helpers/application_helper.rb
+- test/dummy/app/assets/stylesheets/application.css
+- test/dummy/app/assets/javascripts/application.js
+- test/dummy/public/422.html
+- test/dummy/public/404.html
+- test/dummy/public/favicon.ico
+- test/dummy/public/500.html
 - test/dummy/bin/bundle
+- test/dummy/bin/setup
 - test/dummy/bin/rails
 - test/dummy/bin/rake
-- test/dummy/db/schema.rb
-- test/dummy/README.rdoc
-- test/integration/navigation_test.rb
+- test/aranha_test.rb
 - test/test_helper.rb
+- test/integration/navigation_test.rb