aranha 0.14.5 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b89c63a38f3c2f658263dde445f10b45628ecd19069d6a78a0d5e838a1a6127
-  data.tar.gz: 15cfb49d733aba80bd316813d21c587c47c4799af6a60d9a6147bed1e08f132f
+  metadata.gz: cc769d0b380643afe3740301d894fd70e93d3df19d3f6cf4bde1d7633d059d4f
+  data.tar.gz: 8d019e0d77e967ca19ff7325dc0081aa19bcdb114ea1812e49e245d121ca57c2
 SHA512:
-  metadata.gz: fe5b484b9709033c8f87db0f8070c83d5dd328109d2d6bfb12a7ef356ba7268ec45cf728f5554dbd3015e28f60d14be2b81d6ac33f836365024fb4e88f1ed7e1
-  data.tar.gz: 9456fafe876c0894f810575fe41f078703ce2f9b75930c967ed4ffd9d33b93eeeb80e4b899e9bc12f31612d397fb1d462ab29520e7df4f71956fb64c489628f6
+  metadata.gz: 16fd673ddb8be560b62a3dbab2e4e81e4b7b66842d5594ee4eb63639ba7a3bec6a43f239d69918c0417ed7a132069eea9bdf6ac741e41988106cb8f0ec1e0e56
+  data.tar.gz: 7b5c39602e3694ffbca23db1578cb87fc7dab222dfe67dd3b87e4c19fc36277efa339f10692bd2fd77c3d3218fd4201e31229d6d37b75ad209b200cc87b1229b

data/README.rdoc

@@ -1,3 +1,3 @@
 = Aranha
 
-Rails utilities for web crawling.
+Ruby utilities for web crawling.

data/lib/aranha.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-require 'httpclient'
-require 'active_support/dependencies'
-require 'active_scaffold'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
-  require 'aranha/default_processor'
-  require 'aranha/dom_elements_traverser'
-  require 'aranha/engine'
-  require 'aranha/fixtures'
-  require 'aranha/processor'
-  require 'aranha/parsers'
-  require 'aranha/selenium'
+  require_sub __FILE__
 end

data/lib/aranha/address_processor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class AddressProcessor
+    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
+    CORE_EXCEPTIONS = [::SocketError].freeze
+    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
+    HTTPCLIENT_EXCEPTIONS = [
+      ::HTTPClient::BadResponseError,
+      ::HTTPClient::ConnectTimeoutError,
+      ::HTTPClient::ReceiveTimeoutError
+    ].freeze
+    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
+
+    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
+                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
+
+    class << self
+      def rescuable_error?(error)
+        return true if NETWORK_EXCEPTIONS.any? { |klass| error.is_a?(klass) }
+
+        error.cause.present? ? network_error?(error.cause) : false
+      end
+    end
+
+    enable_simple_cache
+    common_constructor :address
+
+    def successful?
+      error.blank?
+    end
+
+    def rescuable_error?
+      self.class.rescuable_error?(error)
+    end
+
+    private
+
+    def error_uncached
+      address.process
+      nil
+    rescue ::StandardError => e
+      e
+    end
+  end
+end

data/lib/aranha/default_processor.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
 require 'addressable'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   class DefaultProcessor
-    attr_reader :source_uri, :extra_data
-
     class << self
       def sanitize_uri(uri)
         return uri if uri.is_a?(Hash)
@@ -15,17 +14,14 @@ module Aranha
       end
     end
 
-    def initialize(source_uri, extra_data)
-      @source_uri = self.class.sanitize_uri(source_uri)
-      @extra_data = extra_data
+    common_constructor :source_uri, :extra_data do
+      self.source_uri = self.class.sanitize_uri(source_uri)
     end
 
     def process
       raise 'Implement method process'
     end
 
-    protected
-
     def target_uri
       source_uri
     end

data/lib/aranha/manager.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/abstract_methods'
+
+module Aranha
+  class Manager
+    include ::EacRubyUtils::AbstractMethods
+
+    class << self
+      attr_accessor :default
+    end
+
+    def addresses_count
+      raise_abstract_method(__method__)
+    end
+
+    def add_address(_uri, _processor_class, _extra_data = nil)
+      raise_abstract_method(__method__)
+    end
+
+    def add_start_point(uri, processor_class, extra_data = nil)
+      start_points_var << ::EacRubyUtils::Struct.new(
+        uri: uri, processor_class: processor_class, extra_data: extra_data
+      )
+    end
+
+    def clear_expired_addresses
+      raise_abstract_method(__method__)
+    end
+
+    def init
+      clear_expired_addresses
+      start_points_to_addresses
+    end
+
+    def log_info(_message)
+      raise_abstract_method(__method__)
+    end
+
+    def log_warn(_message)
+      raise_abstract_method(__method__)
+    end
+
+    def start_points
+      start_points_var.to_enum
+    end
+
+    def start_points_to_addresses
+      start_points_var.each do |sp|
+        add_address(sp.uri, sp.processor_class, sp.extra_data)
+      end
+    end
+
+    def unprocessed_addresses
+      raise_abstract_method(__method__)
+    end
+
+    private
+
+    def start_points_var
+      @start_points_var ||= []
+    end
+  end
+end

data/lib/aranha/processor.rb

@@ -1,30 +1,21 @@
 # frozen_string_literal: true
 
 require 'net/http'
+require 'httpclient'
 require 'aranha/parsers/invalid_state_exception'
+require 'aranha/manager'
 
 module Aranha
   class Processor
-    ARANHA_EXCEPTIONS = [::Aranha::Parsers::InvalidStateException].freeze
-    CORE_EXCEPTIONS = [::SocketError].freeze
-    ERRNO_EXCEPTIONS = [Errno::ECONNREFUSED, ::Errno::ECONNRESET].freeze
-    HTTPCLIENT_EXCEPTIONS = [
-      ::HTTPClient::BadResponseError,
-      ::HTTPClient::ConnectTimeoutError,
-      ::HTTPClient::ReceiveTimeoutError
-    ].freeze
-    NET_EXCEPTIONS = [::Net::HTTPFatalError, ::Net::HTTPServerException, ::Net::OpenTimeout].freeze
-
-    NETWORK_EXCEPTIONS = ARANHA_EXCEPTIONS + CORE_EXCEPTIONS + ERRNO_EXCEPTIONS +
-                         HTTPCLIENT_EXCEPTIONS + NET_EXCEPTIONS
-
     DEFAULT_MAX_TRIES = 3
 
-    def initialize
-      ::Aranha::Address.clear_expired
-      ::Aranha::Address.add_start_points
+    attr_reader :manager
+
+    def initialize(manager = nil)
+      @manager = manager || ::Aranha::Manager.default
       @failed = {}
       @try = 0
+      self.manager.init
       process_loop
       raise "Addresses failed: #{@failed.count}" if @failed.any?
     end
@@ -32,7 +23,7 @@ module Aranha
     private
 
     def process_loop
-      Rails.logger.info("Max tries: #{max_tries_s}")
+      manager.log_info("Max tries: #{max_tries_s}")
       loop do
         break if process_next_address
       end
@@ -52,22 +43,22 @@ module Aranha
     end
 
     def process_address(address)
-      Rails.logger.info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
-          " Unprocessed: #{unprocessed.count}/#{Aranha::Address.count})")
-      begin
-        address.process
-        @failed.delete(address.id)
-      rescue StandardError => e
-        process_exception(address, e)
+      manager.log_info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
+          " Unprocessed: #{unprocessed.count}/#{::Aranha::Manager.default.addresses_count})")
+      ap = ::Aranha::AddressProcessor.new(address)
+      if ap.successful?
+        @failed.delete(ap.address.id)
+      else
+        process_exception(ap)
       end
     end
 
-    def process_exception(address, exception)
-      raise exception unless network_exception?(exception)
+    def process_exception(address_processor)
+      raise address_processor.error unless address_processor.rescuable_error?
 
-      @failed[address.id] ||= 0
-      @failed[address.id] += 1
-      Rails.logger.warn(exception)
+      @failed[address_processor.address.id] ||= 0
+      @failed[address_processor.address.id] += 1
+      manager.log_warn(address_processor.error)
     end
 
     def next_address
@@ -75,13 +66,7 @@ module Aranha
     end
 
     def unprocessed
-      ::Aranha::Address.unprocessed
-    end
-
-    def network_exception?(exception)
-      return true if NETWORK_EXCEPTIONS.any? { |klass| exception.is_a?(klass) }
-
-      exception.cause.present? ? network_exception?(exception.cause) : false
+      ::Aranha::Manager.default.unprocessed_addresses
     end
 
     def not_try_ids

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.14.5'
+  VERSION = '0.16.0'
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.14.5
+  version: 0.16.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-14 00:00:00.000000000 Z
+date: 2021-07-22 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: active_scaffold
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.41.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.41.1
 - !ruby/object:Gem::Dependency
   name: aranha-parsers
   requirement: !ruby/object:Gem::Requirement
@@ -58,34 +44,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.2
-- !ruby/object:Gem::Dependency
-  name: eac_rails_utils
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.11'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: eac_ruby_utils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.35'
+        version: '0.52'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.35'
+        version: '0.52'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -100,48 +72,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.6'
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.11.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.11.3
 - !ruby/object:Gem::Dependency
   name: eac_ruby_gem_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
-- !ruby/object:Gem::Dependency
-  name: sqlite3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 description: 
 email:
 - eduardobogoni@gmail.com
@@ -151,29 +95,12 @@ extra_rdoc_files: []
 files:
 - MIT-LICENSE
 - README.rdoc
-- app/assets/javascripts/aranha/application.js
-- app/assets/stylesheets/aranha/application.css
-- app/controllers/aranha/addresses_controller.rb
-- app/helpers/aranha/application_helper.rb
-- app/models/aranha/address.rb
-- app/views/layouts/aranha/application.html.erb
-- config/locales/en.yml
-- config/locales/pt-BR.yml
-- config/routes.rb
-- db/migrate/20171201021251_create_aranha_addresses.rb
-- db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb
 - lib/aranha.rb
+- lib/aranha/address_processor.rb
 - lib/aranha/default_processor.rb
-- lib/aranha/dom_elements_traverser.rb
-- lib/aranha/dom_elements_traverser/conditions.rb
-- lib/aranha/dom_elements_traverser/cursor.rb
-- lib/aranha/dom_elements_traverser/data.rb
-- lib/aranha/engine.rb
-- lib/aranha/fixtures.rb
-- lib/aranha/fixtures/download.rb
+- lib/aranha/manager.rb
 - lib/aranha/processor.rb
 - lib/aranha/version.rb
-- lib/tasks/aranha_tasks.rake
 homepage: 
 licenses:
 - MIT
@@ -193,8 +120,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
-summary: Rails utilities for web crawling.
+summary: Ruby utilities for web crawling.
 test_files: []

data/app/assets/javascripts/aranha/application.js

@@ -1,14 +0,0 @@
-// This is a manifest file that'll be compiled into application.js, which will include all the files
-// listed below.
-//
-// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
-// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
-//
-// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
-// compiled file.
-//
-// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
-// about supported directives.
-//
-//= require_tree .
-//= require active_scaffold

data/app/assets/stylesheets/aranha/application.css

@@ -1,16 +0,0 @@
-/*
- * This is a manifest file that'll be compiled into application.css, which will include all the files
- * listed below.
- *
- * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
- * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
- *
- * You're free to add application-wide styles to this file and they'll appear at the bottom of the
- * compiled file so the styles you add here take precedence over styles defined in any styles
- * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
- * file per style scope.
- *
- *= require_tree .
- *= require_self
- *= require active_scaffold
- */

data/app/controllers/aranha/addresses_controller.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class AddressesController < ::ApplicationController
-    active_scaffold :'aranha/address' do |_conf|
-    end
-  end
-end

data/app/helpers/aranha/application_helper.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  module ApplicationHelper
-  end
-end

data/app/models/aranha/address.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-require 'eac_ruby_utils/yaml'
-
-module Aranha
-  class Address < ActiveRecord::Base
-    include ::EacRailsUtils::Models::InequalityQueries
-
-    add_inequality_queries(:created_at)
-
-    class << self
-      def set_start_point(url, processor)
-        start_points[url] = processor
-      end
-
-      def add_start_points
-        ::Rails.logger.info("Start points: #{start_points.count}")
-        start_points.each do |url, processor|
-          add(url, processor)
-        end
-      end
-
-      def add(url, processor, extra_data = nil)
-        a = find_or_initialize_by(url: sanitize_url(url))
-        a.processor = processor
-        a.extra_data = extra_data.to_yaml
-        a.save!
-      end
-
-      def clear_expired
-        q = by_created_at_lt(Time.zone.now - 12.hours)
-        Rails.logger.info("Addresses expired: #{q.count}")
-        q.destroy_all
-      end
-
-      private
-
-      def sanitize_url(url)
-        if url.is_a?(Hash)
-          url.to_yaml
-        else
-          url.to_s
-        end
-      end
-
-      def start_points
-        @start_points ||= {}
-      end
-    end
-
-    validates :url, presence: true, uniqueness: true
-    validates :processor, presence: true
-
-    scope :unprocessed, lambda {
-      where(processed_at: nil)
-    }
-
-    def to_s
-      "#{processor}|#{url}"
-    end
-
-    def process
-      ActiveRecord::Base.transaction do
-        instanciate_processor.process
-        self.processed_at = Time.zone.now
-        save!
-      end
-    end
-
-    private
-
-    def instanciate_processor
-      processor_instancier.call(*processor_instancier_arguments)
-    end
-
-    def url_to_process
-      ::EacRubyUtils::Yaml.load(url)
-    end
-
-    def processor_instancier
-      processor.constantize.method(:new)
-    end
-
-    def processor_instancier_arguments
-      if processor_instancier_arity == 2 || processor_instancier_arity.negative?
-        [url_to_process, EacRubyUtils::Yaml.load(extra_data)]
-      elsif processor_instancier_arity == 1
-        [processor_instancier.call(url_to_process)]
-      else
-        raise("#{processor}.initialize should has 1 or 2 or * arguments")
-      end
-    end
-
-    def processor_instancier_arity
-      processor.constantize.instance_method(:initialize).arity
-    end
-  end
-end

data/app/views/layouts/aranha/application.html.erb

@@ -1,12 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <title>Aranha</title>
-    <%= stylesheet_link_tag    "aranha/application", media: "all" %>
-    <%= javascript_include_tag "aranha/application" %>
-    <%= csrf_meta_tags %>
-  </head>
-  <body>
-    <%= yield %>
-  </body>
-</html>

data/config/locales/en.yml

@@ -1,6 +0,0 @@
-en:
-  activerecord:
-    models:
-      aranha/address:
-        one: Aranha address
-        other: Aranha addresses

data/config/locales/pt-BR.yml

@@ -1,6 +0,0 @@
-pt-BR:
-  activerecord:
-    models:
-      aranha/address:
-        one: Endereço Aranha
-        other: Endereços Aranha

data/config/routes.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-Aranha::Engine.routes.draw do
-  concern :active_scaffold, ActiveScaffold::Routing::Basic.new(association: true)
-  resources(:addresses, concerns: :active_scaffold)
-end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-class CreateAranhaAddresses < (
-    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
-  )
-  def change
-    create_table :aranha_addresses do |t|
-      t.string :url
-      t.string :processor
-      t.timestamp :processed_at
-
-      t.timestamps null: false
-    end
-  end
-end

data/db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-class AddExtraDataToAranhaAddresses < (
-    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
-  )
-  def change
-    add_column :aranha_addresses, :extra_data, :text
-  end
-end

data/lib/aranha/dom_elements_traverser.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/dom_elements_traverser/conditions'
-require 'aranha/dom_elements_traverser/data'
-require 'aranha/dom_elements_traverser/cursor'
-
-module Aranha
-  class DomElementsTraverser
-    include ::Aranha::DomElementsTraverser::Conditions
-    include ::Aranha::DomElementsTraverser::Cursor
-    include ::Aranha::DomElementsTraverser::Data
-
-    class << self
-      def traverse(options, &block)
-        new(elements_from_options(options), &block)
-      end
-
-      def empty
-        new([])
-      end
-
-      private
-
-      def elements_from_options(options)
-        options = ::EacRubyUtils::OptionsConsumer.new(options)
-        elements = nil
-        options.consume(:children_of) { |v| elements = v.children.to_a }
-        raise 'None option of [:children_of] defined' unless elements
-
-        options.validate
-        elements
-      end
-    end
-
-    private
-
-    def initialize(elements, &block)
-      @elements = elements
-      @index = 0
-      @data = {}
-      instance_eval(&block) if block
-    end
-  end
-end

data/lib/aranha/dom_elements_traverser/conditions.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class DomElementsTraverser
-    module Conditions
-      private
-
-      def match_conditions?(conditions)
-        raise "No element (Conditions: #{conditions})" unless current
-
-        conditions.all? { |key, value| match_condition?(key, value) }
-      end
-
-      def match_condition?(key, value)
-        case key.to_sym
-        when :text then match_text_condition?(value)
-        when :name then match_name_condition?(value)
-        else raise "Unknown key condition: (#{key})"
-        end
-      end
-
-      def match_name_condition?(tag_name)
-        current.name.casecmp(tag_name.to_s).zero?
-      end
-
-      def match_text_condition?(texts)
-        texts = [texts.to_s] unless texts.is_a?(Array)
-        texts.all? { |t| current.text.downcase.include?(t.downcase) }
-      end
-    end
-  end
-end

data/lib/aranha/dom_elements_traverser/cursor.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require 'eac_ruby_utils/options_consumer'
-
-module Aranha
-  class DomElementsTraverser
-    module Cursor
-      private
-
-      def current
-        @elements[@index]
-      end
-
-      def skip
-        @index += 1
-      end
-
-      def skip_until(options)
-        oc = ::EacRubyUtils::OptionsConsumer.new(options)
-        optional = oc.consume(:optional, false)
-        while current
-          break if match_conditions?(oc.left_data)
-
-          skip
-        end
-        raise "No element found for conditions #{oc.left_data}" unless current || optional
-
-        current
-      end
-
-      def skip_until_after(conditions)
-        skip_until(conditions)
-        skip
-        current
-      end
-
-      def if_found(conditions, &block)
-        marked = @index
-        skip_until({ optional: true }.merge(conditions))
-        if current
-          instance_eval(&block) if block
-        else
-          @index = marked
-        end
-      end
-    end
-  end
-end

data/lib/aranha/dom_elements_traverser/data.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class DomElementsTraverser
-    module Data
-      def data
-        @data.dup
-      end
-
-      private
-
-      def store(key, options = {}, &converter)
-        validate(options)
-        value = store_value(options, converter)
-        @data[key] = value
-        r = current
-        skip
-        r
-      end
-
-      def store_value(options, converter)
-        value = if options.key?(:attribute)
-                  current.attribute(options[:attribute]).value
-                else
-                  current.text.strip
-                end
-        converter ? converter.call(value) : value
-      end
-
-      def validate(options)
-        return unless options.key?(:validate)
-        return if match_conditions?(options[:validate])
-
-        raise "Element does not match conditions #{options[:validate]}" \
-          " (Element: |#{current}|#{current.name}|)"
-      end
-    end
-  end
-end

data/lib/aranha/engine.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class Engine < ::Rails::Engine
-    isolate_namespace Aranha
-
-    initializer :append_migrations do |app|
-      config.paths['db/migrate'].expanded.each do |expanded_path|
-        app.config.paths['db/migrate'] << expanded_path
-      end
-    end
-  end
-end

data/lib/aranha/fixtures.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  module Fixtures
-    require 'aranha/fixtures/download'
-  end
-end

data/lib/aranha/fixtures/download.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/base'
-require 'aranha/parsers/source_address'
-require 'aranha/parsers/source_target_fixtures'
-
-module Aranha
-  module Fixtures
-    class Download
-      attr_reader :pending
-
-      def initialize(options)
-        @prefix = options.fetch(:prefix)
-        @prefix = '' if @prefix.blank?
-        @download = options.fetch(:download)
-        @pending = options.fetch(:pending)
-      end
-
-      def run
-        url_files.each do |f|
-          Rails.logger.info(relative_path(f))
-          download(url(f), target(f)) if @download
-        end
-      end
-
-      private
-
-      def url_files
-        Dir["#{fixtures_root}/**/*.url"].select { |path| select_path?(path) }
-      end
-
-      def select_path?(path)
-        return false unless match_prefix_pattern(path)
-
-        !pending || !source_exist?(path)
-      end
-
-      def match_prefix_pattern(path)
-        relative_path(path).start_with?(@prefix)
-      end
-
-      def fixtures_root
-        Rails.root.to_s
-      end
-
-      def download(url, target)
-        Rails.logger.info "Baixando \"#{url}\"..."
-        content = ::Aranha::Parsers::Base.new(url).content
-        raise "Content is blank for \"#{url}\"" if content.blank?
-
-        File.open(target, 'wb') { |file| file.write(content) }
-      end
-
-      def url(file)
-        ::Aranha::Parsers::SourceAddress.from_file(file)
-      end
-
-      def target(file)
-        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
-      end
-
-      def relative_path(path)
-        path.sub(%r{^#{Regexp.quote(fixtures_root)}/}, '')
-      end
-
-      def source_exist?(path)
-        stf = ::Aranha::Parsers::SourceTargetFixtures.new(::File.dirname(path))
-        stf.source_file(::File.basename(path, '.url')).present?
-      end
-    end
-  end
-end

data/lib/tasks/aranha_tasks.rake

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-namespace(:aranha) do
-  task process: :environment do
-    ::Aranha::Processor.new
-  end
-
-  task clear: :environment do
-    Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
-  end
-
-  namespace :fixtures do
-    desc 'Download remote content for fixtures.'
-    task download: :environment do
-      ::Aranha::Fixtures::Download.new(
-        prefix: ENV['PREFIX'],
-        download: ENV['DOWNLOAD'].present?,
-        pending: ENV['PENDING'].present?
-      ).run
-    end
-  end
-end