aranha 0.14.5 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b89c63a38f3c2f658263dde445f10b45628ecd19069d6a78a0d5e838a1a6127
-  data.tar.gz: 15cfb49d733aba80bd316813d21c587c47c4799af6a60d9a6147bed1e08f132f
+  metadata.gz: c5fac0411750b6def655452009d3d091801905c696158499d8b4b26f99ccc426
+  data.tar.gz: cb8cd648b8603cfd1f578ba92e5017e37d6e1e19427b176d33e097e413c17baf
 SHA512:
-  metadata.gz: fe5b484b9709033c8f87db0f8070c83d5dd328109d2d6bfb12a7ef356ba7268ec45cf728f5554dbd3015e28f60d14be2b81d6ac33f836365024fb4e88f1ed7e1
-  data.tar.gz: 9456fafe876c0894f810575fe41f078703ce2f9b75930c967ed4ffd9d33b93eeeb80e4b899e9bc12f31612d397fb1d462ab29520e7df4f71956fb64c489628f6
+  metadata.gz: c4864a35aa117b9bb00d544c013ae6caaf356b3d674657d5b1d5a3371f644b949b4fa32b6ec97d5f44fa58f65ea71db0499f18208ed885103a4f93d891d41309
+  data.tar.gz: c20f17bb1c3d04ce9b7678d2bec181c74dcf3b930202f9c838f5046cd68b326eb660ad490c4ad4c92647f60474eefc705e6e2734a62f3b42541760d689ab9686

data/README.rdoc

@@ -1,3 +1,3 @@
 = Aranha
 
-Rails utilities for web crawling.
+Ruby utilities for web crawling.

data/lib/aranha.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-require 'httpclient'
-require 'active_support/dependencies'
-require 'active_scaffold'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
-  require 'aranha/default_processor'
-  require 'aranha/dom_elements_traverser'
-  require 'aranha/engine'
-  require 'aranha/fixtures'
-  require 'aranha/processor'
-  require 'aranha/parsers'
-  require 'aranha/selenium'
+  require_sub __FILE__
 end

data/lib/aranha/manager.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/abstract_methods'
+
+module Aranha
+  class Manager
+    include ::EacRubyUtils::AbstractMethods
+
+    class << self
+      attr_accessor :default
+    end
+
+    def addresses_count
+      raise_abstract_method(__method__)
+    end
+
+    def add_address(_uri, _processor_class, _extra_data = nil)
+      raise_abstract_method(__method__)
+    end
+
+    def add_start_point(uri, processor_class, extra_data = nil)
+      start_points_var << ::EacRubyUtils::Struct.new(
+        uri: uri, processor_class: processor_class, extra_data: extra_data
+      )
+    end
+
+    def clear_expired_addresses
+      raise_abstract_method(__method__)
+    end
+
+    def init
+      clear_expired_addresses
+      start_points_to_addresses
+    end
+
+    def log_info(_message)
+      raise_abstract_method(__method__)
+    end
+
+    def log_warn(_message)
+      raise_abstract_method(__method__)
+    end
+
+    def start_points
+      start_points_var.to_enum
+    end
+
+    def start_points_to_addresses
+      start_points_var.each do |sp|
+        add_address(sp.uri, sp.processor_class, sp.extra_data)
+      end
+    end
+
+    def unprocessed_addresses
+      raise_abstract_method(__method__)
+    end
+
+    private
+
+    def start_points_var
+      @start_points_var ||= []
+    end
+  end
+end

data/lib/aranha/processor.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require 'net/http'
+require 'httpclient'
 require 'aranha/parsers/invalid_state_exception'
+require 'aranha/manager'
 
 module Aranha
   class Processor
@@ -20,11 +22,13 @@ module Aranha
 
     DEFAULT_MAX_TRIES = 3
 
-    def initialize
-      ::Aranha::Address.clear_expired
-      ::Aranha::Address.add_start_points
+    attr_reader :manager
+
+    def initialize(manager = nil)
+      @manager = manager || ::Aranha::Manager.default
       @failed = {}
       @try = 0
+      self.manager.init
       process_loop
       raise "Addresses failed: #{@failed.count}" if @failed.any?
     end
@@ -32,7 +36,7 @@ module Aranha
     private
 
     def process_loop
-      Rails.logger.info("Max tries: #{max_tries_s}")
+      manager.log_info("Max tries: #{max_tries_s}")
       loop do
         break if process_next_address
       end
@@ -52,8 +56,8 @@ module Aranha
     end
 
     def process_address(address)
-      Rails.logger.info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
-          " Unprocessed: #{unprocessed.count}/#{Aranha::Address.count})")
+      manager.log_info("Processing #{address} (Try: #{@try}/#{max_tries_s}," \
+          " Unprocessed: #{unprocessed.count}/#{::Aranha::Manager.default.addresses_count})")
       begin
         address.process
         @failed.delete(address.id)
@@ -67,7 +71,7 @@ module Aranha
 
       @failed[address.id] ||= 0
       @failed[address.id] += 1
-      Rails.logger.warn(exception)
+      manager.log_warn(exception)
     end
 
     def next_address
@@ -75,7 +79,7 @@ module Aranha
     end
 
     def unprocessed
-      ::Aranha::Address.unprocessed
+      ::Aranha::Manager.default.unprocessed_addresses
     end
 
     def network_exception?(exception)

data/lib/aranha/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aranha
-  VERSION = '0.14.5'
+  VERSION = '0.15.0'
 end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: aranha
 version: !ruby/object:Gem::Version
-  version: 0.14.5
+  version: 0.15.0
 platform: ruby
 authors:
 - Eduardo H. Bogoni
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-14 00:00:00.000000000 Z
+date: 2020-11-28 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: active_scaffold
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.41.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.4.41.1
 - !ruby/object:Gem::Dependency
   name: aranha-parsers
   requirement: !ruby/object:Gem::Requirement
@@ -58,34 +44,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.2
-- !ruby/object:Gem::Dependency
-  name: eac_rails_utils
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.11'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.11'
 - !ruby/object:Gem::Dependency
   name: eac_ruby_utils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.35'
+        version: '0.52'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.35'
+        version: '0.52'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -100,20 +72,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.6'
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.11.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 4.2.11.3
 - !ruby/object:Gem::Dependency
   name: eac_ruby_gem_support
   requirement: !ruby/object:Gem::Requirement
@@ -128,20 +86,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-- !ruby/object:Gem::Dependency
-  name: sqlite3
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: 
 email:
 - eduardobogoni@gmail.com
@@ -151,29 +95,11 @@ extra_rdoc_files: []
 files:
 - MIT-LICENSE
 - README.rdoc
-- app/assets/javascripts/aranha/application.js
-- app/assets/stylesheets/aranha/application.css
-- app/controllers/aranha/addresses_controller.rb
-- app/helpers/aranha/application_helper.rb
-- app/models/aranha/address.rb
-- app/views/layouts/aranha/application.html.erb
-- config/locales/en.yml
-- config/locales/pt-BR.yml
-- config/routes.rb
-- db/migrate/20171201021251_create_aranha_addresses.rb
-- db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb
 - lib/aranha.rb
 - lib/aranha/default_processor.rb
-- lib/aranha/dom_elements_traverser.rb
-- lib/aranha/dom_elements_traverser/conditions.rb
-- lib/aranha/dom_elements_traverser/cursor.rb
-- lib/aranha/dom_elements_traverser/data.rb
-- lib/aranha/engine.rb
-- lib/aranha/fixtures.rb
-- lib/aranha/fixtures/download.rb
+- lib/aranha/manager.rb
 - lib/aranha/processor.rb
 - lib/aranha/version.rb
-- lib/tasks/aranha_tasks.rake
 homepage: 
 licenses:
 - MIT
@@ -196,5 +122,5 @@ requirements: []
 rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
-summary: Rails utilities for web crawling.
+summary: Ruby utilities for web crawling.
 test_files: []

data/app/assets/javascripts/aranha/application.js

@@ -1,14 +0,0 @@
-// This is a manifest file that'll be compiled into application.js, which will include all the files
-// listed below.
-//
-// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
-// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
-//
-// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
-// compiled file.
-//
-// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
-// about supported directives.
-//
-//= require_tree .
-//= require active_scaffold

data/app/assets/stylesheets/aranha/application.css

@@ -1,16 +0,0 @@
-/*
- * This is a manifest file that'll be compiled into application.css, which will include all the files
- * listed below.
- *
- * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
- * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
- *
- * You're free to add application-wide styles to this file and they'll appear at the bottom of the
- * compiled file so the styles you add here take precedence over styles defined in any styles
- * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
- * file per style scope.
- *
- *= require_tree .
- *= require_self
- *= require active_scaffold
- */

data/app/controllers/aranha/addresses_controller.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class AddressesController < ::ApplicationController
-    active_scaffold :'aranha/address' do |_conf|
-    end
-  end
-end

data/app/helpers/aranha/application_helper.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  module ApplicationHelper
-  end
-end

data/app/models/aranha/address.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-require 'eac_ruby_utils/yaml'
-
-module Aranha
-  class Address < ActiveRecord::Base
-    include ::EacRailsUtils::Models::InequalityQueries
-
-    add_inequality_queries(:created_at)
-
-    class << self
-      def set_start_point(url, processor)
-        start_points[url] = processor
-      end
-
-      def add_start_points
-        ::Rails.logger.info("Start points: #{start_points.count}")
-        start_points.each do |url, processor|
-          add(url, processor)
-        end
-      end
-
-      def add(url, processor, extra_data = nil)
-        a = find_or_initialize_by(url: sanitize_url(url))
-        a.processor = processor
-        a.extra_data = extra_data.to_yaml
-        a.save!
-      end
-
-      def clear_expired
-        q = by_created_at_lt(Time.zone.now - 12.hours)
-        Rails.logger.info("Addresses expired: #{q.count}")
-        q.destroy_all
-      end
-
-      private
-
-      def sanitize_url(url)
-        if url.is_a?(Hash)
-          url.to_yaml
-        else
-          url.to_s
-        end
-      end
-
-      def start_points
-        @start_points ||= {}
-      end
-    end
-
-    validates :url, presence: true, uniqueness: true
-    validates :processor, presence: true
-
-    scope :unprocessed, lambda {
-      where(processed_at: nil)
-    }
-
-    def to_s
-      "#{processor}|#{url}"
-    end
-
-    def process
-      ActiveRecord::Base.transaction do
-        instanciate_processor.process
-        self.processed_at = Time.zone.now
-        save!
-      end
-    end
-
-    private
-
-    def instanciate_processor
-      processor_instancier.call(*processor_instancier_arguments)
-    end
-
-    def url_to_process
-      ::EacRubyUtils::Yaml.load(url)
-    end
-
-    def processor_instancier
-      processor.constantize.method(:new)
-    end
-
-    def processor_instancier_arguments
-      if processor_instancier_arity == 2 || processor_instancier_arity.negative?
-        [url_to_process, EacRubyUtils::Yaml.load(extra_data)]
-      elsif processor_instancier_arity == 1
-        [processor_instancier.call(url_to_process)]
-      else
-        raise("#{processor}.initialize should has 1 or 2 or * arguments")
-      end
-    end
-
-    def processor_instancier_arity
-      processor.constantize.instance_method(:initialize).arity
-    end
-  end
-end

data/app/views/layouts/aranha/application.html.erb

@@ -1,12 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <title>Aranha</title>
-    <%= stylesheet_link_tag    "aranha/application", media: "all" %>
-    <%= javascript_include_tag "aranha/application" %>
-    <%= csrf_meta_tags %>
-  </head>
-  <body>
-    <%= yield %>
-  </body>
-</html>

data/config/locales/en.yml

@@ -1,6 +0,0 @@
-en:
-  activerecord:
-    models:
-      aranha/address:
-        one: Aranha address
-        other: Aranha addresses

data/config/locales/pt-BR.yml

@@ -1,6 +0,0 @@
-pt-BR:
-  activerecord:
-    models:
-      aranha/address:
-        one: Endereço Aranha
-        other: Endereços Aranha

data/config/routes.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-Aranha::Engine.routes.draw do
-  concern :active_scaffold, ActiveScaffold::Routing::Basic.new(association: true)
-  resources(:addresses, concerns: :active_scaffold)
-end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-class CreateAranhaAddresses < (
-    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
-  )
-  def change
-    create_table :aranha_addresses do |t|
-      t.string :url
-      t.string :processor
-      t.timestamp :processed_at
-
-      t.timestamps null: false
-    end
-  end
-end

data/db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-class AddExtraDataToAranhaAddresses < (
-    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
-  )
-  def change
-    add_column :aranha_addresses, :extra_data, :text
-  end
-end

data/lib/aranha/dom_elements_traverser.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/dom_elements_traverser/conditions'
-require 'aranha/dom_elements_traverser/data'
-require 'aranha/dom_elements_traverser/cursor'
-
-module Aranha
-  class DomElementsTraverser
-    include ::Aranha::DomElementsTraverser::Conditions
-    include ::Aranha::DomElementsTraverser::Cursor
-    include ::Aranha::DomElementsTraverser::Data
-
-    class << self
-      def traverse(options, &block)
-        new(elements_from_options(options), &block)
-      end
-
-      def empty
-        new([])
-      end
-
-      private
-
-      def elements_from_options(options)
-        options = ::EacRubyUtils::OptionsConsumer.new(options)
-        elements = nil
-        options.consume(:children_of) { |v| elements = v.children.to_a }
-        raise 'None option of [:children_of] defined' unless elements
-
-        options.validate
-        elements
-      end
-    end
-
-    private
-
-    def initialize(elements, &block)
-      @elements = elements
-      @index = 0
-      @data = {}
-      instance_eval(&block) if block
-    end
-  end
-end

data/lib/aranha/dom_elements_traverser/conditions.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class DomElementsTraverser
-    module Conditions
-      private
-
-      def match_conditions?(conditions)
-        raise "No element (Conditions: #{conditions})" unless current
-
-        conditions.all? { |key, value| match_condition?(key, value) }
-      end
-
-      def match_condition?(key, value)
-        case key.to_sym
-        when :text then match_text_condition?(value)
-        when :name then match_name_condition?(value)
-        else raise "Unknown key condition: (#{key})"
-        end
-      end
-
-      def match_name_condition?(tag_name)
-        current.name.casecmp(tag_name.to_s).zero?
-      end
-
-      def match_text_condition?(texts)
-        texts = [texts.to_s] unless texts.is_a?(Array)
-        texts.all? { |t| current.text.downcase.include?(t.downcase) }
-      end
-    end
-  end
-end

data/lib/aranha/dom_elements_traverser/cursor.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require 'eac_ruby_utils/options_consumer'
-
-module Aranha
-  class DomElementsTraverser
-    module Cursor
-      private
-
-      def current
-        @elements[@index]
-      end
-
-      def skip
-        @index += 1
-      end
-
-      def skip_until(options)
-        oc = ::EacRubyUtils::OptionsConsumer.new(options)
-        optional = oc.consume(:optional, false)
-        while current
-          break if match_conditions?(oc.left_data)
-
-          skip
-        end
-        raise "No element found for conditions #{oc.left_data}" unless current || optional
-
-        current
-      end
-
-      def skip_until_after(conditions)
-        skip_until(conditions)
-        skip
-        current
-      end
-
-      def if_found(conditions, &block)
-        marked = @index
-        skip_until({ optional: true }.merge(conditions))
-        if current
-          instance_eval(&block) if block
-        else
-          @index = marked
-        end
-      end
-    end
-  end
-end

data/lib/aranha/dom_elements_traverser/data.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class DomElementsTraverser
-    module Data
-      def data
-        @data.dup
-      end
-
-      private
-
-      def store(key, options = {}, &converter)
-        validate(options)
-        value = store_value(options, converter)
-        @data[key] = value
-        r = current
-        skip
-        r
-      end
-
-      def store_value(options, converter)
-        value = if options.key?(:attribute)
-                  current.attribute(options[:attribute]).value
-                else
-                  current.text.strip
-                end
-        converter ? converter.call(value) : value
-      end
-
-      def validate(options)
-        return unless options.key?(:validate)
-        return if match_conditions?(options[:validate])
-
-        raise "Element does not match conditions #{options[:validate]}" \
-          " (Element: |#{current}|#{current.name}|)"
-      end
-    end
-  end
-end

data/lib/aranha/engine.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  class Engine < ::Rails::Engine
-    isolate_namespace Aranha
-
-    initializer :append_migrations do |app|
-      config.paths['db/migrate'].expanded.each do |expanded_path|
-        app.config.paths['db/migrate'] << expanded_path
-      end
-    end
-  end
-end

data/lib/aranha/fixtures.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module Aranha
-  module Fixtures
-    require 'aranha/fixtures/download'
-  end
-end

data/lib/aranha/fixtures/download.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-require 'aranha/parsers/base'
-require 'aranha/parsers/source_address'
-require 'aranha/parsers/source_target_fixtures'
-
-module Aranha
-  module Fixtures
-    class Download
-      attr_reader :pending
-
-      def initialize(options)
-        @prefix = options.fetch(:prefix)
-        @prefix = '' if @prefix.blank?
-        @download = options.fetch(:download)
-        @pending = options.fetch(:pending)
-      end
-
-      def run
-        url_files.each do |f|
-          Rails.logger.info(relative_path(f))
-          download(url(f), target(f)) if @download
-        end
-      end
-
-      private
-
-      def url_files
-        Dir["#{fixtures_root}/**/*.url"].select { |path| select_path?(path) }
-      end
-
-      def select_path?(path)
-        return false unless match_prefix_pattern(path)
-
-        !pending || !source_exist?(path)
-      end
-
-      def match_prefix_pattern(path)
-        relative_path(path).start_with?(@prefix)
-      end
-
-      def fixtures_root
-        Rails.root.to_s
-      end
-
-      def download(url, target)
-        Rails.logger.info "Baixando \"#{url}\"..."
-        content = ::Aranha::Parsers::Base.new(url).content
-        raise "Content is blank for \"#{url}\"" if content.blank?
-
-        File.open(target, 'wb') { |file| file.write(content) }
-      end
-
-      def url(file)
-        ::Aranha::Parsers::SourceAddress.from_file(file)
-      end
-
-      def target(file)
-        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
-      end
-
-      def relative_path(path)
-        path.sub(%r{^#{Regexp.quote(fixtures_root)}/}, '')
-      end
-
-      def source_exist?(path)
-        stf = ::Aranha::Parsers::SourceTargetFixtures.new(::File.dirname(path))
-        stf.source_file(::File.basename(path, '.url')).present?
-      end
-    end
-  end
-end

data/lib/tasks/aranha_tasks.rake

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-namespace(:aranha) do
-  task process: :environment do
-    ::Aranha::Processor.new
-  end
-
-  task clear: :environment do
-    Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
-  end
-
-  namespace :fixtures do
-    desc 'Download remote content for fixtures.'
-    task download: :environment do
-      ::Aranha::Fixtures::Download.new(
-        prefix: ENV['PREFIX'],
-        download: ENV['DOWNLOAD'].present?,
-        pending: ENV['PENDING'].present?
-      ).run
-    end
-  end
-end