aranha-rails 0.5.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49f04cc5c0a957bc1aec7617dc1a6e736d860d80905afa04844587b8c57dd2bd
-  data.tar.gz: 18c6a92f2c28731b7224ea7a99a3521c684c56e33446f29601fcab4bf2de3a68
+  metadata.gz: a2f81d25c96765dabe391eba3b55e7e41ba034f7e0c513f9c3e3d41e324b6a84
+  data.tar.gz: 1b17888a4c5ba75da243691599f99c48c1fcd403922650af9402770854e48df4
 SHA512:
-  metadata.gz: a0cf36bd155c0ccdea28701d96d385eb28d3e80a09a9a37e273b1b372f1598c27030a89d040492a707eefea98caf043972e86dc00b90e68f034cb60ef0cd59ab
-  data.tar.gz: 7947c879299d9ee6f0104fe7b6577a8ecc544f6bbccb59cc92f3799b951b98c2f8cd1fc7579b348d69eaa3780f70399008b901a196b42a00a15a7394bb85a2f8
+  metadata.gz: b50cda8b00fdd6408787ec8c115de71b202751aaa472d005711b2c4f47f52851a9711a87db4d60afcf584438132b98073b15b3958e82fcad0dfe73e343b0f579
+  data.tar.gz: 82640e9f247086ff8c47c66c68cdc90629e76dbbadeb828829da43531f845f985bca13f1427c708db0aa763ee85892e5ce94a5e503bc37c25cc740535faa5b4e

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,25 @@
+require:
+  - rubocop-rails
+  - rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.4
+  TargetRailsVersion: 5.1
+
+Layout/LineLength:
+  Max: 100
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/app/assets/javascripts/aranha/application.js

@@ -0,0 +1,2 @@
+//= require_tree .
+//= require eac_active_scaffold

data/app/assets/stylesheets/aranha/application.css

@@ -0,0 +1,5 @@
+/*
+ *= require_tree .
+ *= require_self
+ *= require eac_active_scaffold
+ */

data/app/controllers/aranha/addresses_controller.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Aranha
+  class AddressesController < ::ApplicationController
+    active_scaffold :'aranha/address' do |_conf|
+    end
+  end
+end

data/app/controllers/aranha/processor_configurations_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aranha
+  class ProcessorConfigurationsController < ::ApplicationController
+    before_action :configure_processor_class_options
+
+    active_scaffold :'aranha/processor_configuration' do |conf|
+      conf.columns[:processor_class].form_ui = :select
+    end
+
+    def configure_processor_class_options
+      active_scaffold_config.columns[:processor_class].options =
+        { options: ::Aranha::ProcessorConfiguration.processor_class_options }
+    end
+  end
+end

data/app/controllers/aranha/start_points_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aranha
+  class StartPointsController < ::ApplicationController
+    before_action :configure_processor_class_options
+
+    active_scaffold :'aranha/start_point' do |conf|
+      conf.columns[:processor_class].form_ui = :select
+    end
+
+    def configure_processor_class_options
+      active_scaffold_config.columns[:processor_class].options =
+        { options: ::Aranha::StartPoint.processor_class_options }
+    end
+  end
+end

data/app/models/aranha/address/delayed_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    class DelayedJob
+      common_constructor :address_id
+
+      def perform
+        address_processor.successful? ? perform_on_success : perform_on_error
+      end
+
+      private
+
+      def address
+        address_processor.address
+      end
+
+      def perform_on_success
+        # Do nothing
+      end
+
+      def perform_on_error
+        address.update!(
+          tries_count: address.tries_count + 1,
+          last_error: address_processor.error.to_yaml
+        )
+        address.check_scheduling
+      end
+
+      def address_processor
+        @address_processor ||= ::Aranha::AddressProcessor.new(
+          ::Aranha::Address.find(address_id)
+        )
+      end
+    end
+  end
+end

data/app/models/aranha/address/processor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    module Processor
+      common_concern
+
+      module ClassMethods
+        # @return [Aranha::ProcessorConfiguration]
+        def default_processor_configuration
+          @default_processor_configuration ||= ::Aranha::ProcessorConfiguration.new
+        end
+      end
+
+      # @return [Aranha::ProcessorConfiguration]
+      def processor_configuration
+        ::Aranha::ProcessorConfiguration.find_by(processor_class: processor) ||
+          self.class.default_processor_configuration
+      end
+    end
+  end
+end

data/app/models/aranha/address/scheduling.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    module Scheduling
+      common_concern
+
+      module ClassMethods
+        # @return [Array<Aranha::Address>]
+        def expired(time = ::Time.zone.now)
+          all.select { |record| record.expired?(time) }
+        end
+      end
+
+      def check_scheduling
+        ::ActiveRecord::Base.transaction do
+          return unless schedule?
+
+          job = ::Delayed::Job.enqueue(
+            ::Aranha::Address::DelayedJob.new(id),
+            queue: ::Aranha::Rails::Process::QUEUE
+          )
+          update!(delayed_job: job)
+        end
+      end
+
+      def expired?(time = ::Time.zone.now)
+        time >= (created_at + timeout)
+      end
+
+      def init_scheduling
+        update!(tries_count: 0, last_error: nil) unless processed?
+        check_scheduling
+      end
+
+      def allow_retry?
+        tries_count < ::Aranha::Processor::DEFAULT_MAX_TRIES
+      end
+
+      def schedule?
+        processed_at.blank? && allow_retry? && delayed_job.blank? && enabled?
+      end
+
+      delegate :enabled?, :timeout, to: :processor_configuration
+    end
+  end
+end

data/app/models/aranha/address.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+require 'delayed/backend/active_record'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    include ::EacRailsUtils::Models::InequalityQueries
+    include ::Aranha::Address::Processor
+    include ::Aranha::Address::Scheduling
+
+    add_inequality_queries(:created_at)
+
+    class << self
+      def sanitize_url(url)
+        if url.is_a?(Hash)
+          ::EacRubyUtils::Yaml.dump(url)
+        else
+          url.to_s
+        end
+      end
+    end
+
+    belongs_to :delayed_job, class_name: 'Delayed::Backend::ActiveRecord::Job', dependent: :destroy,
+                             optional: true
+
+    validates :url, presence: true, uniqueness: true
+    validates :processor, presence: true
+    validates :tries_count, presence: true, numericality: { only_integer: true,
+                                                            greater_or_equal: 0 }
+
+    scope :failed, lambda {
+      where(processed_at: nil).where.not(last_error: nil)
+    }
+
+    def to_s
+      "#{processor}|#{url}"
+    end
+
+    def process
+      ActiveRecord::Base.transaction do
+        instanciate_processor.process
+        self.processed_at = Time.zone.now
+        save!
+      end
+    end
+
+    def processed?
+      processed_at.present?
+    end
+
+    private
+
+    def instanciate_processor
+      processor_instancier.call(*processor_instancier_arguments)
+    end
+
+    def url_to_process
+      ::EacRubyUtils::Yaml.load(url)
+    end
+
+    def processor_instancier
+      processor.constantize.method(:new)
+    end
+
+    def processor_instancier_arguments
+      if processor_instancier_arity == 2 || processor_instancier_arity.negative?
+        [url_to_process, EacRubyUtils::Yaml.load(extra_data)]
+      elsif processor_instancier_arity == 1
+        [processor_instancier.call(url_to_process)]
+      else
+        raise("#{processor}.initialize should has 1 or 2 or * arguments")
+      end
+    end
+
+    def processor_instancier_arity
+      processor.constantize.instance_method(:initialize).arity
+    end
+  end
+end

data/app/models/aranha/processor_configuration.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+
+module Aranha
+  class ProcessorConfiguration < ::ActiveRecord::Base
+    DEFAULT_TIMEOUT = 12.hours
+
+    class << self
+      # @return [Enumerator<String>]
+      def processor_class_list
+        ::Set.new(processor_class_list_from_addresses + processor_class_list_from_start_points)
+             .to_enum.sort
+      end
+
+      # @return [Hash<String, String>]
+      def processor_class_options
+        processor_class_list.map { |e| [e, e] }.to_h
+      end
+
+      private
+
+      # @return [Array<String>]
+      def processor_class_list_from_addresses
+        ::Aranha::Address.distinct.pluck(:processor)
+      end
+
+      # @return [Array<String>]
+      def processor_class_list_from_start_points
+        ::Aranha::StartPoint.processor_class_list.to_a
+      end
+    end
+
+    validates :processor_class, presence: true
+    validate :processor_class_in_list
+    validates :timeout_seconds, allow_blank: true,
+                                numericality: { integer_only: true, greater_than_or_equal_to: 1 }
+
+    def processor_class_in_list
+      return if processor_class.blank?
+      return if self.class.processor_class_list.include?(processor_class)
+
+      errors.add(:processor_class, 'Not in list')
+    end
+
+    # @return [ActiveSupport::Duration]
+    def timeout
+      timeout_seconds.if_present(DEFAULT_TIMEOUT, &:seconds)
+    end
+  end
+end

data/app/models/aranha/start_point.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+
+module Aranha
+  class StartPoint < ::ActiveRecord::Base
+    class << self
+      def add_processor_class(klass)
+        processor_class_list_var.add(klass.to_s)
+      end
+
+      def processor_class_list
+        processor_class_list_var.to_enum
+      end
+
+      def processor_class_options
+        processor_class_list.map { |e| [e, e] }.to_h
+      end
+
+      private
+
+      def processor_class_list_var
+        @processor_class_list_var ||= Set.new
+      end
+    end
+
+    validates :uri, presence: true, uniqueness: true,
+                    format: { with: ::URI::DEFAULT_PARSER.make_regexp }
+    validates :processor_class, presence: true
+    validate :processor_class_in_list
+
+    def extra_data
+      extra_data_yaml.nil? ? nil : ::EacRubyUtils::Yaml.load(extra_data_yaml)
+    end
+
+    def extra_data=(value)
+      self.extra_data_yaml = ::EacRubyUtils::Yaml.dump(value)
+    end
+
+    def processor_class_in_list
+      return if processor_class.blank?
+      return if self.class.processor_class_list.include?(processor_class)
+
+      errors.add(:processor_class, 'Not in list')
+    end
+  end
+end

data/config/initializers/aranha.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'aranha/rails/manager'
+
+::Aranha::Manager.default = ::Aranha::Rails::Manager.instance

data/config/locales/en.yml

@@ -0,0 +1,12 @@
+en:
+  activerecord:
+    models:
+      aranha/address:
+        one: Aranha address
+        other: Aranha addresses
+      aranha/processor_configuration:
+        one: Aranha processor configuration
+        other: Aranha processors configurations
+      aranha/start_point:
+        one: Aranha start point
+        other: Aranha start points

data/config/locales/pt-BR.yml

@@ -0,0 +1,12 @@
+pt-BR:
+  activerecord:
+    models:
+      aranha/address:
+        one: Endereço Aranha
+        other: Endereços Aranha
+      aranha/processor_configuration:
+        one: Configuração de processador Aranha
+        other: Configuração de processadores Aranha
+      aranha/start_point:
+        one: Ponto de Partida Aranha
+        other: Pontos de Partida Aranha

data/config/routes.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+::Aranha::Rails::Engine.routes.draw do
+  resources(:addresses, concerns: active_scaffold)
+  resources(:processor_configurations, concerns: active_scaffold)
+  resources(:start_points, concerns: active_scaffold)
+end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateAranhaAddresses < (
+    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+  )
+  def change
+    create_table :aranha_addresses do |t|
+      t.string :url
+      t.string :processor
+      t.timestamp :processed_at
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddExtraDataToAranhaAddresses < (
+    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+  )
+  def change
+    add_column :aranha_addresses, :extra_data, :text
+  end
+end

data/db/migrate/20201128202212_create_aranha_start_points.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateAranhaStartPoints < ActiveRecord::Migration[5.1]
+  def change
+    create_table :aranha_start_points do |t|
+      t.string :uri
+      t.string :processor_class
+      t.string :extra_data_yaml
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20210720023620_add_delayed_job_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddDelayedJobToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_reference :aranha_addresses, :delayed_job, foreign_key: false
+  end
+end

data/db/migrate/20210721171416_add_tries_count_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddTriesCountToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_addresses, :tries_count, :integer, null: false, default: 0
+  end
+end

data/db/migrate/20210721185222_add_last_error_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddLastErrorToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_addresses, :last_error, :text
+  end
+end

data/db/migrate/20210814220540_create_aranha_processor_configurations.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateAranhaProcessorConfigurations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :aranha_processor_configurations do |t|
+      t.string :processor_class
+      t.integer :timeout_seconds, null: true
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20210815160543_add_enabled_to_aranha_processor_configurations.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddEnabledToAranhaProcessorConfigurations < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_processor_configurations, :enabled, :boolean, default: true, null: false
+  end
+end

data/lib/aranha/rails/engine.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'active_scaffold'
+require 'eac_active_scaffold'
 require 'eac_rails_delayed_job/engine'
 
 module Aranha

data/lib/aranha/rails/manager.rb

@@ -22,9 +22,9 @@ module Aranha
       end
 
       def clear_expired_addresses
-        q = ::Aranha::Address.by_created_at_lt(Time.zone.now - 12.hours)
+        q = ::Aranha::Address.expired(::Time.zone.now)
         ::Rails.logger.info("Addresses expired: #{q.count}")
-        q.destroy_all
+        q.each(&:destroy)
       end
 
       def log_info(message)
@@ -43,7 +43,7 @@ module Aranha
       end
 
       def unprocessed_addresses
-        ::Aranha::Address.unprocessed
+        ::Aranha::Address.all.select(&:schedule?)
       end
     end
   end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.5.0'
+    VERSION = '0.7.2'
   end
 end

data/spec/rubocop_spec.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+::EacRubyUtils::Rspec.default_setup.describe_rubocop

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/rspec/default_setup'
+::EacRubyUtils::Rspec.default_setup_create(::File.expand_path('..', __dir__))

metadata

@@ -1,63 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-08 00:00:00.000000000 Z
+date: 2021-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: active_scaffold
+  name: aranha
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: '0.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: '0.15'
 - !ruby/object:Gem::Dependency
-  name: aranha
+  name: aranha-parsers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: aranha-parsers
+  name: eac_active_scaffold
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_rails_delayed_job
   requirement: !ruby/object:Gem::Requirement
@@ -110,16 +104,22 @@ dependencies:
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.7
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.7
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: eac_ruby_gem_support
   requirement: !ruby/object:Gem::Requirement
@@ -146,6 +146,32 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- app/assets/javascripts/aranha/application.js
+- app/assets/stylesheets/aranha/application.css
+- app/controllers/aranha/addresses_controller.rb
+- app/controllers/aranha/processor_configurations_controller.rb
+- app/controllers/aranha/start_points_controller.rb
+- app/models/aranha/address.rb
+- app/models/aranha/address/delayed_job.rb
+- app/models/aranha/address/processor.rb
+- app/models/aranha/address/scheduling.rb
+- app/models/aranha/processor_configuration.rb
+- app/models/aranha/start_point.rb
+- config/initializers/aranha.rb
+- config/locales/en.yml
+- config/locales/pt-BR.yml
+- config/routes.rb
+- db/migrate/20171201021251_create_aranha_addresses.rb
+- db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb
+- db/migrate/20201128202212_create_aranha_start_points.rb
+- db/migrate/20210720023620_add_delayed_job_to_aranha_addresses.rb
+- db/migrate/20210721171416_add_tries_count_to_aranha_addresses.rb
+- db/migrate/20210721185222_add_last_error_to_aranha_addresses.rb
+- db/migrate/20210814220540_create_aranha_processor_configurations.rb
+- db/migrate/20210815160543_add_enabled_to_aranha_processor_configurations.rb
 - lib/aranha/rails.rb
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
@@ -154,6 +180,8 @@ files:
 - lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
+- spec/rubocop_spec.rb
+- spec/spec_helper.rb
 homepage: 
 licenses: []
 metadata: {}
@@ -176,4 +204,9 @@ rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.
-test_files: []
+test_files:
+- spec/rubocop_spec.rb
+- spec/spec_helper.rb
+- ".rspec"
+- Gemfile
+- ".rubocop.yml"