aranha-rails 0.6.0 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c2bfb595a37e39eb60a336df45502b5efdd52ef26b1c488fb60500cc2ca93c1
-  data.tar.gz: 983c14439b792dbd495578a57fcc8960468cb6d126d07f7635b030cc78aa12aa
+  metadata.gz: a322e7787c81f714d13888862ffdf6a30a4ac0d56847091dd119c67c0ca4ca1e
+  data.tar.gz: c67dea8842bf7f4213bd8cc25a4ad45f569f28681bed528d8d67786e49d984e8
 SHA512:
-  metadata.gz: 8e56125aa6b80634a9e9f43db75184868edf4081245157a1949b559c7aaf8e919fa63d6f2a1d210eb5229368eec478918d429d83960719db1bb0d4a28f09c4a5
-  data.tar.gz: 55d5cedac52545c011fdcab381029bdb680db791ee8855e7dd15b9cd4b7aa1a0aa290f713ae8be27bdb9d4cd84cb0116a4d6b90a04e2224c9dfddd41ba66608e
+  metadata.gz: c56775f370ea486705fcfaa35449910933b0468179592bdd94f1585c5f553480a7b8e10c52af0cc86e1b8f3283ebc1e419c6f0ad12cc55c6d5fc1db5676ba2b6
+  data.tar.gz: 9fe98cf105b3d38c13bf5310bca70edd8a5740b7352b96dfbd52b79f6d770c4a458708c7fd8d563aae71c37ada4ac1bfa28e911dacd38f237e08fdc0273f65fb

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,25 @@
+require:
+  - rubocop-rails
+  - rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.4
+  TargetRailsVersion: 5.1
+
+Layout/LineLength:
+  Max: 100
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/app/assets/javascripts/aranha/application.js

@@ -0,0 +1,2 @@
+//= require_tree .
+//= require eac_active_scaffold

data/app/assets/stylesheets/aranha/application.css

@@ -0,0 +1,5 @@
+/*
+ *= require_tree .
+ *= require_self
+ *= require eac_active_scaffold
+ */

data/app/controllers/aranha/addresses_controller.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Aranha
+  class AddressesController < ::ApplicationController
+    active_scaffold :'aranha/address' do |_conf|
+    end
+  end
+end

data/app/controllers/aranha/processor_configurations_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aranha
+  class ProcessorConfigurationsController < ::ApplicationController
+    before_action :configure_processor_class_options
+
+    active_scaffold :'aranha/processor_configuration' do |conf|
+      conf.columns[:processor_class].form_ui = :select
+    end
+
+    def configure_processor_class_options
+      active_scaffold_config.columns[:processor_class].options =
+        { options: ::Aranha::ProcessorConfiguration.processor_class_options }
+    end
+  end
+end

data/app/controllers/aranha/start_points_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aranha
+  class StartPointsController < ::ApplicationController
+    before_action :configure_processor_class_options
+
+    active_scaffold :'aranha/start_point' do |conf|
+      conf.columns[:processor_class].form_ui = :select
+    end
+
+    def configure_processor_class_options
+      active_scaffold_config.columns[:processor_class].options =
+        { options: ::Aranha::StartPoint.processor_class_options }
+    end
+  end
+end

data/app/models/aranha/address/delayed_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    class DelayedJob
+      common_constructor :address_id
+
+      def perform
+        address_processor.successful? ? perform_on_success : perform_on_error
+      end
+
+      private
+
+      def address
+        address_processor.address
+      end
+
+      def perform_on_success
+        # Do nothing
+      end
+
+      def perform_on_error
+        address.update!(
+          tries_count: address.tries_count + 1,
+          last_error: address_processor.error.to_yaml
+        )
+        address.check_scheduling
+      end
+
+      def address_processor
+        @address_processor ||= ::Aranha::AddressProcessor.new(
+          ::Aranha::Address.find(address_id)
+        )
+      end
+    end
+  end
+end

data/app/models/aranha/address/processor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    module Processor
+      common_concern
+
+      module ClassMethods
+        # @return [Aranha::ProcessorConfiguration]
+        def default_processor_configuration
+          @default_processor_configuration ||= ::Aranha::ProcessorConfiguration.new
+        end
+      end
+
+      # @return [Aranha::ProcessorConfiguration]
+      def processor_configuration
+        ::Aranha::ProcessorConfiguration.find_by(processor_class: processor) ||
+          self.class.default_processor_configuration
+      end
+    end
+  end
+end

data/app/models/aranha/address/scheduling.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    module Scheduling
+      common_concern
+
+      module ClassMethods
+        # @return [Array<Aranha::Address>]
+        def expired(time = ::Time.zone.now)
+          all.select { |record| record.expired?(time) }
+        end
+      end
+
+      def check_scheduling
+        ::ActiveRecord::Base.transaction do
+          return unless schedule?
+
+          job = ::Delayed::Job.enqueue(
+            ::Aranha::Address::DelayedJob.new(id),
+            queue: ::Aranha::Rails::Process::QUEUE
+          )
+          update!(delayed_job: job)
+        end
+      end
+
+      def expired?(time = ::Time.zone.now)
+        time >= (created_at + timeout)
+      end
+
+      def init_scheduling
+        update!(tries_count: 0, last_error: nil) unless processed?
+        check_scheduling
+      end
+
+      def allow_retry?
+        tries_count < ::Aranha::Processor::DEFAULT_MAX_TRIES
+      end
+
+      def schedule?
+        processed_at.blank? && allow_retry? && delayed_job.blank? && enabled?
+      end
+
+      delegate :enabled?, :timeout, to: :processor_configuration
+    end
+  end
+end

data/app/models/aranha/address.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+require 'delayed/backend/active_record'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    include ::EacRailsUtils::Models::InequalityQueries
+    include ::Aranha::Address::Processor
+    include ::Aranha::Address::Scheduling
+
+    add_inequality_queries(:created_at)
+
+    class << self
+      def sanitize_url(url)
+        if url.is_a?(Hash)
+          ::EacRubyUtils::Yaml.dump(url)
+        else
+          url.to_s
+        end
+      end
+    end
+
+    belongs_to :delayed_job, class_name: 'Delayed::Backend::ActiveRecord::Job', dependent: :destroy,
+                             optional: true
+
+    validates :url, presence: true, uniqueness: true
+    validates :processor, presence: true
+    validates :tries_count, presence: true, numericality: { only_integer: true,
+                                                            greater_or_equal: 0 }
+
+    scope :failed, lambda {
+      where(processed_at: nil).where.not(last_error: nil)
+    }
+
+    def to_s
+      "#{processor}|#{url}"
+    end
+
+    def process
+      ActiveRecord::Base.transaction do
+        instanciate_processor.process
+        self.processed_at = Time.zone.now
+        save!
+      end
+    end
+
+    def processed?
+      processed_at.present?
+    end
+
+    private
+
+    def instanciate_processor
+      processor_instancier.call(*processor_instancier_arguments)
+    end
+
+    def url_to_process
+      ::EacRubyUtils::Yaml.load(url)
+    end
+
+    def processor_instancier
+      processor.constantize.method(:new)
+    end
+
+    def processor_instancier_arguments
+      if processor_instancier_arity == 2 || processor_instancier_arity.negative?
+        [url_to_process, EacRubyUtils::Yaml.load(extra_data)]
+      elsif processor_instancier_arity == 1
+        [processor_instancier.call(url_to_process)]
+      else
+        raise("#{processor}.initialize should has 1 or 2 or * arguments")
+      end
+    end
+
+    def processor_instancier_arity
+      processor.constantize.instance_method(:initialize).arity
+    end
+  end
+end

data/app/models/aranha/processor_configuration.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+
+module Aranha
+  class ProcessorConfiguration < ::ActiveRecord::Base
+    DEFAULT_TIMEOUT = 12.hours
+
+    class << self
+      # @return [Enumerator<String>]
+      def processor_class_list
+        ::Set.new(processor_class_list_from_addresses + processor_class_list_from_start_points)
+             .to_enum.sort
+      end
+
+      # @return [Hash<String, String>]
+      def processor_class_options
+        processor_class_list.map { |e| [e, e] }.to_h
+      end
+
+      private
+
+      # @return [Array<String>]
+      def processor_class_list_from_addresses
+        ::Aranha::Address.distinct.pluck(:processor)
+      end
+
+      # @return [Array<String>]
+      def processor_class_list_from_start_points
+        ::Aranha::StartPoint.processor_class_list.to_a
+      end
+    end
+
+    validates :processor_class, presence: true
+    validate :processor_class_in_list
+    validates :timeout_seconds, allow_blank: true,
+                                numericality: { integer_only: true, greater_than_or_equal_to: 1 }
+
+    def processor_class_in_list
+      return if processor_class.blank?
+      return if self.class.processor_class_list.include?(processor_class)
+
+      errors.add(:processor_class, 'Not in list')
+    end
+
+    # @return [ActiveSupport::Duration]
+    def timeout
+      timeout_seconds.if_present(DEFAULT_TIMEOUT, &:seconds)
+    end
+  end
+end

data/app/models/aranha/start_point.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+
+module Aranha
+  class StartPoint < ::ActiveRecord::Base
+    class << self
+      def add_processor_class(klass)
+        processor_class_list_var.add(klass.to_s)
+      end
+
+      def processor_class_list
+        processor_class_list_var.to_enum
+      end
+
+      def processor_class_options
+        processor_class_list.map { |e| [e, e] }.to_h
+      end
+
+      private
+
+      def processor_class_list_var
+        @processor_class_list_var ||= Set.new
+      end
+    end
+
+    validates :uri, presence: true, uniqueness: true,
+                    format: { with: ::URI::DEFAULT_PARSER.make_regexp }
+    validates :processor_class, presence: true
+    validate :processor_class_in_list
+
+    def extra_data
+      extra_data_yaml.nil? ? nil : ::EacRubyUtils::Yaml.load(extra_data_yaml)
+    end
+
+    def extra_data=(value)
+      self.extra_data_yaml = ::EacRubyUtils::Yaml.dump(value)
+    end
+
+    def processor_class_in_list
+      return if processor_class.blank?
+      return if self.class.processor_class_list.include?(processor_class)
+
+      errors.add(:processor_class, 'Not in list')
+    end
+  end
+end

data/config/initializers/aranha.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'aranha/rails/manager'
+
+::Aranha::Manager.default = ::Aranha::Rails::Manager.instance

data/config/locales/en.yml

@@ -0,0 +1,12 @@
+en:
+  activerecord:
+    models:
+      aranha/address:
+        one: Aranha address
+        other: Aranha addresses
+      aranha/processor_configuration:
+        one: Aranha processor configuration
+        other: Aranha processors configurations
+      aranha/start_point:
+        one: Aranha start point
+        other: Aranha start points

data/config/locales/pt-BR.yml

@@ -0,0 +1,12 @@
+pt-BR:
+  activerecord:
+    models:
+      aranha/address:
+        one: Endereço Aranha
+        other: Endereços Aranha
+      aranha/processor_configuration:
+        one: Configuração de processador Aranha
+        other: Configuração de processadores Aranha
+      aranha/start_point:
+        one: Ponto de Partida Aranha
+        other: Pontos de Partida Aranha

data/config/routes.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+::Aranha::Rails::Engine.routes.draw do
+  resources(:addresses, concerns: active_scaffold)
+  resources(:processor_configurations, concerns: active_scaffold)
+  resources(:start_points, concerns: active_scaffold)
+end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateAranhaAddresses < (
+    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+  )
+  def change
+    create_table :aranha_addresses do |t|
+      t.string :url
+      t.string :processor
+      t.timestamp :processed_at
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddExtraDataToAranhaAddresses < (
+    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+  )
+  def change
+    add_column :aranha_addresses, :extra_data, :text
+  end
+end

data/db/migrate/20201128202212_create_aranha_start_points.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateAranhaStartPoints < ActiveRecord::Migration[5.1]
+  def change
+    create_table :aranha_start_points do |t|
+      t.string :uri
+      t.string :processor_class
+      t.string :extra_data_yaml
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20210720023620_add_delayed_job_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddDelayedJobToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_reference :aranha_addresses, :delayed_job, foreign_key: false
+  end
+end

data/db/migrate/20210721171416_add_tries_count_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddTriesCountToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_addresses, :tries_count, :integer, null: false, default: 0
+  end
+end

data/db/migrate/20210721185222_add_last_error_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddLastErrorToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_addresses, :last_error, :text
+  end
+end

data/db/migrate/20210814220540_create_aranha_processor_configurations.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateAranhaProcessorConfigurations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :aranha_processor_configurations do |t|
+      t.string :processor_class
+      t.integer :timeout_seconds, null: true
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20210815160543_add_enabled_to_aranha_processor_configurations.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddEnabledToAranhaProcessorConfigurations < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_processor_configurations, :enabled, :boolean, default: true, null: false
+  end
+end

data/lib/aranha/rails/engine.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'active_scaffold'
+require 'eac_active_scaffold'
 require 'eac_rails_delayed_job/engine'
 
 module Aranha

data/lib/aranha/rails/manager.rb

@@ -43,7 +43,7 @@ module Aranha
       end
 
       def unprocessed_addresses
-        ::Aranha::Address.unprocessed
+        ::Aranha::Address.all.select(&:schedule?)
       end
     end
   end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.6.0'
+    VERSION = '0.7.3'
   end
 end

data/spec/rubocop_spec.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+::EacRubyUtils::Rspec.default_setup.describe_rubocop

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/rspec/default_setup'
+::EacRubyUtils::Rspec.default_setup_create(::File.expand_path('..', __dir__))

metadata

@@ -1,63 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.3
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: active_scaffold
+  name: aranha
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: '0.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: '0.15'
 - !ruby/object:Gem::Dependency
-  name: aranha
+  name: aranha-parsers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: aranha-parsers
+  name: eac_active_scaffold
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_rails_delayed_job
   requirement: !ruby/object:Gem::Requirement
@@ -110,42 +104,68 @@ dependencies:
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.7
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.7
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: eac_ruby_gem_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: '0.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: '0.4'
 description: 
 email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- app/assets/javascripts/aranha/application.js
+- app/assets/stylesheets/aranha/application.css
+- app/controllers/aranha/addresses_controller.rb
+- app/controllers/aranha/processor_configurations_controller.rb
+- app/controllers/aranha/start_points_controller.rb
+- app/models/aranha/address.rb
+- app/models/aranha/address/delayed_job.rb
+- app/models/aranha/address/processor.rb
+- app/models/aranha/address/scheduling.rb
+- app/models/aranha/processor_configuration.rb
+- app/models/aranha/start_point.rb
+- config/initializers/aranha.rb
+- config/locales/en.yml
+- config/locales/pt-BR.yml
+- config/routes.rb
+- db/migrate/20171201021251_create_aranha_addresses.rb
+- db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb
+- db/migrate/20201128202212_create_aranha_start_points.rb
+- db/migrate/20210720023620_add_delayed_job_to_aranha_addresses.rb
+- db/migrate/20210721171416_add_tries_count_to_aranha_addresses.rb
+- db/migrate/20210721185222_add_last_error_to_aranha_addresses.rb
+- db/migrate/20210814220540_create_aranha_processor_configurations.rb
+- db/migrate/20210815160543_add_enabled_to_aranha_processor_configurations.rb
 - lib/aranha/rails.rb
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
@@ -154,6 +174,8 @@ files:
 - lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
+- spec/rubocop_spec.rb
+- spec/spec_helper.rb
 homepage: 
 licenses: []
 metadata: {}
@@ -176,4 +198,9 @@ rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.
-test_files: []
+test_files:
+- spec/rubocop_spec.rb
+- spec/spec_helper.rb
+- ".rspec"
+- Gemfile
+- ".rubocop.yml"