aranha-rails 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36ba45ae4efcdde7c87ec2826a8ae37457c791a7384b4fd3cbdd349f32fe9767
-  data.tar.gz: fbda8a52654c3640017c0bf7e62cc5c7ef2dbf66d2c5f59d9083685629c78314
+  metadata.gz: a7f0c7237e3e9ca7d3f5ddc61973513d2a86750e88d96640879c5cd91a46be91
+  data.tar.gz: 59b99c087566e49e93e116d51a05f2c29406d8376640112d52f7c59184543de8
 SHA512:
-  metadata.gz: 5aa3e496c87dae2659430b1fe08388a4b32087a5ff3cf5835c7327e43e2fd0e2b5c945a8e4911c29cdd99bd8aada31d0e1289bb11e9f962c170ec6c230e2b7a4
-  data.tar.gz: cc60ef5294d2af9e2b0066d1cae61ec1c8aecd0d5eccfc005b61509282c7f39911fb9aef399c86b46c833c5a1ad7001f3fc75fa784bac60be17fb5e25864558f
+  metadata.gz: 80ec1e16b92e7ae0cb463f64c4f6f35b93876fb0987fc0a6ed9d3a57cb509fba35fcddeae40b6b16ba8901576e3f6da125c79e40b3d246b77887c90e8ba15bee
+  data.tar.gz: 82a5d76749efb28e26388b1af4cb2d9208915920124e6d6b881d58a1e598a56de989fb539da363a26d09827e96d6cd985f516815ba633ed38069e2eeefffa92e

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,25 @@
+require:
+  - rubocop-rails
+  - rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.4
+  TargetRailsVersion: 5.1
+
+Layout/LineLength:
+  Max: 100
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/app/assets/javascripts/aranha/application.js

@@ -0,0 +1,2 @@
+//= require_tree .
+//= require active_scaffold

data/app/assets/stylesheets/aranha/application.css

@@ -0,0 +1,5 @@
+/*
+ *= require_tree .
+ *= require_self
+ *= require active_scaffold
+ */

data/app/controllers/aranha/addresses_controller.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Aranha
+  class AddressesController < ::ApplicationController
+    active_scaffold :'aranha/address' do |_conf|
+    end
+  end
+end

data/app/controllers/aranha/processor_configurations_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aranha
+  class ProcessorConfigurationsController < ::ApplicationController
+    before_action :configure_processor_class_options
+
+    active_scaffold :'aranha/processor_configuration' do |conf|
+      conf.columns[:processor_class].form_ui = :select
+    end
+
+    def configure_processor_class_options
+      active_scaffold_config.columns[:processor_class].options =
+        { options: ::Aranha::ProcessorConfiguration.processor_class_options }
+    end
+  end
+end

data/app/controllers/aranha/start_points_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Aranha
+  class StartPointsController < ::ApplicationController
+    before_action :configure_processor_class_options
+
+    active_scaffold :'aranha/start_point' do |conf|
+      conf.columns[:processor_class].form_ui = :select
+    end
+
+    def configure_processor_class_options
+      active_scaffold_config.columns[:processor_class].options =
+        { options: ::Aranha::StartPoint.processor_class_options }
+    end
+  end
+end

data/app/models/aranha/address/delayed_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    class DelayedJob
+      common_constructor :address_id
+
+      def perform
+        address_processor.successful? ? perform_on_success : perform_on_error
+      end
+
+      private
+
+      def address
+        address_processor.address
+      end
+
+      def perform_on_success
+        # Do nothing
+      end
+
+      def perform_on_error
+        address.update!(
+          tries_count: address.tries_count + 1,
+          last_error: address_processor.error.to_yaml
+        )
+        address.check_scheduling
+      end
+
+      def address_processor
+        @address_processor ||= ::Aranha::AddressProcessor.new(
+          ::Aranha::Address.find(address_id)
+        )
+      end
+    end
+  end
+end

data/app/models/aranha/address/processor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    module Processor
+      common_concern
+
+      module ClassMethods
+        # @return [Aranha::ProcessorConfiguration]
+        def default_processor_configuration
+          @default_processor_configuration ||= ::Aranha::ProcessorConfiguration.new
+        end
+      end
+
+      # @return [Aranha::ProcessorConfiguration]
+      def processor_configuration
+        ::Aranha::ProcessorConfiguration.find_by(processor_class: processor) ||
+          self.class.default_processor_configuration
+      end
+    end
+  end
+end

data/app/models/aranha/address/scheduling.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    module Scheduling
+      common_concern
+
+      module ClassMethods
+        # @return [Array<Aranha::Address>]
+        def expired(time = ::Time.zone.now)
+          all.select { |record| record.expired?(time) }
+        end
+      end
+
+      def check_scheduling
+        ::ActiveRecord::Base.transaction do
+          return unless schedule?
+
+          job = ::Delayed::Job.enqueue(
+            ::Aranha::Address::DelayedJob.new(id),
+            queue: ::Aranha::Rails::Process::QUEUE
+          )
+          update!(delayed_job: job)
+        end
+      end
+
+      def expired?(time = ::Time.zone.now)
+        time >= (created_at + timeout)
+      end
+
+      def init_scheduling
+        update!(tries_count: 0, last_error: nil) unless processed?
+        check_scheduling
+      end
+
+      def allow_retry?
+        tries_count < ::Aranha::Processor::DEFAULT_MAX_TRIES
+      end
+
+      def schedule?
+        processed_at.blank? && allow_retry? && delayed_job.blank? && enabled?
+      end
+
+      delegate :enabled?, :timeout, to: :processor_configuration
+    end
+  end
+end

data/app/models/aranha/address.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+require 'delayed/backend/active_record'
+
+module Aranha
+  class Address < ::ActiveRecord::Base
+    include ::EacRailsUtils::Models::InequalityQueries
+    include ::Aranha::Address::Processor
+    include ::Aranha::Address::Scheduling
+
+    add_inequality_queries(:created_at)
+
+    class << self
+      def sanitize_url(url)
+        if url.is_a?(Hash)
+          ::EacRubyUtils::Yaml.dump(url)
+        else
+          url.to_s
+        end
+      end
+    end
+
+    belongs_to :delayed_job, class_name: 'Delayed::Backend::ActiveRecord::Job', dependent: :destroy,
+                             optional: true
+
+    validates :url, presence: true, uniqueness: true
+    validates :processor, presence: true
+    validates :tries_count, presence: true, numericality: { only_integer: true,
+                                                            greater_or_equal: 0 }
+
+    scope :failed, lambda {
+      where(processed_at: nil).where.not(last_error: nil)
+    }
+
+    def to_s
+      "#{processor}|#{url}"
+    end
+
+    def process
+      ActiveRecord::Base.transaction do
+        instanciate_processor.process
+        self.processed_at = Time.zone.now
+        save!
+      end
+    end
+
+    def processed?
+      processed_at.present?
+    end
+
+    private
+
+    def instanciate_processor
+      processor_instancier.call(*processor_instancier_arguments)
+    end
+
+    def url_to_process
+      ::EacRubyUtils::Yaml.load(url)
+    end
+
+    def processor_instancier
+      processor.constantize.method(:new)
+    end
+
+    def processor_instancier_arguments
+      if processor_instancier_arity == 2 || processor_instancier_arity.negative?
+        [url_to_process, EacRubyUtils::Yaml.load(extra_data)]
+      elsif processor_instancier_arity == 1
+        [processor_instancier.call(url_to_process)]
+      else
+        raise("#{processor}.initialize should has 1 or 2 or * arguments")
+      end
+    end
+
+    def processor_instancier_arity
+      processor.constantize.instance_method(:initialize).arity
+    end
+  end
+end

data/app/models/aranha/processor_configuration.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+
+module Aranha
+  class ProcessorConfiguration < ::ActiveRecord::Base
+    DEFAULT_TIMEOUT = 12.hours
+
+    class << self
+      # @return [Enumerator<String>]
+      def processor_class_list
+        ::Set.new(processor_class_list_from_addresses + processor_class_list_from_start_points)
+             .to_enum.sort
+      end
+
+      # @return [Hash<String, String>]
+      def processor_class_options
+        processor_class_list.map { |e| [e, e] }.to_h
+      end
+
+      private
+
+      # @return [Array<String>]
+      def processor_class_list_from_addresses
+        ::Aranha::Address.distinct.pluck(:processor)
+      end
+
+      # @return [Array<String>]
+      def processor_class_list_from_start_points
+        ::Aranha::StartPoint.processor_class_list.to_a
+      end
+    end
+
+    validates :processor_class, presence: true
+    validate :processor_class_in_list
+    validates :timeout_seconds, allow_blank: true,
+                                numericality: { integer_only: true, greater_than_or_equal_to: 1 }
+
+    def processor_class_in_list
+      return if processor_class.blank?
+      return if self.class.processor_class_list.include?(processor_class)
+
+      errors.add(:processor_class, 'Not in list')
+    end
+
+    # @return [ActiveSupport::Duration]
+    def timeout
+      timeout_seconds.if_present(DEFAULT_TIMEOUT, &:seconds)
+    end
+  end
+end

data/app/models/aranha/start_point.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/yaml'
+
+module Aranha
+  class StartPoint < ::ActiveRecord::Base
+    class << self
+      def add_processor_class(klass)
+        processor_class_list_var.add(klass.to_s)
+      end
+
+      def processor_class_list
+        processor_class_list_var.to_enum
+      end
+
+      def processor_class_options
+        processor_class_list.map { |e| [e, e] }.to_h
+      end
+
+      private
+
+      def processor_class_list_var
+        @processor_class_list_var ||= Set.new
+      end
+    end
+
+    validates :uri, presence: true, uniqueness: true,
+                    format: { with: ::URI::DEFAULT_PARSER.make_regexp }
+    validates :processor_class, presence: true
+    validate :processor_class_in_list
+
+    def extra_data
+      extra_data_yaml.nil? ? nil : ::EacRubyUtils::Yaml.load(extra_data_yaml)
+    end
+
+    def extra_data=(value)
+      self.extra_data_yaml = ::EacRubyUtils::Yaml.dump(value)
+    end
+
+    def processor_class_in_list
+      return if processor_class.blank?
+      return if self.class.processor_class_list.include?(processor_class)
+
+      errors.add(:processor_class, 'Not in list')
+    end
+  end
+end

data/config/initializers/aranha.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'aranha/rails/manager'
+
+::Aranha::Manager.default = ::Aranha::Rails::Manager.instance

data/config/locales/en.yml

@@ -0,0 +1,12 @@
+en:
+  activerecord:
+    models:
+      aranha/address:
+        one: Aranha address
+        other: Aranha addresses
+      aranha/processor_configuration:
+        one: Aranha processor configuration
+        other: Aranha processors configurations
+      aranha/start_point:
+        one: Aranha start point
+        other: Aranha start points

data/config/locales/pt-BR.yml

@@ -0,0 +1,12 @@
+pt-BR:
+  activerecord:
+    models:
+      aranha/address:
+        one: Endereço Aranha
+        other: Endereços Aranha
+      aranha/processor_configuration:
+        one: Configuração de processador Aranha
+        other: Configuração de processadores Aranha
+      aranha/start_point:
+        one: Ponto de Partida Aranha
+        other: Pontos de Partida Aranha

data/config/routes.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+::Aranha::Rails::Engine.routes.draw do
+  concern :active_scaffold, ActiveScaffold::Routing::Basic.new(association: true)
+  resources(:addresses, concerns: :active_scaffold)
+  resources(:processor_configurations, concerns: :active_scaffold)
+  resources(:start_points, concerns: :active_scaffold)
+end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateAranhaAddresses < (
+    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+  )
+  def change
+    create_table :aranha_addresses do |t|
+      t.string :url
+      t.string :processor
+      t.timestamp :processed_at
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddExtraDataToAranhaAddresses < (
+    Rails.version < '5' ? ActiveRecord::Migration : ActiveRecord::Migration[4.2]
+  )
+  def change
+    add_column :aranha_addresses, :extra_data, :text
+  end
+end

data/db/migrate/20201128202212_create_aranha_start_points.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateAranhaStartPoints < ActiveRecord::Migration[5.1]
+  def change
+    create_table :aranha_start_points do |t|
+      t.string :uri
+      t.string :processor_class
+      t.string :extra_data_yaml
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20210720023620_add_delayed_job_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddDelayedJobToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_reference :aranha_addresses, :delayed_job, foreign_key: false
+  end
+end

data/db/migrate/20210721171416_add_tries_count_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddTriesCountToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_addresses, :tries_count, :integer, null: false, default: 0
+  end
+end

data/db/migrate/20210721185222_add_last_error_to_aranha_addresses.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddLastErrorToAranhaAddresses < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_addresses, :last_error, :text
+  end
+end

data/db/migrate/20210814220540_create_aranha_processor_configurations.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateAranhaProcessorConfigurations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :aranha_processor_configurations do |t|
+      t.string :processor_class
+      t.integer :timeout_seconds, null: true
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20210815160543_add_enabled_to_aranha_processor_configurations.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddEnabledToAranhaProcessorConfigurations < ActiveRecord::Migration[5.1]
+  def change
+    add_column :aranha_processor_configurations, :enabled, :boolean, default: true, null: false
+  end
+end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.7.0'
+    VERSION = '0.7.1'
   end
 end

data/spec/rubocop_spec.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+::EacRubyUtils::Rspec.default_setup.describe_rubocop

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/rspec/default_setup'
+::EacRubyUtils::Rspec.default_setup_create(::File.expand_path('..', __dir__))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-15 00:00:00.000000000 Z
+date: 2021-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -146,6 +146,32 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- app/assets/javascripts/aranha/application.js
+- app/assets/stylesheets/aranha/application.css
+- app/controllers/aranha/addresses_controller.rb
+- app/controllers/aranha/processor_configurations_controller.rb
+- app/controllers/aranha/start_points_controller.rb
+- app/models/aranha/address.rb
+- app/models/aranha/address/delayed_job.rb
+- app/models/aranha/address/processor.rb
+- app/models/aranha/address/scheduling.rb
+- app/models/aranha/processor_configuration.rb
+- app/models/aranha/start_point.rb
+- config/initializers/aranha.rb
+- config/locales/en.yml
+- config/locales/pt-BR.yml
+- config/routes.rb
+- db/migrate/20171201021251_create_aranha_addresses.rb
+- db/migrate/20181125042102_add_extra_data_to_aranha_addresses.rb
+- db/migrate/20201128202212_create_aranha_start_points.rb
+- db/migrate/20210720023620_add_delayed_job_to_aranha_addresses.rb
+- db/migrate/20210721171416_add_tries_count_to_aranha_addresses.rb
+- db/migrate/20210721185222_add_last_error_to_aranha_addresses.rb
+- db/migrate/20210814220540_create_aranha_processor_configurations.rb
+- db/migrate/20210815160543_add_enabled_to_aranha_processor_configurations.rb
 - lib/aranha/rails.rb
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
@@ -154,6 +180,8 @@ files:
 - lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
+- spec/rubocop_spec.rb
+- spec/spec_helper.rb
 homepage: 
 licenses: []
 metadata: {}
@@ -176,4 +204,9 @@ rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.
-test_files: []
+test_files:
+- spec/rubocop_spec.rb
+- spec/spec_helper.rb
+- ".rspec"
+- Gemfile
+- ".rubocop.yml"