aranha-rails 0.3.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5ce68bb8da1755e2c5b3667c314ea10e556c1a8050152348d386900dfd5383e
-  data.tar.gz: 8bedbf6e02d127b69e49c02d11cec3d9173f0b8cc4eff02bf1128ad4ad2900f2
+  metadata.gz: 36ba45ae4efcdde7c87ec2826a8ae37457c791a7384b4fd3cbdd349f32fe9767
+  data.tar.gz: fbda8a52654c3640017c0bf7e62cc5c7ef2dbf66d2c5f59d9083685629c78314
 SHA512:
-  metadata.gz: 9cc382aefdf04bcd68f22557fda034377564e349e31f97d168ad541de8c79312edffa87fc1f7bd1198a2362bc4eb7874f32d3a946eb2d11c4b40f274580b1a68
-  data.tar.gz: 4f7adf4bf7353a07c54a05ebba718b26c6469b3d657dcbf355aec33baccac3110e3c7f1ac7db112b22d5941e3311f9ccfadace9c42b4d6b6a1805cbf37dc5e86
+  metadata.gz: 5aa3e496c87dae2659430b1fe08388a4b32087a5ff3cf5835c7327e43e2fd0e2b5c945a8e4911c29cdd99bd8aada31d0e1289bb11e9f962c170ec6c230e2b7a4
+  data.tar.gz: cc60ef5294d2af9e2b0066d1cae61ec1c8aecd0d5eccfc005b61509282c7f39911fb9aef399c86b46c833c5a1ad7001f3fc75fa784bac60be17fb5e25864558f

data/lib/aranha/rails/engine.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 require 'active_scaffold'
+require 'eac_rails_delayed_job/engine'
 
 module Aranha
   module Rails
     class Engine < ::Rails::Engine
-      isolate_namespace ::Aranha
+      include ::EacRailsUtils::EngineHelper
 
-      initializer :append_migrations do |app|
-        config.paths['db/migrate'].expanded.each do |expanded_path|
-          app.config.paths['db/migrate'] << expanded_path
-        end
-      end
+      isolate_namespace ::Aranha
     end
   end
 end
+
+require 'aranha/rails/patches/delayed_job'

data/lib/aranha/rails/manager.rb

@@ -9,10 +9,12 @@ module Aranha
       include ::Singleton
 
       def add_address(url, processor, extra_data = nil)
+        ::Rails.logger.info("Adding \"#{url}\"")
         a = ::Aranha::Address.find_or_initialize_by(url: ::Aranha::Address.sanitize_url(url))
         a.processor = processor
         a.extra_data = extra_data.to_yaml
         a.save!
+        a.check_scheduling
       end
 
       def addresses_count
@@ -20,9 +22,9 @@ module Aranha
       end
 
       def clear_expired_addresses
-        q = ::Aranha::Address.by_created_at_lt(Time.zone.now - 12.hours)
+        q = ::Aranha::Address.expired(::Time.zone.now)
         ::Rails.logger.info("Addresses expired: #{q.count}")
-        q.destroy_all
+        q.each(&:destroy)
       end
 
       def log_info(message)
@@ -41,7 +43,7 @@ module Aranha
       end
 
       def unprocessed_addresses
-        ::Aranha::Address.unprocessed
+        ::Aranha::Address.all.select(&:schedule?)
       end
     end
   end

data/lib/aranha/rails/patches/delayed_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'delayed_job_active_record'
+
+model = ::Delayed::Backend::ActiveRecord::Job
+model.has_one :aranha_address, class_name: 'Aranha::Address', foreign_key: :delayed_job_id,
+                               dependent: :nullify, inverse_of: :delayed_job

data/lib/aranha/rails/process.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Rails
+    class Process
+      QUEUE = 'aranha'
+      QUEUES = [QUEUE].freeze
+
+      def run
+        run_init
+        run_jobs_workoff
+        run_close
+      end
+
+      private
+
+      def run_close
+        ::Aranha::Address.failed do |a|
+          ::Rails.logger.info "Failed \"#{a.url}\": #{a.last_error}"
+        end
+        raise 'Some address failed' if ::Aranha::Address.failed.any?
+      end
+
+      def run_init
+        ::Aranha::Manager.default.init
+        ::Aranha::Address.all.each(&:init_scheduling)
+      end
+
+      def run_jobs_workoff
+        ::Delayed::Worker.new(exit_on_complete: true, queues: QUEUES).start
+      end
+    end
+  end
+end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.3.0'
+    VERSION = '0.7.0'
   end
 end

data/lib/tasks/aranha.rake

@@ -3,7 +3,7 @@
 namespace(:aranha) do
   desc 'Process Aranha\'s addresses.'
   task process: :environment do
-    ::Aranha::Processor.new
+    ::Aranha::Rails::Process.new.run
   end
 
   desc 'Remove all registered Aranha\'s addresses.'
@@ -11,6 +11,13 @@ namespace(:aranha) do
     Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
   end
 
+  namespace :address do
+    desc 'Process a arbitrary address.'
+    task :process, %i[uri processor] => :environment do |_t, args|
+      args.processor.constantize.new(args.uri, {}).process
+    end
+  end
+
   namespace :fixtures do
     desc 'Download remote content for fixtures.'
     task download: :environment do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-15 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: eac_rails_delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_rails_utils
   requirement: !ruby/object:Gem::Requirement
@@ -84,14 +98,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -112,14 +126,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
 description: 
 email: 
 executables: []
@@ -130,6 +150,8 @@ files:
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
 - lib/aranha/rails/manager.rb
+- lib/aranha/rails/patches/delayed_job.rb
+- lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
 homepage: 
@@ -150,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.