aranha-rails 0.2.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 276592968b7aa54fad42ac7b5d55f12ef4297f47ca17edd442d86e5583302569
-  data.tar.gz: 4dfce89c1c0ea962fc03d49fac0ef655b17bbf0bdafd8e31cd1fbcd3ce1bc44b
+  metadata.gz: 9c2bfb595a37e39eb60a336df45502b5efdd52ef26b1c488fb60500cc2ca93c1
+  data.tar.gz: 983c14439b792dbd495578a57fcc8960468cb6d126d07f7635b030cc78aa12aa
 SHA512:
-  metadata.gz: 7899f2898f8540357265f99278a8acdf311c252dc1e4b0521ee8f6103bad982dc4daea4186d685bd5a0b3d9387451b2ec74d6893df80fbc2b152406667258449
-  data.tar.gz: adb0ab27e55c6b2b44ccd1ccfd256a428c1a2a3f3008ea94f9eb84442c397952bbb91fdd1d08baba3700194638bc3889568b790addc7b50d18aeb13931aba090
+  metadata.gz: 8e56125aa6b80634a9e9f43db75184868edf4081245157a1949b559c7aaf8e919fa63d6f2a1d210eb5229368eec478918d429d83960719db1bb0d4a28f09c4a5
+  data.tar.gz: 55d5cedac52545c011fdcab381029bdb680db791ee8855e7dd15b9cd4b7aa1a0aa290f713ae8be27bdb9d4cd84cb0116a4d6b90a04e2224c9dfddd41ba66608e

data/lib/aranha/rails/engine.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 require 'active_scaffold'
+require 'eac_rails_delayed_job/engine'
 
 module Aranha
   module Rails
     class Engine < ::Rails::Engine
-      isolate_namespace ::Aranha
+      include ::EacRailsUtils::EngineHelper
 
-      initializer :append_migrations do |app|
-        config.paths['db/migrate'].expanded.each do |expanded_path|
-          app.config.paths['db/migrate'] << expanded_path
-        end
-      end
+      isolate_namespace ::Aranha
     end
   end
 end
+
+require 'aranha/rails/patches/delayed_job'

data/lib/aranha/rails/fixtures_download.rb

@@ -3,23 +3,40 @@
 require 'aranha/parsers/base'
 require 'aranha/parsers/source_address'
 require 'aranha/parsers/source_target_fixtures'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Rails
     class FixturesDownload
-      attr_reader :pending
+      DEFAULT_EXTENSION = '.html'
 
-      def initialize(options)
-        @prefix = options.fetch(:prefix)
-        @prefix = '' if @prefix.blank?
-        @download = options.fetch(:download)
-        @pending = options.fetch(:pending)
+      enable_listable
+      lists.add_symbol :option, :extension, :prefix, :download, :pending
+
+      common_constructor :options do
+        self.options = self.class.lists.option.hash_keys_validate!(options)
+      end
+
+      def download?
+        options[OPTION_DOWNLOAD] ? true : false
+      end
+
+      def extension
+        options[OPTION_EXTENSION].if_present(DEFAULT_EXTENSION)
+      end
+
+      def pending?
+        options[OPTION_PENDING] ? true : false
+      end
+
+      def prefix
+        options[OPTION_PREFIX].if_present('')
       end
 
       def run
         url_files.each do |f|
           ::Rails.logger.info(relative_path(f))
-          download(url(f), target(f)) if @download
+          download(url(f), target(f)) if download?
         end
       end
 
@@ -32,11 +49,11 @@ module Aranha
       def select_path?(path)
         return false unless match_prefix_pattern(path)
 
-        !pending || !source_exist?(path)
+        !pending? || !source_exist?(path)
       end
 
       def match_prefix_pattern(path)
-        relative_path(path).start_with?(@prefix)
+        relative_path(path).start_with?(prefix)
       end
 
       def fixtures_root
@@ -56,7 +73,7 @@ module Aranha
       end
 
       def target(file)
-        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
+        File.expand_path(File.basename(file, '.url') + '.source' + extension, File.dirname(file))
       end
 
       def relative_path(path)

data/lib/aranha/rails/manager.rb

@@ -9,10 +9,12 @@ module Aranha
       include ::Singleton
 
       def add_address(url, processor, extra_data = nil)
+        ::Rails.logger.info("Adding \"#{url}\"")
         a = ::Aranha::Address.find_or_initialize_by(url: ::Aranha::Address.sanitize_url(url))
         a.processor = processor
         a.extra_data = extra_data.to_yaml
         a.save!
+        a.check_scheduling
       end
 
       def addresses_count
@@ -20,9 +22,9 @@ module Aranha
       end
 
       def clear_expired_addresses
-        q = ::Aranha::Address.by_created_at_lt(Time.zone.now - 12.hours)
+        q = ::Aranha::Address.expired(::Time.zone.now)
         ::Rails.logger.info("Addresses expired: #{q.count}")
-        q.destroy_all
+        q.each(&:destroy)
       end
 
       def log_info(message)

data/lib/aranha/rails/patches/delayed_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'delayed_job_active_record'
+
+model = ::Delayed::Backend::ActiveRecord::Job
+model.has_one :aranha_address, class_name: 'Aranha::Address', foreign_key: :delayed_job_id,
+                               dependent: :nullify, inverse_of: :delayed_job

data/lib/aranha/rails/process.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Rails
+    class Process
+      QUEUE = 'aranha'
+      QUEUES = [QUEUE].freeze
+
+      def run
+        run_init
+        run_jobs_workoff
+        run_close
+      end
+
+      private
+
+      def run_close
+        ::Aranha::Address.failed do |a|
+          ::Rails.logger.info "Failed \"#{a.url}\": #{a.last_error}"
+        end
+        raise 'Some address failed' if ::Aranha::Address.failed.any?
+      end
+
+      def run_init
+        ::Aranha::Manager.default.init
+        ::Aranha::Address.all.each(&:init_scheduling)
+      end
+
+      def run_jobs_workoff
+        ::Delayed::Worker.new(exit_on_complete: true, queues: QUEUES).start
+      end
+    end
+  end
+end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.2.1'
+    VERSION = '0.6.0'
   end
 end

data/lib/tasks/aranha.rake

@@ -3,7 +3,7 @@
 namespace(:aranha) do
   desc 'Process Aranha\'s addresses.'
   task process: :environment do
-    ::Aranha::Processor.new
+    ::Aranha::Rails::Process.new.run
   end
 
   desc 'Remove all registered Aranha\'s addresses.'
@@ -11,13 +11,21 @@ namespace(:aranha) do
     Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
   end
 
+  namespace :address do
+    desc 'Process a arbitrary address.'
+    task :process, %i[uri processor] => :environment do |_t, args|
+      args.processor.constantize.new(args.uri, {}).process
+    end
+  end
+
   namespace :fixtures do
     desc 'Download remote content for fixtures.'
     task download: :environment do
       ::Aranha::Rails::FixturesDownload.new(
-        prefix: ENV['PREFIX'],
-        download: ENV['DOWNLOAD'].present?,
-        pending: ENV['PENDING'].present?
+        ::Aranha::Rails::FixturesDownload::OPTION_EXTENSION => ENV['EXTENSION'],
+        ::Aranha::Rails::FixturesDownload::OPTION_PREFIX => ENV['PREFIX'],
+        ::Aranha::Rails::FixturesDownload::OPTION_DOWNLOAD => ENV['DOWNLOAD'].present?,
+        ::Aranha::Rails::FixturesDownload::OPTION_PENDING => ENV['PENDING'].present?
       ).run
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-28 00:00:00.000000000 Z
+date: 2021-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: eac_rails_delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_rails_utils
   requirement: !ruby/object:Gem::Requirement
@@ -84,14 +98,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -112,14 +126,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.1
 description: 
 email: 
 executables: []
@@ -130,6 +150,8 @@ files:
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
 - lib/aranha/rails/manager.rb
+- lib/aranha/rails/patches/delayed_job.rb
+- lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
 homepage: 
@@ -150,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.