aranha-rails 0.2.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 223ccb25f6cb914fe5342b32936934560f6020a9d3fbb16d649bfa2f96ce7459
-  data.tar.gz: bc4a7ef62b9a13a53f00fb81f1613173f9b20050d37367bfa112cfbd467d06a5
+  metadata.gz: 49f04cc5c0a957bc1aec7617dc1a6e736d860d80905afa04844587b8c57dd2bd
+  data.tar.gz: 18c6a92f2c28731b7224ea7a99a3521c684c56e33446f29601fcab4bf2de3a68
 SHA512:
-  metadata.gz: 464d6ad5cbeba036d0db9835fe2fb06ad38ec4338a2cc6ced2bf0157cddce35c38f6a073fb74eab90ab94e05593a80d2e3cf8496f61ed150ca43a7a29d264597
-  data.tar.gz: 303773ff9b8abf14431ccdb13eb070799834f8d8d8b3e78e4e4523959afdd29b69b39da823581bf7e0fb7afab611c56cfe39a59eea230fcd4b19a64779b95e40
+  metadata.gz: a0cf36bd155c0ccdea28701d96d385eb28d3e80a09a9a37e273b1b372f1598c27030a89d040492a707eefea98caf043972e86dc00b90e68f034cb60ef0cd59ab
+  data.tar.gz: 7947c879299d9ee6f0104fe7b6577a8ecc544f6bbccb59cc92f3799b951b98c2f8cd1fc7579b348d69eaa3780f70399008b901a196b42a00a15a7394bb85a2f8

data/lib/aranha/rails/engine.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 require 'active_scaffold'
+require 'eac_rails_delayed_job/engine'
 
 module Aranha
   module Rails
     class Engine < ::Rails::Engine
-      isolate_namespace ::Aranha
+      include ::EacRailsUtils::EngineHelper
 
-      initializer :append_migrations do |app|
-        config.paths['db/migrate'].expanded.each do |expanded_path|
-          app.config.paths['db/migrate'] << expanded_path
-        end
-      end
+      isolate_namespace ::Aranha
     end
   end
 end
+
+require 'aranha/rails/patches/delayed_job'

data/lib/aranha/rails/fixtures_download.rb

@@ -3,23 +3,40 @@
 require 'aranha/parsers/base'
 require 'aranha/parsers/source_address'
 require 'aranha/parsers/source_target_fixtures'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Rails
     class FixturesDownload
-      attr_reader :pending
+      DEFAULT_EXTENSION = '.html'
 
-      def initialize(options)
-        @prefix = options.fetch(:prefix)
-        @prefix = '' if @prefix.blank?
-        @download = options.fetch(:download)
-        @pending = options.fetch(:pending)
+      enable_listable
+      lists.add_symbol :option, :extension, :prefix, :download, :pending
+
+      common_constructor :options do
+        self.options = self.class.lists.option.hash_keys_validate!(options)
+      end
+
+      def download?
+        options[OPTION_DOWNLOAD] ? true : false
+      end
+
+      def extension
+        options[OPTION_EXTENSION].if_present(DEFAULT_EXTENSION)
+      end
+
+      def pending?
+        options[OPTION_PENDING] ? true : false
+      end
+
+      def prefix
+        options[OPTION_PREFIX].if_present('')
       end
 
       def run
         url_files.each do |f|
           ::Rails.logger.info(relative_path(f))
-          download(url(f), target(f)) if @download
+          download(url(f), target(f)) if download?
         end
       end
 
@@ -32,11 +49,11 @@ module Aranha
       def select_path?(path)
         return false unless match_prefix_pattern(path)
 
-        !pending || !source_exist?(path)
+        !pending? || !source_exist?(path)
       end
 
       def match_prefix_pattern(path)
-        relative_path(path).start_with?(@prefix)
+        relative_path(path).start_with?(prefix)
       end
 
       def fixtures_root
@@ -56,7 +73,7 @@ module Aranha
       end
 
       def target(file)
-        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
+        File.expand_path(File.basename(file, '.url') + '.source' + extension, File.dirname(file))
       end
 
       def relative_path(path)

data/lib/aranha/rails/manager.rb

@@ -9,10 +9,12 @@ module Aranha
       include ::Singleton
 
       def add_address(url, processor, extra_data = nil)
+        ::Rails.logger.info("Adding \"#{url}\"")
         a = ::Aranha::Address.find_or_initialize_by(url: ::Aranha::Address.sanitize_url(url))
         a.processor = processor
         a.extra_data = extra_data.to_yaml
         a.save!
+        a.check_scheduling
       end
 
       def addresses_count

data/lib/aranha/rails/patches/delayed_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'delayed_job_active_record'
+
+model = ::Delayed::Backend::ActiveRecord::Job
+model.has_one :aranha_address, class_name: 'Aranha::Address', foreign_key: :delayed_job_id,
+                               dependent: :nullify, inverse_of: :delayed_job

data/lib/aranha/rails/process.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Rails
+    class Process
+      QUEUE = 'aranha'
+      QUEUES = [QUEUE].freeze
+
+      def run
+        run_init
+        run_jobs_workoff
+        run_close
+      end
+
+      private
+
+      def run_close
+        ::Aranha::Address.failed do |a|
+          ::Rails.logger.info "Failed \"#{a.url}\": #{a.last_error}"
+        end
+        raise 'Some address failed' if ::Aranha::Address.failed.any?
+      end
+
+      def run_init
+        ::Aranha::Manager.default.init
+        ::Aranha::Address.all.each(&:init_scheduling)
+      end
+
+      def run_jobs_workoff
+        ::Delayed::Worker.new(exit_on_complete: true, queues: QUEUES).start
+      end
+    end
+  end
+end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.2.0'
+    VERSION = '0.5.0'
   end
 end

data/lib/tasks/aranha.rake

@@ -3,7 +3,7 @@
 namespace(:aranha) do
   desc 'Process Aranha\'s addresses.'
   task process: :environment do
-    ::Aranha::Processor.new
+    ::Aranha::Rails::Process.new.run
   end
 
   desc 'Remove all registered Aranha\'s addresses.'
@@ -11,13 +11,21 @@ namespace(:aranha) do
     Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
   end
 
+  namespace :address do
+    desc 'Process a arbitrary address.'
+    task :process, %i[uri processor] => :environment do |_t, args|
+      args.processor.constantize.new(args.uri, {}).process
+    end
+  end
+
   namespace :fixtures do
     desc 'Download remote content for fixtures.'
     task download: :environment do
       ::Aranha::Rails::FixturesDownload.new(
-        prefix: ENV['PREFIX'],
-        download: ENV['DOWNLOAD'].present?,
-        pending: ENV['PENDING'].present?
+        ::Aranha::Rails::FixturesDownload::OPTION_EXTENSION => ENV['EXTENSION'],
+        ::Aranha::Rails::FixturesDownload::OPTION_PREFIX => ENV['PREFIX'],
+        ::Aranha::Rails::FixturesDownload::OPTION_DOWNLOAD => ENV['DOWNLOAD'].present?,
+        ::Aranha::Rails::FixturesDownload::OPTION_PENDING => ENV['PENDING'].present?
       ).run
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-29 00:00:00.000000000 Z
+date: 2021-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: eac_rails_delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_rails_utils
   requirement: !ruby/object:Gem::Requirement
@@ -84,14 +98,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.52'
+        version: '0.72'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -112,20 +126,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.3.1
 description: 
 email: 
 executables: []
@@ -136,6 +150,8 @@ files:
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
 - lib/aranha/rails/manager.rb
+- lib/aranha/rails/patches/delayed_job.rb
+- lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
 homepage: 
@@ -156,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.