aranha-rails 0.1.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea03bcdec3dd45159d259277c82e9caa7d6aa8469499abfff9567101a7d23e6f
-  data.tar.gz: e25011f06e74adc37e3192e6a5fa80e849430b4b8dd0ebc7ddb0e1af3cc7ff98
+  metadata.gz: bbe5992eacaebe66f42310ee30ba8198d8f7533f8b7eccf6fb7cbe552884cb4a
+  data.tar.gz: 74be91626416ddf93d1d9702ef3e69bf91a91ea2f8bafb73cefd35745441d54c
 SHA512:
-  metadata.gz: 458bf075dec14c64b8c63065a81bfde264420940161f7d7475ee84bf1712a86ddb04dc10f45d06d346c0fe9c9963e19d1a756898cd02505c07fe078b017576b4
-  data.tar.gz: 6be3e41fb8f7abd3cf950bcfd55f9d0c36778c124d34e1cd0e12bd44ef5944864b164299358f176157488842228cf8d98650330718d6fa771cfa1b8b4bf468ed
+  metadata.gz: 4a2ffc48a555639bfc1c9960378e700359ba14319b9727a423e8aa59b901b5945bece2e68f720d87bf0c1439965a2523edc89eb1abc7d04ab09fc017bbfa18eb
+  data.tar.gz: 854def0c575004ee01d630d69fe0975deeaa62d486b2ebfdc3ca775144cf88e3f3bcbd67d93eb6f85da11b203c6a0239cdde626a2d2fa8e460a3c82d9f144cea

data/lib/aranha/rails/engine.rb

@@ -1,17 +1,16 @@
 # frozen_string_literal: true
 
 require 'active_scaffold'
+require 'eac_rails_delayed_job/engine'
 
 module Aranha
   module Rails
     class Engine < ::Rails::Engine
-      isolate_namespace ::Aranha
+      include ::EacRailsUtils::EngineHelper
 
-      initializer :append_migrations do |app|
-        config.paths['db/migrate'].expanded.each do |expanded_path|
-          app.config.paths['db/migrate'] << expanded_path
-        end
-      end
+      isolate_namespace ::Aranha
     end
   end
 end
+
+require 'aranha/rails/patches/delayed_job'

data/lib/aranha/rails/fixtures_download.rb

@@ -3,23 +3,40 @@
 require 'aranha/parsers/base'
 require 'aranha/parsers/source_address'
 require 'aranha/parsers/source_target_fixtures'
+require 'eac_ruby_utils/core_ext'
 
 module Aranha
   module Rails
     class FixturesDownload
-      attr_reader :pending
+      DEFAULT_EXTENSION = '.html'
 
-      def initialize(options)
-        @prefix = options.fetch(:prefix)
-        @prefix = '' if @prefix.blank?
-        @download = options.fetch(:download)
-        @pending = options.fetch(:pending)
+      enable_listable
+      lists.add_symbol :option, :extension, :prefix, :download, :pending
+
+      common_constructor :options do
+        self.options = self.class.lists.option.hash_keys_validate!(options)
+      end
+
+      def download?
+        options[OPTION_DOWNLOAD] ? true : false
+      end
+
+      def extension
+        options[OPTION_EXTENSION].if_present(DEFAULT_EXTENSION)
+      end
+
+      def pending?
+        options[OPTION_PENDING] ? true : false
+      end
+
+      def prefix
+        options[OPTION_PREFIX].if_present('')
       end
 
       def run
         url_files.each do |f|
-          Rails.logger.info(relative_path(f))
-          download(url(f), target(f)) if @download
+          ::Rails.logger.info(relative_path(f))
+          download(url(f), target(f)) if download?
         end
       end
 
@@ -32,19 +49,19 @@ module Aranha
       def select_path?(path)
         return false unless match_prefix_pattern(path)
 
-        !pending || !source_exist?(path)
+        !pending? || !source_exist?(path)
       end
 
       def match_prefix_pattern(path)
-        relative_path(path).start_with?(@prefix)
+        relative_path(path).start_with?(prefix)
       end
 
       def fixtures_root
-        Rails.root.to_s
+        ::Rails.root.to_s
       end
 
       def download(url, target)
-        Rails.logger.info "Baixando \"#{url}\"..."
+        ::Rails.logger.info "Baixando \"#{url}\"..."
         content = ::Aranha::Parsers::Base.new(url).content
         raise "Content is blank for \"#{url}\"" if content.blank?
 
@@ -56,7 +73,7 @@ module Aranha
       end
 
       def target(file)
-        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
+        File.expand_path(File.basename(file, '.url') + '.source' + extension, File.dirname(file))
       end
 
       def relative_path(path)

data/lib/aranha/rails/manager.rb

@@ -9,10 +9,12 @@ module Aranha
       include ::Singleton
 
       def add_address(url, processor, extra_data = nil)
+        ::Rails.logger.info("Adding \"#{url}\"")
         a = ::Aranha::Address.find_or_initialize_by(url: ::Aranha::Address.sanitize_url(url))
         a.processor = processor
         a.extra_data = extra_data.to_yaml
         a.save!
+        a.check_scheduling
       end
 
       def addresses_count
@@ -33,6 +35,13 @@ module Aranha
         ::Rails.logger.warn(message)
       end
 
+      def start_points_to_addresses
+        super
+        ::Aranha::StartPoint.all.each do |sp|
+          add_address(sp.uri, sp.processor_class, sp.extra_data)
+        end
+      end
+
       def unprocessed_addresses
         ::Aranha::Address.unprocessed
       end

data/lib/aranha/rails/patches/delayed_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'delayed_job_active_record'
+
+model = ::Delayed::Backend::ActiveRecord::Job
+model.has_one :aranha_address, class_name: 'Aranha::Address', foreign_key: :delayed_job_id,
+                               dependent: :nullify, inverse_of: :delayed_job

data/lib/aranha/rails/process.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Rails
+    class Process
+      QUEUE = 'aranha'
+      QUEUES = [QUEUE].freeze
+
+      def run
+        run_init
+        run_jobs_workoff
+        run_close
+      end
+
+      private
+
+      def run_close
+        ::Aranha::Address.failed do |a|
+          ::Rails.logger.info "Failed \"#{a.url}\": #{a.last_error}"
+        end
+        raise 'Some address failed' if ::Aranha::Address.failed.any?
+      end
+
+      def run_init
+        ::Aranha::Manager.default.init
+        ::Aranha::Address.all.each(&:init_scheduling)
+      end
+
+      def run_jobs_workoff
+        ::Delayed::Worker.new(exit_on_complete: true, queues: QUEUES).start
+      end
+    end
+  end
+end

data/lib/aranha/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Aranha
   module Rails
-    VERSION = '0.1.0'
+    VERSION = '0.4.0'
   end
 end

data/lib/tasks/aranha.rake

@@ -3,7 +3,7 @@
 namespace(:aranha) do
   desc 'Process Aranha\'s addresses.'
   task process: :environment do
-    ::Aranha::Processor.new
+    ::Aranha::Rails::Process.new.run
   end
 
   desc 'Remove all registered Aranha\'s addresses.'
@@ -15,9 +15,10 @@ namespace(:aranha) do
     desc 'Download remote content for fixtures.'
     task download: :environment do
       ::Aranha::Rails::FixturesDownload.new(
-        prefix: ENV['PREFIX'],
-        download: ENV['DOWNLOAD'].present?,
-        pending: ENV['PENDING'].present?
+        ::Aranha::Rails::FixturesDownload::OPTION_EXTENSION => ENV['EXTENSION'],
+        ::Aranha::Rails::FixturesDownload::OPTION_PREFIX => ENV['PREFIX'],
+        ::Aranha::Rails::FixturesDownload::OPTION_DOWNLOAD => ENV['DOWNLOAD'].present?,
+        ::Aranha::Rails::FixturesDownload::OPTION_PENDING => ENV['PENDING'].present?
       ).run
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aranha-rails
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Put here the authors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-28 00:00:00.000000000 Z
+date: 2021-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_scaffold
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: eac_rails_delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: eac_rails_utils
   requirement: !ruby/object:Gem::Requirement
@@ -112,20 +126,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0.2'
 description: 
 email: 
 executables: []
@@ -136,6 +144,8 @@ files:
 - lib/aranha/rails/engine.rb
 - lib/aranha/rails/fixtures_download.rb
 - lib/aranha/rails/manager.rb
+- lib/aranha/rails/patches/delayed_job.rb
+- lib/aranha/rails/process.rb
 - lib/aranha/rails/version.rb
 - lib/tasks/aranha.rake
 homepage: 
@@ -156,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Put here de description.