aranha-rails 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ea03bcdec3dd45159d259277c82e9caa7d6aa8469499abfff9567101a7d23e6f
+  data.tar.gz: e25011f06e74adc37e3192e6a5fa80e849430b4b8dd0ebc7ddb0e1af3cc7ff98
+SHA512:
+  metadata.gz: 458bf075dec14c64b8c63065a81bfde264420940161f7d7475ee84bf1712a86ddb04dc10f45d06d346c0fe9c9963e19d1a756898cd02505c07fe078b017576b4
+  data.tar.gz: 6be3e41fb8f7abd3cf950bcfd55f9d0c36778c124d34e1cd0e12bd44ef5944864b164299358f176157488842228cf8d98650330718d6fa771cfa1b8b4bf468ed

data/lib/aranha/rails.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'eac_ruby_utils/core_ext'
+
+module Aranha
+  module Rails
+    require_sub __FILE__
+  end
+end

data/lib/aranha/rails/engine.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'active_scaffold'
+
+module Aranha
+  module Rails
+    class Engine < ::Rails::Engine
+      isolate_namespace ::Aranha
+
+      initializer :append_migrations do |app|
+        config.paths['db/migrate'].expanded.each do |expanded_path|
+          app.config.paths['db/migrate'] << expanded_path
+        end
+      end
+    end
+  end
+end

data/lib/aranha/rails/fixtures_download.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'aranha/parsers/base'
+require 'aranha/parsers/source_address'
+require 'aranha/parsers/source_target_fixtures'
+
+module Aranha
+  module Rails
+    class FixturesDownload
+      attr_reader :pending
+
+      def initialize(options)
+        @prefix = options.fetch(:prefix)
+        @prefix = '' if @prefix.blank?
+        @download = options.fetch(:download)
+        @pending = options.fetch(:pending)
+      end
+
+      def run
+        url_files.each do |f|
+          Rails.logger.info(relative_path(f))
+          download(url(f), target(f)) if @download
+        end
+      end
+
+      private
+
+      def url_files
+        Dir["#{fixtures_root}/**/*.url"].select { |path| select_path?(path) }
+      end
+
+      def select_path?(path)
+        return false unless match_prefix_pattern(path)
+
+        !pending || !source_exist?(path)
+      end
+
+      def match_prefix_pattern(path)
+        relative_path(path).start_with?(@prefix)
+      end
+
+      def fixtures_root
+        Rails.root.to_s
+      end
+
+      def download(url, target)
+        Rails.logger.info "Baixando \"#{url}\"..."
+        content = ::Aranha::Parsers::Base.new(url).content
+        raise "Content is blank for \"#{url}\"" if content.blank?
+
+        File.open(target, 'wb') { |file| file.write(content) }
+      end
+
+      def url(file)
+        ::Aranha::Parsers::SourceAddress.from_file(file)
+      end
+
+      def target(file)
+        File.expand_path(File.basename(file, '.url') + '.source.html', File.dirname(file))
+      end
+
+      def relative_path(path)
+        path.sub(%r{^#{Regexp.quote(fixtures_root)}/}, '')
+      end
+
+      def source_exist?(path)
+        stf = ::Aranha::Parsers::SourceTargetFixtures.new(::File.dirname(path))
+        stf.source_file(::File.basename(path, '.url')).present?
+      end
+    end
+  end
+end

data/lib/aranha/rails/manager.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'aranha/manager'
+require 'singleton'
+
+module Aranha
+  module Rails
+    class Manager < ::Aranha::Manager
+      include ::Singleton
+
+      def add_address(url, processor, extra_data = nil)
+        a = ::Aranha::Address.find_or_initialize_by(url: ::Aranha::Address.sanitize_url(url))
+        a.processor = processor
+        a.extra_data = extra_data.to_yaml
+        a.save!
+      end
+
+      def addresses_count
+        ::Aranha::Address.count
+      end
+
+      def clear_expired_addresses
+        q = ::Aranha::Address.by_created_at_lt(Time.zone.now - 12.hours)
+        ::Rails.logger.info("Addresses expired: #{q.count}")
+        q.destroy_all
+      end
+
+      def log_info(message)
+        ::Rails.logger.info(message)
+      end
+
+      def log_warn(message)
+        ::Rails.logger.warn(message)
+      end
+
+      def unprocessed_addresses
+        ::Aranha::Address.unprocessed
+      end
+    end
+  end
+end

data/lib/aranha/rails/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Aranha
+  module Rails
+    VERSION = '0.1.0'
+  end
+end

data/lib/tasks/aranha.rake

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+namespace(:aranha) do
+  desc 'Process Aranha\'s addresses.'
+  task process: :environment do
+    ::Aranha::Processor.new
+  end
+
+  desc 'Remove all registered Aranha\'s addresses.'
+  task clear: :environment do
+    Rails.logger.info("Addresses deleted: #{::Aranha::Address.destroy_all.count}")
+  end
+
+  namespace :fixtures do
+    desc 'Download remote content for fixtures.'
+    task download: :environment do
+      ::Aranha::Rails::FixturesDownload.new(
+        prefix: ENV['PREFIX'],
+        download: ENV['DOWNLOAD'].present?,
+        pending: ENV['PENDING'].present?
+      ).run
+    end
+  end
+end

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: aranha-rails
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Put here the authors
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: active_scaffold
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.5
+- !ruby/object:Gem::Dependency
+  name: aranha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+- !ruby/object:Gem::Dependency
+  name: aranha-parsers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: eac_rails_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: eac_ruby_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.52'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.52'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.7
+- !ruby/object:Gem::Dependency
+  name: eac_ruby_gem_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/aranha/rails.rb
+- lib/aranha/rails/engine.rb
+- lib/aranha/rails/fixtures_download.rb
+- lib/aranha/rails/manager.rb
+- lib/aranha/rails/version.rb
+- lib/tasks/aranha.rake
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: Put here de description.
+test_files: []