aranha 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7776940b9f7bcb2d542e4993240fd10c18ff2024
+  data.tar.gz: d6579b4134a9a4ce5ec98cf335db1d11ab1b7038
+SHA512:
+  metadata.gz: 8d4924f70f27bb9a4809c2c56034fcaba0013b50925e8b204391034fa0cb88d5c2a3dc6de0fc365b8da8db5c81f45458c5b87b34ed058ab037dcc604d4f4fe0b
+  data.tar.gz: f5774605515bc18a2b57c4c794e1a0d050026dd96d16d13e7eb8683492199a48032281995a773f8b937b8e9c90ae0514fd304759b4a27e830629caefd59e19c4

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2017 Eduardo H. Bogoni
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= Aranha
+
+Rails utilities for web crawling.

data/Rakefile

@@ -0,0 +1,37 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Aranha'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+load 'rails/tasks/statistics.rake'
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/app/assets/javascripts/aranha/application.js

@@ -0,0 +1,14 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .
+//= require active_scaffold

data/app/assets/stylesheets/aranha/application.css

@@ -0,0 +1,16 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ *= require active_scaffold
+ */

data/app/controllers/aranha/addresses_controller.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+require_dependency 'aranha/application_controller'
+
+module Aranha
+  class AddressesController < ApplicationController
+    active_scaffold :'aranha/address' do |_conf|
+    end
+  end
+end

data/app/controllers/aranha/application_controller.rb

@@ -0,0 +1,5 @@
+module Aranha
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+  end
+end

data/app/helpers/aranha/application_helper.rb

@@ -0,0 +1,4 @@
+module Aranha
+  module ApplicationHelper
+  end
+end

data/app/models/aranha/address.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+module Aranha
+  class Address < ActiveRecord::Base
+    include ::Eac::InequalityQueries
+
+    add_inequality_queries(:created_at)
+
+    class << self
+      def set_start_point(url, processor)
+        start_points[url] = processor
+      end
+
+      def add_start_points
+        start_points.each do |url, processor|
+          add(url, processor)
+        end
+      end
+
+      def add(url, processor)
+        a = find_or_initialize_by(url: url)
+        a.processor = processor
+        a.save!
+      end
+
+      def clear_expired
+        q = by_created_at_lt(Time.zone.now - 12.hours)
+        Rails.logger.info("Addresses expired: #{q.count}")
+        q.destroy_all
+      end
+
+      private
+
+      def start_points
+        @start_points ||= {}
+      end
+    end
+
+    validates :url, presence: true, uniqueness: true
+    validates :processor, presence: true
+
+    scope :unprocessed, lambda {
+      where(processed_at: nil)
+    }
+
+    def to_s
+      "#{processor}|#{url}"
+    end
+
+    def process
+      ActiveRecord::Base.transaction do
+        processor.constantize.new(url).process
+        self.processed_at = Time.zone.now
+        save!
+      end
+    end
+  end
+end

data/app/views/layouts/aranha/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Aranha</title>
+  <%= stylesheet_link_tag    "aranha/application", media: "all" %>
+  <%= javascript_include_tag "aranha/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,3 @@
+Aranha::Engine.routes.draw do
+  resources(:addresses) { as_routes }
+end

data/db/migrate/20171201021251_create_aranha_addresses.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+class CreateAranhaAddresses < ActiveRecord::Migration
+  def change
+    create_table :aranha_addresses do |t|
+      t.string :url
+      t.string :processor
+      t.timestamp :processed_at
+
+      t.timestamps null: false
+    end
+  end
+end

data/lib/aranha.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+require 'active_support/dependencies'
+require_dependency 'aranha/engine'
+require_dependency 'active_scaffold'
+
+module Aranha
+end
+
+require_dependency 'aranha/processor'

data/lib/aranha/engine.rb

@@ -0,0 +1,11 @@
+module Aranha
+  class Engine < ::Rails::Engine
+    isolate_namespace Aranha
+
+    initializer :append_migrations do |app|
+      config.paths['db/migrate'].expanded.each do |expanded_path|
+        app.config.paths['db/migrate'] << expanded_path
+      end
+    end
+  end
+end

data/lib/aranha/processor.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+module Aranha
+  class Processor
+    def initialize
+      ::Aranha::Address.clear_expired
+      ::Aranha::Address.add_start_points
+      loop do
+        a = ::Aranha::Address.unprocessed.first
+        break unless a
+        Rails.logger.info("Processing #{a}")
+        a.process
+      end
+    end
+  end
+end

data/lib/aranha/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Aranha
+  VERSION = '0.0.1'
+end

data/lib/tasks/aranha_tasks.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+namespace(:aranha) do
+  task process: :environment do
+    ::Aranha::Processor.new
+  end
+end

data/test/aranha_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class AranhaTest < ActiveSupport::TestCase
+  test "truth" do
+    assert_kind_of Module, Aranha
+  end
+end

data/test/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/test/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    'application', media: 'all', 'data-turbolinks-track' => true %>
+  <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/test/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

data/test/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application', __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/test/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/test/dummy/bin/setup

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+# path to your application root.
+APP_ROOT = Pathname.new File.expand_path('../../',  __FILE__)
+
+Dir.chdir APP_ROOT do
+  # This script is a starting point to setup your application.
+  # Add necessary setup steps to this file:
+
+  puts "== Installing dependencies =="
+  system "gem install bundler --conservative"
+  system "bundle check || bundle install"
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?("config/database.yml")
+  #   system "cp config/database.yml.sample config/database.yml"
+  # end
+
+  puts "\n== Preparing database =="
+  system "bin/rake db:setup"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system "rm -f log/*"
+  system "rm -rf tmp/cache"
+
+  puts "\n== Restarting application server =="
+  system "touch tmp/restart.txt"
+end

data/test/dummy/config.ru

@@ -0,0 +1,4 @@
+# This file is used by Rack-based servers to start the application.
+
+require ::File.expand_path('../config/environment', __FILE__)
+run Rails.application

data/test/dummy/config/application.rb

@@ -0,0 +1,26 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'rails/all'
+
+Bundler.require(*Rails.groups)
+require "aranha"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+
+    # Do not swallow errors in after_commit/after_rollback callbacks.
+    config.active_record.raise_in_transactional_callbacks = true
+  end
+end
+