daimon_skycrawlers 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01cf8887ce71dff448a5130eadb47e702643a042
-  data.tar.gz: 280aca91923bbb0f40af2996074b1d2f32a7e5ed
+  metadata.gz: 3c9d264378336af1acadb1bec4dc2dba783d9089
+  data.tar.gz: c532cd7213889299f369ba2e8e682ea69a39172a
 SHA512:
-  metadata.gz: 81fe7008849bb2ef7936e2b169671c275669692d77440b2ebada247870fc0a67d08ccad2b9c1c5255c253a90ae22a81c221c163f1dc1b007918f4cc3f90f89fe
-  data.tar.gz: ea67f734ed38fb348d4385ec7b603b88385b2c9f5161ba4765980fa8e355e754b08b680106e2da5e75a4badf4a98287c8f1ecb0b12ce60f140bbeeeb4487bc0f
+  metadata.gz: 86bcf270ee971a26cd029ab9bd9372cfbe544e15640b3275e897e8b0ec2131976db379330a047ef5ea03bf9f568c111e083a135ae1fddd3d0ead38a373086e33
+  data.tar.gz: a9aef67c6bcb70488a1ce0c5e54ed565f65e7a5490b2e0b7db87feb2960801bb829467d991ae4fb3ab52ddef145d25315ba03ca1e2178256b5147b7b52a043a2

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -37,6 +37,9 @@ module DaimonSkycrawlers
         when headers.key?("etag") && page.etag
           headers["etag"] != page.etag
         when headers.key?("last-modified") && page.last_modified_at
+          if headers["last-modified"] < page.last_modified_at
+            log.warn("#{url} returns old contents. #{headers["last-modified"]} < #{page.last_modified_at}")
+          end
           headers["last-modified"] > page.last_modified_at
         else
           true

data/lib/daimon_skycrawlers/generator/new.rb

@@ -27,7 +27,7 @@ module DaimonSkycrawlers
           timestamps: true
         }
         invoke(MigrationGenerator, [
-                 "CreatePage",
+                 "CreatePages",
                  "url:string",
                  "headers:text",
                  "body:binary",
@@ -37,6 +37,22 @@ module DaimonSkycrawlers
                migration_options)
       end
 
+      def insert_index
+        Dir.glob(File.join(destination_root, name, "db/migrate/*_create_pages.rb")) do |entry|
+          source = File.read(entry)
+          replaced_source = source.gsub(/(^ +)t.timestamps$/) do |_match; indent|
+            indent = $1
+            <<-CODE.chomp
+#{indent}t.timestamps
+
+#{indent}t.index [:url]
+#{indent}t.index [:url, :updated_at]
+            CODE
+          end
+          File.write(entry, replaced_source)
+        end
+      end
+
       def copy_files
         [
           "Gemfile",

data/lib/daimon_skycrawlers/storage/rdb.rb

@@ -9,7 +9,7 @@ module DaimonSkycrawlers
     class RDB < Base
       def initialize(config_path = "config/database.yml")
         super()
-        Base.configurations = YAML.load_file(config_path)
+        Base.configurations = YAML.load(ERB.new(::File.read(config_path)).result)
         Base.establish_connection(DaimonSkycrawlers.env.to_sym)
       end
 
@@ -34,7 +34,7 @@ module DaimonSkycrawlers
       # @param [String] url identity of the page
       #
       def find(url)
-        Page.where(url: url).order(last_modified_at: :desc).limit(1).first
+        Page.where(url: url).order(updated_at: :desc).limit(1).first
       end
 
       class Base < ActiveRecord::Base

data/lib/daimon_skycrawlers/tasks/database_tasks.rake

@@ -43,7 +43,7 @@ Rake::Task.define_task("db:load_config") do
     config.migrations_paths       = ["db/migrate"]
     config.fixtures_path          = "test/fixtures"
     config.seed_loader            = seed_loader.new
-    config.database_configuration = YAML.load_file("config/database.yml")
+    config.database_configuration = YAML.load(ERB.new(::File.read("config/database.yml")).result)
   end
   environment = ENV["SKYCRAWLERS_ENV"] || "development"
   ActiveRecord::Base.configurations = ActiveRecord::Tasks::DatabaseTasks.database_configuration

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.7.2"
+  VERSION = "0.8.0"
 end

data/sample/itp-crawler/Gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: ../../
   specs:
-    daimon_skycrawlers (0.6.0)
+    daimon_skycrawlers (0.7.2)
       activerecord
-      bundler (~> 1.11)
       faraday
       faraday_middleware
       nokogiri
@@ -45,7 +44,7 @@ GEM
     amq-protocol (2.0.1)
     arel (7.1.4)
     builder (3.2.2)
-    bunny (2.6.0)
+    bunny (2.6.1)
       amq-protocol (>= 2.0.1)
     concurrent-ruby (1.0.2)
     erubis (2.7.0)
@@ -89,8 +88,8 @@ GEM
     thread_safe (0.3.5)
     timers (4.1.1)
       hitimes
-    typhoeus (0.8.0)
-      ethon (>= 0.8.0)
+    typhoeus (1.1.0)
+      ethon (>= 0.9.0)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     webrobots (0.1.2)

data/sample/itp-crawler/db/migrate/{20161018044144_create_page.rb → 20161018044144_create_pages.rb}

@@ -1,13 +1,15 @@
-class CreatePage < ActiveRecord::Migration[5.0]
+class CreatePages < ActiveRecord::Migration[5.0]
   def change
     create_table :pages do |t|
-      t.string :url
+      t.string :url, index: true
       t.text :headers
       t.binary :body
       t.datetime :last_modified_at
       t.string :etag
 
       t.timestamps
+
+      t.index [:url, :updated_at]
     end
   end
 end

data/sample/itp-crawler/db/schema.rb

@@ -23,6 +23,8 @@ ActiveRecord::Schema.define(version: 20161018044144) do
     t.string   "etag"
     t.datetime "created_at",       null: false
     t.datetime "updated_at",       null: false
+    t.index ["url", "updated_at"], name: "index_pages_on_url_and_updated_at", using: :btree
+    t.index ["url"], name: "index_pages_on_url", using: :btree
   end
 
 end

data/sample/spider/db/migrate/20160830155803_create_pages.rb

@@ -1,13 +1,15 @@
 class CreatePages < ActiveRecord::Migration
   def change
     create_table :pages do |t|
-      t.string :url
+      t.string :url, index: true
       t.text :headers
       t.binary :body, limit: 10 * 1024 ** 2 # 10MiB
       t.datetime :last_modified_at
       t.string :etag
 
       t.timestamps null: false
+
+      t.index [:url, :updated_at]
     end
   end
 end

data/sample/spider/db/schema.rb

@@ -23,6 +23,8 @@ ActiveRecord::Schema.define(version: 20160830155803) do
     t.string   "etag"
     t.datetime "created_at",       null: false
     t.datetime "updated_at",       null: false
+    t.index ["url", "updated_at"], name: "index_pages_on_url_and_updated_at", using: :btree
+    t.index ["url"], name: "index_pages_on_url", using: :btree
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-25 00:00:00.000000000 Z
+date: 2016-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -363,7 +363,7 @@ files:
 - sample/itp-crawler/config/database.yml
 - sample/itp-crawler/config/database_itp.yml
 - sample/itp-crawler/config/init.rb
-- sample/itp-crawler/db/migrate/20161018044144_create_page.rb
+- sample/itp-crawler/db/migrate/20161018044144_create_pages.rb
 - sample/itp-crawler/db/schema.rb
 - sample/itp-crawler/db_itp/migrate/20161020044144_create_shop.rb
 - sample/itp-crawler/db_itp/schema.rb