daimon_skycrawlers 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58dd7f91f6e9da8f9388a3364731ab0a543c01cd
-  data.tar.gz: b815a1cdad154eaf1b828568a697887df02bfcb6
+  metadata.gz: 7c8534d5ada19111a31249998b4ff7c935dc44c4
+  data.tar.gz: 77a7c9d78db52e9d40e412c9ced08d024caa1499
 SHA512:
-  metadata.gz: 1ffbdd022a6e6a3a80d292bae8927a716f33f339e19e12f67da536812f9118615be0d8886fc143b529b7854f3b4ef0731962f3102aa6c759790d86dd2df54914
-  data.tar.gz: 1beb4753e8602224c95081651a4da6276659bfe132cdb9dc5fe55e8ae9f2c524681bd9be8137d4e29d52380d0f137ea2630097337bc07aea31de55e3f2785d3c
+  metadata.gz: 205234df54f6c985aa14f97fa5e5a967e18c923d7bf0fc4b29fe3d3c40c85d88667987b6bc7ca9d389dd513b9cb235b699b4741f228ab1151aa7dcb1a5802ebb
+  data.tar.gz: e2090d4983e1ffb7434eef92b76c30795e495f79c89fe9fe3567ba4e7e8f8b650b33870b0ab718d5d1afd799de45d35d7791fbc29e43f4f76bfb23ce510b32e9

data/daimon_skycrawlers.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit-notify"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "tapp"
+  spec.add_development_dependency "simplecov"
   spec.add_development_dependency "sqlite3"
   spec.add_development_dependency "yard"
 end

data/lib/daimon_skycrawlers/callbacks.rb

@@ -0,0 +1,26 @@
+module DaimonSkycrawlers
+  module Callbacks
+    def initialize
+      super
+      @before_process_callbacks = []
+    end
+
+    def before_process(callback = nil, &block)
+      if block_given?
+        @before_process_callbacks << block
+      else
+        @before_process_callbacks << callback if callback.respond_to?(:call)
+      end
+    end
+
+    def run_before_callbacks(message)
+      @before_process_callbacks.all? do |callback|
+        callback.call(message)
+      end
+    end
+
+    def clear_before_process_callbacks
+      @before_process_callbacks = []
+    end
+  end
+end

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -3,6 +3,7 @@ require "faraday"
 
 require "daimon_skycrawlers/logger"
 require "daimon_skycrawlers/config"
+require "daimon_skycrawlers/callbacks"
 require "daimon_skycrawlers/storage"
 require "daimon_skycrawlers/processor"
 require "daimon_skycrawlers/filter/update_checker"
@@ -16,6 +17,7 @@ module DaimonSkycrawlers
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
+      include DaimonSkycrawlers::Callbacks
 
       # @!attribute [w] storage
       #   Set storage to crawler instance.
@@ -80,19 +82,24 @@ module DaimonSkycrawlers
       end
 
       def process(message, &block)
-        url = message.delete(:url)
-
         @skipped = false
         @n_processed_urls += 1
-        # url can be a path
-        url = connection.url_prefix + url
 
-        apply_filters(url)
+        setup_default_filters
 
-        unless skipped?
-          @prepare.call(connection)
-          fetch(url, message, &block)
+        proceeding = run_before_callbacks(message)
+        unless proceeding
+          @skipped = true
+          skip(message[:url])
+          return
         end
+
+        # url can be a path
+        url = message.delete(:url)
+        url = (URI(connection.url_prefix) + url).to_s
+
+        @prepare.call(connection)
+        fetch(url, message, &block)
       end
 
       def fetch(path, message = {})
@@ -109,18 +116,22 @@ module DaimonSkycrawlers
 
       private
 
-      def apply_filters(url)
+      def setup_default_filters
         if @options[:obey_robots_txt]
-          robots_txt_checker = DaimonSkycrawlers::Filter::RobotsTxtChecker.new(base_url: @base_url)
-          unless robots_txt_checker.allowed?({ url: url })
-            skip(url)
-            return
+          before_process do |m|
+            robots_txt_checker = DaimonSkycrawlers::Filter::RobotsTxtChecker.new(base_url: @base_url)
+            allowed = robots_txt_checker.allowed?(m)
+            log.debug("Not allowed: #{m[:url]}") unless allowed
+            allowed
           end
         end
-        update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
-        unless update_checker.updated?({ url: url.to_s }, connection: connection)
-          skip(url)
-          return
+        before_process do |m|
+          update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
+          updated = update_checker.updated?(m, connection: connection)
+          unless updated
+            log.debug("Not updated: #{m[:url]}")
+          end
+          updated
         end
       end
 

data/lib/daimon_skycrawlers/filter/base.rb

@@ -27,6 +27,13 @@ module DaimonSkycrawlers
       def call(message)
         raise NotImplementedError, "Must implement this method in subclass"
       end
+
+      private
+
+      def normalize_url(url)
+        return url unless @base_url
+        (URI(@base_url) + url).to_s
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -21,10 +21,7 @@ module DaimonSkycrawlers
       # @return [true|false] Return false when duplicated, otherwise return true.
       #
       def call(message)
-        url = message[:url]
-        unless URI(url).absolute?
-          url = (@base_url + url).to_s
-        end
+        url = normalize_url(message[:url])
         return false if @urls.include?(url)
         @urls << url
         true

data/lib/daimon_skycrawlers/filter/robots_txt_checker.rb

@@ -20,10 +20,7 @@ module DaimonSkycrawlers
       # @return [true|false] Return true when web site allows to fetch the URL, otherwise return false
       #
       def call(message)
-        url = message[:url]
-        unless URI(url).absolute?
-          url = (@base_url + url).to_s
-        end
+        url = normalize_url(message[:url])
         @webrobots.allowed?(url)
       end
 

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -22,10 +22,7 @@ module DaimonSkycrawlers
       # @return [true|false] Return true when need update, otherwise return false
       #
       def call(message, connection: nil)
-        url = message[:url]
-        unless URI(url).absolute?
-          url = (@base_url + url).to_s
-        end
+        url = normalize_url(message[:url])
         page = storage.find(url)
         return true unless page
         if connection

data/lib/daimon_skycrawlers/generator/templates/new/Dockerfile

@@ -1,6 +1,6 @@
 FROM ruby:2.3.1-alpine
 
-RUN apk --no-cache --update add build-base ruby-dev libxml2-dev postgresql-dev libcurl git
+RUN apk --no-cache --update add build-base ruby-dev libxml2-dev postgresql-dev libcurl openssl git
 
 RUN adduser -D -h /home/crawler -g "DaimonSkycrawlers user" -s /bin/sh crawler crawler
 
@@ -8,7 +8,12 @@ ARG SKYCRAWLERS_ENV=production
 ARG SKYCRAWLERS_MAIN=crawler
 ENV SKYCRAWLERS_ENV=$SKYCRAWLERS_ENV \
     SKYCRAWLERS_MAIN=$SKYCRAWLERS_MAIN \
-    BUNDLE_JOBS=4
+    BUNDLE_JOBS=4 \
+    DOCKERIZE_VERSION=v0.3.0
+
+RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
 USER crawler
 WORKDIR /home/crawler

data/lib/daimon_skycrawlers/generator/templates/new/docker-compose.yml.erb

@@ -2,11 +2,15 @@ version: "2"
 services:
   <%= name %>-rabbitmq:
     image: rabbitmq
+    volumes:
+      - <%= name %>-rabbitmq-storage:/var/lib/rabbitmq
 
   <%= name %>-db:
     build:
       context: .
       dockerfile: Dockerfile.db
+    volumes:
+      - <%= name %>-db-storage:/var/lib/postgresql/data
     env_file: .env.db
 
   <%= name %>-common: &common
@@ -33,3 +37,7 @@ services:
     depends_on:
       - <%= name %>-crawler
     command: processor
+
+volumes:
+  <%= name %>-db-storage:
+  <%= name %>-rabbitmq-storage:

data/lib/daimon_skycrawlers/generator/templates/new/env.erb

@@ -1,2 +1,5 @@
 SKYCRAWLERS_RABBITMQ_HOST=<%= name %>-rabbitmq
-DATABASE_URL=postgres://crawler:<%= config[:password] %>@<%= name %>-db/xxx_development
+SKYCRAWLERS_RABBITMQ_PORT=5672
+DATABASE_HOST=<%= name %>-db
+DATABASE_PORT=5432
+DATABASE_URL=postgres://crawler:<%= config[:password] %>@<%= name %>-db/<%= name %>_development

data/lib/daimon_skycrawlers/generator/templates/new/services/common/docker-entrypoint.sh

@@ -3,10 +3,13 @@
 set -x
 
 MAIN=$1
+dockerize -timeout 60s \
+          -wait tcp://${DATABASE_HOST}:${DATABASE_PORT} \
+          -wait tcp://${SKYCRAWLERS_RABBITMQ_HOST}:${SKYCRAWLERS_RABBITMQ_PORT}
 case $MAIN in
     crawler)
         bundle check || bundle install --retry=3 --path=vendor/bundle \
-                && bundle exec rake db:schema:load || bundle exec rake db:migrate
+                && bundle exec rake db:migrate
         bundle exec daimon_skycrawlers exec $MAIN
         ;;
     processor)
@@ -14,13 +17,16 @@ case $MAIN in
             sleep 5
         done
         bundle check || bundle install --retry=3 --path=vendor/bundle \
-                && bundle exec rake db:schema:load || bundle exec rake db:migrate
+                && bundle exec rake db:migrate
         bundle exec daimon_skycrawlers exec $MAIN
         ;;
     setup)
-        bundle install --path=vendor/bundle
+        bundle install --retry=3 --path=vendor/bundle
         bundle exec rake db:schema:load
         ;;
+    migrate)
+        bundle exec rake db:migrate
+        ;;
     none)
         echo NOP
         ;;

data/lib/daimon_skycrawlers/generator/templates/processor.rb.erb

@@ -8,6 +8,5 @@ class <%= config[:class_name] %> < DaimonSkycrawlers::Processor::Base
   end
 end
 
-base_url = ""
-processor = <%= config[:class_name] %>.new(base_url)
+processor = <%= config[:class_name] %>.new
 DaimonSkycrawlers.register_processor(processor)

data/lib/daimon_skycrawlers/processor/base.rb

@@ -1,27 +1,17 @@
 require "daimon_skycrawlers/logger"
 require "daimon_skycrawlers/config"
+require "daimon_skycrawlers/callbacks"
 
 module DaimonSkycrawlers
   module Processor
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
-
-      def initialize
-        super
-        @before_process_filters = []
-      end
-
-      def before_process(filter = nil, &block)
-        if block_given?
-          @before_process_filters << block
-        else
-          @before_process_filters << filter if filter.respond_to?(:call)
-        end
-      end
+      include DaimonSkycrawlers::Callbacks
 
       def process(message)
-        return unless apply_before_filters(message)
+        proceeding = run_before_callbacks(message)
+        return unless proceeding
         call(message)
       end
 
@@ -32,14 +22,6 @@ module DaimonSkycrawlers
       def storage
         @storage ||= DaimonSkycrawlers::Storage::RDB.new
       end
-
-      private
-
-      def apply_before_filters(message)
-        @before_process_filters.all? do |filter|
-          filter.call(message)
-        end
-      end
     end
   end
 end

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.10.0"
+  VERSION = "0.11.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-29 00:00:00.000000000 Z
+date: 2016-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -262,6 +262,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -307,6 +321,7 @@ files:
 - daimon_skycrawlers.gemspec
 - db/schema.rb
 - lib/daimon_skycrawlers.rb
+- lib/daimon_skycrawlers/callbacks.rb
 - lib/daimon_skycrawlers/cli.rb
 - lib/daimon_skycrawlers/commands/enqueue.rb
 - lib/daimon_skycrawlers/commands/runner.rb
@@ -407,7 +422,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: This is a crawler framework.