seiya 0.0.7.5 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d611112f85f93780736302a06e67091bb3114a4
-  data.tar.gz: 941172b9cd341b05e0af011c1a44aadef7aa3ee2
+  metadata.gz: 2226d4f76aa277059e8da84420def305d0ab0be2
+  data.tar.gz: 652ac53f990da56b552d42ec18b5d54d887e75d0
 SHA512:
-  metadata.gz: d1ee4335d3b8d2c4534117e56b0a874bdca536ed1779813a9aa613f8e36fb53feb167d1b0a6ed4b85f7ff252d608a5af4f0e120898978773f474876ccddb2dbf
-  data.tar.gz: 92a7195643cc5231e23f639b0409289e174a6e3f80051b3bc8471ca034e5a200a9e02c8ba84a740680b46920a62ebb9d96c2035feab209c616fbb74d87b74e61
+  metadata.gz: f466b0e929feeee9042c3576994cfedcd53c00433f69860d503e5ab1d3593d5e5cd1bf44a2aae1512e7133006f63a3b9f02403e588c4cd7887c6429396bd4d2a
+  data.tar.gz: 3726f0b6bbcc902fc5512fbea2e63ad38b8ac00dfa726b1bdefa8a4cf7e94d88b406dfd7d2b929cb50f3156c6780dba7afe4ca6896dff572c04f5c987c4d0e6a

data/lib/seiya/contrib/commands/crawl.rb

@@ -1,15 +1,16 @@
 require 'seiya/command'
 require 'optparse'
 
-module Contrib
-  module Commands
-    class Crawl < Seiya::Command
-      def summary
-        'Crawl a task'
-      end
+module Seiya
+  module Contrib
+    module Commands
+      class Crawl < Seiya::Command
+        def summary
+          'Crawl a task'
+        end
 
-      def usage
-        'Usage
+        def usage
+          'Usage
 =====
   seiya crawl <task_name> [options]
 
@@ -20,52 +21,53 @@ Options
 --help, -h      show this help message and exit
 -a NAME=VALUE   set task argument (may be repeated)
 --list, -l      show task list in this project'
-      end
+        end
 
-      def print_task_list
-        task_list = Seiya.tasks.map do |k, v|
-          '%-14s%-30s' % [k, v]
-        end.join("\n")
-        puts "Task list
+        def print_task_list
+          task_list = Seiya.tasks.map do |k, v|
+            '%-14s%-30s' % [k, v]
+          end.join("\n")
+          puts "Task list
 =========
 
 #{task_list}
 "
-      end
+        end
 
-      def run(*args)
-        options = {}
-        OptionParser.new do |opts|
-          opts.banner = 'Usage: seiya [options]'
+        def run(*args)
+          options = {}
+          OptionParser.new do |opts|
+            opts.banner = 'Usage: seiya [options]'
 
-          opts.on '-aArg', '--argument=Arg', 'send argument to seiya task' do |a|
-            options[:args] = [] unless options[:args]
-            options[:args] << a
-          end
+            opts.on '-aArg', '--argument=Arg', 'send argument to seiya task' do |a|
+              options[:args] = [] unless options[:args]
+              options[:args] << a
+            end
 
-          opts.on '-l', '--list', 'list tasks' do |l|
-            options[:list] = l
-          end
-        end.parse!
+            opts.on '-l', '--list', 'list tasks' do |l|
+              options[:list] = l
+            end
+          end.parse!
 
-        if options[:list]
-          print_task_list
-          exit 0
-        end
+          if options[:list]
+            print_task_list
+            exit 0
+          end
 
-        task_name = args.shift
-        if task_name.nil?
-          puts 'Need a task_name'
-          exit!
-        end
-        task_class = Seiya.get_task_class task_name
+          task_name = args.shift
+          if task_name.nil?
+            puts 'Need a task_name'
+            exit!
+          end
+          task_class = Seiya.get_task_class task_name
 
 
-        _args = options[:args] ? options[:args] : []
+          _args = options[:args] ? options[:args] : []
 
-        task = task_class.new *_args
+          task = task_class.new *_args
 
-        task.run
+          task.run
+        end
       end
     end
   end

data/lib/seiya/contrib/commands/create.rb

@@ -1,13 +1,15 @@
 require 'seiya/command'
-module Contrib
-  module Commands
-    class Create < Seiya::Command
-      def summary
-        'Create a new project'
-      end
 
-      def usage
-        'Usage
+module Seiya
+  module Contrib
+    module Commands
+      class Create < Seiya::Command
+        def summary
+          'Create a new project'
+        end
+
+        def usage
+          'Usage
 =====
   seiya create <project_name>
 
@@ -16,15 +18,16 @@ Create new project
 Options
 =======
 --help, -h      show this help message and exit'
-      end
+        end
 
-      def run(*args)
-        project_name = args.shift
-        if project_name.nil?
-          puts 'Need a project_name!'
-          exit!
+        def run(*args)
+          project_name = args.shift
+          if project_name.nil?
+            puts 'Need a project_name!'
+            exit!
+          end
+          Seiya.gen_project_file project_name
         end
-        Seiya.gen_project_file project_name
       end
     end
   end

data/lib/seiya/contrib/commands/gentask.rb

@@ -1,13 +1,15 @@
 require 'seiya/command'
-module Contrib
-  module Commands
-    class Gentask < Seiya::Command
-      def summary
-        'Generate new task using pre-defined templates'
-      end
 
-      def usage
-        'Usage
+module Seiya
+  module Contrib
+    module Commands
+      class Gentask < Seiya::Command
+        def summary
+          'Generate new task using pre-defined templates'
+        end
+
+        def usage
+          'Usage
 =====
   seiya gentask [options] <name> <domain>
 
@@ -16,16 +18,17 @@ Generate new task using pre-defined templates
 Options
 =======
 --help, -h      show this help message and exit'
-      end
+        end
 
-      def run(*args)
-        task_name = args.shift
-        task_domain = args.shift
-        if task_name.nil?
-          puts 'Need a task_name!'
-          exit!
+        def run(*args)
+          task_name = args.shift
+          task_domain = args.shift
+          if task_name.nil?
+            puts 'Need a task_name!'
+            exit!
+          end
+          Seiya.gen_task_file task_name, task_domain
         end
-        Seiya.gen_task_file task_name, task_domain
       end
     end
   end

data/lib/seiya/contrib/middlewares.rb

@@ -0,0 +1,16 @@
+require 'seiya/middleware'
+require 'seiya/settings'
+
+module Seiya
+  module Contrib
+    module RequestMiddlewares
+      class RandomUserAgentMiddleware < Seiya::RequestMiddleware
+        def process_request(request)
+          headers = request.headers
+          headers = {} unless headers.is_a? Hash
+          headers['User-Agent'] = Seiya::Settings::USER_AGENTS.sample
+        end
+      end
+    end
+  end
+end

data/lib/seiya/contrib/pipelines.rb

@@ -1,10 +1,12 @@
 require 'seiya/pipeline'
 
-module Contrib
-  module Pipelines
-    class BasePipeline < Seiya::Pipeline
-      def process_item(item)
-        item
+module Seiya
+  module Contrib
+    module Pipelines
+      class BasePipeline < Seiya::Pipeline
+        def process_item(item)
+          item
+        end
       end
     end
   end

data/lib/seiya/contrib.rb

@@ -1,2 +1,3 @@
 require 'seiya/contrib/pipelines'
 require 'seiya/contrib/commands'
+require 'seiya/contrib/middlewares'

data/lib/seiya/middleware.rb

@@ -0,0 +1,7 @@
+module Seiya
+  class RequestMiddleware
+    def process_request(request)
+      request
+    end
+  end
+end

data/lib/seiya/processer.rb

@@ -0,0 +1,13 @@
+module Seiya
+  def process_item(item)
+    @pipelines.each do |p|
+      item = p.process_item item
+    end
+  end
+
+  def process_request(request)
+    @request_middlewares.each do |rm|
+      rm.process_request request
+    end
+  end
+end

data/lib/seiya/request.rb

@@ -5,15 +5,20 @@ require 'seiya/response'
 
 module Seiya
   class Request
-    def initialize(url, *args, method: 'get')
+    attr_reader :url
+    attr_accessor :params, :headers
+
+    def initialize(url, *args, params: {}, headers: {}, method: 'get')
       @url = url
       @method = method.upcase
       @args = args
+      @params = params
+      @headers = headers
       @httpclient = HTTPClient.new
     end
 
     def get_response
-      Response.new @httpclient.send(@method.downcase, @url, *@args)
+      Response.new @httpclient.send(@method.downcase, @url, @params, @headers, *@args)
     end
 
     def register(&block)

data/lib/seiya/response.rb

@@ -5,7 +5,15 @@ module Seiya
     end
 
     def url
-      @resp.http_header.request_uri.to_s
+      @resp.header.request_uri.to_s
+    end
+
+    def header
+      @resp.header
+    end
+
+    def headers
+      @resp.headers
     end
 
     def body

data/lib/seiya/scheduler.rb

@@ -11,7 +11,9 @@ module Seiya
 
     def add_requests(requests)
       requests.each do |request|
-        @request_q << request if request.registered?
+        next unless request.registered?
+        Seiya.process_request request
+        @request_q << request
       end
       run unless @run
     end

data/lib/seiya/settings.rb

@@ -1,8 +1,34 @@
 module Seiya
   module Settings
     PIPELINES = {
-        'seiya/contrib|Contrib::Pipelines::BasePipeline' => 2
+        'seiya/contrib|Seiya::Contrib::Pipelines::BasePipeline' => 0
     }
-    COMMANDS = 'seiya/contrib|Contrib::Commands'
+    REQUEST_MIDDLEWARES = {
+        'seiya/contrib|Seiya::Contrib::RequestMiddlewares::RandomUserAgentMiddleware' => 0
+    }
+    COMMANDS = 'seiya/contrib|Seiya::Contrib::Commands'
+    USER_AGENTS = [
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7',
+        'Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1',
+        'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6',
+        'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6',
+        'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5',
+        'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3',
+        'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24',
+        'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24',
+    ]
   end
 end

data/lib/seiya/util.rb

@@ -8,6 +8,21 @@ module Seiya
       end
     end
 
+    def argument_to_hash(args, *field)
+      return nil if args.empty?
+      if args.size == 1 and Hash === args[0]
+        h = args[0]
+        if field.any? { |f| h.key?(f) }
+          return h
+        end
+      end
+      h = {}
+      field.each_with_index do |e, idx|
+        h[e] = args[idx]
+      end
+      h
+    end
+
     def processors_in_use
       procs=[]
       Dir.glob('/proc/*/stat') do |filename|

data/lib/seiya/version.rb

@@ -1,3 +1,3 @@
 module Seiya
-  VERSION = '0.0.7.5'
+  VERSION = '0.0.8'
 end

data/lib/seiya.rb

@@ -1,22 +1,20 @@
 require 'fileutils'
+require 'inifile'
+require 'seiya/util'
 require 'seiya/version'
 require 'seiya/request'
 require 'seiya/task'
 require 'seiya/item'
 require 'seiya/pipeline'
+require 'seiya/middleware'
 require 'seiya/settings'
 require 'seiya/command'
 require 'seiya/support'
+require 'seiya/processer'
 
 module Seiya
   extend self
 
-  def process_item(item)
-    @pipelines.each do |p|
-      item = p.process_item item
-    end
-  end
-
   def get_const!(require_str, const_str)
     begin
       require require_str
@@ -78,34 +76,52 @@ module Seiya
     $:.unshift load_path unless load_path.nil?
   end
 
+  def component_instance_variables(*variable_names)
+    variable_names.each do |variable_name|
+      variable_name = variable_name.to_s
+      const_name = variable_name.upcase
+      super_class = Seiya.const_get variable_name.sub(/s$/, '').camelize
+      vars = Settings.const_get const_name
+      begin
+        vars.merge! Util.get_const "#{@settings_const_str}::#{const_name}"
+      rescue NameError
+        # ignored
+      end
+
+      vars = {} unless vars.is_a? Hash
+
+      vars = vars.select do |_, v|
+        v >= 0
+      end.sort_by do |_, v|
+        v
+      end.to_h
+
+      vars = vars.keys.map do |k|
+        require_str, const_str = k.split '|'
+        klass = get_const require_str, const_str
+        klass.new
+      end.select do |p|
+        p.is_a? super_class
+      end
+
+      instance_variable_set '@' << variable_name, vars
+    end
+  end
+
   def setup(conf_file: 'seiya.ini')
     settings_const_str = ''
     if File.exist? conf_file
-      load_path = File.dirname File.expand_path(conf_file)
-      extend_load_path load_path
+      path = File.dirname File.expand_path(conf_file)
+      extend_load_path path
 
-      require 'inifile'
-      require 'seiya/util'
       conf = IniFile.load conf_file
       settings_file = conf.to_h.fetch('global', {}).fetch('settings', 'settings')
       settings_require_str, settings_const_str = settings_file.split '|'
+      @settings_const_str = settings_const_str
       require settings_require_str
     end
 
-    pipelines = Settings::PIPELINES
-    begin
-      pipelines.merge! Util.get_const "#{settings_const_str}::PIPELINES"
-    rescue NameError
-      # ignored
-    end
-
-    pipelines = pipelines.sort_by { |_, v| v }.to_h
-
-    @pipelines = pipelines.keys.map do |k|
-      require_str, const_str = k.split '|'
-      klass = get_const require_str, const_str
-      klass.new
-    end
+    component_instance_variables :pipelines, :request_middlewares
 
     commands = [Settings::COMMANDS]
     begin

data/sample/test/aa/middlewares.rb

@@ -0,0 +1,16 @@
+require 'paint'
+require 'seiya'
+
+module RequestMiddlewares
+  class MyMiddleware < Seiya::RequestMiddleware
+    def process_request(request)
+      puts Paint['I am MyMiddleware!', :red]
+      puts Paint["User-Agent: #{request.headers['User-Agent']}", :random]
+    end
+  end
+  class MySubMiddleware < Seiya::RequestMiddleware
+    def process_request(request)
+      puts Paint['I am MySubMiddleware!', :green]
+    end
+  end
+end

data/sample/test/aa/pipelines/t.rb

@@ -1,17 +1,18 @@
+require 'paint'
 require 'seiya'
 
 module Pipelines
   class A < Seiya::Pipeline
     def process_item(item)
-      p 'I am A Pipeline'
+      puts Paint['I am A Pipeline', :yellow]
       item[:pipeline] = 'A'
       item
     end
   end
   class B < Seiya::Pipeline
     def process_item(item)
-      p 'I am B Pipeline'
-      p item
+      puts Paint['I am B Pipeline', :blue]
+      puts item.to_json
       item
     end
   end

data/sample/test/aa/settings.rb

@@ -3,5 +3,9 @@ module Settings
       'aa/pipelines/t|Pipelines::A' => 1,
       'aa/pipelines/t|Pipelines::B' => 2,
   }
+  REQUEST_MIDDLEWARES = {
+      'aa/middlewares|RequestMiddlewares::MyMiddleware' => 2,
+      'aa/middlewares|RequestMiddlewares::MySubMiddleware' => 1,
+  }
   COMMANDS = 'aa/commands|Commands'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: seiya
 version: !ruby/object:Gem::Version
-  version: 0.0.7.5
+  version: 0.0.8
 platform: ruby
 authors:
 - yetone
@@ -26,9 +26,12 @@ files:
 - lib/seiya/contrib/commands/crawl.rb
 - lib/seiya/contrib/commands/create.rb
 - lib/seiya/contrib/commands/gentask.rb
+- lib/seiya/contrib/middlewares.rb
 - lib/seiya/contrib/pipelines.rb
 - lib/seiya/item.rb
+- lib/seiya/middleware.rb
 - lib/seiya/pipeline.rb
+- lib/seiya/processer.rb
 - lib/seiya/request.rb
 - lib/seiya/response.rb
 - lib/seiya/scheduler.rb
@@ -40,6 +43,7 @@ files:
 - sample/test/aa/commands.rb
 - sample/test/aa/commands/sing.rb
 - sample/test/aa/items.rb
+- sample/test/aa/middlewares.rb
 - sample/test/aa/pipelines/t.rb
 - sample/test/aa/settings.rb
 - sample/test/aa/tasks.rb