seiya 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d5ca500e310f4b98f85a00120047c72d64cf611
-  data.tar.gz: 63db6b3c6d4258678fb808a3b1c2e765af3592f2
+  metadata.gz: b46494f0bfaec4f0a78fd9cf6cc9ac06c306b9f7
+  data.tar.gz: 4fbc8f4febaddf6f20f7f04ac4ae7dd43d86c20e
 SHA512:
-  metadata.gz: cdfcef0a38a291788696f3b220b0700e0546e3169ac2c2853b47317bd8af4129d8f3b33e30c2b5b051bab4e6e4c5dfb926dafe1dc2488273abe09be2ff115ea7
-  data.tar.gz: 24cdc21f897340a4e51854f79c4a36a069cb42284fa1a40fca260371933ae85dda20d7051d9c68953f4de4384dafe7a567c59d5d786153db36cb74a52c9fb11c
+  metadata.gz: 43afa2ca3d3c85cf28b8ac0985f37cd56b2c0001a1d69ba108dd4b7679410ebbbeb8cf2d62a543c19d2f29aa38ec0ef9248ac18cf2942d703fc143b09b447dee
+  data.tar.gz: 797a0a441cf7d7b129e5958d09fc20902ed75040b320a5b555cd2cf174ef5b58c62c2540b2ca8f65b31409e73bae3a54987b029191a88d2762674f13717e95a1

data/bin/seiya

@@ -3,71 +3,14 @@
 $:.unshift File.expand_path '.'
 $:.unshift File.expand_path '../../lib', __FILE__
 require 'seiya'
-require 'optparse'
 
-options = {}
+command = ARGV.shift
 
-OptionParser.new do |opts|
-  opts.banner = 'Usage: seiya [options]'
-
-  opts.on '-v', '--version', 'seiya version' do |v|
-    options[:version] = v
-  end
-
-  opts.on '-tTask', '--task=Task', 'run a seiya task' do |t|
-    options[:task] = t
-  end
-
-  opts.on '-aArg', '--argument=Arg', 'send argument to seiya task' do |a|
-    options[:args] = [] unless options[:args]
-    options[:args] << a
-  end
-
-  opts.on '-cTask', '--create=Task', 'create a seiya task' do |t|
-    options[:create] = t
-  end
-end.parse!
-
-def extend_load_path(path)
-  Dir.foreach path do |f|
-    unless %w(. ..).include? f
-      if File.directory? File.join path, f
-        new_path = File.join path, f
-        extend_load_path new_path
-      elsif f == 'tasks.rb'
-        $:.unshift path
-        break
-      end
-    end
-  end
-end
-
-if options.key? :task
-  extend_load_path Dir.pwd
-
-  begin
-    require 'tasks'
-  rescue
-    puts 'No tasks!'
-    exit!
-  end
-
-  Seiya.setup
-  task_name = options[:task]
-  begin
-    task = Seiya.get_task task_name
-  rescue NameError
-    puts "No task named: #{task_name}"
-    exit!
-  end
-  task.run
-end
-
-if options.key? :version
-  puts Seiya::VERSION
-end
-
-if options.key? :create
-  task_name = options[:create]
-  Seiya.gen_task_file task_name
+case command
+  when '-v', '--version'
+    puts Seiya::VERSION
+  when '-h', '--help'
+    Seiya.usage
+  else
+    Seiya.run_command command, *ARGV
 end

data/lib/seiya/command.rb

@@ -0,0 +1,22 @@
+module Seiya
+  class Command
+    def summary
+      'I am a seiya command'
+    end
+
+    def usage
+      'no usage'
+    end
+
+    def run
+    end
+
+    def run!(*args)
+      if args.include? '-h' or args.include? '--help'
+        puts usage
+        return
+      end
+      run *args
+    end
+  end
+end

data/lib/seiya/contrib/commands/crawl.rb

@@ -0,0 +1,52 @@
+require 'seiya/command'
+require 'optparse'
+
+module Contrib
+  module Commands
+    class Crawl < Seiya::Command
+      def summary
+        'Crawl a task'
+      end
+
+      def usage
+        'Usage
+=====
+  seiya crawl <task_name> [options]
+
+Run a task
+
+Options
+=======
+--help, -h      show this help message and exit
+-a NAME=VALUE   set task argument (may be repeated)'
+      end
+
+      def run(*args)
+        require 'tasks'
+
+        task_name = args.shift
+        if task_name.nil?
+          puts 'Need a task_name'
+          exit!
+        end
+        task_class = Seiya.get_task_class task_name
+
+        options = {}
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: seiya [options]'
+
+          opts.on '-aArg', '--argument=Arg', 'send argument to seiya task' do |a|
+            options[:args] = [] unless options[:args]
+            options[:args] << a
+          end
+        end.parse!
+
+        _args = options[:args] ? options[:args] : []
+
+        task = task_class.new *_args
+
+        task.run
+      end
+    end
+  end
+end

data/lib/seiya/contrib/commands/create.rb

@@ -0,0 +1,31 @@
+require 'seiya/command'
+module Contrib
+  module Commands
+    class Create < Seiya::Command
+      def summary
+        'Create a new project'
+      end
+
+      def usage
+        'Usage
+=====
+  seiya create <project_name>
+
+Create new project
+
+Options
+=======
+--help, -h      show this help message and exit'
+      end
+
+      def run(*args)
+        project_name = args.shift
+        if project_name.nil?
+          puts 'Need a project_name!'
+          exit!
+        end
+        Seiya.gen_project_file project_name
+      end
+    end
+  end
+end

data/lib/seiya/contrib/commands.rb

@@ -0,0 +1,2 @@
+require_relative './commands/create'
+require_relative './commands/crawl'

data/lib/seiya/contrib.rb

@@ -1 +1,2 @@
 require 'seiya/contrib/pipelines'
+require 'seiya/contrib/commands'

data/lib/seiya/settings.rb

@@ -3,5 +3,6 @@ module Seiya
     PIPELINES = {
         'seiya/contrib|Contrib::Pipelines::BasePipeline' => 2
     }
+    COMMANDS = 'seiya/contrib|Contrib::Commands'
   end
 end

data/lib/seiya/support.rb

@@ -0,0 +1,9 @@
+class String
+  def underscore
+    self.gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        tr('-', '_').
+        downcase
+  end
+end

data/lib/seiya/version.rb

@@ -1,3 +1,3 @@
 module Seiya
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
 end

data/lib/seiya.rb

@@ -5,6 +5,8 @@ require 'seiya/task'
 require 'seiya/item'
 require 'seiya/pipeline'
 require 'seiya/settings'
+require 'seiya/command'
+require 'seiya/support'
 
 module Seiya
   extend self
@@ -15,7 +17,40 @@ module Seiya
     end
   end
 
-  def setup(conf_file: 'seiya.ini')
+  def get_const(require_str, const_str)
+    begin
+      require require_str
+    rescue LoadError => e
+      puts e
+      puts "Cannot load #{require_str}"
+      exit!
+    end
+
+    begin
+      Util.get_const const_str
+    rescue NameError
+      puts "Cannot get #{const_str}"
+      exit!
+    end
+  end
+
+  def extend_load_path(path)
+    Dir.foreach path do |f|
+      unless %w(. ..).include? f
+        if File.directory? File.join path, f
+          new_path = File.join path, f
+          extend_load_path new_path
+        elsif f == 'tasks.rb'
+          $:.unshift path
+          break
+        end
+      end
+    end
+  end
+
+  def setup(conf_file: 'seiya.ini', load_path: Dir.pwd)
+    extend_load_path load_path
+
     require 'inifile'
     require 'seiya/util'
     conf = IniFile.load conf_file
@@ -23,49 +58,101 @@ module Seiya
     settings_require_str, settings_const_str = settings_file.split '|'
     require settings_require_str
 
+    pipelines = Settings::PIPELINES
     begin
-      pipelines = Util.get_const "#{settings_const_str.empty? ? 'Settings' : settings_const_str}::PIPELINES"
+      pipelines.merge! Util.get_const "#{settings_const_str}::PIPELINES"
     rescue NameError
-      pipelines = {}
+      # ignored
     end
 
-    pipelines.merge! Settings::PIPELINES
-
     pipelines = pipelines.sort_by { |_, v| v }.to_h
 
     @pipelines = pipelines.keys.map do |k|
-      require_str, class_name = k.split('|')
-      begin
-        require require_str
-        clazz = Util.get_const class_name
-        clazz.new
-      rescue LoadError
-        puts 'current directory is not a spider directory!'
-        exit!
+      require_str, const_str = k.split '|'
+      clazz = get_const require_str, const_str
+      clazz.new
+    end
+
+    commands = [Settings::COMMANDS]
+    begin
+      commands << Util.get_const("#{settings_const_str}::COMMANDS")
+    rescue NameError
+      # ignored
+    end
+
+    @commands = commands.map do |dir|
+      require_str, const_str = dir.split '|'
+      _module = get_const require_str, const_str
+      clazz_symbols = _module.constants.select do |c|
+        const = _module.const_get(c)
+        const.is_a? Class and const < Command
+      end
+      clazz_symbols.map do |c|
+        clazz = _module.const_get c
+        [clazz.name.underscore.split('/').last.to_sym, clazz.new]
       end
+    end.flatten(1).to_h
+  end
+
+  def run_command(command, *args)
+    if @commands.nil?
+      setup
+    end
+
+    command_sym = command.to_sym
+    unless @commands.key? command_sym
+      puts "No command: #{command}"
+      exit!
+    end
+
+    @commands[command_sym].run! *args
+  end
+
+  def usage
+    if @commands.nil?
+      setup
     end
+    available_commands = @commands.map do |k, v|
+      '  %-14s%-30s' % [k.to_s, v.summary]
+    end.join("\n")
+
+    puts %(Seiya #{VERSION}
+
+Usage:
+  seiya <command> [options] [args]
+
+Available commands:
+#{available_commands}
+
+Use "seiya <command> -h" to see more info about a command
+)
   end
 
-  def get_task(task_name)
+  def get_task_class(task_name)
     task_name = 'Tasks::' << task_name unless task_name.include? '::'
-    clazz = Util::get_const task_name
-    clazz.new
+    begin
+      Util::get_const task_name
+    rescue NameError
+      p $:
+      puts "Cannot get task: #{task_name}"
+      exit!
+    end
   end
 
-  def gen_task_file(task_name)
-    base_path = "#{task_name}/#{task_name}"
+  def gen_project_file(project_name)
+    base_path = "#{project_name}/#{project_name}"
     FileUtils.mkpath "#{base_path}"
     FileUtils.mkpath "#{base_path}/tasks"
     FileUtils.mkpath "#{base_path}/items"
     FileUtils.mkpath "#{base_path}/pipelines"
-    File.write("#{task_name}/seiya.ini",
+    File.write("#{project_name}/seiya.ini",
 %([global]
-settings = #{task_name}/settings|Settings
+settings = #{project_name}/settings|Settings
 ))
     File.write("#{base_path}/settings.rb",
 %(module Settings
   PIPELINES = {
-      '#{task_name}/pipelines|Pipelines::Test' => 10,
+      '#{project_name}/pipelines|Pipelines::Test' => 10,
   }
 end))
     File.write("#{base_path}/items.rb", %(require 'items/test'))

data/sample/test/aa/commands/sing.rb

@@ -0,0 +1,8 @@
+require 'seiya'
+module Commands
+  class Sing <Seiya::Command
+    def run
+      puts 'I am a singer!'
+    end
+  end
+end

data/sample/test/aa/commands.rb

@@ -0,0 +1 @@
+require_relative './commands/sing'

data/sample/test/aa/settings.rb

@@ -3,4 +3,5 @@ module Settings
       'aa/pipelines/t|Pipelines::A' => 1,
       'aa/pipelines/t|Pipelines::B' => 2,
   }
+  COMMANDS = 'aa/commands|Commands'
 end

data/sample/test/aa/tasks/test.rb

@@ -3,7 +3,8 @@ require_relative '../items'
 
 module Tasks
   class Test < Seiya::Task
-    def initialize
+    def initialize *args
+      puts args
       @start_urls = ('a'..'d').map do |w|
         'http://www.baidu.com/?key=' << w
       end
@@ -13,7 +14,7 @@ module Tasks
       item = Items::Test.new
       item[:url] = response.url
       enum.yield item
-      request = Seiya::Request.new 'http://www.collipa.com'
+      request = Seiya::Request.new 'http://www.weibo.com'
       request.register &method(:other_parse)
       enum.yield request
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seiya
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - yetone
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-05 00:00:00.000000000 Z
+date: 2015-02-06 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: i@yetone.net
@@ -20,7 +20,11 @@ files:
 - README.md
 - bin/seiya
 - lib/seiya.rb
+- lib/seiya/command.rb
 - lib/seiya/contrib.rb
+- lib/seiya/contrib/commands.rb
+- lib/seiya/contrib/commands/crawl.rb
+- lib/seiya/contrib/commands/create.rb
 - lib/seiya/contrib/pipelines.rb
 - lib/seiya/item.rb
 - lib/seiya/pipeline.rb
@@ -28,9 +32,12 @@ files:
 - lib/seiya/response.rb
 - lib/seiya/scheduler.rb
 - lib/seiya/settings.rb
+- lib/seiya/support.rb
 - lib/seiya/task.rb
 - lib/seiya/util.rb
 - lib/seiya/version.rb
+- sample/test/aa/commands.rb
+- sample/test/aa/commands/sing.rb
 - sample/test/aa/items.rb
 - sample/test/aa/pipelines/t.rb
 - sample/test/aa/settings.rb