seiya 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f514e3349566ec80c0e81e09852949553b1f2ffc
+  data.tar.gz: 6eb2642bd8c0b5c1687a9d8fd24fdad9e019c17b
+SHA512:
+  metadata.gz: 10b5496b4f8c5b725ba5a143dedec113984edf3903ee90997398269bc225d4e20e717c82ddd5ba7944264ed7119688a9d477550e4c8dd7450003eb1c7fa846f5
+  data.tar.gz: ebaf58dbff82691e387d6bfea6294f04a4eafe28c219b962a70b97d30ccfcd8dd43a94f7875b5ace3b43ba4b64ebd3984e8e15d2057ca9b0d9ac90129f971148

data/README.md

@@ -0,0 +1 @@
+A ruby spider like scrapy-python.

data/bin/seiya

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.expand_path '.'
+$:.unshift File.expand_path '../../lib', __FILE__
+require 'optparse'
+
+options = {}
+
+OptionParser.new do |opts|
+  opts.banner = 'Usage: seiya [options]'
+
+  opts.on '-v', '--version', 'seiya version' do |v|
+    options[:version] = v
+  end
+
+  opts.on '-tTask', '--task=Task', 'seiya task' do |t|
+    options[:task] = t
+  end
+
+  opts.on '-aArg', '--argument=Arg', 'send argument to seiya task' do |a|
+    options[:args] = [] unless options[:args]
+    options[:args] << a
+  end
+end.parse!
+
+def extend_load_path(path)
+  Dir.foreach path do |f|
+    unless %w(. ..).include? f
+      if File.directory? File.join path, f
+        new_path = File.join path, f
+        extend_load_path new_path
+      elsif f == 'tasks.rb'
+        $:.unshift path
+        break
+      end
+    end
+  end
+end
+pwd = Dir.pwd
+extend_load_path pwd
+
+begin
+  require 'tasks'
+rescue
+  raise 'No spiders!'
+end
+
+Seiya.setup
+
+if options.key? :task
+  task_name = options[:task]
+  task = Seiya.get_task task_name
+  task.run
+end
+
+if options.key? :version
+  p Seiya::VERSION
+end

data/lib/seiya/contrib/pipelines.rb

@@ -0,0 +1,11 @@
+require 'seiya/pipeline'
+
+module Contrib
+  module Pipelines
+    class BasePipeline < Seiya::Pipeline
+      def process_item(item)
+        item
+      end
+    end
+  end
+end

data/lib/seiya/contrib.rb

@@ -0,0 +1 @@
+require 'seiya/contrib/pipelines'

data/lib/seiya/item.rb

@@ -0,0 +1,20 @@
+module Seiya
+  class Item
+    def initialize(h = {})
+      @data = {}
+      @data.merge!(h)
+    end
+
+    def [](key)
+      @data[key]
+    end
+
+    def []=(key, value)
+      @data[key] = value
+    end
+
+    def inspect
+      @data.inspect
+    end
+  end
+end

data/lib/seiya/pipeline.rb

@@ -0,0 +1,7 @@
+module Seiya
+  class Pipeline
+    def process_item(item)
+      item
+    end
+  end
+end

data/lib/seiya/request.rb

@@ -0,0 +1,35 @@
+require 'httpclient'
+require 'nokogiri'
+require 'singleton'
+require 'seiya/response'
+
+module Seiya
+  class Request
+    def initialize(url, *args, method: 'get')
+      @url = url
+      @method = method.upcase
+      @args = args
+      @httpclient = HTTPClient.new
+    end
+
+    def get_response
+      Response.new @httpclient.send(@method.downcase, @url, *@args)
+    end
+
+    def register(&block)
+      @handler = proc do
+        Enumerator.new do |enum|
+          block.call(get_response, enum)
+        end
+      end
+    end
+
+    def registered?
+      !@handler.nil?
+    end
+
+    def fire
+      @handler.call
+    end
+  end
+end

data/lib/seiya/response.rb

@@ -0,0 +1,45 @@
+module Seiya
+  class Response
+    def initialize(resp)
+      @resp = resp
+    end
+
+    def url
+      @resp.http_header.request_uri.to_s
+    end
+
+    def body
+      @resp.body
+    end
+
+    def doc
+      begin
+        return @doc unless @doc.nil?
+        @doc = Nokogiri::HTML body
+        @has_doc = true
+      rescue
+        @has_doc = false
+      end
+    end
+
+    def json
+      begin
+        return @json unless @json.nil?
+        @json = JSON.parse body
+        @has_json = true
+      rescue
+        @has_json = false
+      end
+    end
+
+    def has_doc?
+      doc
+      !!@has_doc
+    end
+
+    def has_json?
+      json
+      !!@has_json
+    end
+  end
+end

data/lib/seiya/scheduler.rb

@@ -0,0 +1,83 @@
+require 'singleton'
+
+module Seiya
+  class Scheduler
+    include Singleton
+
+    def initialize
+      @request_q = Queue.new
+      @run = false
+    end
+
+    def add_requests(requests)
+      requests.each do |request|
+        @request_q << request if request.registered?
+      end
+      run unless @run
+    end
+
+  private
+
+    def run
+      @run = true
+      @thread = Thread.new do
+        requests = []
+        until @request_q.empty?
+          requests << @request_q.pop
+        end
+        multi_run requests
+      end
+      @thread.join
+      if @request_q.empty?
+        stop
+      else
+        run
+      end
+    end
+
+    def num_processors
+      return @num_processors unless @num_processors.nil?
+      @num_processors = Util.num_processors
+    end
+
+    def multi_run(requests)
+      count = requests.count / num_processors + 1
+      threads = []
+      requests.each_slice(count) do |slice|
+        threads << Thread.new do
+          process_requests slice
+        end
+      end
+      threads.each do |t|
+        t.join
+      end
+    end
+
+    def process_requests(requests)
+      requests.each do |request|
+        gen = request.fire
+        process_gen gen
+      end
+    end
+
+    def process_gen(gen)
+      gen.each do |e|
+        if e.is_a? Array
+          e.each do |_e|
+            process_gen _e
+          end
+        elsif e.is_a? Seiya::Item
+          Seiya.process_item e
+        elsif e.is_a? Request
+          add_requests [e]
+        elsif e.is_a? Enumerator
+          process_gen e
+        end
+      end
+    end
+
+    def stop
+      @run = false
+    end
+  end
+end

data/lib/seiya/settings.rb

@@ -0,0 +1,7 @@
+module Seiya
+  module Settings
+    PIPELINES = {
+        'seiya/contrib|Contrib::Pipelines::BasePipeline' => 2
+    }
+  end
+end

data/lib/seiya/task.rb

@@ -0,0 +1,35 @@
+require 'seiya/request'
+require 'seiya/item'
+require 'seiya/pipeline'
+require 'seiya/util'
+require 'seiya/scheduler'
+
+def process_gen(gen)
+  gen.each do |e|
+    if e.is_a? Seiya::Item
+      Seiya.process_item e
+    elsif e.is_a? Enumerator
+      process_gen e
+    end
+  end
+end
+
+
+module Seiya
+  class Task
+    def initialize
+      @start_urls = []
+    end
+
+    def run
+      return unless @start_urls.is_a? Array
+      requests = @start_urls.map do |url|
+        request = Request.new url
+        handler = method :parse
+        request.register &handler
+        request
+      end
+      Scheduler.instance.add_requests requests
+    end
+  end
+end

data/lib/seiya/util.rb

@@ -0,0 +1,41 @@
+module Seiya
+  module Util
+    extend self
+
+    def get_const(str)
+      str.split('::').inject(Object) do |o, c|
+        o.const_get c
+      end
+    end
+
+    def processors_in_use
+      procs=[]
+      Dir.glob('/proc/*/stat') do |filename|
+        next if File.directory?(filename)
+        this_proc=[]
+        File.open(filename) { |file| this_proc = file.gets.split.values_at(2, 38) }
+        procs << this_proc[1].to_i if this_proc[0] == 'R'
+      end
+      procs.uniq.length
+    end
+
+    def num_processors
+      IO.readlines('/proc/cpuinfo').delete_if { |x| x.index('processor') == nil }.length
+    end
+
+    def num_free_processors
+      num_processors - processors_in_use
+    end
+
+    def estimate_free_cpus(count, wait_time)
+      results = []
+      count.times {
+        results << num_free_processors
+        sleep(wait_time)
+      }
+      sum = 0
+      results.each { |x| sum += x }
+      (sum.to_f / results.length).round
+    end
+  end
+end

data/lib/seiya/version.rb

@@ -0,0 +1,3 @@
+module Seiya
+  VERSION = '0.0.1'
+end

data/lib/seiya.rb

@@ -0,0 +1,47 @@
+require 'seiya/version'
+require 'seiya/request'
+require 'seiya/task'
+require 'seiya/item'
+require 'seiya/pipeline'
+require 'seiya/settings'
+
+module Seiya
+  extend self
+
+  def process_item(item)
+    @pipelines.each do |p|
+      item = p.process_item item
+    end
+  end
+
+  def setup(conf_file: 'seiya.ini')
+    require 'inifile'
+    require 'seiya/util'
+    conf = IniFile.load conf_file
+    settings_file = conf.to_h.fetch('global', {}).fetch('settings', 'settings')
+    require settings_file
+
+    begin
+      pipelines = ::Settings::PIPELINES
+    rescue NameError
+      pipelines = {}
+    end
+
+    pipelines.merge! Settings::PIPELINES
+
+    pipelines = pipelines.sort_by { |_, v| v }.to_h
+
+    @pipelines = pipelines.keys.map do |k|
+      require_str, class_name = k.split('|')
+      require require_str
+      clazz = Util.get_const class_name
+      clazz.new
+    end
+  end
+
+  def get_task(task_name)
+    task_name = 'Tasks::' << task_name unless task_name.include? '::'
+    clazz = Util::get_const task_name
+    clazz.new
+  end
+end

data/sample/test/aa/items.rb

@@ -0,0 +1,9 @@
+require 'seiya'
+
+module Items
+  class Test < Seiya::Item
+    def to_s
+      'fuck'
+    end
+  end
+end

data/sample/test/aa/pipelines/t.rb

@@ -0,0 +1,18 @@
+require 'seiya'
+
+module Pipelines
+  class A < Seiya::Pipeline
+    def process_item(item)
+      p 'I am A Pipeline'
+      item[:pipeline] = 'A'
+      item
+    end
+  end
+  class B < Seiya::Pipeline
+    def process_item(item)
+      p 'I am B Pipeline'
+      p item
+      item
+    end
+  end
+end

data/sample/test/aa/settings.rb

@@ -0,0 +1,6 @@
+module Settings
+  PIPELINES = {
+      'aa/pipelines/t|Pipelines::A' => 1,
+      'aa/pipelines/t|Pipelines::B' => 2,
+  }
+end

data/sample/test/aa/tasks/test.rb

@@ -0,0 +1,26 @@
+require 'seiya'
+require_relative '../items'
+
+module Tasks
+  class Test < Seiya::Task
+    def initialize
+      @start_urls = ('a'..'d').map do |w|
+        'http://www.baidu.com/?key=' << w
+      end
+    end
+    
+    def parse(response, enum)
+      item = Items::Test.new
+      item[:url] = response.url
+      enum.yield item
+      request = Seiya::Request.new 'http://www.collipa.com'
+      request.register &method(:other_parse)
+      enum.yield request
+    end
+
+    def other_parse(response, enum)
+      item = Items::Test.new name: 'yetone', url: response.url
+      enum.yield item
+    end
+  end
+end

data/sample/test/aa/tasks.rb

@@ -0,0 +1 @@
+require 'tasks/test'

data/sample/test/seiya.ini

@@ -0,0 +1,2 @@
+[global]
+settings = aa/settings

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: seiya
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- yetone
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-05 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: i@yetone.net
+executables:
+- seiya
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/seiya
+- lib/seiya.rb
+- lib/seiya/contrib.rb
+- lib/seiya/contrib/pipelines.rb
+- lib/seiya/item.rb
+- lib/seiya/pipeline.rb
+- lib/seiya/request.rb
+- lib/seiya/response.rb
+- lib/seiya/scheduler.rb
+- lib/seiya/settings.rb
+- lib/seiya/task.rb
+- lib/seiya/util.rb
+- lib/seiya/version.rb
+- sample/test/aa/items.rb
+- sample/test/aa/pipelines/t.rb
+- sample/test/aa/settings.rb
+- sample/test/aa/tasks.rb
+- sample/test/aa/tasks/test.rb
+- sample/test/seiya.ini
+homepage: https://github.com/yetone/seiya
+licenses:
+- WTFPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A ruby spider like scrapy-python
+test_files: []