samao 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 561ab598e5442161a48b2878887296f8047aa269
-  data.tar.gz: 76d3daa2ca3c1407286690404809ee2b58115764
+  metadata.gz: a483278b39eb8b7c7f68d9ccd02b9d8fa7bb5cec
+  data.tar.gz: 307181057f12f81fca2a408a85dfa9de6fdedeaf
 SHA512:
-  metadata.gz: acab6baf781e25d75cb34d7214ac78cdb79936cca735fa554f152342fd285ed562e30c2ac5983a0c21cd6b3154dacd1830871637fd08be89dfbde109569591ca
-  data.tar.gz: 5a2f6e9d1a9ee336fe2e4234ec8d6336f00748cb5d39a06932987a557ec342c7f223471700502a64d3b02296920e91f3c2484078b23432b189ec2b09fb529abc
+  metadata.gz: e4608bbab36feb41b82f5779a7c2be74c45349b349b2243cbe30ccf4666c83a95e1407b7439a36c9f4ce0fb3eabc5bd0d2a9cca5b0c6529bde85355ae84afea0
+  data.tar.gz: f181681e70dd14f9607f9a8002e2656522b6f4cb483aa3aa18153f6c2d5bc74619436997e3e1eac5a7d2e3558dfb66fae355bd2c3e73b80a6f39cb40ef8b38c7

data/README.md

@@ -29,28 +29,32 @@ Or install it yourself as:
     samao = Samao::Detector.new
 
     # set base url and start page
-    samao.base_url 'https://github.com'
+    samao.baseurl 'https://github.com'
     samao.from '/Lax?tab=repositories'
     # the following line have the same effect
     #samao.from  'https://github.com/Lax?tab=repositories'
 
     # tell samao how to find the next page
-    samao.match :next, 'div.pagination a.next_page'
+    samao.find :next, 'div.pagination a.next_page'
+    samao.max_page 1
 
     # tell samao how to find items.
     # further more, set the data from matched HTML node/element.
-    samao.add_item 'div#user-repositories-list li a[itemprop="name codeRepository"]' do |item|
+    samao.find_item 'div#user-repositories-list li a[itemprop="name codeRepository"]' do |item|
       item.set_url :url, item.raw(:item)['href']
       item.set :title, item.raw(:item).text.strip
     end
 
+    samao.find_item 'div#user-repositories-list li' do |item|
+      item.find(:url, 'a[itemprop="name codeRepository"]') {|value| [:set_url, :url, value.first['href']] }
+      item.find(:title, 'a[itemprop="name codeRepository"]') {|value| [:set, value.first.text.strip] }
+    end
+
     # if it need to open content page for more information
     # default key is :url
     samao.add_detail :url do |detail|
     #samao.add_detail do |detail|
-      detail.match :author, 'h1.public .author a' do |item|
-        item.set :author, item.raw(:author).first.text.strip
-      end
+      detail.find(:author, 'h1.public .author a') {|value| value.first.text.strip }
     end
 
     # run the detector

data/lib/samao/catcher.rb

@@ -10,7 +10,8 @@ module Samao
       @code = 0
 
       @headers = {}
-      @headers["Referer"] = params[:base_url] if params[:base_url]
+      @headers["Referer"] = params[:baseurl].to_s if params[:baseurl]
+      @headers["User-Agent"] = "Samao/%s; Ruby/%s" % [Samao::VERSION, RUBY_VERSION]
 
       self
     end

data/lib/samao/detail.rb

@@ -7,8 +7,8 @@ module Samao
 
       @item = params[:item]
       @url = params[:url]
-      @base_url = params[:base_url]
-      @catcher = Catcher.new(url:@url, base_url: @base_url)
+      @baseurl = params[:baseurl]
+      @catcher = Catcher.new(url:@url, baseurl: @baseurl)
 
       yield self if block_given?
 
@@ -18,8 +18,7 @@ module Samao
     def run
       if @catcher and @catcher.run.success? and doc = @catcher.doc
         @selector.each do |name, sel|
-          @item.set_raw name, doc.css(sel)
-          @on[name].call @item if @on[name]
+          found(name, doc.css(sel), @item)
         end
       end
 

data/lib/samao/detector.rb

@@ -5,10 +5,16 @@ module Samao
     def initialize(params={})
       matchable
 
-      @current_url = @base_url = @from = nil
+      @current_url = @baseurl = @from = @max_page = nil
       @pages = []
       @items = []
 
+      @queue_of_items = Queue.new
+
+      @semaphore = Queue.new
+      @max_concurrent = params[:max_concurrent] || 5
+      @max_concurrent.times { @semaphore.push(1) }
+
       yield self if block_given?
 
       self
@@ -16,31 +22,55 @@ module Samao
 
     # return Detector self
     def run
-      while @from and @from.run.success? and @current_doc = @from.doc
+      threads = []
+      while @from
+        break unless @from.run.success?
+        @current_doc = @from.doc
+
         # find items in current_page
         if found = @current_doc.css(@selector[:item]) and found.size >= 1
-          @items += found.map do |raw_item|
-            item = Item.new(base_url: @current_url, raw_item:raw_item) do |item|
-              @on[:item].call(item) if @on[:item]
-            end.run
-
-            if @detail_key
-              detail = Detail.new(item: item, url: item.prop(@detail_key)) do |detail|
-                @on[:detail].call(detail) if @on[:detail]
-              end.run
-            end
-
-            item.prop
-          end
-        end
+          found.each do |raw_item|
+            threads << Thread.new do
+              @semaphore.pop
+              # puts "#{Time.now} #{@semaphore.size} available tokens. #{@semaphore.num_waiting} threads waiting."
+
+              begin
+                item = Item.new(baseurl: @current_url, raw_item:raw_item) do |item|
+                  @on[:item].call(item) if @on[:item]
+                end.run
+
+                if @detail_key
+                  detail = Detail.new(item: item, url: item.prop(@detail_key)) do |detail|
+                    @on[:detail].call(detail) if @on[:detail]
+                  end.run
+                end
+
+                @queue_of_items.push item.prop
+              rescue => e
+                p e
+              ensure
+                @semaphore.push(1)
+              end
+            end # end Thread
+          end # end found.each loop
+        end # end if found
 
         # find next page[s] in current page
-        if @selector[:next] and next_url = @current_doc.at_css(@selector[:next]) and next_url = URI.join(@current_url, next_url['href'])
+        if @max_page and @pages.size >= @max_page
+          stop
+        elsif @selector[:next] and next_url = @current_doc.at_css(@selector[:next]) and next_url = URI.join(@current_url, next_url['href'])
           @on[:next].call(next_url) if @on[:next]
           from next_url
         else
           stop
         end
+      end # end while @from
+
+      threads.each(&:join)
+
+      threads.size.times do
+        item = @queue_of_items.pop
+        @items << item
       end
 
       self
@@ -51,18 +81,18 @@ module Samao
       @on[:detail] = block if block
     end
 
-    def add_item(selector, &block)
-      match(:item, selector, &block)
+    def find_item(selector, &block)
+      find(:item, selector, &block)
     end
 
     # set front page
     def from(url)
-      if prev_url = @current_url || @base_url
+      if prev_url = @current_url || @baseurl
         url = URI.join(prev_url, url)
       end
       url = URI(url) if ! url.is_a? URI
 
-      @from = Catcher.new(url:url, base_url:@current_url)
+      @from = Catcher.new(url:url, baseurl:@current_url)
       @pages << url
       @current_url = url
 
@@ -70,8 +100,22 @@ module Samao
     end
 
     # set base url
-    def base_url(url)
-      @base_url = url
+    def baseurl(url)
+      @baseurl = url
+
+      self
+    end
+
+    # set max page
+    def max_page(max)
+      @max_page = max
+
+      self
+    end
+
+    # set max page
+    def max_page(max)
+      @max_page = max
 
       self
     end

data/lib/samao/item.rb

@@ -8,7 +8,8 @@ module Samao
       @prop = {}   # usefull properties
       @raw = {}    # nodes go here.
 
-      @base_url = params[:base_url]
+      @baseurl = params[:baseurl].to_s
+
       set_raw :item, params[:raw_item] if params[:raw_item]
 
       yield self if block_given?
@@ -18,8 +19,7 @@ module Samao
 
     def extract
       @selector.each do |name, sel|
-        set_raw name, @raw[:item].css(sel)
-        @on[name].call self if @on[name]
+        found(name, @raw[:item].css(sel))
       end
 
       self
@@ -31,10 +31,14 @@ module Samao
     end
 
     def set_url(name, value)
-      value = URI.join @base_url, value if @base_url
+      value = URI.join @baseurl, value if @baseurl
       set(name, value.to_s)
     end
 
+    def set_baseurl(baseurl=nil)
+      @prop[:baseurl] = baseurl || @baseurl
+    end
+
     def prop(name=nil)
       if name
         return @prop[name]

data/lib/samao/matchable.rb

@@ -5,17 +5,38 @@ module Samao
     end
 
     def matchable
+      @cmd_sets = [:set, :set_url] ## target class should inplements methods as: set(name, value)
       @selector = {}
       @on = {}
     end
 
-    def match(name, selector, &block)
+    def find(name, selector, &block)
       @selector[name] = selector
 
       @on[name] = block if block
 
       self
     end
+    alias match find
+
+    def found(name, value, target=self)
+      cmd = :set
+
+      if @on[name]
+        value = @on[name].call value
+        if value.is_a? Array and @cmd_sets.include?(value[0].to_sym)
+          case value.length
+          when 2
+            cmd, value = value
+          when 3
+            cmd, name, value = value
+          end
+        end
+        target.send cmd, name, value
+      end
+
+      target.send cmd, name, value
+    end
 
   end
 end

data/lib/samao/version.rb

@@ -1,3 +1,3 @@
 module Samao
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: samao
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Liu Lantao
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-18 00:00:00.000000000 Z
+date: 2016-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri