maiha-dm-ys 0.2.2 → 0.3

data/README

@@ -24,4 +24,29 @@ Example
   => ["Gem", "Maintainer", "Name"]
 
 
+Pagination
+==========
+
+Special syntax "uri*" means pagination mode that enables recursive retrieving.
+
+  class Plugin
+    include DataMapper::YunkerStar
+    uri "http://merbi.st/plugins/index?page=1"
+  end
+
+  irb(main):001:0> Plugin.count
+  => 20
+
+This parses only specified uri.
+Append "*" to uri if you want pagination mode.
+  
+  class Plugin
+    include DataMapper::YunkerStar
+    uri "http://merbi.st/plugins/index?page=1*"
+  end
+
+  irb(main):001:0> Plugin.count
+  => 36
+
+
 Copyright (c) 2008 maiha@wota.jp, released under the MIT license

data/Rakefile

@@ -33,7 +33,7 @@ AUTHOR = "maiha"
 EMAIL = "maiha@wota.jp"
 HOMEPAGE = "http://github.com/maiha/dm-ys"
 SUMMARY = "a DataMapper extension that uses html table as its schema and data powerfully like YunkerStar"
-GEM_VERSION = "0.2.2"
+GEM_VERSION = "0.3"
 
 spec = Gem::Specification.new do |s|
 #  s.rubyforge_project = 'merb'

data/lib/dm-ys/base.rb

@@ -26,5 +26,14 @@ module DataMapper
     def self.descendants
       @descendants ||= Set.new
     end
+
+    # @api public
+    def self.[](uri)
+      klass = Class.new do
+        include DataMapper::YunkerStar
+      end
+      klass.uri uri
+      return klass
+    end
   end
 end

data/lib/dm-ys/proxy.rb

@@ -24,16 +24,7 @@ module DataMapper
 
       module ClassMethods
         def proxy
-          @proxy ||= lazy_load
-        end
-
-        def lazy_load
-          loader = Scraper.new(self)
-          loader.names.each do |name|
-            type = String         # TODO
-            property name.intern, type
-          end
-          return loader
+          @proxy ||= Scraper.load(self)
         end
 
         def names
@@ -54,9 +45,9 @@ module DataMapper
 
         def all
           count = 0
-          @all ||= proxy.entries.map{|array|
+          @all ||= entries.map{|array|
             count += 1
-            new(Hash[*proxy.names.zip(array).flatten].merge(:id=>count))
+            new(Hash[*names.zip(array).flatten].merge(:id=>count))
           }
         end
 

data/lib/dm-ys/scraper.rb

@@ -8,96 +8,219 @@ module DataMapper
     require 'open-uri'
     require 'hpricot'
 
-    class Scraper
-      include CachedAccessor
-
+    module Scraper
       class TableNotFound < RuntimeError; end
 
-      attr_reader :html
+      def self.paginate?(model)
+        model.uri.to_s[-1] == ?*
+      end
 
-      def initialize(model)
-        raise ArgumentError, "missing model" unless model
-        raise ArgumentError, "missing uri"   unless model.uri
-        @model = model
-        @html  = NKF.nkf('-w', open(model.uri).read)
-        @invalid_name_count = 0
+      def self.lookup(model)
+        scraper = paginate?(model) ? Composite : Page
+        scraper.new(model)
       end
 
-      def guess_table
-        [max_table_from("table"), max_table_from("table > tbody")].sort_by(&:first).last.last or
-          raise TableNotFound, "set 'table' or 'tbody' manually"
+      def self.load(model)
+        loader = lookup(model)
+        loader.register_properties!
+        return loader
       end
 
-      cached_accessor do
-        doc     {Hpricot(@html)}
-        table   {specified(:table) or guess_table}
-        thead   {specified(:thead) or table.search("> thead").first or table}
-        tbody   {specified(:tbody) or table.search("> tbody").first or table}
-        names   {labels.map{|i| label2name(i)}}
-        labels  {thead.search("> tr").first.search("> td|th").map{|i|strip_tags(i.inner_html)}}
-        entries {tbody.search("> tr").map{|tr| tr.search("> td").map{|i|strip_tags(i.inner_html)}}.delete_if{|i|i.blank?}}
+      ######################################################################
+      ### Utils
+
+      module Utils
+        def constantize(label)
+          require 'cgi'
+          label = CGI.unescapeHTML(label.to_s)
+          label.gsub!(/&[a-z]+;/, '')
+          label.gsub!(/\r?\n/, '')
+          label.gsub!(/\s+/,'')
+          label.delete!('!"#$%&()=~|`{}^-[]/<>:;,.\\-')
+          label.delete!("'")
+          return label
+        end
+
+        module_function :constantize
       end
 
-      private
+      ######################################################################
+      ### Base Scraper
 
-        def max_table_from(entry)
-          table = nil
-          count = -1
-          doc.search(entry).each do |t|
-            size = t.search("> tr").size
-            if size > count
-              count = size
-              table = t
-            end
-          end
-          [count, table]
+      class Base
+        include CachedAccessor
+
+        def initialize(model, *args)
+          raise ArgumentError, "missing model" unless model
+          raise ArgumentError, "missing uri"   unless model.uri
+          @model = model
+        end
+        [:names, :labels, :entries].each do |method|
+          define_method(method) {raise NotImplementedError, method.to_s}
         end
 
-        def specified(name)
-          @model.respond_to?(name)    or raise ArgumentError, "invalid selector name: #{name}"
-          css = @model.__send__(name) or return nil
-
-          element = doc.search(css)
-          case element
-          when Hpricot::Elem
-            return element
-          when Hpricot::Elements
-            return element.first
-          else
-            return nil
+        def uri
+          @uri || @model.uri.to_s.chomp('*')
+        end
+
+        def register_properties!
+          names.each do |name|
+            type = String         # TODO
+            @model.property name.intern, type
           end
         end
+      end
 
-        def label2name(label)
-          require 'cgi'
-          label = CGI.unescapeHTML(label)
-          label.gsub!(/&nbsp;/, '')
-          label.gsub!(/\r?\n/, '')
-          label.delete!('!"#$%&()=~|`{}^-[]/<>:; \\')
-          label.delete!("'")
-          label.strip!
+      ######################################################################
+      ### Page Scraper
+
+      class Page < Base
+        attr_reader :html
+
+        def initialize(model, uri = nil)
+          super
+          @uri  = uri
+          @html = NKF.nkf('-w', open(self.uri).read)
+          @invalid_name_count = 0
+        end
+
+        def guess_table
+          max_table or
+            raise TableNotFound, "set 'table' or 'tbody' manually"
+        end
+
+        def pagination_links
+          base = URI.parse(uri.split('?').first)
+          urls = (doc / "a").map{|i| i[:href] =~ /^http/ ? i[:href] : (base+i[:href]).to_s}.uniq
+          urls.select{|url| /^#{Regexp.escape(base.to_s)}/ === url}
+        end
+
+        def inspect
+          attrs = [
+            [ :html,      "#{html.size}bytes" ],
+            [ :names,     names ],
+            [ :entries,   entries.size ],
+          ]
+          "#<#{self.class.name} #{attrs.map { |(k,v)| "@#{k}=#{v.inspect}" } * ' '}>"
+        end
+
+        cached_accessor do
+          doc     {Hpricot(@html)}
+          table   {specified(:table) or guess_table}
+          thead   {specified(:thead) or table.search("> thead").first or table}
+          tbody   {specified(:tbody) or table.search("> tbody").first or table}
+          names   {labels.map{|i| label2name(i)}}
+          labels  {thead.search("> tr").first.search("> td|th").map{|i|strip_tags(i.inner_html)}}
+          entries {tbody.search("> tr").map{|tr| tr.search("> td").map{|i|strip_tags(i.inner_html)}}.delete_if{|i|i.blank?}}
+          count   {entries.size}
+        end
 
-          if /^([A-Z])/ === label and Object.const_defined?(label)
-            label = "_#{label}"
+        private
+
+          def max_table
+            table = nil
+            count = -1
+            doc.search("table").each do |t|
+              size = [t.search("> tr").size, t.search("> tbody > tr").size].max
+              if size > count
+                count = size
+                table = t
+              end
+            end
+            return table
+          end
+
+          def specified(name)
+            @model.respond_to?(name)    or raise ArgumentError, "invalid selector name: #{name}"
+            css = @model.__send__(name) or return nil
+
+            element = doc.search(css)
+            case element
+            when Hpricot::Elem
+              return element
+            when Hpricot::Elements
+              return element.first
+            else
+              return nil
+            end
           end
-          if label.blank? or @model.respond_to?(label, true)
-            new_name_for(label)
-          elsif /^[0-9]/ === label
-            "_#{label}"
-          else
-            label
+
+          def label2name(label)
+            label = Utils.constantize(label)
+
+            if /^([A-Z])/ === label and Object.const_defined?(label)
+              label = "_#{label}"
+            end
+            if label.blank? or @model.respond_to?(label, true)
+              new_name_for(label)
+            elsif /^[0-9]/ === label
+              "_#{label}"
+            else
+              label
+            end
           end
+
+          def new_name_for(label)
+            @invalid_name_count += 1
+            "col_#{@invalid_name_count}"
+          end
+
+          def strip_tags(html)
+            html.gsub(/<.*?>/, '').strip
+          end
+      end
+
+      ######################################################################
+      ### Composite Scraper
+
+      class Composite < Base
+        def pages
+          @pages ||= execute
         end
 
-        def new_name_for(label)
-          @invalid_name_count += 1
-          "col_#{@invalid_name_count}"
+        def count
+          pages.map(&:count).inject(0){|i,v| i+v}
         end
 
-        def strip_tags(html)
-          html.gsub(/<.*?>/, '').strip
+        def names
+          pages.first.names
         end
 
+        def labels
+          pages.first.labels
+        end
+
+        def entries
+          pages.inject([]){|a,p| a+p.entries}
+        end
+
+        private
+          def execute
+            visit(uri)
+            valid_pages
+          end
+
+          def valid_pages
+            loaded_pages.values.compact
+          end
+
+          def loaded_pages
+            @loaded_pages ||= {} # url => page object
+          end
+
+          def visit(uri)
+            return if loaded_pages[uri]
+            page = Page.new(@model, uri)
+            base = valid_pages.first
+            if !base or base.names == page.names
+              loaded_pages[uri] = page
+            else
+              loaded_pages[uri] = nil
+            end
+            page.pagination_links.each{|uri| visit(uri)}
+          end
+      end
+
     end
   end
 end

data/spec/anonymous_spec.rb

@@ -0,0 +1,26 @@
+require File.join( File.dirname(__FILE__), "spec_helper" )
+
+describe DataMapper::YunkerStar do
+  it "should provide []" do
+    DataMapper::YunkerStar.should respond_to(:[])
+  end
+
+  describe "[]" do
+    before(:each) do
+      @uri = "http://merbi.st/plugins/index?page=1"
+      @ys  = DataMapper::YunkerStar[@uri]
+    end
+
+    it "should return a new class" do
+      @ys.should be_kind_of(Class)
+    end
+
+    it "should include DataMapper::YunkerStar" do
+      @ys.ancestors.should be_include(DataMapper::YunkerStar)
+    end
+
+    it "should set uri" do
+      @ys.uri.should == @uri
+    end
+  end
+end

data/spec/composite_scraper_spec.rb

@@ -0,0 +1,69 @@
+require File.join( File.dirname(__FILE__), "spec_helper" )
+
+describe DataMapper::YunkerStar::Scraper::Composite do
+  before(:each) do
+    @scraper = DataMapper::YunkerStar::Scraper::Composite.new(Plugin)
+  end
+
+  it "should provide #uri" do
+    @scraper.should respond_to(:uri)
+  end
+
+  describe "#uri" do
+    it "should strip last '*'" do
+      @scraper.uri.should == "http://merbi.st/plugins/index?page=1"
+    end
+  end
+
+  it "should provide #pages" do
+    @scraper.should respond_to(:pages)
+  end
+
+  describe "#pages" do
+    it "should return page objects" do
+      @scraper.pages.should be_kind_of(Array)
+      @scraper.pages.map(&:uri).sort.should == 
+        ["http://merbi.st/plugins/index?page=1","http://merbi.st/plugins/index?page=2"]
+    end
+  end
+
+  it "should provide #count" do
+    @scraper.should respond_to(:count)
+  end
+
+  describe "#count" do
+    it "should return sum of page counts" do
+      @scraper.count.should == 36
+    end
+  end
+
+  it "should provide #names" do
+    @scraper.should respond_to(:names)
+  end
+
+  describe "#names" do
+    it "should return same value as Plugin" do
+      @scraper.names.should == DataMapper::YunkerStar::Scraper::Page.new(Plugin1).names
+    end
+  end
+
+  it "should provide #labels" do
+    @scraper.should respond_to(:labels)
+  end
+
+  describe "#labels" do
+    it "should return same value as Plugin" do
+      @scraper.labels.should == DataMapper::YunkerStar::Scraper::Page.new(Plugin1).labels
+    end
+  end
+
+  it "should provide #entries" do
+    @scraper.should respond_to(:entries)
+  end
+
+  describe "#entries" do
+    it "should return same value as Plugin" do
+      @scraper.entries.should == (Plugin1.entries + Plugin2.entries)
+    end
+  end
+end