maiha-dm-ys 0.3 → 0.3.1

data/Rakefile

@@ -33,7 +33,7 @@ AUTHOR = "maiha"
 EMAIL = "maiha@wota.jp"
 HOMEPAGE = "http://github.com/maiha/dm-ys"
 SUMMARY = "a DataMapper extension that uses html table as its schema and data powerfully like YunkerStar"
-GEM_VERSION = "0.3"
+GEM_VERSION = "0.3.1"
 
 spec = Gem::Specification.new do |s|
 #  s.rubyforge_project = 'merb'

data/lib/dm-ys.rb

@@ -4,6 +4,7 @@ require 'dm-core'
 require 'dsl_accessor'
 
 __DIR__ = File.dirname(__FILE__)
+require __DIR__ + '/dm-ys/config'
 require __DIR__ + '/dm-ys/base'
 require __DIR__ + '/dm-ys/cached_accessor'
 require __DIR__ + '/dm-ys/memory_repository'

data/lib/dm-ys/config.rb

@@ -0,0 +1,30 @@
+module DataMapper
+  module YunkerStar
+    class Config
+      def self.default
+        {:max_pages=>100, :uniq=>true}
+      end
+ 
+      def initialize(options = nil)
+        @options = options
+        @options = self.class.default unless @options.is_a?(Hash)
+      end
+
+      def [](key)
+        @options[key]
+      end
+
+      def []=(key, val)
+        @options[key] = val
+      end
+
+      def uniq_page?
+        !!self[:uniq]
+      end
+
+      def uniq_entry?
+        self[:uniq] == true or self[:uniq] == :entry
+      end
+    end
+  end
+end

data/lib/dm-ys/proxy.rb

@@ -18,10 +18,13 @@ module DataMapper
           dsl_accessor :table
           dsl_accessor :tbody
           dsl_accessor :thead
+          dsl_accessor :ys, :default=>proc{|*a| DataMapper::YunkerStar::Config.new}
           property :id, DataMapper::Types::Serial
         end
       end
 
+      class MaxPagesOverflow < RuntimeError; end
+
       module ClassMethods
         def proxy
           @proxy ||= Scraper.load(self)

data/lib/dm-ys/scraper.rb

@@ -7,6 +7,7 @@ module DataMapper
     require 'nkf'
     require 'open-uri'
     require 'hpricot'
+    require 'digest/sha1'
 
     module Scraper
       class TableNotFound < RuntimeError; end
@@ -59,6 +60,10 @@ module DataMapper
           define_method(method) {raise NotImplementedError, method.to_s}
         end
 
+        def count
+          entries.size
+        end
+
         def uri
           @uri || @model.uri.to_s.chomp('*')
         end
@@ -99,11 +104,16 @@ module DataMapper
           attrs = [
             [ :html,      "#{html.size}bytes" ],
             [ :names,     names ],
-            [ :entries,   entries.size ],
+            [ :entries,   count ],
           ]
           "#<#{self.class.name} #{attrs.map { |(k,v)| "@#{k}=#{v.inspect}" } * ' '}>"
         end
 
+        def page_hash
+          body = entries.flatten.join("\t")
+          Digest::SHA1.hexdigest(body)
+        end
+
         cached_accessor do
           doc     {Hpricot(@html)}
           table   {specified(:table) or guess_table}
@@ -112,7 +122,6 @@ module DataMapper
           names   {labels.map{|i| label2name(i)}}
           labels  {thead.search("> tr").first.search("> td|th").map{|i|strip_tags(i.inner_html)}}
           entries {tbody.search("> tr").map{|tr| tr.search("> td").map{|i|strip_tags(i.inner_html)}}.delete_if{|i|i.blank?}}
-          count   {entries.size}
         end
 
         private
@@ -178,10 +187,6 @@ module DataMapper
           @pages ||= execute
         end
 
-        def count
-          pages.map(&:count).inject(0){|i,v| i+v}
-        end
-
         def names
           pages.first.names
         end
@@ -191,33 +196,66 @@ module DataMapper
         end
 
         def entries
-          pages.inject([]){|a,p| a+p.entries}
+          records = []
+          digests = Set.new
+          pages.each do |page|
+            page.entries.each do |entry|
+              if config.uniq_entry?
+                sha1 = Digest::SHA1.hexdigest(entry.join("\t"))
+                next if digests.include?(sha1)
+                digests << sha1
+              end
+              records << entry
+            end
+          end
+          return records
         end
 
         private
           def execute
             visit(uri)
-            valid_pages
+            uniq_pages
+          end
+
+          def config
+            @model.ys
           end
 
-          def valid_pages
-            loaded_pages.values.compact
+          def uniq_pages
+            return loaded_pages unless config.uniq_page?
+
+            digests = Set.new
+            loaded_pages.select do |page|
+              sha1 = page.page_hash
+              if digests.include?(sha1)
+                false
+              else
+                digests << sha1
+                true
+              end
+            end
           end
 
           def loaded_pages
-            @loaded_pages ||= {} # url => page object
+            loaded_pages_hash.values.compact
+          end
+
+          def loaded_pages_hash
+            @loaded_pages_hash ||= {} # url => page object
           end
 
-          def visit(uri)
-            return if loaded_pages[uri]
+          def visit(uri, runtime_options = {:count => 0})
+            return if loaded_pages_hash[uri]
+            raise Proxy::MaxPagesOverflow if (runtime_options[:count]+=1) > @model.ys[:max_pages]
+            
             page = Page.new(@model, uri)
-            base = valid_pages.first
+            base = loaded_pages.first
             if !base or base.names == page.names
-              loaded_pages[uri] = page
+              loaded_pages_hash[uri] = page
             else
-              loaded_pages[uri] = nil
+              loaded_pages_hash[uri] = nil
             end
-            page.pagination_links.each{|uri| visit(uri)}
+            page.pagination_links.each{|uri| visit(uri, runtime_options)}
           end
       end
 

data/spec/composite_scraper_spec.rb

@@ -66,4 +66,56 @@ describe DataMapper::YunkerStar::Scraper::Composite do
       @scraper.entries.should == (Plugin1.entries + Plugin2.entries)
     end
   end
+
+  describe "SortedPlugin" do
+    it "should return 6 pages" do
+      SortedPlugin.proxy.pages.size.should == 6
+    end
+
+    describe "#entries" do
+      it "should return duplicate entries" do
+        SortedPlugin.entries.sort.should == ((SortedPlugin1.entries + SortedPlugin2.entries)*3).sort
+      end
+    end
+
+    describe "#count" do
+      it "should return duplicate entries" do
+        SortedPlugin.count.should == (SortedPlugin1.count + SortedPlugin2.count)*3
+      end
+    end
+  end
+
+  describe "SortedPlugin with uniq page option" do
+    it "should return 2 pages" do
+      SortedPluginWithUniqPage.proxy.pages.size.should == 2
+    end
+
+    describe "#entries" do
+      it "should return same value as Plugin" do
+        SortedPluginWithUniqPage.entries.should == (SortedPlugin1.entries + SortedPlugin2.entries)
+      end
+    end
+
+    describe "#count" do
+      it "should return same value as Plugin" do
+        SortedPluginWithUniqPage.count.should == (SortedPlugin1.count + SortedPlugin2.count)
+      end
+    end
+  end
+
+  describe "UniqPlugin" do
+    it "should return 2 pages" do
+      UniqPlugin.proxy.pages.size.should == 2
+    end
+
+    describe "#count" do
+      it "should return same value as Plugin" do
+        UniqPlugin1.count.should == 20
+        UniqPlugin2.count.should == 4
+        UniqPlugin .count.should == 22
+      end
+    end
+  end
+
+
 end

data/spec/config_spec.rb

@@ -0,0 +1,68 @@
+require File.join( File.dirname(__FILE__), "spec_helper" )
+
+describe DataMapper::YunkerStar::Config do
+  before(:each) do
+    @config = DataMapper::YunkerStar::Config.new
+  end
+
+  it "should provide []" do
+    @config.should respond_to(:[])
+  end
+
+  describe "[:uniq]" do
+    it "should has true as default value" do
+      @config[:uniq].should == true
+    end
+  end
+
+  it "should provide #uniq_page?" do
+    @config.should respond_to(:uniq_page?)
+  end
+
+  describe "#uniq_page?" do
+    it "should return true as default value" do
+      @config.uniq_page?.should == true
+    end
+
+    it "should return true when :uniq is set to :page" do
+      config = DataMapper::YunkerStar::Config.new(:uniq=>:page)
+      config.uniq_page?.should == true
+    end
+
+    it "should return true when :uniq is set to :entry" do
+      config = DataMapper::YunkerStar::Config.new(:uniq=>:entry)
+      config.uniq_page?.should == true
+    end
+
+    it "should return false when :uniq is set to false" do
+      config = DataMapper::YunkerStar::Config.new(:uniq=>false)
+      config.uniq_page?.should == false
+    end
+  end
+
+  it "should provide #uniq_entry?" do
+    @config.should respond_to(:uniq_entry?)
+  end
+
+  describe "#uniq_entry?" do
+    it "should return true as default value" do
+      @config.uniq_entry?.should == true
+    end
+
+    it "should return false when :uniq is set to :page" do
+      config = DataMapper::YunkerStar::Config.new(:uniq=>:page)
+      config.uniq_entry?.should == false
+    end
+
+    it "should return true when :uniq is set to :entry" do
+      config = DataMapper::YunkerStar::Config.new(:uniq=>:entry)
+      config.uniq_entry?.should == true
+    end
+
+    it "should return false when :uniq is set to :page" do
+      config = DataMapper::YunkerStar::Config.new(:uniq=>false)
+      config.uniq_entry?.should == false
+    end
+
+  end
+end

data/spec/data/sorted1.html

@@ -0,0 +1,262 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-us" lang="en-us">
+  <head>
+    <title>Merbist Plugins</title>
+    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+    <meta name="verify-v1"
+     content="QqJ1Kmvs51kF+1Sn+7JUqkXTXbnmLVKzFctoGLRDLE8=" />
+    <link rel="stylesheet" href="/stylesheets/master.css" type="text/css"
+     media="screen" charset="utf-8" />
+    <script src="/javascripts/jquery.js" type="text/javascript"></script>
+      <link rel="alternate" type="application/rss+xml" title="Atom" href="/plugins.atom"></link>
+
+  </head>
+  <body>
+    <div id="base">
+      <div id="header">
+        <div id="navi">
+  <a href="/">Top</a>
+  <a href="/plugins">Plugins</a>
+  <a href="/talks">Talks</a>
+  <a href="/sites">Sites</a>
+  <a href="/users">Users</a>
+</div>
+<div id="menu">
+    <a href="/login">Login</a>
+    <a href="/users/new">Sing up</a>
+</div>
+<div class="clear"><!----></div>
+
+      </div>
+      <div id="body">
+        <div class="command">
+  <a href="/plugins">Index</a>
+</div>
+
+
+<h1>Plugin List</h1>
+
+<table class="plugin-list" cellspacing="1" border="0">
+  <thead>
+    <tr>
+      <th>
+        <a href="/plugins/sorted?page=1&sort=name1">▲</a>
+        Name
+        <a href="/plugins/sorted?page=1&sort=name2">▼</a
+      </th>
+      <th>Repos</th>
+      <th>Registered by</th>
+      <th>Description</th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr class="even">
+      <td><a href="/plugins/36">eventmachine-0.12.5</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>EventMachine: fast, simple event-processing library for Ruby programs</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/35">dm-last-0.0.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A DataMapper plugin that offers a short-hand for Model.all.last as Model.last</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/34">sweet_merb_fixtures-0.0.4</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>The way to store records from YAML file.</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/33">merb-slices-search-path-fix-0.0.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A merb plugin that fixes </td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/32">rubyforge-1.0.2</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A script which automates a limited set of rubyforge operations.
+</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/31">ruby-git</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>Ruby/Git is a Ruby library that can be used to create, read and manipulate Git repositories by wrapping system calls to the git binary.</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/30">fastthread-1.0.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>fastthread that is compatible to ruby191</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/29">mime-types-1.15.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>This library allows for the identification of a file’s likely MIME content type.</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/28">memcached-0.14</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A Ruby interface to the libmemcached C client</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/27">gem_plugin-0.2.3</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>Gem Based Plugin System</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/26">cgi_multipart_eof_fix-2.5.0</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>cgi_multipart_eof_fix on github</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/25">mongrel-1.1.2</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>Mongrel</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/24">hpricot-0.6.207</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A swift, liberal HTML parser with a fantastic library</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/23">methopara-0.3.0</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>Method#parameters for ruby-1.9.1</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/22">irb_rocket-0.1.3</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>irb plugin that makes irb #=&gt; rocket</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/21">ruby-terminfo-0.1.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>ruby-terminfo is a terminfo binding for Ruby</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/20">extlib-present-0.0.2</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A plugin that provides Object#present?</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/19">merb_render_filter-0.0.2</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>A plugin that provides {before|after}_render filters for Controller.</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/18">bcrypt-ruby-2.0.3</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>bcrypt-ruby is a Ruby binding for the OpenBSD bcrypt() password hashing algorithm, allowing you to easily store a secure hash of your users' passwords.
+</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/17">json-1.1.4.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>This library can parse JSON texts and generate them from ruby data structures. Install the extension variant (in C) with &quot;gem install json&quot; or install the pure Ruby variant with &quot;gem install json_pure&quot;.</td>
+      <td>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+<div class="pagination"><span class="prev disabled">&laquo; Prev</span>
+<span class="current disabled">1</span>
+<span class="disabled"><a href="/plugins/sorted?page=2">2</a></span>
+<a class="next" rel="next" href="/plugins/sorted?page=2">Next &raquo;</a></div>
+
+<div class="footnote">
+  &dagger; You can install these gems by
+  <code>
+    gem install gem-name --source http://merbi.st
+  </code>
+</div>
+
+
+      </div>
+      <div id="footer">
+        <div id="footer">
+  2008 
+  <a href="http://wota.jp/ac/">maiha</a>,
+  <a href="http://d.jong.gr.jp/shachi">shachi</a>
+  and
+  <a href="http://blog.s21g.com/genki">genki</a>
+  (<a href="http://www.s21g.com/">s21g LLC</a>).
+  <span class="powered-by">
+    Powered by
+    Merb-1.0.9
+    (Ruby-1.9.1)
+  </span>
+</div>
+
+      </div>
+    </div>
+    
+    <script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+try {
+var pageTracker = _gat._getTracker("UA-2733799-11");
+pageTracker._trackPageview();
+} catch(err) {}</script>
+
+  </body>
+</html>