maiha-dm-ys 0.4 → 0.4.1

data/README

@@ -64,8 +64,31 @@ Append "*" to uri if you want pagination mode.
   => 36
 
 
+Raw Element
+===========
+
+"element_for" method returns raw scraped elements those are currently hpricot elements.
+
+  >> Plugin.names               #=> ["Name", "Repos", "Registeredby", "Description", "col_1"]
+  >> record = Plugin.first      #=> #<Plugin id=1 Name="eventmachine-0.12.5" Repos=...>
+  >> record.Name                #=> "eventmachine-0.12.5"
+  >> record.element_for("Name") #=> {elem td {elem a href"/plugins/36" "eventmachine-0.12.5" a} td}
+  >> record.element_for("Name").class #=> Hpricot::Elem
+
+"link_for" is syntax sugar for extracting href tag from its element.
+
+  >> record.link_for("Name")         #=> "http://merbi.st/plugins/36"
+  >> record.link_for("Registeredby") #=> "http://merbi.st/users/1"
+
+Furthermore, :only_path option can control the fully qualified URL or not.
+
+  >> Plugin.ys[:only_path] = true
+  >> record.link_for("Name")         #=> "/plugins/36"
+  >> record.link_for("Registeredby") #=> "/users/1"
+
+
 TODO
 ====
-* store link information for each columns
+* Feel free to request what you want! :)
 
 Copyright (c) 2008 maiha@wota.jp, released under the MIT license

data/Rakefile

@@ -33,7 +33,7 @@ AUTHOR = "maiha"
 EMAIL = "maiha@wota.jp"
 HOMEPAGE = "http://github.com/maiha/dm-ys"
 SUMMARY = "a DataMapper extension that uses html table as its schema and data powerfully like YunkerStar"
-GEM_VERSION = "0.4"
+GEM_VERSION = "0.4.1"
 
 spec = Gem::Specification.new do |s|
 #  s.rubyforge_project = 'merb'

data/lib/dm-ys/config.rb

@@ -2,7 +2,7 @@ module DataMapper
   module YS
     class Config
       def self.default
-        {:max_pages=>100, :uniq=>true}
+        {:max_pages=>100, :uniq=>true, :only_path=>false}
       end
  
       def initialize(options = nil)
@@ -25,6 +25,10 @@ module DataMapper
       def uniq_entry?
         self[:uniq] == true or self[:uniq] == :entry
       end
+
+      def only_path?
+        !!self[:only_path]
+      end
     end
   end
 end

data/lib/dm-ys/element_property.rb

@@ -0,0 +1,47 @@
+module DataMapper
+  module YS
+
+    # ==== Example
+    #
+    #   Class Foo
+    #     include DataMapper::YS
+    #     uri ...
+    #
+    #   # <tr><th>name</th>...
+    #   # <tr><td><a href="/plugins/36">dm-ys</a></td>...
+    #
+    #   foo = Foo.first
+    #   foo.link_for(:name) # => "/plugins/36"
+
+    module ElementProperty
+      def link_for(key)
+        links_for(key).first
+      end
+
+      def links_for(key)
+        key = normalized_property_for(key)
+        (@links[key.to_s] || []).map do |url|
+          if self.class.ys.only_path?
+            url
+          else
+            (self.class.proxy.base_uri + url).to_s
+          end
+        end
+      end
+
+      def element_for(key)
+        key = normalized_property_for(key)
+        @elements[key.to_s]
+      end
+
+      def links=(value)
+        @links = value
+      end
+
+      def elements=(value)
+        @elements = value
+      end
+    end
+
+  end
+end

data/lib/dm-ys/scraper.rb

@@ -68,6 +68,10 @@ module DataMapper
           @uri || @model.uri.to_s.chomp('*')
         end
 
+        def base_uri
+          URI.parse(uri.split('?').first)
+        end
+
         def register_properties!
           names.each do |name|
             type = String         # TODO
@@ -95,7 +99,7 @@ module DataMapper
         end
 
         def pagination_links
-          base = URI.parse(uri.split('?').first)
+          base = base_uri
           urls = (doc / "a").map{|i| i[:href] =~ /^http/ ? i[:href] : (base+i[:href]).to_s}.uniq
           urls.select{|url| /^#{Regexp.escape(base.to_s)}/ === url}
         end
@@ -122,12 +126,17 @@ module DataMapper
           labels   {thead.search("> tr").first.search("> td|th").map{|i|strip_tags(i.inner_html)}}
           records  {
             tbody.search("> tr").map do |tr|
-              elems  = tr.search("> td")
-              values = elems.map{|i|strip_tags(i.inner_html)}
+              elems = tr.search("> td")
+              next if elems.blank? # ignored because this should be TH columns
+
+              values   = elems.map{|i|strip_tags(i.inner_html)}
+              elements = Hash[*names.zip(elems).flatten]
+
               record = @model.new(Hash[*names.zip(values).flatten])
-              record.elements = Hash[*names.zip(elems).flatten]
+              record.elements = elements
+              record.links = names.inject({}){|h,name| h[name] = links_for(elements[name]); h}
               record
-            end
+            end.compact
           }
         end
 
@@ -184,6 +193,17 @@ module DataMapper
           def strip_tags(html)
             html.gsub(/<.*?>/, '').strip
           end
+
+          def links_for(element)
+            case element
+            when Hpricot::Elem
+              return Array(element.search("a")).map{|i| i[:href]}
+            when Hpricot::Elements
+              return element.map{|e| links_for(e)}.flatten
+            else
+              return []
+            end
+          end
       end
 
       ######################################################################

data/spec/attribute_spec.rb

@@ -0,0 +1,42 @@
+require File.join( File.dirname(__FILE__), "spec_helper" )
+
+describe DataMapper::YS, "(a record)" do
+  class OnlyPath
+    include DataMapper::YS
+    uri "http://merbi.st/plugins/"
+    ys[:only_path] = true
+  end
+
+  before(:each) do
+    @record = Plugin.first
+  end
+
+  it "should provide #link_for" do
+    @record.should respond_to(:link_for)
+  end
+
+  describe "#link_for" do
+    it "should return first link if its element has href attributes" do
+      @record.link_for("Name").should == "http://merbi.st/plugins/36"
+    end
+
+    it "should return nil if its element has no href attributes" do
+      @record.link_for("Description").should == nil
+    end
+
+    it "should return only path link when :only_path is true" do
+      OnlyPath.first.link_for("Name").should == "/plugins/36"
+    end
+  end
+
+  it "should provide #element_for" do
+    @record.should respond_to(:element_for)
+  end
+
+  describe "#element_for" do
+    it "should return first link if its element has href attributes" do
+      @record.element_for("Name").to_s.should == "<td><a href=\"/plugins/36\">eventmachine-0.12.5</a></td>"
+    end
+  end
+
+end

data/spec/config_spec.rb

@@ -63,6 +63,15 @@ describe DataMapper::YS::Config do
       config = DataMapper::YS::Config.new(:uniq=>false)
       config.uniq_entry?.should == false
     end
+  end
+
+  it "should provide #only_path?" do
+    @config.should respond_to(:only_path?)
+  end
 
+  describe "[:only_path]" do
+    it "should has false as default value" do
+      @config[:only_path].should == false
+    end
   end
 end

data/spec/data/th.html

@@ -0,0 +1,108 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-us" lang="en-us">
+  <head>
+    <title>Merbist Plugins</title>
+    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
+    <meta name="verify-v1"
+     content="QqJ1Kmvs51kF+1Sn+7JUqkXTXbnmLVKzFctoGLRDLE8=" />
+    <link rel="stylesheet" href="/stylesheets/master.css" type="text/css"
+     media="screen" charset="utf-8" />
+    <script src="/javascripts/jquery.js" type="text/javascript"></script>
+      <link rel="alternate" type="application/rss+xml" title="Atom" href="/plugins.atom"></link>
+
+  </head>
+  <body>
+    <div id="base">
+      <div id="header">
+        <div id="navi">
+  <a href="/">Top</a>
+  <a href="/plugins">Plugins</a>
+  <a href="/talks">Talks</a>
+  <a href="/sites">Sites</a>
+  <a href="/users">Users</a>
+</div>
+<div id="menu">
+    <a href="/login">Login</a>
+    <a href="/users/new">Sing up</a>
+</div>
+<div class="clear"><!----></div>
+
+      </div>
+      <div id="body">
+        <div class="command">
+  <a href="/plugins">Index</a>
+</div>
+
+
+<h1>Plugin List</h1>
+
+<table class="plugin-list" cellspacing="1" border="0">
+    <tr>
+      <th>Name</th>
+      <th>Repos</th>
+      <th>Registered by</th>
+      <th>Description</th>
+      <th></th>
+    </tr>
+    <tr class="even">
+      <td><a href="/plugins/36">eventmachine-0.12.5</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>EventMachine</td>
+      <td>
+      </td>
+    </tr>
+    <tr class="">
+      <td><a href="/plugins/35">dm-last-0.0.1</a></td>
+      <td>&dagger;</td>
+      <td><a href="/users/1">genki</a></td>
+      <td>Model.last</td>
+      <td>
+      </td>
+    </tr>
+</table>
+
+<div class="pagination"><span class="prev disabled">&laquo; Prev</span>
+<span class="current disabled">1</span>
+<span class="disabled"><a href="/plugins/uniq?page=2">2</a></span>
+<a class="next" rel="next" href="/plugins/uniq?page=2">Next &raquo;</a></div>
+
+<div class="footnote">
+  &dagger; You can install these gems by
+  <code>
+    gem install gem-name --source http://merbi.st
+  </code>
+</div>
+
+
+      </div>
+      <div id="footer">
+        <div id="footer">
+  2008 
+  <a href="http://wota.jp/ac/">maiha</a>,
+  <a href="http://d.jong.gr.jp/shachi">shachi</a>
+  and
+  <a href="http://blog.s21g.com/genki">genki</a>
+  (<a href="http://www.s21g.com/">s21g LLC</a>).
+  <span class="powered-by">
+    Powered by
+    Merb-1.0.9
+    (Ruby-1.9.1)
+  </span>
+</div>
+
+      </div>
+    </div>
+    
+    <script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+try {
+var pageTracker = _gat._getTracker("UA-2733799-11");
+pageTracker._trackPageview();
+} catch(err) {}</script>
+
+  </body>
+</html>

data/spec/models/style.rb

@@ -0,0 +1,24 @@
+class BlankHtml
+  include DataMapper::YS
+  uri spec_data_path("blank.html")
+end
+
+class BlankStyle
+  include DataMapper::YS
+  uri spec_data_path("plugins1.html")
+end
+
+class TableStyle < BlankStyle
+  uri spec_data_path("plugins1.html")
+  table "table.main"
+end
+
+class TheadStyle < BlankStyle
+  uri spec_data_path("plugins1.html")
+  thead "table.main"
+end
+
+class ThStyle
+  include DataMapper::YS
+  uri spec_data_path("th.html")
+end

data/spec/proxy_spec.rb

@@ -1,26 +1,6 @@
 require File.join( File.dirname(__FILE__), "spec_helper" )
 
 describe DataMapper::YS do
-  class ::BlankHtml
-    include DataMapper::YS
-    uri spec_data_path("blank.html")
-  end
-
-  class ::BlankStyle
-    include DataMapper::YS
-    uri spec_data_path("plugins1.html")
-  end
-
-  class ::TableStyle < BlankStyle
-    uri spec_data_path("plugins1.html")
-    table "table.main"
-  end
-
-  class ::TheadStyle < BlankStyle
-    uri spec_data_path("plugins1.html")
-    thead "table.main"
-  end
-
   ######################################################################
   ### Config
 
@@ -120,6 +100,5 @@ describe DataMapper::YS do
           ["Name", "Repos", "Registered by", "Description", ""]
       end
     end
-
   end
 end

data/spec/scraper_spec.rb

@@ -39,4 +39,10 @@ describe DataMapper::YS::Proxy  do
     end
   end
 
+  ######################################################################
+  ### Guess tbody
+
+  it "should ignore th columns" do
+    ThStyle.count.should == 2
+  end
 end

data/spec/spec_helper.rb

@@ -32,6 +32,8 @@ rescue LoadError
 end
 
 mapping = {
+  "http://merbi.st/plugins/" => spec_data_path("plugins1.html"),
+
   # plugin (paginated)
   "http://merbi.st/plugins/index?page=1" => spec_data_path("plugins1.html"),
   "http://merbi.st/plugins/index?page=2" => spec_data_path("plugins2.html"),

data/spec/uniq_record_spec.rb

@@ -0,0 +1,17 @@
+require File.join( File.dirname(__FILE__), "spec_helper" )
+
+describe DataMapper::YS::Scraper::Composite do
+  describe "UniqPlugin" do
+    it "should return 2 pages" do
+      UniqPlugin.proxy.pages.size.should == 2
+    end
+
+    describe "#count" do
+      it "should return same value as Plugin" do
+        UniqPlugin1.count.should == 2
+        UniqPlugin2.count.should == 2
+        UniqPlugin .count.should == 3
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: maiha-dm-ys
 version: !ruby/object:Gem::Version 
-  version: "0.4"
+  version: 0.4.1
 platform: ruby
 authors: 
 - maiha
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-07 00:00:00 -08:00
+date: 2009-03-08 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -59,6 +59,7 @@ files:
 - lib/dm-ys
 - lib/dm-ys/base.rb
 - lib/dm-ys/config.rb
+- lib/dm-ys/element_property.rb
 - lib/dm-ys/indexed_property.rb
 - lib/dm-ys/memory_repository.rb
 - lib/dm-ys/scraper.rb
@@ -69,15 +70,19 @@ files:
 - spec/data/uniq2.html
 - spec/data/sorted2.html
 - spec/data/sorted1.html
+- spec/data/th.html
 - spec/data/plugins1.html
 - spec/data/gem_maintainers.html
 - spec/data/plugins2.html
 - spec/data/uniq1.html
 - spec/data/blank.html
 - spec/models
+- spec/models/style.rb
 - spec/models/gem_maintainer.rb
 - spec/models/plugin.rb
 - spec/composite_scraper_spec.rb
+- spec/uniq_record_spec.rb
+- spec/attribute_spec.rb
 - spec/anonymous_spec.rb
 - spec/scraper_spec.rb
 - spec/indexed_property_spec.rb