maiha-dm-ys 0.1 → 0.2

data/README

@@ -6,18 +6,22 @@ a DataMapper extension that uses html table as its schema and data powerfully li
 Example
 =======
 
-  class Monster
+  class Maintainer
     include DataMapper::YunkerStar
-    
-    uri "http://ds.gkwiki2.com/47.html"
-    thead "table.style_table thead tr"
-    tbody "table.style_table tbody tr"
+    uri "http://datamapper.org/doku.php?id=gem_maintainers"
   end
- 
-  irb(main):001:0> Monster.count
-  => 120
-  irb(main):002:0> Monster.first
-  => #<Monster id=nil LV="2" 種族="妖精" 名称="ピクシー" HP="30" MP="27" ...
+
+  irb(main):001:0> Maintainer.count
+  => 31
+
+  irb(main):002:0> Maintainer.first
+  => #<Maintainer id=nil _Gem="data_objects" _Maintainer="dbussink" Name="Dirkjan Bussink">
+
+  irb(main):003:0> Maintainer.names
+  => ["_Gem", "_Maintainer", "Name"]
+
+  irb(main):004:0> Maintainer.labels
+  => ["Gem", "Maintainer", "Name"]
 
 
 Copyright (c) 2008 maiha@wota.jp, released under the MIT license

data/Rakefile

@@ -33,7 +33,7 @@ AUTHOR = "maiha"
 EMAIL = "maiha@wota.jp"
 HOMEPAGE = "http://github.com/maiha/dm-ys"
 SUMMARY = "a DataMapper extension that uses html table as its schema and data powerfully like YunkerStar"
-GEM_VERSION = "0.1"
+GEM_VERSION = "0.2"
 
 spec = Gem::Specification.new do |s|
 #  s.rubyforge_project = 'merb'

data/lib/dm-ys/cached_accessor.rb

@@ -21,7 +21,9 @@ module DataMapper
         end
 
         def method_missing(symbol, &block)
-          @klass.send(:define_method, symbol, &block)
+          cached = "__cached__#{symbol}"
+          @klass.send(:define_method, cached, &block)
+          @klass.class_eval("def #{symbol}; @#{cached} ||= #{cached}; end", "(__CACHED_ACCESSOR__)", 1)
         end
       end
     end

data/lib/dm-ys/proxy.rb

@@ -15,6 +15,7 @@ module DataMapper
         model.class_eval do
           extend ClassMethods
           dsl_accessor :uri
+          dsl_accessor :table
           dsl_accessor :tbody
           dsl_accessor :thead
           property :id, DataMapper::Types::Serial
@@ -28,13 +29,17 @@ module DataMapper
 
         def lazy_load
           loader = Scraper.new(self)
-          loader.labels.each do |name|
+          loader.names.each do |name|
             type = String         # TODO
             property name.intern, type
           end
           return loader
         end
 
+        def names
+          proxy.names
+        end
+
         def labels
           proxy.labels
         end
@@ -49,7 +54,7 @@ module DataMapper
 
         def all
           @all ||= proxy.entries.map{|array|
-            new(Hash[*proxy.labels.zip(array).flatten])
+            new(Hash[*proxy.names.zip(array).flatten])
           }
         end
 

data/lib/dm-ys/scraper.rb

@@ -11,17 +11,93 @@ module DataMapper
     class Scraper
       include CachedAccessor
 
+      class TableNotFound < RuntimeError; end
+
+      attr_reader :html
+
       def initialize(model)
+        raise ArgumentError, "missing model" unless model
+        raise ArgumentError, "missing uri"   unless model.uri
         @model = model
-        @html  = NKF.nkf('-w', open(@model.uri).read)
+        @html  = NKF.nkf('-w', open(model.uri).read)
+        @invalid_name_count = 0
+      end
+
+      def guess_table
+        [max_table_from("table"), max_table_from("table > tbody")].sort_by(&:first).last.last or
+          raise TableNotFound, "set 'table' or 'tbody' manually"
       end
 
       cached_accessor do
         doc     {Hpricot(@html)}
-        labels  {doc.search(@model.thead).first.children.map(&:inner_html)}
-        entries {doc.search(@model.tbody).map{|tr| tr.children.map(&:inner_html)}}
+        table   {specified(:table) or guess_table}
+        thead   {specified(:thead) or table.search("> thead").first or table}
+        tbody   {specified(:tbody) or table.search("> tbody").first or table}
+        names   {labels.map{|i| label2name(i)}}
+        labels  {thead.search("> tr").first.search("> td|th").map{|i|strip_tags(i.inner_html)}}
+        entries {tbody.search("> tr").map{|tr| tr.search("> td").map{|i|strip_tags(i.inner_html)}}.delete_if{|i|i.blank?}}
       end
-    end
 
+      private
+
+        def max_table_from(entry)
+          table = nil
+          count = -1
+          doc.search(entry).each do |t|
+            size = t.search("> tr").size
+            if size > count
+              count = size
+              table = t
+            end
+          end
+          [count, table]
+        end
+
+        def specified(name)
+          @model.respond_to?(name)    or raise ArgumentError, "invalid selector name: #{name}"
+          css = @model.__send__(name) or return nil
+
+          element = doc.search(css)
+          case element
+          when Hpricot::Elem
+            return element
+          when Hpricot::Elements
+            return element.first
+          else
+            return nil
+          end
+        end
+
+        def label2name(label)
+          require 'cgi'
+          label = CGI.unescapeHTML(label)
+          label.gsub!(/&nbsp;/, '')
+          label.gsub!(/\r?\n/, '')
+          label.delete!('!"#$%&()=~|`{}^-[]/<>:; \\')
+          label.delete!("'")
+          label.strip!
+
+          if /^([A-Z])/ === label and Object.const_defined?(label)
+            label = "_#{label}"
+          end
+          if label.blank? or @model.respond_to?(label, true)
+            new_name_for(label)
+          elsif /^[0-9]/ === label
+            "_#{label}"
+          else
+            label
+          end
+        end
+
+        def new_name_for(label)
+          @invalid_name_count += 1
+          "col_#{@invalid_name_count}"
+        end
+
+        def strip_tags(html)
+          html.gsub(/<.*?>/, '').strip
+        end
+
+    end
   end
 end

data/spec/guess_spec.rb

@@ -0,0 +1,90 @@
+require File.join( File.dirname(__FILE__), "spec_helper" )
+
+describe DataMapper::YunkerStar do
+  class BlankHtml
+    include DataMapper::YunkerStar
+    uri spec_data_path("blank.html")
+  end
+
+  class BlankStyle
+    include DataMapper::YunkerStar
+    uri spec_data_path("ki.html")
+  end
+
+  class TableStyle < BlankStyle
+    uri spec_data_path("ki.html")
+    table "table.main"
+  end
+
+  class TheadStyle < BlankStyle
+    uri spec_data_path("ki.html")
+    thead "table.main"
+  end
+
+  it "should provide proxy" do
+    BlankStyle.should respond_to(:proxy)
+  end
+
+  describe ".proxy" do
+    it "should provide guess_table" do
+      BlankStyle.proxy.should respond_to(:guess_table)
+    end
+
+    describe "#guess_table" do
+      it "should return a Hpricot::Elem" do
+        BlankStyle.proxy.guess_table.class.should == Hpricot::Elem
+      end
+
+      it "should return a right element" do
+        BlankStyle.proxy.guess_table["class"].should == "main"
+      end
+
+      it "should raise when the html contains no tables" do
+        lambda {
+          BlankHtml.proxy.guess_table
+        }.should raise_error(DataMapper::YunkerStar::Scraper::TableNotFound)
+      end
+    end
+
+    it "should provide table" do
+      BlankStyle.proxy.should respond_to(:table)
+
+      describe "#table" do
+        it "should raise when the html contains no tables" do
+          lambda {
+            BlankHtml.proxy.table
+          }.should raise_error(DataMapper::YunkerStar::Scraper::TableNotFound)
+        end
+
+        it "should return specified table" do
+          table = TableStyle.proxy.table
+          table.class.should == Hpricot::Elem
+          table[:class].should == "main"
+        end
+      end
+    end
+
+    it "should provide thead" do
+      BlankStyle.proxy.should respond_to(:thead)
+
+      describe "#thead" do
+        it "should raise when the html contains no tables" do
+          lambda {
+            BlankHtml.proxy.thead
+          }.should raise_error(DataMapper::YunkerStar::Scraper::TableNotFound)
+        end
+      end
+    end
+
+    it "should provide labels" do
+      BlankStyle.proxy.should respond_to(:labels)
+
+      describe "#labels" do
+        it "should return th values" do
+          BlankStyle.proxy.labels.map(&:strip).should == %w( col1 col2 col3 col4 )
+        end
+      end
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require "rubygems"
+require "spec"
+
+require File.dirname(__FILE__) + "/../lib/dm-ys"
+
+def spec_data_path(name)
+  Pathname(File.dirname(__FILE__) + "/data/#{name}")
+end
+
+def spec_data(name)
+  spec_data_path(name).read
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: maiha-dm-ys
 version: !ruby/object:Gem::Version 
-  version: "0.1"
+  version: "0.2"
 platform: ruby
 authors: 
 - maiha
@@ -59,6 +59,11 @@ files:
 - lib/dm-ys/scraper.rb
 - lib/dm-ys/cached_accessor.rb
 - lib/dm-ys/proxy.rb
+- spec/guess_spec.rb
+- spec/data
+- spec/data/ki.html
+- spec/data/blank.html
+- spec/spec_helper.rb
 has_rdoc: true
 homepage: http://github.com/maiha/dm-ys
 post_install_message: