sinew 1.0.0 → 1.0.1

data/Rakefile

@@ -1,6 +1,7 @@
 require "bundler"
 require "bundler/setup"
 require "rake"
+require "rake/testtask"
 
 $LOAD_PATH << File.expand_path("../lib", __FILE__)
 require "sinew/version"
@@ -24,7 +25,15 @@ task :release => :build do
   system "gem push sinew-#{Sinew::VERSION}.gem"
 end
 
-task :default => :gem
+#
+# minitest
+#
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << "test"
+end
+
+task :default => :test
 
 # to test:
 # block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew

data/bin/sinew

@@ -3,10 +3,14 @@
 require "sinew"
 require "trollop"
 
+raise "Sinew requires Ruby 1.9 or higher" if RUBY_VERSION < "1.9"
+
 # ARGV
 options = Trollop.options do
   banner "Usage: sinew [options] <gub.sinew>"
+  opt :cache, "Set the cache directory (defaults to ~/.sinew)"
   opt :verbose, "Dump every row"
+  opt :quiet, "Be quiet"
 end
 Trollop.die "need a .sinew file to run against" if ARGV.blank?
 

data/lib/sinew/main.rb

@@ -13,73 +13,17 @@ module Sinew
 
     def initialize(options)
       @options = options.dup
-      @csv = @path = nil
-
-      @curler = Curler.new(user_agent: "sinew/#{VERSION}")
-
-      file = @options[:file]
-      if !File.exists?(file)
-        Util.fatal("#{file} not found")
-      end
-
-      tm = Time.now
-      instance_eval(File.read(file, mode: "rb"), file)
-      if @path
-        Util.banner("Finished #{@path} in #{(Time.now - tm).to_i}s.")
-      else
-        Util.banner("Finished in #{(Time.now - tm).to_i}s.")
-      end
+      _run if !@options[:test]
     end
 
     def get(url, params = nil)
-      http(url, params, :get)
+      _http(url, params, :get)
     end
     
     def post(url, params = nil)
-      http(url, params, :post)
+      _http(url, params, :post)
     end
 
-    def http(url, params, method)
-      url = url.to_s
-      raise "invalid url #{url.inspect}" if url !~ /^http/i
-
-      # decode entities
-      url = CODER.decode(url)
-
-      # handle params
-      body = nil
-      if params
-        q = params.map { |key, value| [CGI.escape(key.to_s), CGI.escape(value.to_s)] }.sort
-        q = q.map { |key, value| "#{key}=#{value}" }.join("&")
-        if method == :get
-          separator = url.include?(??) ? "&" : "?"
-          url = "#{url}#{separator}#{q}"
-        else
-          body = q
-        end
-      end
-
-      begin
-        if method == :get
-          path = @curler.get(url)
-        else
-          path = @curler.post(url, body)
-        end
-        @raw = File.read(path, mode: "rb")
-      rescue Curler::Error => e
-        $stderr.puts "xxx #{e.message}"
-        @raw = ""
-      end
-
-      # setup local variables
-      @url, @uri = @curler.url, @curler.uri
-      @html = nil
-      @clean = nil
-      @noko = nil
-
-      nil
-    end
-    
     #
     # lazy accessors for cleaned up version
     #
@@ -136,10 +80,92 @@ module Sinew
       @csv = CSV.open(file, "wb")
       @csv_keys = args
       @csv << @csv_keys
-      Util.banner("Writing to #{@path}...")
+      _banner("Writing to #{@path}...")
     end
 
-    def normalize(key, s)
+    def csv_emit(row, options = {})
+      csv_header(row.keys.sort) if !@csv
+
+      print = { }
+      row = @csv_keys.map do |i|
+        s = _normalize(row[i], i)
+        print[i] = s if !s.empty?
+        s
+      end
+      $stderr.puts print.ai if @options[:verbose]
+      @csv << row    
+      @csv.flush
+    end
+
+    protected
+
+    def _curler
+      @curler ||= begin
+        # curler
+        options = { user_agent: "sinew/#{VERSION}" }
+        options[:dir] = @options[:cache] if @options[:cache]
+        options[:verbose] = false if @options[:quiet]
+        Curler.new(options)
+      end
+    end
+
+    def _run
+      @csv = @path = nil
+      
+      file = @options[:file]
+      if !File.exists?(file)
+        Util.fatal("#{file} not found")
+      end
+
+      tm = Time.now
+      instance_eval(File.read(file, mode: "rb"), file)
+      if @path
+        _banner("Finished #{@path} in #{(Time.now - tm).to_i}s.")
+      else
+        _banner("Finished in #{(Time.now - tm).to_i}s.")
+      end
+    end
+
+    def _http(url, params, method)
+      url = url.to_s
+      raise "invalid url #{url.inspect}" if url !~ /^http/i
+
+      # decode entities
+      url = CODER.decode(url)
+
+      # handle params
+      body = nil
+      if params
+        q = params.map { |key, value| [CGI.escape(key.to_s), CGI.escape(value.to_s)] }.sort
+        q = q.map { |key, value| "#{key}=#{value}" }.join("&")
+        if method == :get
+          separator = url.include?(??) ? "&" : "?"
+          url = "#{url}#{separator}#{q}"
+        else
+          body = q
+        end
+      end
+
+      begin
+        if method == :get
+          path = _curler.get(url)
+        else
+          path = _curler.post(url, body)
+        end
+        @raw = File.read(path, mode: "rb")
+      rescue Curler::Error => e
+        $stderr.puts "xxx #{e.message}"
+        @raw = ""
+      end
+
+      # setup local variables
+      @url, @uri = _curler.url, _curler.uri
+      @html = nil
+      @clean = nil
+      @noko = nil
+    end
+    
+    def _normalize(s, key = nil)
       case s
       when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
         s = s.inner_html
@@ -155,18 +181,8 @@ module Sinew
       s
     end
 
-    def csv_emit(row, options = {})
-      csv_header(row.keys.sort) if !@csv
-
-      print = { }
-      row = @csv_keys.map do |i|
-        s = normalize(i, row[i])
-        print[i] = s if !s.empty?
-        s
-      end
-      $stderr.puts print.ai if @options[:verbose]
-      @csv << row    
-      @csv.flush
+    def _banner(s)
+      Util.banner(s) if !@options[:quiet]
     end
   end
 end

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

data/test/helper.rb

@@ -0,0 +1,64 @@
+require "active_support/core_ext"
+require "test/unit"
+require "sinew"
+
+module Sinew
+  class TestCase < Test::Unit::TestCase
+    TMP = "/tmp/_test_sinew"
+    HTML_FILE = File.expand_path("#{File.dirname(__FILE__)}/test.html")
+    HTML = File.read(HTML_FILE)
+
+    #
+    # for mocking curl
+    #
+
+    def mock_curl_200
+      Proc.new do |cmd, args|
+        mock_curl(args, HTML, "HTTP/1.1 200 OK")
+      end
+    end
+
+    def mock_curl_302
+      Proc.new do |cmd, args|
+        mock_curl(args, "", "HTTP/1.1 302 Moved Temporarily\r\nLocation: http://www.gub.com")
+      end
+    end
+
+    def mock_curl_500
+      Proc.new do |cmd, args|
+        raise Util::RunError, "curl error"
+      end
+    end
+
+    def mock_curl(args, body, head)
+      File.write(args[args.index("--output") + 1], body)
+      File.write(args[args.index("--dump-header") + 1], "#{head}\r\n\r\n")
+    end
+  end
+end
+
+#
+# from MiniTest, but not in the gem yet
+#
+
+class Object
+  def stub name, val_or_callable, &block
+    new_name = "__minitest_stub__#{name}"
+
+    metaclass = class << self; self; end
+    metaclass.send :alias_method, new_name, name
+    metaclass.send :define_method, name do |*args|
+      if val_or_callable.respond_to? :call then
+        val_or_callable.call(*args)
+      else
+        val_or_callable
+      end
+    end
+
+    yield
+  ensure
+    metaclass.send :undef_method, name
+    metaclass.send :alias_method, name, new_name
+    metaclass.send :undef_method, new_name
+  end
+end

data/test/test.html

@@ -0,0 +1,39 @@
+<html>
+  <head>
+    <title>Title</title>
+    <script>
+      alert("alert 1");
+      alert("alert 2");      
+    </script>
+  </head>
+  
+  <body>
+    <div id="main">
+      <span class="class1"> text1 </span>
+      <span class="class2"> text2 </span>
+
+      <!-- for test_normalize -->
+      <div id="element">text</div>
+      <div class="e">text1</div>
+      <div class="e">text2</div>            
+    </div>
+    
+    <div id="nokogiri_ext">
+      <ul>
+        <li>hello</li>
+        <li>world</li>
+      </ul>
+      <div>
+        a
+        <p>b<span>c</span></p>
+        <p>b<span>c</span></p>
+      </div>
+    </div>
+
+    <div id="text_util">
+      <!-- a comment that should be removed -->
+      <div class="will_be_removed"/>
+      <a class="will_be_preserved"/>
+    </div>
+  </body>
+</html>

data/test/test_curler.rb

@@ -0,0 +1,70 @@
+require "helper"
+
+module Sinew
+  class TestCurler < TestCase
+    def setup
+      # create TMP dir
+      FileUtils.rm_rf(TMP) if File.exists?(TMP)
+      FileUtils.mkdir_p(TMP)
+
+      # curler, pointed at TMP
+      @curler = Curler.new(dir: TMP, verbose: false)
+    end
+
+    #
+    # tests
+    #
+
+    def test_200
+      Util.stub(:run, mock_curl_200) do
+        path = @curler.get("http://www.example.com")
+        assert_equal(HTML, File.read(path))
+      end
+    end
+
+    def test_500
+      assert_raises(Curler::Error) do
+        Util.stub(:run, mock_curl_500) do
+          @curler.get("http://www.example.com")
+        end
+      end
+    end
+
+    def test_cached
+      Util.stub(:run, mock_curl_200) do
+        assert_equal(HTML, File.read(@curler.get("http://www.example.com")))
+      end
+      # the file is cached, so this shouldn't produce an error
+      Util.stub(:run, mock_curl_500) do
+        @curler.get("http://www.example.com")
+      end
+    end
+
+    def test_302
+      Util.stub(:run, mock_curl_302) do
+        @curler.get("http://www.example.com")
+        assert_equal("http://www.gub.com", @curler.url)
+      end
+    end
+
+    def test_rate_limit
+      slept = false
+      
+      # change Kernel#sleep to not really sleep!
+      Kernel.send(:alias_method, :old_sleep, :sleep)
+      Kernel.send(:define_method, :sleep) do |x|
+        slept = true
+      end
+
+      Util.stub(:run, mock_curl_200) do
+        @curler.get("http://www.example.com/1")
+        @curler.get("http://www.example.com/2")
+      end
+      assert(slept)
+
+      # restore old Kernel#sleep
+      Kernel.send(:alias_method, :sleep, :old_sleep)
+      Kernel.send(:undef_method, :old_sleep)
+    end
+  end
+end

data/test/test_main.rb

@@ -0,0 +1,101 @@
+# encoding: UTF-8
+
+require "helper"
+
+module Sinew
+  class TestMain < TestCase
+    RECIPE = "#{TMP}/test.sinew"
+    CSV    = "#{TMP}/test.csv"    
+
+    def setup
+      # create TMP dir
+      FileUtils.rm_rf(TMP) if File.exists?(TMP)
+      FileUtils.mkdir_p(TMP)
+    end
+
+    def run_recipe(recipe)
+      File.write(RECIPE, recipe)
+      Util.stub(:run, mock_curl_200) do
+        Sinew::Main.new(cache: TMP, file: RECIPE, quiet: true)
+      end
+    end
+
+    def test_noko
+      run_recipe <<'EOF'
+get "http://www.example.com"
+csv_header(:class, :text)
+noko.css("#main span").each do |span|
+  csv_emit(class: span[:class], text: span.text)
+end
+EOF
+      assert_equal("class,text\nclass1,text1\nclass2,text2\n", File.read(CSV))
+    end
+
+    def test_raw
+      # test javascript, which is only crawlable with raw
+      run_recipe <<'EOF'
+get "http://www.example.com"
+raw.scan(/alert\("([^"]+)/) do
+  csv_emit(alert: $1)
+end
+EOF
+      assert_equal("alert\nalert 1\nalert 2\n", File.read(CSV))
+    end
+
+    def test_html
+      # note the cleaned up whitespace
+      run_recipe <<'EOF'
+get "http://www.example.com"
+csv_header(:class, :text)
+html.scan(/<span class="(\w+)">(\w+)/) do
+  csv_emit(class: $1, text: $2)
+end
+EOF
+      assert_equal("class,text\nclass1,text1\nclass2,text2\n", File.read(CSV))
+    end
+
+    def test_clean
+      # note the removed attributes from span
+      run_recipe <<'EOF'
+get "http://www.example.com"
+clean.scan(/<span>(text\d)/) do
+  csv_emit(text: $1)
+end
+EOF
+      assert_equal("text\ntext1\ntext2\n", File.read(CSV))      
+    end
+
+    def test_normalize
+      s = Sinew::Main.new(test: true)
+
+      #
+      # non-strings
+      #
+      
+      noko = Nokogiri::HTML(HTML).css("#main")      
+      # node => text
+      assert_equal("text", s.send(:_normalize, noko.css("#element")))
+      # nodes => text joined with space
+      assert_equal("text1 text2", s.send(:_normalize, noko.css(".e")))
+      # array => text joined with pipe
+      assert_equal("1|2", s.send(:_normalize, [1,2]))
+
+      #
+      # string cleanups
+      #
+      
+      # untag
+      assert_equal("gub", s.send(:_normalize, "<tag>gub</tag>"))
+      # convert_accented_entities
+      assert_equal("a", s.send(:_normalize, "&aacute;"))
+      # unent
+      assert_equal("<>", s.send(:_normalize, "&lt;&gt;"))
+      # to_ascii
+      assert_equal("cafe", s.send(:_normalize, "caf\xc3\xa9"))
+      # squish
+      assert_equal("hello world", s.send(:_normalize, "\nhello \t \rworld"))
+    end
+  end
+end
+
+

data/test/test_nokogiri_ext.rb

@@ -0,0 +1,19 @@
+require "helper"
+
+module Sinew
+  class TestNokogiriExt < TestCase
+    def setup
+      @noko = Nokogiri::HTML(HTML).css("#nokogiri_ext")
+    end
+    
+    def test_inner_text
+      assert_equal("hello world", @noko.css("li").inner_text)
+      assert_equal("<li>hello</li> <li>world</li>", @noko.css("ul").inner_html.squish)
+    end
+
+    def test_just_me
+      assert_equal("a", @noko.css("div").text_just_me.squish)
+      assert_equal("b b", @noko.css("p").text_just_me.squish)    
+    end
+  end
+end

data/test/test_text_util.rb

@@ -0,0 +1,23 @@
+require "helper"
+
+module Sinew
+  class TestTextUtil < TestCase
+    def test_tidy
+      tidy = TextUtil.html_tidy(HTML)
+      # tags removed?
+      assert(tidy !~ /script|meta/)
+      # squished?
+      assert(tidy !~ /  /)
+      # comments removed?
+      assert(tidy !~ /<!--/)      
+    end
+
+    def test_clean
+      clean = TextUtil.html_clean(HTML)
+      # attributes removed
+      assert(clean !~ /will_be_removed/)
+      # attributes preserved
+      assert(clean =~ /will_be_preserved/)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-04 00:00:00.000000000 Z
+date: 2012-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -146,6 +146,12 @@ files:
 - lib/sinew/version.rb
 - sample.sinew
 - sinew.gemspec
+- test/helper.rb
+- test/test.html
+- test/test_curler.rb
+- test/test_main.rb
+- test/test_nokogiri_ext.rb
+- test/test_text_util.rb
 homepage: http://github.com/gurgeous/sinew
 licenses: []
 post_install_message: 
@@ -160,7 +166,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 106543959769779396
+      hash: 2227650352747651089
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -169,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 106543959769779396
+      hash: 2227650352747651089
 requirements: []
 rubyforge_project: sinew
 rubygems_version: 1.8.21