sinew 1.0.0

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+rdoc
+sample.csv

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Adam Doppelt
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,34 @@
+## Welcome to Sinew
+
+Sinew collects structured data from web sites (screen scraping). It provides a Ruby DSL built for crawling, a robust caching system, and integration with [Nokogiri](http://nokogiri.org). Though small, this project is the culmination of years of effort based on crawling systems built at several different companies.
+
+Sinew requires Ruby 1.9, [HTML Tidy](http://tidy.sourceforge.net) and [Curl](http://curl.haxx.se).
+
+## Example
+
+Here's an example for collecting Amazon's bestseller list:
+
+```ruby
+# get the url
+get "http://www.amazon.com/gp/bestsellers/books/ref=sv_b_3"
+
+# use nokogiri to find books
+noko.css(".zg_itemRow").each do |item|
+  # pull out the stuff we care about using nokogiri
+  row = { }
+  row[:url] = item.css(".zg_title a").first[:href]
+  row[:title] = item.css(".zg_title")
+  row[:img] = item.css(".zg_itemImage_normal img").first[:src]
+  
+  # append a row to the csv
+  csv_emit(row)
+end
+```
+
+If you paste this into a file called `bestsellers.sinew` and run `sinew bestsellers.sinew`, it will create a `bestsellers.csv` file containing the url, title and img for each bestseller.
+
+## Full Documentation
+
+Full docs are in the wiki:
+
+https://github.com/gurgeous/sinew/wiki

data/Rakefile

@@ -0,0 +1,30 @@
+require "bundler"
+require "bundler/setup"
+require "rake"
+
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+require "sinew/version"
+
+#
+# gem
+#
+
+task :gem => :build
+task :build do
+  system "gem build --quiet sinew.gemspec"
+end
+
+task :install => :build do
+  system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
+end
+
+task :release => :build do
+  system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
+  system "git push --tags"
+  system "gem push sinew-#{Sinew::VERSION}.gem"
+end
+
+task :default => :gem
+
+# to test:
+# block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew

data/bin/sinew

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require "sinew"
+require "trollop"
+
+# ARGV
+options = Trollop.options do
+  banner "Usage: sinew [options] <gub.sinew>"
+  opt :verbose, "Dump every row"
+end
+Trollop.die "need a .sinew file to run against" if ARGV.blank?
+
+# now run!
+ARGV.each do |i|
+  Sinew::Main.new(options.merge(file: i))
+end

data/lib/sinew.rb

@@ -0,0 +1,6 @@
+require "sinew/version"
+require "sinew/util"
+require "sinew/curler"
+require "sinew/nokogiri_ext"
+require "sinew/text_util"
+require "sinew/main"

data/lib/sinew/curler.rb

@@ -0,0 +1,173 @@
+require "uri"
+
+module Sinew
+  class Curler
+    class Error < StandardError ; end
+    
+    DEFAULT_OPTIONS = {
+      :cache_errors => true,
+      :max_time => 30,
+      :retry => 3,
+      :verbose => true,
+    }
+    
+    attr_reader :url, :uri, :root
+
+    def initialize(options = {})
+      @options = DEFAULT_OPTIONS.merge(options)
+      @curl_args = ["--silent", "--fail", "--user-agent", @options[:user_agent], "--max-time", @options[:max_time], "--retry", @options[:retry], "--location", "--max-redirs", "3"]
+      @last_request = Time.at(0)      
+
+      @root = @options[:dir]
+      if !@root
+        if File.exists?(ENV["HOME"]) && File.stat(ENV["HOME"]).writable?
+          @root = "#{ENV["HOME"]}/.sinew"
+        else
+          @root = "/tmp/sinew"
+        end
+      end
+    end
+
+    def get(url)
+      curl(url, nil)
+    end
+
+    def post(url, body)
+      curl(url, body)
+    end
+    
+    def curl(url, body)
+      #
+      # prepare url/uri and calculate paths
+      #
+
+      @uri = url.is_a?(URI) ? url : Curler.url_to_uri(url.to_s)
+      @url = @uri.to_s
+
+      path = fullpath(@uri)
+      path = "#{path},#{Util.pathify(body)}" if body
+
+      # shorten long paths
+      if path.length > 250
+        dir, base = File.dirname(path), File.basename(path)
+        path = "#{dir}/#{Util.md5(base)}"
+      end
+      
+      head = "#{File.dirname(path)}/head/#{File.basename(path)}"
+
+      if !File.exists?(path)
+        verbose(body ? "curl #{@url} (POST)" : "curl #{@url}")
+        tmp = "/tmp/curler_#{Util.random_text(6)}"
+        tmph = "#{tmp}.head"
+        begin
+          rate_limit
+          Util.mkdir_if_necessary(File.dirname(path))
+          Util.mkdir_if_necessary(File.dirname(head))        
+          begin
+            command = []
+            command += @curl_args
+            if body
+              command += ["--data-binary", body]
+              command += ["--header", "Content-Type: application/x-www-form-urlencoded"]
+            end
+            command += ["--output", tmp]
+            command += ["--dump-header", tmph]
+            command << @url
+            
+            Util.run("curl", command)
+
+            # empty response?
+            if !File.exists?(tmp)
+              Util.touch(tmp)
+              Util.touch(tmph)            
+            end
+          rescue Util::RunError => e
+            message = "curl error"
+            if e.message =~ /(\d+)$/
+              message = "#{message} (#{$1})"
+            end
+            
+            # cache the error?
+            if @options[:cache_errors]
+              File.open(path, "w") { |f| f.puts "" }
+              File.open(head, "w") { |f| f.puts "CURLER_ERROR\t#{message}" }
+            end
+            
+            raise Error, message
+          end
+          Util.mv(tmp, path)
+          Util.mv(tmph, head)
+        ensure
+          Util.rm_if_necessary(tmp)
+          Util.rm_if_necessary(tmph)
+        end
+      end
+      
+      #
+      # handle redirects (recalculate @uri/@url)
+      #
+
+      if File.exists?(head)
+        head_contents = File.read(head)
+        # handle cached errors
+        if head_contents =~ /^CURLER_ERROR\t(.*)/
+          raise Error, $1
+        end
+        original = @uri
+        head_contents.scan(/\A(HTTP\/\d\.\d (\d+).*?\r\n\r\n)/m) do |i|
+          headers, code = $1, $2
+          if code =~ /^3/
+            if redir = headers[/^Location: ([^\r\n]+)/, 1]
+              @uri += redir
+              @url = @uri.to_s
+            end
+          end
+        end
+        # kill unnecessary head files
+        if original == @uri
+          Util.rm(head)
+        end
+      end
+      
+      path
+    end
+    
+    def verbose(s)
+      $stderr.puts s if @options[:verbose]
+    end
+
+    #
+    # helpers
+    #
+
+    def fullpath(uri)
+      "#{@root}/#{Curler.uri_to_path(uri)}"  
+    end
+
+    def uncache!(url)
+      Util.rm_if_necessary("#{@root}/#{Curler.url_to_path(url)}")
+    end
+
+    def self.url_to_uri(url)
+      url = url.gsub(" ", "%20")
+      url = url.gsub("'", "%27")
+      URI.parse(url)
+    end
+    
+    def self.url_to_path(url)
+      uri_to_path(url_to_uri(url))
+    end
+
+    def self.uri_to_path(uri)
+      s = uri.path
+      s = "#{s}?#{uri.query}" if uri.query
+      "#{Util.pathify(uri.host)}/#{Util.pathify(s)}"
+    end
+    
+    def rate_limit
+      sleep = (@last_request + 1) - Time.now
+      sleep(sleep) if sleep > 0
+      @last_request = Time.now
+    end
+  end
+end

data/lib/sinew/main.rb

@@ -0,0 +1,172 @@
+require "nokogiri" # must be loaded before awesome_print
+require "awesome_print"
+require "cgi"
+require "csv"
+require "htmlentities"
+require "stringex"
+
+module Sinew
+  class Main
+    CODER = HTMLEntities.new
+
+    attr_accessor :url, :uri, :raw
+
+    def initialize(options)
+      @options = options.dup
+      @csv = @path = nil
+
+      @curler = Curler.new(user_agent: "sinew/#{VERSION}")
+
+      file = @options[:file]
+      if !File.exists?(file)
+        Util.fatal("#{file} not found")
+      end
+
+      tm = Time.now
+      instance_eval(File.read(file, mode: "rb"), file)
+      if @path
+        Util.banner("Finished #{@path} in #{(Time.now - tm).to_i}s.")
+      else
+        Util.banner("Finished in #{(Time.now - tm).to_i}s.")
+      end
+    end
+
+    def get(url, params = nil)
+      http(url, params, :get)
+    end
+    
+    def post(url, params = nil)
+      http(url, params, :post)
+    end
+
+    def http(url, params, method)
+      url = url.to_s
+      raise "invalid url #{url.inspect}" if url !~ /^http/i
+
+      # decode entities
+      url = CODER.decode(url)
+
+      # handle params
+      body = nil
+      if params
+        q = params.map { |key, value| [CGI.escape(key.to_s), CGI.escape(value.to_s)] }.sort
+        q = q.map { |key, value| "#{key}=#{value}" }.join("&")
+        if method == :get
+          separator = url.include?(??) ? "&" : "?"
+          url = "#{url}#{separator}#{q}"
+        else
+          body = q
+        end
+      end
+
+      begin
+        if method == :get
+          path = @curler.get(url)
+        else
+          path = @curler.post(url, body)
+        end
+        @raw = File.read(path, mode: "rb")
+      rescue Curler::Error => e
+        $stderr.puts "xxx #{e.message}"
+        @raw = ""
+      end
+
+      # setup local variables
+      @url, @uri = @curler.url, @curler.uri
+      @html = nil
+      @clean = nil
+      @noko = nil
+
+      nil
+    end
+    
+    #
+    # lazy accessors for cleaned up version
+    #
+
+    def html
+      @html ||= begin
+        s = TextUtil.html_tidy(@raw)
+        nelements = @raw.count("<")
+        if nelements > 1
+          # is there a problem with tidy?
+          percent = 100 * s.count("<") / nelements
+          if percent < 80
+            # bad xml processing instruction? Try fixing it.
+            maybe = TextUtil.html_tidy(@raw.gsub(/<\?[^>]*?>/, ""))
+            new_percent = 100 * maybe.count("<") / nelements
+            if new_percent > 80
+              # yes!
+              s = maybe
+            else
+              Util.warning "Hm - it looks like tidy ate some of your file (#{percent}%)" if percent < 90
+            end
+          end
+        end
+        s
+      end
+    end
+
+    def clean
+      @clean ||= TextUtil.html_clean_from_tidy(self.html)
+    end
+
+    def noko
+      @noko ||= Nokogiri::HTML(html)
+    end
+
+    #
+    # csv
+    #
+
+    def csv_header(*args)
+      args = args.flatten
+      if args.first.is_a?(String)
+        file = args.shift
+        if file !~ /^\//
+          file = "#{File.dirname(@options[:file])}/#{file}"
+        end
+      else
+        file = @options[:file]
+      end
+      ext = File.extname(file)
+      file = ext.empty? ? "#{file}.csv" : file.gsub(ext, ".csv")
+
+      @path = file
+      @csv = CSV.open(file, "wb")
+      @csv_keys = args
+      @csv << @csv_keys
+      Util.banner("Writing to #{@path}...")
+    end
+
+    def normalize(key, s)
+      case s
+      when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
+        s = s.inner_html
+      when Array
+        s = s.map { |j| j.to_s }.join("|")
+      else
+        s = s.to_s
+      end
+      s = TextUtil.untag(s)
+      s = s.convert_accented_entities
+      s = TextUtil.unent(s)
+      s = s.to_ascii.squish
+      s
+    end
+
+    def csv_emit(row, options = {})
+      csv_header(row.keys.sort) if !@csv
+
+      print = { }
+      row = @csv_keys.map do |i|
+        s = normalize(i, row[i])
+        print[i] = s if !s.empty?
+        s
+      end
+      $stderr.puts print.ai if @options[:verbose]
+      @csv << row    
+      @csv.flush
+    end
+  end
+end

data/lib/sinew/nokogiri_ext.rb

@@ -0,0 +1,27 @@
+require "nokogiri"
+
+# modify NodeSet to join with SPACE instead of empty string
+class Nokogiri::XML::NodeSet
+  alias :old_inner_html :inner_html
+  alias :old_inner_text :inner_text  
+  
+  def inner_text
+    collect { |i| i.inner_text }.join(" ")
+  end
+  def inner_html *args
+    collect { |i| i.inner_html(*args) }.join(" ")
+  end
+end
+
+# text_just_me
+class Nokogiri::XML::Node
+  def text_just_me
+    t = children.find { |i| i.node_type == Nokogiri::XML::Node::TEXT_NODE }
+    t && t.text
+  end
+end
+class Nokogiri::XML::NodeSet
+  def text_just_me
+    map { |i| i.text_just_me }.join(" ")
+  end
+end

data/lib/sinew/text_util.rb

@@ -0,0 +1,103 @@
+require "active_support/core_ext"
+require "set"
+
+module Sinew
+  module TextUtil
+    extend self
+
+    ATTRS_KEEP = Set.new %w(a img iframe)
+    TIDY_OPTIONS = {
+      "-asxml" => nil,
+      "-bare" => nil,
+      "-quiet" => nil,
+      "-utf8" => nil,    
+      "-wrap" =>  0,
+      "--doctype" => "omit",
+      "--hide-comments" => "yes",
+      "--numeric-entities" => "no",
+      "--preserve-entities" => "yes",
+      "--force-output" => "yes",    
+      "-f" => "/dev/null",
+    }
+
+    XML_ENTITIES = { "&"=>"&amp;", "<"=>"&lt;", ">"=>"&gt;", "'"=>"&apos;", '"'=>"&quot;" }
+    XML_ENTITIES_INV = XML_ENTITIES.invert
+    COMMON_ENTITIES_INV = XML_ENTITIES_INV.merge(
+                                                 "&frac12;" => "1/2",
+                                                 "&frac14;" => "1/4",
+                                                 "&frac34;" => "3/4",
+                                                 "&ldquo;" => '"',
+                                                 "&lsquo;" => "'",
+                                                 "&mdash;" => "-",
+                                                 "&nbsp;" => " ",
+                                                 "&ndash;" => "-",
+                                                 "&rdquo;" => '"',
+                                                 "&rsquo;" => "'",
+                                                 "&tilde;" => "~",
+                                                 "&#34;" => '"',
+                                                 "&#39;" => "'",
+                                                 "&#160;" => " ",
+                                                 "&#8232;" => "\n"
+                                                 )
+    
+    #
+    # tidy/clean
+    #
+    
+    def html_tidy(s)
+      # run tidy
+      args = TIDY_OPTIONS.map { |k, v| "#{k} #{v}" }.join(" ")
+      s = IO.popen("tidy #{args}", "rb+") do |f|
+        f.write(s)
+        f.close_write
+        f.read
+      end
+      raise "could not run tidy" if ($? >> 8) > 2
+
+      # now kill some tags
+      s.sub!(/<html\b[^>]+>/, "<html>")
+      s.gsub!(/<\/?(meta|link)\b[^>]*>/m, "")
+      s.gsub!(/<(style|script)\b[^>]*(\/>|>.*?<\/\1\b>)/m, "")    
+      s.gsub!(/<\?[^>]*>/m, "")
+      s.squish!
+
+      # kill whitespace around tags
+      s.gsub!(/ ?<([^>]+)> ?/, "<\\1>")
+      
+      s
+    end
+
+    def html_clean(s)
+      html_clean_from_tidy(html_tidy(s))
+    end
+
+    def html_clean_from_tidy(s)
+      # then kill most attrs
+      s = s.dup
+      s.gsub!(/<([^\s>]+)[^>]*?(\/)?>/) do |i|
+        ATTRS_KEEP.include?($1) ? i : "<#{$1}#{$2}>"
+      end
+      s
+    end
+
+    #
+    # untag/unent
+    #
+
+    def xml_escape(s)
+      s.gsub(/[&<>'"]/) { |i| XML_ENTITIES[i] }
+    end
+
+    def xml_unescape(s)
+      s.gsub(/&(amp|lt|gt|apos|quot);/) { |i| XML_ENTITIES_INV[i] }
+    end
+
+    def untag(s)
+      s.gsub(/<[^>]+>/, " ")    
+    end
+
+    def unent(s)
+      s.gsub(/&#?[a-z0-9]{2,};/) { |i| COMMON_ENTITIES_INV[i] }
+    end
+  end
+end

data/lib/sinew/util.rb

@@ -0,0 +1,236 @@
+require "digest/md5"
+require "etc"
+require "fileutils"
+
+module Sinew
+  # Helper module for executing commands and printing stuff
+  # out.
+  #
+  # The general idea is to only print commands that are actually
+  # interesting. For example, mkdir_if_necessary won't print anything
+  # if the directory already exists. That way we can scan output and
+  # see what changes were made without getting lost in repetitive
+  # commands that had no actual effect.
+  module Util
+    class RunError < StandardError ; end
+
+    extend self
+    
+    RESET   = "\e[0m"
+    RED     = "\e[1;37;41m"
+    GREEN   = "\e[1;37;42m"
+    YELLOW  = "\e[1;37;43m"
+    BLUE    = "\e[1;37;44m"
+    MAGENTA = "\e[1;37;45m"
+    CYAN    = "\e[1;37;46m"
+
+    #
+    # running commands
+    #
+
+    # Make all commands echo before running.
+    def run_verbose!
+      @run_verbose = true
+    end
+    
+    # Run a command, raise an error upon failure. Output goes to
+    # $stdout/$stderr.
+    def run(command, args = nil)
+      line = nil
+      if args
+        args = args.map(&:to_s)
+        line = "#{command} #{args.join(" ")}"
+        vputs line
+        system(command, *args)
+      else
+        line = command
+        vputs line
+        system(command)
+      end
+      if $? != 0
+        if $?.termsig == Signal.list["INT"]
+          raise "#{line} interrupted"
+        end
+        raise RunError, "#{line} failed : #{$?.to_i / 256}"
+      end
+    end
+
+    # Like mkdir -p. Optionally, set the owner and mode.
+    def mkdir(dir, owner = nil, mode = nil)
+      FileUtils.mkdir_p(dir, :verbose => verbose?)
+      chmod(dir, mode) if mode
+      chown(dir, owner) if owner
+    end
+
+    # mkdir only if the directory doesn't already exist. Optionally,
+    # set the owner and mode.
+    def mkdir_if_necessary(dir, owner = nil, mode = nil)
+      mkdir(dir, owner, mode) if !(File.exists?(dir) || File.symlink?(dir))
+    end
+
+    # rm a dir and recreate it.
+    def rm_and_mkdir(dir)
+      raise "don't do this" if dir == ""
+      run "rm -rf #{dir} && mkdir -p #{dir}"
+    end
+
+    # Are two files different?
+    def different?(a, b)
+      !FileUtils.compare_file(a, b)
+    end
+
+    # Copy file or dir from src to dst. Optionally, set the mode and
+    # owner of dst.
+    def cp(src, dst, owner = nil, mode = nil)
+      FileUtils.cp_r(src, dst, :preserve => true, :verbose => verbose?)
+      if owner && !File.symlink?(dst)      
+        chown(dst, owner) 
+      end
+      if mode
+        chmod(dst, mode)
+      end
+    end
+
+    # Copy file or dir from src to dst, but create the dst directory
+    # first if necessary. Optionally, set the mode and owner of dst.
+    def cp_with_mkdir(src, dst, owner = nil, mode = nil)
+      mkdir_if_necessary(File.dirname(dst))
+      cp(src, dst, owner, mode)
+    end
+
+    # Copy file or dir from src to dst, but ONLY if dst doesn't exist
+    # or has different contents than src. Optionally, set the mode and
+    # owner of dst.
+    def cp_if_necessary(src, dst, owner = nil, mode = nil)
+      if !File.exists?(dst) || different?(src, dst)
+        cp(src, dst, owner, mode)
+        true
+      end
+    end
+
+    # Move src to dst. Because this uses FileUtils, it works even if
+    # dst is on a different partition.
+    def mv(src, dst)
+      FileUtils.mv(src, dst, :verbose => verbose?)
+    end
+
+    # Move src to dst, but create the dst directory first if
+    # necessary.
+    def mv_with_mkdir(src, dst)
+      mkdir_if_necessary(File.dirname(dst))
+      mv(src, dst)
+    end
+
+    # Chown file to be owned by user.
+    def chown(file, user)
+      user = user.to_s
+      # who is the current owner?
+      @uids ||= {}
+      @uids[user] ||= Etc.getpwnam(user).uid
+      uid = @uids[user]
+      if File.stat(file).uid != uid
+        run "chown #{user}:#{user} '#{file}'"        
+      end
+    end
+
+    # Chmod file to a new mode.
+    def chmod(file, mode)
+      if File.stat(file).mode != mode
+        FileUtils.chmod(mode, file, :verbose => verbose?)      
+      end
+    end
+
+    # rm a file
+    def rm(file)
+      FileUtils.rm(file, :force => true, :verbose => verbose?)
+    end
+
+    # rm a file, but only if it exists.
+    def rm_if_necessary(file)
+      if File.exists?(file)
+        rm(file)
+        true
+      end
+    end
+
+    # Create a symlink from src to dst.
+    def ln(src, dst)
+      FileUtils.ln_sf(src, dst, :verbose => verbose?)
+    end
+
+    # Create a symlink from src to dst, but only if it hasn't already
+    # been created.
+    def ln_if_necessary(src, dst)
+      ln = false
+      if !File.symlink?(dst)
+        ln = true
+      elsif File.readlink(dst) != src
+        rm(dst)
+        ln = true
+      end
+      if ln
+        ln(src, dst)
+        true
+      end
+    end
+
+    # Touch a file
+    def touch(file)
+      FileUtils.touch(file)      
+    end
+
+    # A nice printout in green.
+    def banner(s, color = GREEN)
+      s = "#{s} ".ljust(72, " ")      
+      $stderr.write "#{color}[#{Time.new.strftime('%H:%M:%S')}] #{s}#{RESET}\n"
+      $stderr.flush
+    end
+
+    # Print a warning in yellow.
+    def warning(msg)
+      banner("Warning: #{msg}", YELLOW)
+    end
+
+    # Print a fatal error in red, then exit.
+    def fatal(msg)
+      banner(msg, RED)
+      exit(1)
+    end
+
+    # Generate some random text
+    def random_text(len)
+      chars = ("A".."Z").to_a + ("a".."z").to_a + ("0".."9").to_a
+      (1..len).map { chars[rand(chars.length - 1)] }.join("")
+    end
+    
+    # Convert a string into something that could be a path segment
+    def pathify(s)
+      s = s.gsub(/^\//, "")
+      s = s.gsub("..", ",")
+      s = s.gsub(/[?\/&]/, ",")
+      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
+        hex = i.unpack("H2").first
+        "%#{hex}"
+      end
+      s = "_root_" if s.empty?
+      s = s.downcase
+      s
+    end
+
+    # checksum some text    
+    def md5(s)
+      Digest::MD5.hexdigest(s.to_s)
+    end
+
+    private
+
+    # Returns true if verbosity is turned on.
+    def verbose?
+      @run_verbose ||= nil
+    end
+
+    def vputs(s)
+      $stderr.puts s if verbose?
+    end
+  end
+end

data/lib/sinew/version.rb

@@ -0,0 +1,4 @@
+module Sinew
+  # Gem version
+  VERSION = "1.0.0"
+end

data/sample.sinew

@@ -0,0 +1,8 @@
+get "http://www.amazon.com/gp/bestsellers/books/ref=sv_b_3"
+noko.css(".zg_itemRow").each do |item|
+  row = { }
+  row[:url] = item.css(".zg_title a").first[:href]
+  row[:title] = item.css(".zg_title")
+  row[:img] = item.css(".zg_itemImage_normal img").first[:src]
+  csv_emit(row)
+end

data/sinew.gemspec

@@ -0,0 +1,29 @@
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+
+require "sinew/version"
+
+Gem::Specification.new do |s|
+  s.name        = "sinew"
+  s.version     = Sinew::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Adam Doppelt"]
+  s.email       = ["amd@gurge.com"]
+  s.homepage    = "http://github.com/gurgeous/sinew"
+  s.summary     = "Sinew - structured web crawling using recipes."
+  s.description = "Crawl web sites easily using ruby recipes, with caching and nokogiri."
+
+  s.rubyforge_project = "sinew"
+
+  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "awesome_print"
+  s.add_runtime_dependency "htmlentities"
+  s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "stringex"
+  s.add_runtime_dependency "trollop"
+  s.add_development_dependency "rake"
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: sinew
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Adam Doppelt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stringex
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
+email:
+- amd@gurge.com
+executables:
+- sinew
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/sinew
+- lib/sinew.rb
+- lib/sinew/curler.rb
+- lib/sinew/main.rb
+- lib/sinew/nokogiri_ext.rb
+- lib/sinew/text_util.rb
+- lib/sinew/util.rb
+- lib/sinew/version.rb
+- sample.sinew
+- sinew.gemspec
+homepage: http://github.com/gurgeous/sinew
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 106543959769779396
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 106543959769779396
+requirements: []
+rubyforge_project: sinew
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: Sinew - structured web crawling using recipes.
+test_files: []