sinew 1.0.3 → 1.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c99e34e066e5c74889a842032d90dbe867e18ac6
+  data.tar.gz: 4cae139e85a63aecb0a5671a43fc8881219236df
+SHA512:
+  metadata.gz: c341f22f607ff0ac03c3f2646d168b6776a21ab9f8b23875a99e9fae71d6cf972a6b22255ec176a3fad68b6d65d26f8cd6d2f34bdfb8a7b6d949bdb61deea6d2
+  data.tar.gz: 8314c4ab14180443050ddc74b828b5833610a86d93fb1e29e3bc2b320e0e6a12cf6b86b96c6a02bc9104f064fd242c2df7df11faf4d2237fa7756bb1865791c5

data/README.md

@@ -12,20 +12,18 @@ gem install sinew
 
 ## Example
 
-Here's an example for collecting Amazon's bestseller list:
+Here's an example for collecting the links from httpbin.org:
 
 ```ruby
 # get the url
-get "http://www.amazon.com/gp/bestsellers/books/ref=sv_b_3"
+get "http://httpbin.org"
 
-# use nokogiri to find books
-noko.css(".zg_itemRow").each do |item|
-  # pull out the stuff we care about using nokogiri
+# use nokogiri to collect links
+noko.css("ul li a").each do |a|
   row = { }
-  row[:url] = item.css(".zg_title a").first[:href]
-  row[:title] = item.css(".zg_title")
-  row[:img] = item.css(".zg_itemImage_normal img").first[:src]
-  
+  row[:url] = a[:href]
+  row[:title] = a.text
+
   # append a row to the csv
   csv_emit(row)
 end
@@ -33,6 +31,14 @@ end
 
 If you paste this into a file called `bestsellers.sinew` and run `sinew bestsellers.sinew`, it will create a `bestsellers.csv` file containing the url, title and img for each bestseller.
 
+## How does Sinew differ from Mechanize?
+
+I'm not an expert on Mechanize, but this question has come up repeatedly and I'll try to address it. Mechanize is a great toolkit and it's better for some situations. Briefly:
+
+* Sinew caches all HTTP requests on disk. That makes it possible to iterate quickly. Crawl once and then continue to work on your recipe. Run the recipe over and over while you tune your CSS selectors and regular expressions.
+* Sinew runs responses through [HTML Tidy](http://tidy.sourceforge.net). This cleans up dirty HTML and makes it easier to parse in many cases, especially if you have to fallback to regular expressions instead of Nokogiri. Unfortunately, this is a common use case in my experience.
+* Sinew outputs CSV files. It does exactly one thing and it does it well - Sinew crawls a site and outputs a CSV file. Mechanize is a more general toolkit.
+
 ## Full Documentation
 
 Full docs are in the wiki:

data/lib/sinew/main.rb

@@ -19,7 +19,7 @@ module Sinew
     def get(url, params = nil)
       _http(url, params, :get)
     end
-    
+
     def post(url, params = nil)
       _http(url, params, :post)
     end
@@ -93,7 +93,7 @@ module Sinew
         s
       end
       $stderr.puts print.ai if @options[:verbose]
-      @csv << row    
+      @csv << row
       @csv.flush
     end
 
@@ -111,7 +111,7 @@ module Sinew
 
     def _run
       @csv = @path = nil
-      
+
       file = @options[:file]
       if !File.exists?(file)
         Util.fatal("#{file} not found")
@@ -164,7 +164,7 @@ module Sinew
       @clean = nil
       @noko = nil
     end
-    
+
     def _normalize(s, key = nil)
       case s
       when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
@@ -175,7 +175,7 @@ module Sinew
         s = s.to_s
       end
       s = TextUtil.untag(s)
-      s = s.convert_accented_entities
+      s = s.convert_accented_html_entities
       s = TextUtil.unent(s)
       s = s.to_ascii.squish
       s

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = "1.0.3"
+  VERSION = "1.0.4"
 end

data/sample.sinew

@@ -1,8 +1,7 @@
-get "http://www.amazon.com/gp/bestsellers/books/ref=sv_b_3"
-noko.css(".zg_itemRow").each do |item|
+get "http://httpbin.org"
+noko.css("ul li a").each do |a|
   row = { }
-  row[:url] = item.css(".zg_title a").first[:href]
-  row[:title] = item.css(".zg_title")
-  row[:img] = item.css(".zg_itemImage_normal img").first[:src]
+  row[:url] = a[:href]
+  row[:title] = a.text
   csv_emit(row)
 end

data/sinew.gemspec

@@ -14,14 +14,14 @@ Gem::Specification.new do |s|
 
   s.rubyforge_project = "sinew"
 
-  s.add_runtime_dependency "activesupport"
+  s.add_runtime_dependency "activesupport", "~> 3.0"
   s.add_runtime_dependency "awesome_print"
   s.add_runtime_dependency "htmlentities"
   s.add_runtime_dependency "nokogiri"
-  s.add_runtime_dependency "stringex"
+  s.add_runtime_dependency "stringex", "~> 2.0"
   s.add_runtime_dependency "trollop"
   s.add_development_dependency "rake"
-  
+
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }

metadata

@@ -1,126 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 1.0.3
-  prerelease: 
+  version: 1.0.4
 platform: ruby
 authors:
 - Adam Doppelt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-19 00:00:00.000000000 Z
+date: 2013-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: htmlentities
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: stringex
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: trollop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
@@ -154,32 +139,31 @@ files:
 - test/test_text_util.rb
 homepage: http://github.com/gurgeous/sinew
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1578176275991755695
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -1578176275991755695
 requirements: []
 rubyforge_project: sinew
-rubygems_version: 1.8.21
+rubygems_version: 2.0.6
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Sinew - structured web crawling using recipes.
-test_files: []
+test_files:
+- test/helper.rb
+- test/test.html
+- test/test_curler.rb
+- test/test_main.rb
+- test/test_nokogiri_ext.rb
+- test/test_text_util.rb