sinew 1.0.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c99e34e066e5c74889a842032d90dbe867e18ac6
-  data.tar.gz: 4cae139e85a63aecb0a5671a43fc8881219236df
+SHA256:
+  metadata.gz: 3f5a493286f0bfdff9cb26bf85e768750edc57f489aec31e049a3c634bdc9074
+  data.tar.gz: bda8c48fe0a1bccd4caf0c3304c248c218839f0d5547269df977dca56bca43c6
 SHA512:
-  metadata.gz: c341f22f607ff0ac03c3f2646d168b6776a21ab9f8b23875a99e9fae71d6cf972a6b22255ec176a3fad68b6d65d26f8cd6d2f34bdfb8a7b6d949bdb61deea6d2
-  data.tar.gz: 8314c4ab14180443050ddc74b828b5833610a86d93fb1e29e3bc2b320e0e6a12cf6b86b96c6a02bc9104f064fd242c2df7df11faf4d2237fa7756bb1865791c5
+  metadata.gz: e9be078939e5e652c1775473322af55065d4ed0a541ba204b84381ba2d44c22b013e19993f500e02f08c9c686c0ee0f0af95cba9f644edd76058d0c1c05180c8
+  data.tar.gz: 9bd2bde0fa0802a0d3b8467322dae695591a6a8928bf86ac1c7fab11f9754a4b66741e41e2ca029f9f9d1c5307c7e750bd29c64231399866ca4acb4d42087e31

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 pkg/*
 rdoc
 sample.csv
+TAGS

data/.rubocop.yml

@@ -0,0 +1,49 @@
+AllCops:
+  Exclude:
+  TargetRubyVersion: 2.3
+
+# amd: customizations
+Layout/SpaceInsideArrayLiteralBrackets:
+  EnforcedStyle: compact
+Layout/CaseIndentation:
+  EnforcedStyle: end
+Layout/EndAlignment:
+  EnforcedStyleAlignWith: variable
+Style/CollectionMethods:
+  Enabled: true
+  PreferredMethods:
+    reduce: inject
+Style/EmptyMethod:
+  Enabled: false
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+
+# amd: these seem extreme
+Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
+Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
+Naming/BinaryOperatorParameterName: { Enabled: false } # silly
+Naming/HeredocDelimiterNaming: { Enabled: false } # silly
+Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
+Performance/RegexpMatch: { Enabled: false } # =~ is fine
+Performance/TimesMap: { Enabled: false } # silly
+Style/ClassAndModuleChildren: { Enabled: false } # silly
+Style/Documentation: { Enabled: false } # we don't need this
+Style/DoubleNegation: { Enabled: false } # silly
+Style/FormatStringToken: { Enabled: false } # we like printf here
+Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
+Style/GuardClause: { Enabled: false } # confusing
+Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
+Style/NegatedIf: { Enabled: false } # these are fine
+Style/Next: { Enabled: false } # these are fine
+Style/NumericPredicate: { Enabled: false } # silly
+Style/ParallelAssignment: { Enabled: false } # these are fine
+Style/PerlBackrefs: { Enabled: false } # these are fine
+Style/RaiseArgs: { Enabled: false } # silly
+Style/RegexpLiteral: { Enabled: false } # these are fine
+Style/StderrPuts: { Enabled: false } # this is awful
+
+# amd: these Metric rules are annoying, disable
+Metrics:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.7
+  - 2.5.1

data/.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["rebornix.Ruby"]
+}

data/.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+  "editor.formatOnSave": true,
+  "editor.formatOnSaveTimeout": 1500,
+  "editor.tabSize": 2,
+  "editor.wordSeparators": "`~#$%^&*()-=+[{]}\\|;:'\",.<>/",
+  "files.associations": {
+    "*.sinew": "ruby"
+  },
+  "files.insertFinalNewline": true,
+  "files.trimTrailingWhitespace": true,
+  "ruby.format": "rubocop",
+  "ruby.lint": {
+    "rubocop": true
+  }
+}

data/Gemfile

@@ -1,2 +1,2 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 gemspec

data/README.md

@@ -2,15 +2,47 @@
 
 Sinew collects structured data from web sites (screen scraping). It provides a Ruby DSL built for crawling, a robust caching system, and integration with [Nokogiri](http://nokogiri.org). Though small, this project is the culmination of years of effort based on crawling systems built at several different companies.
 
-Sinew requires Ruby 1.9, [HTML Tidy](http://tidy.sourceforge.net) and [Curl](http://curl.haxx.se).
-
 Sinew is distributed as a ruby gem:
 
 ```ruby
 gem install sinew
 ```
 
-## Example
+or in your Gemfile:
+
+```ruby
+gem 'sinew'
+```
+
+## Table of Contents
+
+<!---
+markdown-toc --no-firsth1 --maxdepth 1 readme.md
+-->
+
+* [Sinew 2 (May 2018)](#sinew-2-may-2018)
+* [Quick Example](#quick-example)
+* [How it Works](#how-it-works)
+* [DSL Reference](#dsl-reference)
+* [Hints](#hints)
+* [Limitations](#limitations)
+* [Changelog](#changelog)
+
+## Sinew 2 (May 2018)
+
+I am pleased to announce the release of Sinew 2.0, a complete rewrite of Sinew for the modern era. Enhancements include:
+
+* Remove dependencies on active_support, curl and tidy. We use HTTParty now.
+* Much easier to customize requests in `.sinew` files. For example, setting User-Agent or Bearer tokens.
+* More operations like `post_json` or the generic `http`. These methods are thing wrappers around HTTParty.
+* New end-of-run report.
+* Tests, rubocop, vscode settings, travis, etc.
+
+**Breaking change**
+
+Sinew uses a new format for cached responses. Old Sinew 1 cache directories must be removed before running Sinew again. Sinew 2 might choke on Sinew 1 cache directores when reading `head/`. This is not tested or supported.
+
+## Quick Example
 
 Here's an example for collecting the links from httpbin.org:
 
@@ -29,18 +61,127 @@ noko.css("ul li a").each do |a|
 end
 ```
 
-If you paste this into a file called `bestsellers.sinew` and run `sinew bestsellers.sinew`, it will create a `bestsellers.csv` file containing the url, title and img for each bestseller.
+If you paste this into a file called `sample.sinew` and run `sinew sample.sinew`, it will create a `sample.csv` file containing the href and text for each link.
+
+## How it Works
+
+There are three main features provided by Sinew.
+
+#### The Sinew DSL
+
+Sinew uses recipe files to crawl web sites. Recipes have the `.sinew` extension, but they are plain old Ruby. The [Sinew DSL](#dsl) makes crawling easy. Use `get` to make an HTTP GET:
+
+```ruby
+get "https://www.google.com/search?q=darwin"
+get "https://www.google.com/search", q: "charles darwin"
+```
+
+Once you've done a `get`, you have access to the document in a few different formats. In general, it's easiest to use `noko` to automatically parse and interact with the results. If Nokogiri isn't appropriate, you can fall back to regular expressions run against `raw` or `html`. Use `json` if you are expecting a JSON response.
+
+```ruby
+get "https://www.google.com/search?q=darwin"
+
+# pull out the links with nokogiri
+links = noko.css("a").map { |i| i[:href] }
+puts links.inspect
+
+# or, use a regex
+links = html[/<a[^>]+href="([^"]+)/, 1]
+puts links.inspect
+```
+
+#### CSV Output
+
+Recipes output CSV files. To continue the example above:
+
+```ruby
+get "https://www.google.com/search?q=darwin"
+noko.css("a").each do |i|
+  row = { }
+  row[:href] = i[:href]
+  row[:text] = i.text
+  csv_emit row
+end
+```
+
+Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)` appends a row. The values of your hash are converted to strings:
+
+1.  Nokogiri nodes are converted to text
+1.  Arrays are joined with "|", so you can separate them later
+1.  HTML tags, entities and non-ascii chars are removed
+1.  Whitespace is squished
+
+#### Caching
+
+Requests are made using HTTParty, and all responses are cached on disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
+
+The files in `~/.sinew` have nice names and are designed to be human readable. This helps when writing recipes. Sinew never deletes files from the cache - that's up to you!
+
+Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you build up your recipe.
+
+## DSL Reference
+
+#### Making requests
+
+* `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
+* `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the POST body.
+* `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
+* `http(method, url, options = {})` - use this for more complex requests
+
+#### Parsing the response
+
+* `raw` - the raw response from the last request
+* `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
+* `noko` - a [Nokogiri](http://nokogiri.org) document built from the tidied HTML
+* `json` - parse the response as JSON, with symbolized keys
+* `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
+* `uri` - the URI of the last request. This is useful for resolving relative URLs.
+
+#### Writing CSV
+
+* `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
+* `csv_emit(hash)` - append a row to the CSV file
+
+## Hints
+
+Writing Sinew recipes is fun and easy. The builtin caching means you can iterate quickly, since you won't have to re-fetch the data. Here are some hints for writing idiomatic recipes:
+
+* Sinew doesn't (yet) check robots.txt - please check it manually.
+* Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
+* In Chrome, `$` in the console is your friend.
+* Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
+* Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
+* Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
+* Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
+
+```ruby
+noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
+```
+
+* Debug your recipes using plain old `puts`, or better yet use `ap` from [awesome_print](https://github.com/michaeldv/awesome_print).
+* Run `sinew -v` to get a report on every `csv_emit`. Very handy.
+* Add the CSV files to your git repo. That way you can version them and get diffs!
+
+## Limitations
+
+* Caching is based on URL, so use caution with cookies and other forms of authentication
+* Almost no support for international (non-english) characters
+
+## Changelog
+
+#### 2.0.0 (May 2018)
+
+* Complete rewrite. See above.
 
-## How does Sinew differ from Mechanize?
+#### 1.0.3
 
-I'm not an expert on Mechanize, but this question has come up repeatedly and I'll try to address it. Mechanize is a great toolkit and it's better for some situations. Briefly:
+* Friendlier message if curl or tidy are missing.
 
-* Sinew caches all HTTP requests on disk. That makes it possible to iterate quickly. Crawl once and then continue to work on your recipe. Run the recipe over and over while you tune your CSS selectors and regular expressions.
-* Sinew runs responses through [HTML Tidy](http://tidy.sourceforge.net). This cleans up dirty HTML and makes it easier to parse in many cases, especially if you have to fallback to regular expressions instead of Nokogiri. Unfortunately, this is a common use case in my experience.
-* Sinew outputs CSV files. It does exactly one thing and it does it well - Sinew crawls a site and outputs a CSV file. Mechanize is a more general toolkit.
+#### 1.0.2
 
-## Full Documentation
+* Remove entity options from tidy, which didn't work on MacOS (thanks Rex!)
 
-Full docs are in the wiki:
+#### 1.0.1
 
-https://github.com/gurgeous/sinew/wiki
+* Trying to run on 1.8 produces a fatal error. Onward!
+* Added first batch of unit tests

data/Rakefile

@@ -1,27 +1,26 @@
-require "bundler"
-require "bundler/setup"
-require "rake"
-require "rake/testtask"
+require 'bundler'
+require 'bundler/setup'
 
-$LOAD_PATH << File.expand_path("../lib", __FILE__)
-require "sinew/version"
+require 'rake'
+require 'rake/testtask'
+require 'sinew/version'
 
 #
 # gem
 #
 
-task :gem => :build
+task gem: :build
 task :build do
-  system "gem build --quiet sinew.gemspec"
+  system 'gem build --quiet sinew.gemspec'
 end
 
-task :install => :build do
+task install: :build do
   system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
 end
 
-task :release => :build do
+task release: :build do
   system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
-  system "git push --tags"
+  system 'git push --tags'
   system "gem push sinew-#{Sinew::VERSION}.gem"
 end
 
@@ -29,11 +28,11 @@ end
 # minitest
 #
 
-Rake::TestTask.new(:test) do |test|
-  test.libs << "test"
+Rake::TestTask.new(:test) do |t|
+  t.warning = false
 end
 
-task :default => :test
+task default: :test
 
 # to test:
 # block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew

data/bin/sinew

@@ -1,29 +1,49 @@
 #!/usr/bin/env ruby
 
-require "sinew"
-require "trollop"
+$LOAD_PATH.unshift("#{__dir__}/../lib")
 
-# check requirements
-if RUBY_VERSION < "1.9"
-  Sinew::Util.fatal "Sorry, Sinew requires Ruby 1.9 or higher. Please install it."
-end
-%w(curl tidy).each do |i|
-  system("which #{i} > /dev/null")
-  if $? != 0
-    Sinew::Util.fatal "Sorry, Sinew requires #{i}. Please install it."
+require 'sinew'
+require 'slop'
+
+#
+# options
+#
+
+options = Slop.parse do |o|
+  o.banner = 'Usage: sinew [options] <gub.sinew>'
+  o.bool '-v', '--verbose', 'dump every row'
+  o.bool '--version', 'show version'
+  o.bool '-q', '--quiet', 'suppress some output'
+  o.string '--cache', 'Set the cache directory (defaults to ~/.sinew)', default: "#{ENV['HOME']}/.sinew"
+  o.on '--help' do
+    puts o
+    exit
   end
 end
 
-# ARGV
-options = Trollop.options do
-  banner "Usage: sinew [options] <gub.sinew>"
-  opt :cache, "Set the cache directory (defaults to ~/.sinew)"
-  opt :verbose, "Dump every row"
-  opt :quiet, "Be quiet"
+if options[:version]
+  puts Sinew::VERSION
+  exit
 end
-Trollop.die "need a .sinew file to run against" if ARGV.blank?
 
-# now run!
-ARGV.each do |i|
-  Sinew::Main.new(options.merge(file: i))
+#
+# recipe
+#
+
+recipe = options.arguments.first
+if !recipe
+  Scripto.fatal('need a .sinew file to run against')
 end
+if !File.exist?(recipe)
+  Scripto.fatal("#{recipe} not found")
+end
+if options.arguments.length > 1
+  Scripto.fatal('can only run on one .sinew file')
+end
+options = options.to_h.merge(recipe: recipe)
+
+#
+# main
+#
+
+Sinew::Main.new(options).run

data/lib/sinew.rb

@@ -1,6 +1,10 @@
-require "sinew/version"
-require "sinew/util"
-require "sinew/curler"
-require "sinew/nokogiri_ext"
-require "sinew/text_util"
-require "sinew/main"
+require_relative 'sinew/cache'
+require_relative 'sinew/core_ext'
+require_relative 'sinew/dsl'
+require_relative 'sinew/main'
+require_relative 'sinew/nokogiri_ext'
+require_relative 'sinew/output'
+require_relative 'sinew/request'
+require_relative 'sinew/response'
+require_relative 'sinew/runtime_options'
+require_relative 'sinew/version'

data/lib/sinew/cache.rb

@@ -0,0 +1,79 @@
+require 'fileutils'
+require 'tempfile'
+
+#
+# This class handles the caching of http responses on disk.
+#
+
+module Sinew
+  class Cache
+    attr_reader :sinew
+
+    def initialize(sinew)
+      @sinew = sinew
+    end
+
+    def get(request)
+      body = read_if_exist(body_path(request))
+      return nil if !body
+
+      head = read_if_exist(head_path(request))
+      Response.from_cache(request, body, head)
+    end
+
+    def set(response)
+      body_path = body_path(response.request)
+      head_path = head_path(response.request)
+
+      FileUtils.mkdir_p(File.dirname(body_path))
+      FileUtils.mkdir_p(File.dirname(head_path))
+
+      # write body, and head if necessary
+      atomic_write(body_path, response.body)
+      if head_necessary?(response)
+        head = JSON.pretty_generate(response.head_as_json)
+        atomic_write(head_path, head)
+      end
+    end
+
+    def root_dir
+      sinew.options[:cache]
+    end
+    protected :root_dir
+
+    def head_necessary?(response)
+      response.error? || response.redirected?
+    end
+    protected :head_necessary?
+
+    def body_path(request)
+      "#{root_dir}/#{request.cache_key}"
+    end
+    protected :body_path
+
+    def head_path(request)
+      body_path = body_path(request)
+      dir, base = File.dirname(body_path), File.basename(body_path)
+      "#{dir}/head/#{base}"
+    end
+    protected :head_path
+
+    def read_if_exist(path)
+      if File.exist?(path)
+        IO.read(path, mode: 'r:UTF-8')
+      end
+    end
+    protected :read_if_exist
+
+    def atomic_write(path, data)
+      tmp = Tempfile.new('sinew', encoding: 'UTF-8')
+      tmp.write(data)
+      tmp.close
+      FileUtils.chmod(0o644, tmp.path)
+      FileUtils.mv(tmp.path, path)
+    ensure
+      FileUtils.rm(tmp.path, force: true)
+    end
+    protected :atomic_write
+  end
+end