sinew 3.0.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba5558019816540d71e1bb44029f2733aff53649a3f4da72a9905e0a67d06ad9
-  data.tar.gz: 446c245782cad55f1caa36e01b0e8c98748295b47eb5b1911175fff2f79589b2
+  metadata.gz: c16f880ce1bf6454b10c34dd1f071daf1a758eebb52e598262d6357cebb2e9f2
+  data.tar.gz: 4e57acae70775805a96fd5e6bd7ed00ebd5d74dbc0e6daa3348fd9161118d00d
 SHA512:
-  metadata.gz: cef8c1145a21e84f560b44821071ffc7b57ef965167b633e1c837f7b7d9dbfce340b14d1afb2a35891c7b1ed4aa4f08e47ca7405cf382acca7eae855a47d3a71
-  data.tar.gz: 8dc7b67511fc541cccef23b69463abd9a8081f9c75583c1d6c6756dda0561ce7a91be35ee06f7d886ea7488b9005c969e14fa10794cac92475a3063c8968abec
+  metadata.gz: 99bf3da2db47a04dbd6f18dfb7aa3f2bc5f706bbe460633f3f8e3589c85377ae14b35d43c750d384fbdeac0883247b5dbab696127700c6fee71731818df57e74
+  data.tar.gz: 1ecda4412fc9f2384bf01aa38ed40327ac5855c37eb3c82ce65d21779a8a582880c6afaffa5d68954759ddccaa04cec9a6f36f8460c96849722ecafe4ac2ba6e

data/.gitignore

@@ -1,7 +1,5 @@
+.ruby-version
+.yardoc
 *.gem
-.bundle
-Gemfile.lock
-pkg/*
-rdoc
+doc/
 sample.csv
-TAGS

data/.rubocop.yml

@@ -1,52 +1,34 @@
 AllCops:
-  TargetRubyVersion: 2.7
   NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.7
 
-# amd: customizations
-Layout/SpaceInsideArrayLiteralBrackets:
-  EnforcedStyle: compact
-Layout/CaseIndentation:
-  EnforcedStyle: end
-Layout/EndAlignment:
-  EnforcedStyleAlignWith: variable
-Style/CollectionMethods:
-  Enabled: true
-  PreferredMethods:
-    reduce: inject
-Style/EmptyMethod:
-  Enabled: false
-Style/TrailingCommaInArrayLiteral:
-  EnforcedStyleForMultiline: consistent_comma
-Style/TrailingCommaInHashLiteral:
-  EnforcedStyleForMultiline: consistent_comma
-
-# amd: these seem extreme
-Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
-Lint/SuppressedException: { Enabled: false } # blank rescues are useful
-Naming/BinaryOperatorParameterName: { Enabled: false } # silly
-Naming/HeredocDelimiterNaming: { Enabled: false } # silly
-Naming/MethodParameterName: { Enabled: false } # silly
-Style/AccessorGrouping: { Enabled: false } # silly
-Style/AsciiComments: { Enabled: false } # silly
-Style/ClassAndModuleChildren: { Enabled: false } # silly
-Style/Documentation: { Enabled: false } # we don't need this
-Style/DoubleNegation: { Enabled: false } # silly
-Style/FormatStringToken: { Enabled: false } # we like printf here
-Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
-Style/GuardClause: { Enabled: false } # confusing
-Style/HashTransformValues: { Enabled: false } # breaks code by trying to apply to an array
-Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
-Style/NegatedIf: { Enabled: false } # these are fine
-Style/Next: { Enabled: false } # these are fine
-Style/NumericPredicate: { Enabled: false } # silly
-Style/ParallelAssignment: { Enabled: false } # these are fine
-Style/PerlBackrefs: { Enabled: false } # these are fine
-Style/RaiseArgs: { Enabled: false } # silly
-Style/RedundantAssignment: { Enabled: false } # these are usually on purpose
-Style/RegexpLiteral: { Enabled: false } # these are fine
-Style/SoleNestedConditional: { Enabled: false } # these are fine
-Style/StderrPuts: { Enabled: false } # this is awful
+# this is buggy in 2.7.0
+Style/HashTransformValues: { Enabled: false }
 
-# amd: these Metric rules are annoying, disable
-Metrics:
-  Enabled: false
+# minimal personal preference
+Layout/CaseIndentation: { Enabled: false }
+Layout/EndAlignment: { EnforcedStyleAlignWith: variable }
+Lint/AssignmentInCondition: { Enabled: false }
+Lint/NonLocalExitFromIterator: { Enabled: false }
+Metrics: { Enabled: false }
+Naming/HeredocDelimiterNaming: { Enabled: false }
+Naming/MethodParameterName: { Enabled: false }
+Naming/VariableNumber: { Enabled: false }
+Style/AsciiComments: { Enabled: false }
+Style/ClassVars: { Enabled: false }
+Style/CommentAnnotation: { Enabled: false }
+Style/Documentation: { Enabled: false }
+Style/DoubleNegation: { Enabled: false }
+Style/EmptyCaseCondition: { Enabled: false }
+Style/FormatStringToken: { Enabled: false }
+Style/FrozenStringLiteralComment: { Enabled: false }
+Style/GuardClause: { Enabled: false }
+Style/IfUnlessModifier: { Enabled: false }
+Style/NegatedIf: { Enabled: false }
+Style/NumericPredicate: { Enabled: false }
+Style/ParallelAssignment: { Enabled: false }
+Style/StderrPuts: { Enabled: false }
+Style/StringConcatenation: { Enabled: false }
+Style/TrailingCommaInArrayLiteral: { EnforcedStyleForMultiline: consistent_comma }
+Style/TrailingCommaInHashLiteral: { EnforcedStyleForMultiline: consistent_comma }

data/Gemfile

@@ -1,11 +1,11 @@
 source 'http://rubygems.org'
+gemspec
 
-group :development do
+group :development, :test do
   gem 'minitest'
   gem 'mocha'
+  gem 'pry'
   gem 'rake'
-  gem 'rubocop', '~> 0.91.0', require: false
+  gem 'rubocop', '~> 1.18'
   gem 'webmock'
 end
-
-gemspec

data/Gemfile.lock

@@ -0,0 +1,124 @@
+PATH
+  remote: .
+  specs:
+    sinew (4.0.0)
+      amazing_print (~> 1.3)
+      faraday (~> 1.4)
+      faraday-encoding (~> 0)
+      faraday-rate_limiter (~> 0.0)
+      hashie (~> 4.1)
+      httpdisk (~> 0.5)
+      nokogiri (~> 1.11)
+      slop (~> 4.8)
+      sterile (~> 1.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    amazing_print (1.3.0)
+    ast (2.4.2)
+    coderay (1.1.3)
+    content-type (0.0.1)
+      parslet (~> 1.5)
+    crack (0.4.5)
+      rexml
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (1.5.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
+      http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-encoding (0.0.5)
+      faraday
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
+    faraday-patron (1.0.0)
+    faraday-rate_limiter (0.0.4)
+      faraday
+    faraday_middleware (1.0.0)
+      faraday (~> 1.0)
+    hashdiff (1.0.1)
+    hashie (4.1.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    httpdisk (0.5.2)
+      content-type (~> 0.0)
+      faraday (~> 1.4)
+      faraday-cookie_jar (~> 0.0)
+      faraday_middleware (~> 1.0)
+      slop (~> 4.8)
+    method_source (1.0.0)
+    mini_portile2 (2.5.3)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    multipart-post (2.1.1)
+    nokogiri (1.11.7)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
+    parallel (1.20.1)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    parslet (1.8.2)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.6)
+    racc (1.5.2)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.18.3)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.7.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.7.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.4)
+    slop (4.9.1)
+    sterile (1.0.23)
+      nokogiri (>= 1.11.7)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    unicode-display_width (2.0.0)
+    webmock (3.13.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest
+  mocha
+  pry
+  rake
+  rubocop (~> 1.18)
+  sinew!
+  webmock
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -2,17 +2,22 @@
 
 ## Welcome to Sinew
 
-Sinew collects structured data from web sites (screen scraping). It provides a Ruby DSL built for crawling, a robust caching system, and integration with [Nokogiri](http://nokogiri.org). Though small, this project is the culmination of years of effort based on crawling systems built at several different companies.
+Sinew is a Ruby library for collecting data from web sites (scraping). Though small, this project is the culmination of years of effort based on crawling systems built at several different companies. Sinew has been used to crawl millions of websites.
 
-Sinew is distributed as a ruby gem:
+## Key Features
 
-```sh
-$ gem install sinew
-```
+- Robust crawling with the [Faraday](https://lostisland.github.io/faraday/) HTTP client
+- Aggressive caching with [httpdisk](https://github.com/gurgeous/httpdisk/)
+- Easy parsing with HTML cleanup, Nokogiri, JSON, etc.
+- CSV generation for crawled data
 
-or in your Gemfile:
+## Installation
 
 ```ruby
+# install gem
+$ gem install sinew
+
+# or add to your Gemfile:
 gem 'sinew'
 ```
 
@@ -20,22 +25,22 @@ gem 'sinew'
 
 <!--- markdown-toc --no-firsth1 --maxdepth 1 readme.md -->
 
-- [Sinew 3](#sinew-3-may-2021)
+- [Sinew 4](#sinew-4-june-2021)
 - [Quick Example](#quick-example)
 - [How it Works](#how-it-works)
-- [DSL Reference](#dsl-reference)
+- [Reference](#dsreference)
 - [Hints](#hints)
 - [Limitations](#limitations)
 - [Changelog](#changelog)
 - [License](#license)
 
-## Sinew 3 (May 2021)
-
-I am pleased to announce the release of Sinew 3.0. Sinew has been streamlined and updated to use the [Faraday](https://lostisland.github.io/faraday/) HTTP client with [sinew](https://github.com/gurgeous/sinew/) middleware for caching.
+## Sinew 4 (June 2021)
 
 **Breaking change**
 
-Sinew 3 uses a new format for cached responses. Old Sinew 2 cache directories should be removed before running Sinew again.
+We are pleased to announce the release of Sinew 4. The Sinew DSL exposes a single `sinew` method in lieu of the many methods exposed in Sinew 3. Because of this single entry point, Sinew is now much easier to embed in other applications. Also, each Sinew 4 request returns a full Response object to faciliate parallelism.
+
+Sinew uses the [Faraday](https://lostisland.github.io/faraday/) HTTP client with the [httpdisk](https://github.com/gurgeous/httpdisk/) middleware for aggressive caching of responses.
 
 ## Quick Example
 
@@ -43,16 +48,16 @@ Here's an example for collecting the links from httpbingo.org. Paste this into a
 
 ```ruby
 # get the url
-get "http://httpbingo.org"
+response = sinew.get "https://httpbingo.org"
 
 # use nokogiri to collect links
-noko.css("ul li a").each do |a|
+response.noko.css("ul li a").each do |a|
   row = { }
   row[:url] = a[:href]
   row[:title] = a.text
 
   # append a row to the csv
-  csv_emit(row)
+  sinew.csv_emit(row)
 end
 ```
 
@@ -60,26 +65,26 @@ end
 
 There are three main features provided by Sinew.
 
-#### The Sinew DSL
+#### Recipes
 
-Sinew uses recipe files to crawl web sites. Recipes have the `.sinew` extension, but they are plain old Ruby. The [Sinew DSL](#dsl) makes crawling easy. Use `get` to make an HTTP GET:
+Sinew uses recipe files to crawl web sites. Recipes have the .sinew extension, but they are plain old Ruby. Here's a trivial example that calls `get` to make an HTTP GET request:
 
 ```ruby
-get "https://www.google.com/search?q=darwin"
-get "https://www.google.com/search", q: "charles darwin"
+response = sinew.get "https://www.google.com/search?q=darwin"
+response = sinew.get "https://www.google.com/search", q: "charles darwin"
 ```
 
-Once you've done a `get`, you have access to the document in a few different formats. In general, it's easiest to use `noko` to automatically parse and interact with the results. If Nokogiri isn't appropriate, you can fall back to regular expressions run against `raw` or `html`. Use `json` if you are expecting a JSON response.
+Once you've done a `get`, you can access the document in a few different formats. In general, it's easiest to use `noko` to automatically parse and interact with HTML results. If Nokogiri isn't appropriate, fall back to regular expressions run against `body` or `html`. Use `json` if you are expecting a JSON response.
 
 ```ruby
-get "https://www.google.com/search?q=darwin"
+response = sinew.get "https://www.google.com/search?q=darwin"
 
 # pull out the links with nokogiri
-links = noko.css("a").map { |i| i[:href] }
+links = response.noko.css("a").map { _1[:href] }
 puts links.inspect
 
 # or, use a regex
-links = html[/<a[^>]+href="([^"]+)/, 1]
+links = response.html[/<a[^>]+href="([^"]+)/, 1]
 puts links.inspect
 ```
 
@@ -88,16 +93,16 @@ puts links.inspect
 Recipes output CSV files. To continue the example above:
 
 ```ruby
-get "https://www.google.com/search?q=darwin"
-noko.css("a").each do |i|
+response = sinew.get "https://www.google.com/search?q=darwin"
+response.noko.css("a").each do |i|
   row = { }
   row[:href] = i[:href]
   row[:text] = i.text
-  csv_emit row
+  sinew.csv_emit row
 end
 ```
 
-Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)` appends a row. The values of your hash are converted to strings:
+Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)` appends a row. The values of your hash are cleaned up and converted to strings:
 
 1.  Nokogiri nodes are converted to text
 1.  Arrays are joined with "|", so you can separate them later
@@ -108,35 +113,84 @@ Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)`
 
 Sinew uses [httpdisk](https://github.com/gurgeous/httpdisk/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
 
-Sinew never deletes files from the cache - that's up to you!
+Sinew never deletes files from the cache - that's up to you! Sinew has various command line options to refresh the cache. See `--expires`, `--force` and `--force-errors`.
 
-Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you build up your recipe.
+Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you work on your recipe.
 
-## DSL Reference
+## Running Sinew
 
-#### Making requests
+The `sinew` command line has many useful options. You will be using this command many times as you iterate on your recipe:
 
-- `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
-- `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the URL encoded POST body.
-- `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
-- `http(method, url, options = {})` - use this for more complex requests
+```sh
+$ bin/sinew --help
+Usage: sinew [options] [recipe]
+    -l, --limit     quit after emitting this many rows
+    --proxy         use host[:port] as HTTP proxy
+    --timeout       maximum time allowed for the transfer
+    -s, --silent    suppress some output
+    -v, --verbose   dump emitted rows while running
+From httpdisk:
+    --dir           set custom cache directory
+    --expires       when to expire cached requests (ex: 1h, 2d, 3w)
+    --force         don't read anything from cache (but still write)
+    --force-errors  don't read errors from cache (but still write)
+```
+
+`Sinew` also has many runtime options that can be set by in your recipe. For example:
+
+```ruby
+sinew.options[:headers] = { 'User-Agent' => 'xyz' }
+
+...
+```
+
+Here is the list of available options for `Sinew`:
+
+- **headers** - default HTTP headers to use on every request
+- **ignore_params** - ignore these query params when generating httpdisk cache keys
+- **insecure** - ignore SSL errors
+- **params** - default query parameters to use on every request
+- **rate_limit** - minimum time between network requests
+- **retries** - number of times to retry each failed request
+- **url_prefix** - deafult URL base to use on every request
+
+## Reference
+
+#### Making HTTP requests
+
+- `sinew.get(url, params = nil, headers = nil)` - fetch a url with GET
+- `sinew.post(url, body = nil, headers = nil)` - fetch a url with POST, using `form` as the URL encoded POST body.
+- `sinew.post_json(url, body = nil, headers = nil)` - fetch a url with POST, using `json` as the POST body.
 
 #### Parsing the response
 
-These variables are set after each HTTP request.
+Each request method returns a `Sinew::Response`. The response has several helpers to make parsing easier:
 
-- `raw` - the raw response from the last request
-- `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
-- `noko` - parse the response as HTML and return a [Nokogiri](http://nokogiri.org) document
-- `xml` - parse the response as XML and return a [Nokogiri](http://nokogiri.org) document
-- `json` - parse the response as JSON, with symbolized keys
-- `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
-- `uri` - the URI of the last request. This is useful for resolving relative URLs.
+- `body` - the raw body
+- `html` - like `body`, but with a handful of HTML-specific whitespace cleanups
+- `noko` - parse as HTML and return a [Nokogiri](http://nokogiri.org) document
+- `xml` - parse as XML and return a [Nokogiri](http://nokogiri.org) document
+- `json` - parse as JSON, with symbolized keys
+- `mash` - parse as JSON and return a [Hashie::Mash](https://github.com/hashie/hashie#mash)
+- `url` - the url of the request. If the request goes through a redirect, `url` will reflect the final url.
 
 #### Writing CSV
 
-- `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
-- `csv_emit(hash)` - append a row to the CSV file
+- `sinew.csv_header(columns)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `sinew.csv_emit`.
+- `sinew.csv_emit(hash)` - append a row to the CSV file
+
+#### Advanced: Cache
+
+Sinew has some advanced helpers for checking the httpdisk cache. For the following methods, `body` hashes default to form body type.
+
+- `sinew.cached?(method, url, params = nil, body = nil)` - check if request is cached
+- `sinew.uncache(method, url, params = nil, body = nil)` - remove cache file, if any
+- `sinew.status(method, url, params = nil, body = nil)` - get httpdisk status
+
+Plus some caching helpers in Sinew::Response:
+
+- `diskpath` - the location on disk for the cached httpdisk response
+- `uncache` - remove cache file for this response
 
 ## Hints
 
@@ -145,13 +199,15 @@ Writing Sinew recipes is fun and easy. The builtin caching means you can iterate
 - Sinew doesn't (yet) check robots.txt - please check it manually.
 - Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
 - In Chrome, `$` in the console is your friend.
-- Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
+- Fallback to regular expressions if you're desperate. Depending on the site, use either `body` or `html`. `html` is probably your best bet. `body` is good for crawling Javascript, but it's fragile if the site changes.
 - Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
 - Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
 - Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
 
 ```ruby
-noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
+noko.css("table")[4].css("td").select do
+  _1[:width].to_i > 80
+end.map(&:text)
 ```
 
 - Debug your recipes using plain old `puts`, or better yet use `ap` from [amazing_print](https://github.com/amazing-print/amazing_print).
@@ -165,6 +221,11 @@ noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
 
 ## Changelog
 
+#### 4.0.0 (July 2021)
+
+- Rewritten to use simpler DSL
+- Upgraded to httpdisk 0.5 to take advantage of the new encoding support
+
 #### 3.0.0 (May 2021)
 
 - Major rewrite of network and caching layer. See above.