sinew 2.0.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fde4bbaa95fce45f3a7ae7aeacab1672615ea1ace852845b0395ce9cce32f861
-  data.tar.gz: 5743800570722443f704c5fc7bc421346cc4f2fb116b8fe9f615bf84fb95f826
+  metadata.gz: df55f2168ff4242fceb31d083b8d16f1046139fa7acb8a9c4fc3f06f7884e113
+  data.tar.gz: 520967eba4ea2d8446690736f2c28d34642b452c0f4e5003dcb89ce373c116e5
 SHA512:
-  metadata.gz: 94009061e7f4e36cc23528be3866c6a372df51a83e096144cafbd923259439e6d44a7d656fbdcfe09c2e059b48deb553caca3ec5d332b33845afd1e91550371a
-  data.tar.gz: 5a8baf7fbdba371065c796c9fdce4312039558b27b4a16676b3df16d5138916ce84db0677dce6ede1831be8040df9112a0491e421813af5e5fd0b0b747d49239
+  metadata.gz: 7443bccc5fc4e1bd112ce50b3d17445f0c21f5b351a6b5be586aadd63f36396312370ec6115d8116701165b3af19fcb852f85d18d0fbe7b4bf0d797312d3fa40
+  data.tar.gz: 9ca4f3c424e021100f518ca4f2231f515b38dbeb402ff4fce07c13a2440f19b0fabcc2a2920e51aa3f6228507550d3c94cb70a46fd22711ba3add90e4fc28004

data/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: test
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      max-parallel: 3
+      matrix:
+        os: [ubuntu, macos]
+        ruby-version: [3.0, 2.7]
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - run: bundle install
+      - run: bundle exec rake test

data/.rubocop.yml

@@ -1,6 +1,6 @@
 AllCops:
-  Exclude:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
+  NewCops: enable
 
 # amd: customizations
 Layout/SpaceInsideArrayLiteralBrackets:
@@ -22,18 +22,19 @@ Style/TrailingCommaInHashLiteral:
 
 # amd: these seem extreme
 Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
-Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
+Lint/SuppressedException: { Enabled: false } # blank rescues are useful
 Naming/BinaryOperatorParameterName: { Enabled: false } # silly
 Naming/HeredocDelimiterNaming: { Enabled: false } # silly
-Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
-Performance/RegexpMatch: { Enabled: false } # =~ is fine
-Performance/TimesMap: { Enabled: false } # silly
+Naming/MethodParameterName: { Enabled: false } # silly
+Style/AccessorGrouping: { Enabled: false } # silly
+Style/AsciiComments: { Enabled: false } # silly
 Style/ClassAndModuleChildren: { Enabled: false } # silly
 Style/Documentation: { Enabled: false } # we don't need this
 Style/DoubleNegation: { Enabled: false } # silly
 Style/FormatStringToken: { Enabled: false } # we like printf here
 Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
 Style/GuardClause: { Enabled: false } # confusing
+Style/HashTransformValues: { Enabled: false } # breaks code by trying to apply to an array
 Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
 Style/NegatedIf: { Enabled: false } # these are fine
 Style/Next: { Enabled: false } # these are fine
@@ -41,7 +42,9 @@ Style/NumericPredicate: { Enabled: false } # silly
 Style/ParallelAssignment: { Enabled: false } # these are fine
 Style/PerlBackrefs: { Enabled: false } # these are fine
 Style/RaiseArgs: { Enabled: false } # silly
+Style/RedundantAssignment: { Enabled: false } # these are usually on purpose
 Style/RegexpLiteral: { Enabled: false } # these are fine
+Style/SoleNestedConditional: { Enabled: false } # these are fine
 Style/StderrPuts: { Enabled: false } # this is awful
 
 # amd: these Metric rules are annoying, disable

data/.vscode/settings.json

@@ -1,15 +1,5 @@
 {
-  "editor.formatOnSave": true,
-  "editor.formatOnSaveTimeout": 1500,
-  "editor.tabSize": 2,
-  "editor.wordSeparators": "`~#$%^&*()-=+[{]}\\|;:'\",.<>/",
   "files.associations": {
     "*.sinew": "ruby"
-  },
-  "files.insertFinalNewline": true,
-  "files.trimTrailingWhitespace": true,
-  "ruby.format": "rubocop",
-  "ruby.lint": {
-    "rubocop": true
   }
 }

data/Gemfile

@@ -1,2 +1,11 @@
 source 'http://rubygems.org'
+
+group :development do
+  gem 'minitest'
+  gem 'mocha'
+  gem 'rake'
+  gem 'rubocop', '~> 0.91.0', require: false
+  gem 'webmock'
+end
+
 gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Adam Doppelt
+Copyright (c) 2012-2018 Adam Doppelt
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,11 +1,13 @@
+[![Build Status](https://github.com/gurgeous/sinew/workflows/test/badge.svg?branch=master)](https://github.com/gurgeous/sinew/action)
+
 ## Welcome to Sinew
 
 Sinew collects structured data from web sites (screen scraping). It provides a Ruby DSL built for crawling, a robust caching system, and integration with [Nokogiri](http://nokogiri.org). Though small, this project is the culmination of years of effort based on crawling systems built at several different companies.
 
 Sinew is distributed as a ruby gem:
 
-```ruby
-gem install sinew
+```sh
+$ gem install sinew
 ```
 
 or in your Gemfile:
@@ -16,39 +18,32 @@ gem 'sinew'
 
 ## Table of Contents
 
-<!---
-markdown-toc --no-firsth1 --maxdepth 1 readme.md
--->
+<!--- markdown-toc --no-firsth1 --maxdepth 1 readme.md -->
 
-* [Sinew 2 (May 2018)](#sinew-2-may-2018)
-* [Quick Example](#quick-example)
-* [How it Works](#how-it-works)
-* [DSL Reference](#dsl-reference)
-* [Hints](#hints)
-* [Limitations](#limitations)
-* [Changelog](#changelog)
+- [Sinew 3](#sinew-3-may-2021)
+- [Quick Example](#quick-example)
+- [How it Works](#how-it-works)
+- [DSL Reference](#dsl-reference)
+- [Hints](#hints)
+- [Limitations](#limitations)
+- [Changelog](#changelog)
+- [License](#license)
 
-## Sinew 2 (May 2018)
+## Sinew 3 (May 2021)
 
-I am pleased to announce the release of Sinew 2.0, a complete rewrite of Sinew for the modern era. Enhancements include:
-
-* Remove dependencies on active_support, curl and tidy. We use HTTParty now.
-* Much easier to customize requests in `.sinew` files. For example, setting User-Agent or Bearer tokens.
-* More operations like `post_json` or the generic `http`. These methods are thing wrappers around HTTParty.
-* New end-of-run report.
-* Tests, rubocop, vscode settings, travis, etc.
+I am pleased to announce the release of Sinew 3.0. Sinew has been streamlined and updated to use the [Faraday](https://lostisland.github.io/faraday/) HTTP client with [sinew](https://github.com/gurgeous/sinew/) middleware for caching.
 
 **Breaking change**
 
-Sinew uses a new format for cached responses. Old Sinew 1 cache directories must be removed before running Sinew again. Sinew 2 might choke on Sinew 1 cache directores when reading `head/`. This is not tested or supported.
+Sinew 3 uses a new format for cached responses. Old Sinew 2 cache directories should be removed before running Sinew again.
 
 ## Quick Example
 
-Here's an example for collecting the links from httpbin.org:
+Here's an example for collecting the links from httpbingo.org:
 
 ```ruby
 # get the url
-get "http://httpbin.org"
+get "http://httpbingo.org"
 
 # use nokogiri to collect links
 noko.css("ul li a").each do |a|
@@ -113,9 +108,9 @@ Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)`
 
 #### Caching
 
-Requests are made using HTTParty, and all responses are cached on disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
+Sinew uses [sinew](https://github.com/gurgeous/sinew/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
 
-The files in `~/.sinew` have nice names and are designed to be human readable. This helps when writing recipes. Sinew never deletes files from the cache - that's up to you!
+Sinew never deletes files from the cache - that's up to you!
 
 Because all requests are cached, you can run Sinew repeatedly with confidence. Run it over and over again while you build up your recipe.
 
@@ -123,65 +118,89 @@ Because all requests are cached, you can run Sinew repeatedly with confidence. R
 
 #### Making requests
 
-* `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
-* `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the POST body.
-* `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
-* `http(method, url, options = {})` - use this for more complex requests
+- `get(url, query = {})` - fetch a url with HTTP GET. URL parameters can be added using `query.
+- `post(url, form = {})` - fetch a url with HTTP POST, using `form` as the URL encoded POST body.
+- `post_json(url, json = {})` - fetch a url with HTTP POST, using `json` as the POST body.
+- `http(method, url, options = {})` - use this for more complex requests
 
 #### Parsing the response
 
-* `raw` - the raw response from the last request
-* `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
-* `noko` - a [Nokogiri](http://nokogiri.org) document built from the tidied HTML
-* `json` - parse the response as JSON, with symbolized keys
-* `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
-* `uri` - the URI of the last request. This is useful for resolving relative URLs.
+These variables are set after each HTTP request.
+
+- `raw` - the raw response from the last request
+- `html` - like `raw`, but with a handful of HTML-specific whitespace cleanups
+- `noko` - parse the response as HTML and return a [Nokogiri](http://nokogiri.org) document
+- `xml` - parse the response as XML and return a [Nokogiri](http://nokogiri.org) document
+- `json` - parse the response as JSON, with symbolized keys
+- `url` - the url of the last request. If the request goes through a redirect, `url` will reflect the final url.
+- `uri` - the URI of the last request. This is useful for resolving relative URLs.
 
 #### Writing CSV
 
-* `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
-* `csv_emit(hash)` - append a row to the CSV file
+- `csv_header(keys)` - specify the columns for CSV output. If you don't call this, Sinew will use the keys from the first call to `csv_emit`.
+- `csv_emit(hash)` - append a row to the CSV file
 
 ## Hints
 
 Writing Sinew recipes is fun and easy. The builtin caching means you can iterate quickly, since you won't have to re-fetch the data. Here are some hints for writing idiomatic recipes:
 
-* Sinew doesn't (yet) check robots.txt - please check it manually.
-* Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
-* In Chrome, `$` in the console is your friend.
-* Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
-* Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
-* Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
-* Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
+- Sinew doesn't (yet) check robots.txt - please check it manually.
+- Prefer Nokogiri over regular expressions wherever possible. Learn [CSS selectors](http://www.w3schools.com/cssref/css_selectors.asp).
+- In Chrome, `$` in the console is your friend.
+- Fallback to regular expressions if you're desperate. Depending on the site, use either `raw` or `html`. `html` is probably your best bet. `raw` is good for crawling Javascript, but it's fragile if the site changes.
+- Learn to love `String#[regexp]`, which is an obscure operator but incredibly handy for Sinew.
+- Laziness is useful. Keep your CSS selectors and regular expressions simple, so maybe they'll work again the next time you need to crawl a site.
+- Don't be afraid to mix CSS selectors, regular expressions, and Ruby:
 
 ```ruby
 noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
 ```
 
-* Debug your recipes using plain old `puts`, or better yet use `ap` from [awesome_print](https://github.com/michaeldv/awesome_print).
-* Run `sinew -v` to get a report on every `csv_emit`. Very handy.
-* Add the CSV files to your git repo. That way you can version them and get diffs!
+- Debug your recipes using plain old `puts`, or better yet use `ap` from [amazing_print](https://github.com/amazing-print/amazing_print).
+- Run `sinew -v` to get a report on every `csv_emit`. Very handy.
+- Add the CSV files to your git repo. That way you can version them and get diffs!
 
 ## Limitations
 
-* Caching is based on URL, so use caution with cookies and other forms of authentication
-* Almost no support for international (non-english) characters
+- Caching is based on URL, so use caution with cookies and other forms of authentication
+- Almost no support for international (non-english) characters
 
 ## Changelog
 
-#### 2.0.0 (May 2018)
+#### 3.0.0 (May 2021)
+
+- Major rewrite of network and caching layer. See above.
+- Use Faraday HTTP client with sinew middleware for caching.
+- Supports multiple proxies (`--proxy host1,host2,...`)
+
+#### 2.0.4 (May 2018)
 
-* Complete rewrite. See above.
+- Handle and cache more errors (too many redirects, connection failures, etc.)
+- Support for adding uri.scheme in generate_cache_key
+- Added status `code`, a peer to `uri`, `raw`, etc.
 
-#### 1.0.3
+#### 2.0.3 (May 2018)
+
+- &amp; now normalizes to & (not and)
+
+#### 2.0.2 (May 2018)
+
+- Support for `--limit`, `--proxy` and the `xml` variable
+- Dedup - warn and ignore if row[:url] has already been emitted
+- Auto gunzip if contents are compressed
+
+#### 2.0.1 (May 2018)
+
+- Support for legacy cached `head` files from Sinew 1
+
+#### 2.0.0 (May 2018)
 
-* Friendlier message if curl or tidy are missing.
+- Complete rewrite. See above.
 
-#### 1.0.2
+#### 1.0.3 (June 2012)
 
-* Remove entity options from tidy, which didn't work on MacOS (thanks Rex!)
+...
 
-#### 1.0.1
+## License
 
-* Trying to run on 1.8 produces a fatal error. Onward!
-* Added first batch of unit tests
+This extension is [licensed under the MIT License](LICENSE).

data/Rakefile

@@ -1,38 +1,53 @@
-require 'bundler'
 require 'bundler/setup'
 
-require 'rake'
 require 'rake/testtask'
 require 'sinew/version'
 
+# load the spec, we use it below
+spec = Gem::Specification.load('sinew.gemspec')
+
 #
-# gem
+# testing
+# don't forget about TESTOPTS="--verbose" rake
+# also: rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
 #
 
-task gem: :build
-task :build do
-  system 'gem build --quiet sinew.gemspec'
+# test (default)
+task default: :test
+
+Rake::TestTask.new do
+  _1.libs << 'test'
+  _1.warning = false # sterile has a few issues here
 end
 
-task install: :build do
-  system "sudo gem install --quiet sinew-#{Sinew::VERSION}.gem"
+# Watch rb files, run tests whenever something changes
+task :watch do
+  # https://superuser.com/a/665208 / https://unix.stackexchange.com/a/42288
+  system("while true; do find . -name '*.rb' | entr -c -d rake; test $? -gt 128 && break; done")
 end
 
-task release: :build do
-  system "git tag -a #{Sinew::VERSION} -m 'Tagging #{Sinew::VERSION}'"
-  system 'git push --tags'
-  system "gem push sinew-#{Sinew::VERSION}.gem"
+#
+# rubocop
+#
+
+task :rubocop do
+  system('bundle exec rubocop -A .', exception: true)
 end
 
 #
-# minitest
+# gem
 #
 
-Rake::TestTask.new(:test) do |t|
-  t.warning = false
+task :build do
+  system 'gem build --quiet sinew.gemspec', exception: true
 end
 
-task default: :test
+task install: :build do
+  system "gem install --quiet sinew-#{spec.version}.gem", exception: true
+end
 
-# to test:
-# block ; rake install && rm -rf ~/.sinew/www.amazon.com && /usr/local/bin/sinew sample.sinew
+task release: %i[rubocop test build] do
+  system "git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true
+  system 'git push --tags', exception: true
+  system "gem push sinew-#{spec.version}.gem", exception: true
+end

data/bin/sinew

@@ -11,11 +11,15 @@ require 'slop'
 
 options = Slop.parse do |o|
   o.banner = 'Usage: sinew [options] <gub.sinew>'
-  o.bool '-v', '--verbose', 'dump every row'
-  o.bool '--version', 'show version'
+  o.bool '-v', '--verbose', 'dump emitted rows while running'
   o.bool '-q', '--quiet', 'suppress some output'
-  o.string '--cache', 'Set the cache directory (defaults to ~/.sinew)', default: "#{ENV['HOME']}/.sinew"
-  o.on '--help' do
+  o.integer '-l', '--limit', 'quit after emitting this many rows'
+  o.string '-c', '--cache', 'set custom cache directory', default: "#{ENV['HOME']}/.sinew"
+  o.bool '--force', "don't read anything from cache (but still write)"
+  o.bool '--force-errors', "don't read errors from cache (but still write)"
+  o.string '--proxy', 'use host[:port] as HTTP proxy'
+  o.bool '--version', 'show version and exit'
+  o.on('--help', 'show this help') do
     puts o
     exit
   end

data/lib/sinew.rb

@@ -1,4 +1,3 @@
-require_relative 'sinew/cache'
 require_relative 'sinew/core_ext'
 require_relative 'sinew/dsl'
 require_relative 'sinew/main'

data/lib/sinew/connection.rb

@@ -0,0 +1,52 @@
+require 'faraday'
+require 'faraday-encoding'
+require 'faraday/logging/formatter'
+require 'httpdisk'
+require 'sinew/connection/log_formatter'
+require 'sinew/connection/rate_limit'
+
+module Sinew
+  module Connection
+    def self.create(options:, runtime_options:)
+      connection_options = {}
+      connection_options[:ssl] = { verify: false } if runtime_options.insecure
+
+      Faraday.new(nil, connection_options) do
+        _1.use RateLimit, rate_limit: runtime_options.rate_limit
+
+        # auto-encode form bodies
+        _1.request :url_encoded
+
+        # Before httpdisk so each redirect segment is cached
+        # Keep track of redirect status for logger
+        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
+
+        # set Ruby string encoding based on Content-Type (should be above httpdisk)
+        _1.response :encoding
+
+        # disk caching
+        httpdisk_options = {
+          dir: options[:cache],
+          force: options[:force],
+          force_errors: options[:force_errors],
+        }.merge(runtime_options.httpdisk_options)
+
+        _1.use :httpdisk, httpdisk_options
+
+        # After httpdisk so that only non-cached requests are logged.
+        # Before retry so that we don't log each retry attempt.
+        _1.response :logger, nil, formatter: LogFormatter if !options[:quiet]
+
+        # After httpdisk so transient failures are not cached
+        retry_options = {
+          interval: runtime_options.rate_limit,
+          max: runtime_options.retries,
+          methods: %w[delete get head options patch post put trace],
+          retry_statuses: (500..600).to_a,
+          retry_if: ->(_env, _err) { true },
+        }
+        _1.request :retry, retry_options
+      end
+    end
+  end
+end