sinew 3.0.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df55f2168ff4242fceb31d083b8d16f1046139fa7acb8a9c4fc3f06f7884e113
-  data.tar.gz: 520967eba4ea2d8446690736f2c28d34642b452c0f4e5003dcb89ce373c116e5
+  metadata.gz: ba5558019816540d71e1bb44029f2733aff53649a3f4da72a9905e0a67d06ad9
+  data.tar.gz: 446c245782cad55f1caa36e01b0e8c98748295b47eb5b1911175fff2f79589b2
 SHA512:
-  metadata.gz: 7443bccc5fc4e1bd112ce50b3d17445f0c21f5b351a6b5be586aadd63f36396312370ec6115d8116701165b3af19fcb852f85d18d0fbe7b4bf0d797312d3fa40
-  data.tar.gz: 9ca4f3c424e021100f518ca4f2231f515b38dbeb402ff4fce07c13a2440f19b0fabcc2a2920e51aa3f6228507550d3c94cb70a46fd22711ba3add90e4fc28004
+  metadata.gz: cef8c1145a21e84f560b44821071ffc7b57ef965167b633e1c837f7b7d9dbfce340b14d1afb2a35891c7b1ed4aa4f08e47ca7405cf382acca7eae855a47d3a71
+  data.tar.gz: 8dc7b67511fc541cccef23b69463abd9a8081f9c75583c1d6c6756dda0561ce7a91be35ee06f7d886ea7488b9005c969e14fa10794cac92475a3063c8968abec

data/README.md

@@ -39,7 +39,7 @@ Sinew 3 uses a new format for cached responses. Old Sinew 2 cache directories sh
 
 ## Quick Example
 
-Here's an example for collecting the links from httpbingo.org:
+Here's an example for collecting the links from httpbingo.org. Paste this into a file called `sample.sinew` and run `sinew sample.sinew`. It will create a `sample.csv` file containing the href and text for each link:
 
 ```ruby
 # get the url
@@ -56,8 +56,6 @@ noko.css("ul li a").each do |a|
 end
 ```
 
-If you paste this into a file called `sample.sinew` and run `sinew sample.sinew`, it will create a `sample.csv` file containing the href and text for each link.
-
 ## How it Works
 
 There are three main features provided by Sinew.
@@ -108,7 +106,7 @@ Sinew creates a CSV file with the same name as the recipe, and `csv_emit(hash)`
 
 #### Caching
 
-Sinew uses [sinew](https://github.com/gurgeous/sinew/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
+Sinew uses [httpdisk](https://github.com/gurgeous/httpdisk/) to aggressively cache all HTTP responses to disk in `~/.sinew`. Error responses are cached as well. Each URL will be hit exactly once, and requests are rate limited to one per second. Sinew tries to be polite.
 
 Sinew never deletes files from the cache - that's up to you!
 

data/lib/sinew/main.rb

@@ -34,7 +34,7 @@ module Sinew
     #
 
     def http(method, url, options = {})
-      request = Request.new(self, method, url, options)
+      request = Request.new(method, url, request_options(options))
       response = request.perform(connection)
 
       # always log error messages
@@ -62,6 +62,32 @@ module Sinew
     # helpers
     #
 
+    def request_options(options)
+      options.dup.tap do |req|
+        req[:headers] = {}.tap do |h|
+          [ runtime_options.headers, options[:headers]].each do
+            h.merge!(_1) if _1
+          end
+        end
+        req[:proxy] = random_proxy
+      end
+    end
+    protected :request_options
+
+    PROXY_RE = /\A#{URI::PATTERN::HOST}(:\d+)?\Z/.freeze
+
+    def random_proxy
+      return if !options[:proxy]
+
+      proxy = options[:proxy].split(',').sample
+      if proxy !~ PROXY_RE
+        raise ArgumentError, "invalid proxy #{proxy.inspect}, should be host[:port]"
+      end
+
+      "http://#{proxy}"
+    end
+    protected :random_proxy
+
     def footer
       output.report
       finished = output.count > 0 ? "Finished #{output.filename}" : 'Finished'

data/lib/sinew/request.rb

@@ -1,5 +1,4 @@
-require 'digest/md5'
-require 'htmlentities'
+require 'sterile'
 
 #
 # Process a single HTTP request.
@@ -9,42 +8,29 @@ module Sinew
   class Error < StandardError; end
 
   class Request
-    HTML_ENTITIES = HTMLEntities.new
     VALID_METHODS = %w[get post patch put delete head options].freeze
     METHODS_WITH_BODY = %w[patch post put].freeze
 
-    attr_reader :sinew, :method, :uri, :options
+    attr_reader :method, :options, :uri
 
     # Supported options:
     #  body: Body of http post
     #  headers: Hash of HTTP headers (combined with runtime_options.headers)
     #  query: Hash of query parameters to add to url
-    def initialize(sinew, method, url, options = {})
-      @sinew = sinew
+    def initialize(method, url, options = {})
       @method = method
       @options = options.dup
       @uri = parse_url(url)
     end
 
-    def proxy
-      @proxy ||= begin
-        if proxies = sinew.options[:proxy]
-          proxies.split(',').sample
-        end
-      end
-    end
-
     # run the request, return the result
     def perform(connection)
       validate!
 
-      headers = sinew.runtime_options.headers
-      headers = headers.merge(options[:headers]) if options[:headers]
-
       body = options.delete(:body)
-
-      fday_response = connection.send(method, uri, body, headers) do
-        _1.options[:proxy] = proxy
+      fday_response = connection.send(method, uri, body) do
+        _1.headers.update(options[:headers]) if options[:headers]
+        _1.options[:proxy] = options[:proxy]
       end
 
       Response.from_network(self, fday_response)
@@ -52,10 +38,10 @@ module Sinew
 
     # We accept sloppy urls and attempt to clean them up
     def parse_url(url)
-      s = url
+      s = url.to_s
 
       # remove entities
-      s = HTML_ENTITIES.decode(s)
+      s = Sterile.decode_entities(s)
 
       # fix a couple of common encoding bugs
       s = s.gsub(' ', '%20')
@@ -96,32 +82,5 @@ module Sinew
       headers && headers['Content-Type']
     end
     protected :content_type
-
-    def form?
-      content_type == 'application/x-www-form-urlencoded'
-    end
-    protected :form?
-
-    def pathify(s)
-      # remove leading slash
-      s = s.gsub(/^\//, '')
-      # .. => comma
-      s = s.gsub('..', ',')
-      # query separators => comma
-      s = s.gsub(/[?\/&]/, ',')
-      # ,, => comma
-      s = s.gsub(',,', ',')
-      # encode invalid path chars
-      s = s.gsub(/[^A-Za-z0-9_.,=-]/) do |i|
-        hex = i.unpack1('H2')
-        "%#{hex}"
-      end
-      # handle empty case
-      s = '_root_' if s.blank?
-      # always downcase
-      s = s.downcase
-      s
-    end
-    protected :pathify
   end
 end

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '3.0.0'.freeze
+  VERSION = '3.0.1'.freeze
 end

data/sinew.gemspec

@@ -24,7 +24,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'amazing_print', '~> 1.3'
   s.add_runtime_dependency 'faraday', '~> 1.4'
   s.add_runtime_dependency 'faraday-encoding', '~> 0'
-  s.add_runtime_dependency 'htmlentities', '~> 4.3'
   s.add_runtime_dependency 'httpdisk', '~> 0'
   s.add_runtime_dependency 'nokogiri', '~> 1.11'
   s.add_runtime_dependency 'scripto', '~> 0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.1
 platform: ruby
 authors:
 - Adam Doppelt
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-11 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -53,20 +53,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: htmlentities
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.3'
 - !ruby/object:Gem::Dependency
   name: httpdisk
   requirement: !ruby/object:Gem::Requirement