sinew 3.0.1 → 4.0.1

data/lib/sinew/response.rb

@@ -1,72 +1,61 @@
-require 'stringio'
-require 'zlib'
-
-#
-# An HTTP response.
-#
+require "delegate"
+require "hashie/mash"
+require "json"
+require "nokogiri"
 
 module Sinew
-  class Response
-    attr_accessor :request, :uri, :body, :code, :headers
+  # A wrapper around Faraday::Response, with some parsing helpers.
+  class Response < SimpleDelegator
+    # Like body, but tries to cleanup whitespace around HTML for easier parsing.
+    def html
+      @html ||= body.dup.tap do
+        # fix invalid utf8
+        if _1.encoding == Encoding::UTF_8
+          _1.encode!("UTF-8", invalid: :replace, undef: :replace, replace: "?")
+        end
 
-    #
-    # factory methods
-    #
+        # squish
+        _1.strip!
+        _1.gsub!(/\s+/, " ")
 
-    def self.from_network(request, fday_response)
-      Response.new.tap do
-        _1.request = request
-        _1.uri = fday_response.env.url
-        _1.code = fday_response.status
-        _1.headers = fday_response.headers.to_h
-        _1.body = process_body(fday_response)
+        # kill whitespace around tags
+        _1.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
       end
     end
 
-    # helper for decoding bodies before parsing
-    def self.process_body(response)
-      body = response.body
-
-      # inflate if necessary
-      bits = body[0, 10].force_encoding('BINARY')
-      if bits =~ /\A\x1f\x8b/n
-        body = Zlib::GzipReader.new(StringIO.new(body)).read
-      end
-
-      # force to utf-8 if we think this could be text
-      if body.encoding != Encoding::UTF_8
-        if content_type = response.headers['content-type']
-          if content_type =~ /\b(html|javascript|json|text|xml)\b/
-            body = body.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
-          end
-        end
-      end
+    # Return body as JSON
+    def json
+      @json ||= JSON.parse(body, symbolize_names: true)
+    end
 
-      body
+    # Return JSON body as Hashie::Mash
+    def mash
+      @mash ||= Hashie::Mash.new(json)
     end
 
-    #
-    # accessors
-    #
+    # Return body HTML as Nokogiri document
+    def noko
+      @noko ||= Nokogiri::HTML(html)
+    end
 
-    def error?
-      code >= 400
+    # Return body XML as Nokogiri document
+    def xml
+      @xml ||= Nokogiri::XML(html)
     end
 
-    def error_500?
-      code / 100 >= 5
+    # Return the final URI for the request, after redirects
+    def url
+      env.url
     end
 
-    def redirected?
-      request.uri != uri
+    # Return the cache diskpath for this response
+    def diskpath
+      env[:httpdisk_diskpath]
     end
 
-    def head_as_json
-      {
-        uri: uri,
-        code: code,
-        headers: headers,
-      }
+    # Remove cached response from disk, if any
+    def uncache
+      File.unlink(diskpath) if File.exist?(diskpath)
     end
   end
 end

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '3.0.1'.freeze
+  VERSION = "4.0.1".freeze
 end

data/lib/sinew.rb

@@ -1,9 +1,23 @@
-require_relative 'sinew/core_ext'
-require_relative 'sinew/dsl'
-require_relative 'sinew/main'
-require_relative 'sinew/nokogiri_ext'
-require_relative 'sinew/output'
-require_relative 'sinew/request'
-require_relative 'sinew/response'
-require_relative 'sinew/runtime_options'
-require_relative 'sinew/version'
+# sinew
+require "sinew/args"
+require "sinew/base"
+require "sinew/csv"
+require "sinew/main"
+require "sinew/nokogiri_ext"
+require "sinew/response"
+require "sinew/version"
+
+# custom faraday middleware
+require "sinew/middleware/log_formatter"
+
+module Sinew
+  # flow control for --limit
+  class LimitError < StandardError; end
+
+  # shortcut for Sinew::Base.new
+  class << self
+    def new(**args)
+      Sinew::Base.new(**args)
+    end
+  end
+end

data/sample.rb

@@ -0,0 +1,13 @@
+require_relative "lib/sinew"
+
+sinew = Sinew.new(output: "sample.csv", verbose: true)
+
+response = sinew.get "http://httpbingo.org"
+response.noko.css("ul li a").each do |a|
+  row = {}
+  row[:url] = a[:href]
+  row[:title] = a.text
+  sinew.csv_emit(row)
+end
+
+sinew.get "http://httpbingo.org/redirect/2"

data/sample.sinew

@@ -1,9 +1,9 @@
-get 'http://httpbingo.org'
-noko.css('ul li a').each do |a|
+response = sinew.get 'http://httpbingo.org'
+response.noko.css('ul li a').each do |a|
   row = {}
   row[:url] = a[:href]
   row[:title] = a.text
-  csv_emit(row)
+  sinew.csv_emit(row)
 end
 
-get 'http://httpbingo.org/redirect/2'
+sinew.get 'http://httpbingo.org/redirect/2'

data/sinew.gemspec

@@ -1,32 +1,36 @@
 $LOAD_PATH.unshift("#{__dir__}/lib")
 
-require 'sinew/version'
+require "sinew/version"
 
 Gem::Specification.new do |s|
-  s.name        = 'sinew'
-  s.version     = Sinew::VERSION
-  s.license     = 'MIT'
-  s.authors     = [ 'Adam Doppelt', 'Nathan Kriege' ]
-  s.email       = [ 'amd@gurge.com' ]
-  s.homepage    = 'http://github.com/gurgeous/sinew'
-  s.summary     = 'Sinew - structured web crawling using recipes.'
-  s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
-  s.required_ruby_version = '>= 2.7'
+  s.name = "sinew"
+  s.version = Sinew::VERSION
+  s.authors = ["Adam Doppelt", "Nathan Kriege"]
+  s.email = ["amd@gurge.com"]
+
+  s.summary = "Sinew - structured web crawling using recipes."
+  s.description = "Crawl web sites easily using ruby recipes, with caching and nokogiri."
+  s.homepage = "http://github.com/gurgeous/sinew"
+  s.license = "MIT"
+  s.required_ruby_version = ">= 3.1"
 
   # what's in the gem?
   s.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { _1.match(%r{^test/}) }
   end
-  s.bindir = 'bin'
+  s.bindir = "bin"
   s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
-  s.require_paths = [ 'lib' ]
+  s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'amazing_print', '~> 1.3'
-  s.add_runtime_dependency 'faraday', '~> 1.4'
-  s.add_runtime_dependency 'faraday-encoding', '~> 0'
-  s.add_runtime_dependency 'httpdisk', '~> 0'
-  s.add_runtime_dependency 'nokogiri', '~> 1.11'
-  s.add_runtime_dependency 'scripto', '~> 0'
-  s.add_runtime_dependency 'slop', '~> 4.8'
-  s.add_runtime_dependency 'sterile', '~> 1.0'
+  # gem dependencies
+  s.add_dependency "amazing_print", "~> 1.5"
+  s.add_dependency "faraday", "~> 2.7"
+  s.add_dependency "faraday-encoding", "~> 0.0"
+  s.add_dependency "faraday-rate_limiter", "~> 0.0"
+  s.add_dependency "faraday-retry", "~> 2.0"
+  s.add_dependency "hashie", "~> 5.0"
+  s.add_dependency "httpdisk", "~> 1.0"
+  s.add_dependency "nokogiri", "~> 1.15"
+  s.add_dependency "slop", "~> 4.10"
+  s.add_dependency "sterile", "~> 1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 4.0.1
 platform: ruby
 authors:
 - Adam Doppelt
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-04 00:00:00.000000000 Z
+date: 2023-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -17,98 +17,126 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: faraday-encoding
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: httpdisk
+  name: faraday-rate_limiter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: httpdisk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: scripto
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.8'
+        version: '4.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.8'
+        version: '4.10'
 - !ruby/object:Gem::Dependency
   name: sterile
   requirement: !ruby/object:Gem::Requirement
@@ -134,26 +162,23 @@ files:
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".vscode/extensions.json"
-- ".vscode/settings.json"
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/sinew
+- justfile
 - lib/sinew.rb
-- lib/sinew/connection.rb
-- lib/sinew/connection/log_formatter.rb
-- lib/sinew/connection/rate_limit.rb
-- lib/sinew/core_ext.rb
-- lib/sinew/dsl.rb
+- lib/sinew/args.rb
+- lib/sinew/base.rb
+- lib/sinew/csv.rb
 - lib/sinew/main.rb
+- lib/sinew/middleware/log_formatter.rb
 - lib/sinew/nokogiri_ext.rb
-- lib/sinew/output.rb
-- lib/sinew/request.rb
 - lib/sinew/response.rb
-- lib/sinew/runtime_options.rb
 - lib/sinew/version.rb
+- sample.rb
 - sample.sinew
 - sinew.gemspec
 homepage: http://github.com/gurgeous/sinew
@@ -168,14 +193,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.

data/.vscode/extensions.json

@@ -1,3 +0,0 @@
-{
-  "recommendations": ["rebornix.Ruby"]
-}

data/.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-  "files.associations": {
-    "*.sinew": "ruby"
-  }
-}

data/lib/sinew/connection/rate_limit.rb

@@ -1,29 +0,0 @@
-module Sinew
-  module Connection
-    class RateLimit < Faraday::Middleware
-      attr_reader :rate_limit
-
-      def initialize(app, options = {})
-        super(app)
-
-        @last_request_tm = @current_request_tm = nil
-        @rate_limit = options.fetch(:rate_limit, 1)
-      end
-
-      def on_request(_env)
-        if @last_request_tm
-          sleep = (@last_request_tm + rate_limit) - Time.now
-          sleep(sleep) if sleep > 0
-        end
-
-        @current_request_tm = Time.now
-      end
-
-      def on_complete(env)
-        # Only rate limit on uncached requests
-        @last_request_tm = @current_request_tm unless env[:httpdisk]
-        @current_request_tm = nil
-      end
-    end
-  end
-end

data/lib/sinew/connection.rb

@@ -1,52 +0,0 @@
-require 'faraday'
-require 'faraday-encoding'
-require 'faraday/logging/formatter'
-require 'httpdisk'
-require 'sinew/connection/log_formatter'
-require 'sinew/connection/rate_limit'
-
-module Sinew
-  module Connection
-    def self.create(options:, runtime_options:)
-      connection_options = {}
-      connection_options[:ssl] = { verify: false } if runtime_options.insecure
-
-      Faraday.new(nil, connection_options) do
-        _1.use RateLimit, rate_limit: runtime_options.rate_limit
-
-        # auto-encode form bodies
-        _1.request :url_encoded
-
-        # Before httpdisk so each redirect segment is cached
-        # Keep track of redirect status for logger
-        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
-
-        # set Ruby string encoding based on Content-Type (should be above httpdisk)
-        _1.response :encoding
-
-        # disk caching
-        httpdisk_options = {
-          dir: options[:cache],
-          force: options[:force],
-          force_errors: options[:force_errors],
-        }.merge(runtime_options.httpdisk_options)
-
-        _1.use :httpdisk, httpdisk_options
-
-        # After httpdisk so that only non-cached requests are logged.
-        # Before retry so that we don't log each retry attempt.
-        _1.response :logger, nil, formatter: LogFormatter if !options[:quiet]
-
-        # After httpdisk so transient failures are not cached
-        retry_options = {
-          interval: runtime_options.rate_limit,
-          max: runtime_options.retries,
-          methods: %w[delete get head options patch post put trace],
-          retry_statuses: (500..600).to_a,
-          retry_if: ->(_env, _err) { true },
-        }
-        _1.request :retry, retry_options
-      end
-    end
-  end
-end

data/lib/sinew/core_ext.rb

@@ -1,59 +0,0 @@
-#
-# A few core extensions brought over from ActiveSupport. These are handy for
-# parsing.
-#
-
-class String
-  def squish
-    dup.squish!
-  end
-
-  def squish!
-    strip!
-    gsub!(/\s+/, ' ')
-    self
-  end
-
-  def first(limit = 1)
-    if limit == 0
-      ''
-    elsif limit >= size
-      dup
-    else
-      self[0..limit - 1]
-    end
-  end
-
-  def last(limit = 1)
-    if limit == 0
-      ''
-    elsif limit >= size
-      dup
-    else
-      self[-limit..]
-    end
-  end
-
-  alias starts_with? start_with?
-  alias ends_with? end_with?
-end
-
-#
-# blank?/present?
-#
-
-class Object
-  def blank?
-    respond_to?(:empty?) ? !!empty? : !self
-  end
-
-  def present?
-    !blank?
-  end
-end
-
-class String
-  def blank?
-    !!(self =~ /\A\s*\z/)
-  end
-end