sinew 3.0.1 → 4.0.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Adam Doppelt
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-04 00:00:00.000000000 Z
+date: 2021-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -54,47 +54,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: httpdisk
+  name: faraday-rate_limiter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '4.1'
 - !ruby/object:Gem::Dependency
-  name: scripto
+  name: httpdisk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -134,26 +148,22 @@ files:
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".vscode/extensions.json"
-- ".vscode/settings.json"
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/sinew
 - lib/sinew.rb
-- lib/sinew/connection.rb
-- lib/sinew/connection/log_formatter.rb
-- lib/sinew/connection/rate_limit.rb
-- lib/sinew/core_ext.rb
-- lib/sinew/dsl.rb
+- lib/sinew/args.rb
+- lib/sinew/base.rb
+- lib/sinew/csv.rb
 - lib/sinew/main.rb
+- lib/sinew/middleware/log_formatter.rb
 - lib/sinew/nokogiri_ext.rb
-- lib/sinew/output.rb
-- lib/sinew/request.rb
 - lib/sinew/response.rb
-- lib/sinew/runtime_options.rb
 - lib/sinew/version.rb
+- sample.rb
 - sample.sinew
 - sinew.gemspec
 homepage: http://github.com/gurgeous/sinew

data/.vscode/extensions.json

@@ -1,3 +0,0 @@
-{
-  "recommendations": ["rebornix.Ruby"]
-}

data/.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-  "files.associations": {
-    "*.sinew": "ruby"
-  }
-}

data/lib/sinew/connection.rb

@@ -1,52 +0,0 @@
-require 'faraday'
-require 'faraday-encoding'
-require 'faraday/logging/formatter'
-require 'httpdisk'
-require 'sinew/connection/log_formatter'
-require 'sinew/connection/rate_limit'
-
-module Sinew
-  module Connection
-    def self.create(options:, runtime_options:)
-      connection_options = {}
-      connection_options[:ssl] = { verify: false } if runtime_options.insecure
-
-      Faraday.new(nil, connection_options) do
-        _1.use RateLimit, rate_limit: runtime_options.rate_limit
-
-        # auto-encode form bodies
-        _1.request :url_encoded
-
-        # Before httpdisk so each redirect segment is cached
-        # Keep track of redirect status for logger
-        _1.response :follow_redirects, callback: ->(_old_env, new_env) { new_env[:redirect] = true }
-
-        # set Ruby string encoding based on Content-Type (should be above httpdisk)
-        _1.response :encoding
-
-        # disk caching
-        httpdisk_options = {
-          dir: options[:cache],
-          force: options[:force],
-          force_errors: options[:force_errors],
-        }.merge(runtime_options.httpdisk_options)
-
-        _1.use :httpdisk, httpdisk_options
-
-        # After httpdisk so that only non-cached requests are logged.
-        # Before retry so that we don't log each retry attempt.
-        _1.response :logger, nil, formatter: LogFormatter if !options[:quiet]
-
-        # After httpdisk so transient failures are not cached
-        retry_options = {
-          interval: runtime_options.rate_limit,
-          max: runtime_options.retries,
-          methods: %w[delete get head options patch post put trace],
-          retry_statuses: (500..600).to_a,
-          retry_if: ->(_env, _err) { true },
-        }
-        _1.request :retry, retry_options
-      end
-    end
-  end
-end

data/lib/sinew/connection/rate_limit.rb

@@ -1,29 +0,0 @@
-module Sinew
-  module Connection
-    class RateLimit < Faraday::Middleware
-      attr_reader :rate_limit
-
-      def initialize(app, options = {})
-        super(app)
-
-        @last_request_tm = @current_request_tm = nil
-        @rate_limit = options.fetch(:rate_limit, 1)
-      end
-
-      def on_request(_env)
-        if @last_request_tm
-          sleep = (@last_request_tm + rate_limit) - Time.now
-          sleep(sleep) if sleep > 0
-        end
-
-        @current_request_tm = Time.now
-      end
-
-      def on_complete(env)
-        # Only rate limit on uncached requests
-        @last_request_tm = @current_request_tm unless env[:httpdisk]
-        @current_request_tm = nil
-      end
-    end
-  end
-end

data/lib/sinew/core_ext.rb

@@ -1,59 +0,0 @@
-#
-# A few core extensions brought over from ActiveSupport. These are handy for
-# parsing.
-#
-
-class String
-  def squish
-    dup.squish!
-  end
-
-  def squish!
-    strip!
-    gsub!(/\s+/, ' ')
-    self
-  end
-
-  def first(limit = 1)
-    if limit == 0
-      ''
-    elsif limit >= size
-      dup
-    else
-      self[0..limit - 1]
-    end
-  end
-
-  def last(limit = 1)
-    if limit == 0
-      ''
-    elsif limit >= size
-      dup
-    else
-      self[-limit..]
-    end
-  end
-
-  alias starts_with? start_with?
-  alias ends_with? end_with?
-end
-
-#
-# blank?/present?
-#
-
-class Object
-  def blank?
-    respond_to?(:empty?) ? !!empty? : !self
-  end
-
-  def present?
-    !blank?
-  end
-end
-
-class String
-  def blank?
-    !!(self =~ /\A\s*\z/)
-  end
-end

data/lib/sinew/dsl.rb

@@ -1,115 +0,0 @@
-require 'amazing_print'
-require 'cgi'
-require 'json'
-
-#
-# The DSL available to .sinew files.
-#
-
-module Sinew
-  class DSL
-    # this is used to break out of --limit
-    class LimitError < StandardError; end
-
-    attr_reader :sinew, :uri, :raw, :code, :elapsed
-
-    def initialize(sinew)
-      @sinew = sinew
-    end
-
-    def run
-      tm = Time.now
-      begin
-        recipe = sinew.options[:recipe]
-        instance_eval(File.read(recipe, mode: 'rb'), recipe)
-      rescue LimitError
-        # ignore - this is flow control for --limit
-      end
-      @elapsed = Time.now - tm
-    end
-
-    #
-    # request
-    #
-
-    def get(url, query = {})
-      http('get', url, query: query)
-    end
-
-    def post(url, form = {})
-      body = form
-      headers = {
-        'Content-Type' => 'application/x-www-form-urlencoded',
-      }
-      http('post', url, body: body, headers: headers)
-    end
-
-    def post_json(url, json = {})
-      body = json.to_json
-      headers = {
-        'Content-Type' => 'application/json',
-      }
-      http('post', url, body: body, headers: headers)
-    end
-
-    def http(method, url, options = {})
-      # these need to be cleared before each request
-      %i[@html @noko @xml @json].each do |i|
-        instance_variable_set(i, nil)
-      end
-
-      # fetch and make response available to callers
-      response = sinew.http(method, url, options)
-      @uri, @raw, @code = response.uri, response.body, response.code
-
-      # don't confuse the user
-      nil
-    end
-
-    #
-    # response
-    #
-
-    def html
-      @html ||= begin
-        s = raw.dup
-        # squish!
-        s.squish!
-        # kill whitespace around tags
-        s.gsub!(/ ?<([^>]+)> ?/, '<\\1>')
-        s
-      end
-    end
-
-    def noko
-      @noko ||= Nokogiri::HTML(html)
-    end
-
-    def xml
-      @xml ||= Nokogiri::XML(html)
-    end
-
-    def json
-      @json ||= JSON.parse(raw, symbolize_names: true)
-    end
-
-    def url
-      uri.to_s
-    end
-
-    #
-    # csv
-    #
-
-    def csv_header(*args)
-      sinew.output.header(args)
-    end
-
-    def csv_emit(row)
-      sinew.output.emit(row)
-      if sinew.output.count == sinew.options[:limit]
-        raise LimitError.new
-      end
-    end
-  end
-end

data/lib/sinew/output.rb

@@ -1,133 +0,0 @@
-require 'csv'
-require 'set'
-require 'sterile'
-
-#
-# CSV output.
-#
-
-module Sinew
-  class Output
-    attr_reader :sinew, :columns, :rows, :urls, :csv
-
-    def initialize(sinew)
-      @sinew = sinew
-      @rows = []
-      @urls = Set.new
-    end
-
-    def filename
-      @filename ||= begin
-        recipe = sinew.options[:recipe]
-        ext = File.extname(recipe)
-        if ext.empty?
-          "#{recipe}.csv"
-        else
-          recipe.gsub(ext, '.csv')
-        end
-      end
-    end
-
-    def header(columns)
-      sinew.banner("Writing to #{filename}...") if !sinew.quiet?
-
-      columns = columns.flatten
-      @columns = columns
-
-      # open csv, write header row
-      @csv = CSV.open(filename, 'wb')
-      csv << columns
-    end
-
-    def emit(row)
-      # implicit header if necessary
-      header(row.keys) if !csv
-
-      # don't allow duplicate urls
-      return if dup_url?(row)
-
-      rows << row.dup
-
-      # map columns to row, and normalize along the way
-      print = {}
-      row = columns.map do |i|
-        value = normalize(row[i])
-        print[i] = value if value.present?
-        value
-      end
-
-      # print
-      sinew.vputs print.ai
-
-      csv << row
-      csv.flush
-    end
-
-    def count
-      rows.length
-    end
-
-    def report
-      return if count == 0
-
-      sinew.banner("Got #{count} rows.")
-
-      # calculate counts
-      counts = Hash.new(0)
-      rows.each do |row|
-        row.each_pair { |k, v| counts[k] += 1 if v.present? }
-      end
-      # sort by counts
-      cols = columns.sort_by { |i| [ -counts[i], i ] }
-
-      # report
-      len = cols.map { |i| i.to_s.length }.max
-      fmt = "  %-#{len + 1}s %7d / %-7d %6.1f%%\n"
-      cols.each do |col|
-        $stderr.printf(fmt, col, counts[col], count, counts[col] * 100.0 / count)
-      end
-    end
-
-    def normalize(s)
-      # noko/array/misc => string
-      s = case s
-      when Nokogiri::XML::Element, Nokogiri::XML::NodeSet
-        s.inner_html
-      when Array
-        s.map(&:to_s).join('|')
-      else
-        s.to_s
-      end
-
-      # strip html tags. Note that we replace tags with spaces
-      s = s.gsub(/<[^>]+>/, ' ')
-
-      # Converts MS Word 'smart punctuation' to ASCII
-      s = Sterile.plain_format(s)
-
-      # &aacute; &amp; etc.
-      s = Sterile.decode_entities(s)
-
-      # "šţɽĩɳģ" => "string"
-      s = Sterile.transliterate(s)
-
-      # squish
-      s = s.squish
-
-      s
-    end
-    protected :normalize
-
-    def dup_url?(row)
-      if url = row[:url]
-        if urls.include?(url)
-          sinew.warning("duplicate url: #{url}") if !sinew.quiet?
-          return true
-        end
-        urls << url
-      end
-      false
-    end
-    protected :dup_url?
-  end
-end