sinew 2.0.3 → 4.0.0

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '2.0.3'.freeze
+  VERSION = '4.0.0'.freeze
 end

data/sample.rb

@@ -0,0 +1,13 @@
+require_relative 'lib/sinew'
+
+sinew = Sinew.new(output: 'sample.csv', verbose: true)
+
+response = sinew.get 'http://httpbingo.org'
+response.noko.css('ul li a').each do |a|
+  row = {}
+  row[:url] = a[:href]
+  row[:title] = a.text
+  sinew.csv_emit(row)
+end
+
+sinew.get 'http://httpbingo.org/redirect/2'

data/sample.sinew

@@ -1,9 +1,9 @@
-get 'http://httpbin.org'
-noko.css('ul li a').each do |a|
+response = sinew.get 'http://httpbingo.org'
+response.noko.css('ul li a').each do |a|
   row = {}
   row[:url] = a[:href]
   row[:title] = a.text
-  csv_emit(row)
+  sinew.csv_emit(row)
 end
 
-get 'http://httpbin.org/redirect/2'
+sinew.get 'http://httpbingo.org/redirect/2'

data/sinew.gemspec

@@ -3,32 +3,33 @@ $LOAD_PATH.unshift("#{__dir__}/lib")
 require 'sinew/version'
 
 Gem::Specification.new do |s|
-  s.name        = 'sinew'
-  s.version     = Sinew::VERSION
-  s.platform    = Gem::Platform::RUBY
-  s.license     = 'MIT'
-  s.authors     = [ 'Adam Doppelt' ]
-  s.email       = [ 'amd@gurge.com' ]
-  s.homepage    = 'http://github.com/gurgeous/sinew'
-  s.summary     = 'Sinew - structured web crawling using recipes.'
-  s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
-  s.required_ruby_version = '~> 2.3'
+  s.name = 'sinew'
+  s.version = Sinew::VERSION
+  s.authors = ['Adam Doppelt', 'Nathan Kriege']
+  s.email = ['amd@gurge.com']
 
-  s.rubyforge_project = 'sinew'
+  s.summary = 'Sinew - structured web crawling using recipes.'
+  s.description = 'Crawl web sites easily using ruby recipes, with caching and nokogiri.'
+  s.homepage = 'http://github.com/gurgeous/sinew'
+  s.license = 'MIT'
+  s.required_ruby_version = '>= 2.7'
 
-  s.add_runtime_dependency 'awesome_print', '~> 1.8'
-  s.add_runtime_dependency 'htmlentities', '~> 4.3'
-  s.add_runtime_dependency 'httparty', '~> 0.16'
-  s.add_runtime_dependency 'nokogiri', '~> 1.8'
-  s.add_runtime_dependency 'scripto', '~> 0'
-  s.add_runtime_dependency 'slop', '~> 4.6'
-  s.add_runtime_dependency 'stringex', '~> 2.8'
-  s.add_development_dependency 'minitest', '~> 5.11'
-  s.add_development_dependency 'rake', '~> 12.3'
-  s.add_development_dependency 'webmock', '~> 3.4'
+  # what's in the gem?
+  s.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { _1.match(%r{^test/}) }
+  end
+  s.bindir = 'bin'
+  s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
+  s.require_paths = ['lib']
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- test/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
-  s.require_paths = [ 'lib' ]
+  # gem dependencies
+  s.add_dependency 'amazing_print', '~> 1.3'
+  s.add_dependency 'faraday', '~> 1.4'
+  s.add_dependency 'faraday-encoding', '~> 0'
+  s.add_dependency 'faraday-rate_limiter', '~> 0.0'
+  s.add_dependency 'hashie', '~> 4.1'
+  s.add_dependency 'httpdisk', '~> 0.5'
+  s.add_dependency 'nokogiri', '~> 1.11'
+  s.add_dependency 'slop', '~> 4.8'
+  s.add_dependency 'sterile', '~> 1.0'
 end

metadata

@@ -1,155 +1,142 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 4.0.0
 platform: ruby
 authors:
 - Adam Doppelt
-autorequire: 
+- Nathan Kriege
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-15 00:00:00.000000000 Z
+date: 2021-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: awesome_print
+  name: amazing_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: htmlentities
+  name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
-  name: httparty
+  name: faraday-encoding
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: faraday-rate_limiter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: scripto
+  name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '4.1'
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: httpdisk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.6'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: stringex
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '1.11'
 - !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.11'
-- !ruby/object:Gem::Dependency
-  name: rake
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
-  type: :development
+        version: '4.8'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '4.8'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: sterile
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '1.0'
 description: Crawl web sites easily using ruby recipes, with caching and nokogiri.
 email:
 - amd@gurge.com
@@ -158,97 +145,48 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
-- ".vscode/extensions.json"
-- ".vscode/settings.json"
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/sinew
 - lib/sinew.rb
-- lib/sinew/cache.rb
-- lib/sinew/core_ext.rb
-- lib/sinew/dsl.rb
+- lib/sinew/args.rb
+- lib/sinew/base.rb
+- lib/sinew/csv.rb
 - lib/sinew/main.rb
+- lib/sinew/middleware/log_formatter.rb
 - lib/sinew/nokogiri_ext.rb
-- lib/sinew/output.rb
-- lib/sinew/request.rb
 - lib/sinew/response.rb
-- lib/sinew/runtime_options.rb
 - lib/sinew/version.rb
+- sample.rb
 - sample.sinew
 - sinew.gemspec
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/recipes/array_header.sinew
-- test/recipes/basic.sinew
-- test/recipes/dups.sinew
-- test/recipes/implicit_header.sinew
-- test/recipes/limit.sinew
-- test/recipes/noko.sinew
-- test/recipes/uri.sinew
-- test/recipes/xml.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_recipes.rb
-- test/test_requests.rb
-- test/test_utf8.rb
 homepage: http://github.com/gurgeous/sinew
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "~>"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: sinew
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.
-test_files:
-- test/legacy/eu.httpbin.org/head/redirect,3
-- test/legacy/eu.httpbin.org/head/status,500
-- test/legacy/eu.httpbin.org/redirect,3
-- test/legacy/eu.httpbin.org/status,500
-- test/legacy/legacy.sinew
-- test/recipes/array_header.sinew
-- test/recipes/basic.sinew
-- test/recipes/dups.sinew
-- test/recipes/implicit_header.sinew
-- test/recipes/limit.sinew
-- test/recipes/noko.sinew
-- test/recipes/uri.sinew
-- test/recipes/xml.sinew
-- test/test.html
-- test/test_cache.rb
-- test/test_helper.rb
-- test/test_legacy.rb
-- test/test_main.rb
-- test/test_nokogiri_ext.rb
-- test/test_output.rb
-- test/test_recipes.rb
-- test/test_requests.rb
-- test/test_utf8.rb
+test_files: []

data/.travis.yml

@@ -1,4 +0,0 @@
-language: ruby
-rvm:
-  - 2.3.7
-  - 2.5.1

data/.vscode/extensions.json

@@ -1,3 +0,0 @@
-{
-  "recommendations": ["rebornix.Ruby"]
-}

data/.vscode/settings.json

@@ -1,15 +0,0 @@
-{
-  "editor.formatOnSave": true,
-  "editor.formatOnSaveTimeout": 1500,
-  "editor.tabSize": 2,
-  "editor.wordSeparators": "`~#$%^&*()-=+[{]}\\|;:'\",.<>/",
-  "files.associations": {
-    "*.sinew": "ruby"
-  },
-  "files.insertFinalNewline": true,
-  "files.trimTrailingWhitespace": true,
-  "ruby.format": "rubocop",
-  "ruby.lint": {
-    "rubocop": true
-  }
-}

data/lib/sinew/cache.rb

@@ -1,79 +0,0 @@
-require 'fileutils'
-require 'tempfile'
-
-#
-# This class handles the caching of http responses on disk.
-#
-
-module Sinew
-  class Cache
-    attr_reader :sinew
-
-    def initialize(sinew)
-      @sinew = sinew
-    end
-
-    def get(request)
-      body = read_if_exist(body_path(request))
-      return nil if !body
-
-      head = read_if_exist(head_path(request))
-      Response.from_cache(request, body, head)
-    end
-
-    def set(response)
-      body_path = body_path(response.request)
-      head_path = head_path(response.request)
-
-      FileUtils.mkdir_p(File.dirname(body_path))
-      FileUtils.mkdir_p(File.dirname(head_path))
-
-      # write body, and head if necessary
-      atomic_write(body_path, response.body)
-      if head_necessary?(response)
-        head = JSON.pretty_generate(response.head_as_json)
-        atomic_write(head_path, head)
-      end
-    end
-
-    def root_dir
-      sinew.options[:cache]
-    end
-    protected :root_dir
-
-    def head_necessary?(response)
-      response.error? || response.redirected?
-    end
-    protected :head_necessary?
-
-    def body_path(request)
-      "#{root_dir}/#{request.cache_key}"
-    end
-    protected :body_path
-
-    def head_path(request)
-      body_path = body_path(request)
-      dir, base = File.dirname(body_path), File.basename(body_path)
-      "#{dir}/head/#{base}"
-    end
-    protected :head_path
-
-    def read_if_exist(path)
-      if File.exist?(path)
-        IO.read(path, mode: 'r:UTF-8')
-      end
-    end
-    protected :read_if_exist
-
-    def atomic_write(path, data)
-      tmp = Tempfile.new('sinew', encoding: 'UTF-8')
-      tmp.write(data)
-      tmp.close
-      FileUtils.chmod(0o644, tmp.path)
-      FileUtils.mv(tmp.path, path)
-    ensure
-      FileUtils.rm(tmp.path, force: true)
-    end
-    protected :atomic_write
-  end
-end