sinew 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f5a493286f0bfdff9cb26bf85e768750edc57f489aec31e049a3c634bdc9074
-  data.tar.gz: bda8c48fe0a1bccd4caf0c3304c248c218839f0d5547269df977dca56bca43c6
+  metadata.gz: fde4bbaa95fce45f3a7ae7aeacab1672615ea1ace852845b0395ce9cce32f861
+  data.tar.gz: 5743800570722443f704c5fc7bc421346cc4f2fb116b8fe9f615bf84fb95f826
 SHA512:
-  metadata.gz: e9be078939e5e652c1775473322af55065d4ed0a541ba204b84381ba2d44c22b013e19993f500e02f08c9c686c0ee0f0af95cba9f644edd76058d0c1c05180c8
-  data.tar.gz: 9bd2bde0fa0802a0d3b8467322dae695591a6a8928bf86ac1c7fab11f9754a4b66741e41e2ca029f9f9d1c5307c7e750bd29c64231399866ca4acb4d42087e31
+  metadata.gz: 94009061e7f4e36cc23528be3866c6a372df51a83e096144cafbd923259439e6d44a7d656fbdcfe09c2e059b48deb553caca3ec5d332b33845afd1e91550371a
+  data.tar.gz: 5a8baf7fbdba371065c796c9fdce4312039558b27b4a16676b3df16d5138916ce84db0677dce6ede1831be8040df9112a0491e421813af5e5fd0b0b747d49239

data/lib/sinew/response.rb

@@ -6,6 +6,10 @@ module Sinew
   class Response
     attr_accessor :request, :uri, :body, :code, :headers
 
+    #
+    # factory methods
+    #
+
     def self.from_network(request, party_response)
       Response.new.tap do |response|
         response.request = request
@@ -34,6 +38,9 @@ module Sinew
 
         # overwrite with cached response headers
         if head
+          if head !~ /^{/
+            return from_legacy_head(response, head)
+          end
           head = JSON.parse(head, symbolize_names: true)
           response.uri = URI.parse(head[:uri])
           response.code = head[:code]
@@ -52,6 +59,26 @@ module Sinew
       end
     end
 
+    def self.from_legacy_head(response, head)
+      response.tap do |response|
+        case head
+        when /\ACURLER_ERROR/
+          # error
+          response.code = 999
+        when /\AHTTP/
+          # redirect
+          location = head.scan(/Location: ([^\r\n]+)/).flatten.last
+          response.uri += location
+        else
+          $stderr.puts "unknown cached /head for #{response.uri}"
+        end
+      end
+    end
+
+    #
+    # accessors
+    #
+
     def error?
       code >= 400
     end

data/lib/sinew/version.rb

@@ -1,4 +1,4 @@
 module Sinew
   # Gem version
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.0.1'.freeze
 end

data/test/legacy/eu.httpbin.org/head/redirect,3

@@ -0,0 +1,51 @@
+HTTP/1.1 302 FOUND
+Connection: keep-alive
+Server: gunicorn/19.7.1
+Date: Wed, 02 May 2018 20:55:20 GMT
+Content-Type: text/html; charset=utf-8
+Content-Length: 247
+Location: /relative-redirect/2
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Credentials: true
+X-Powered-By: Flask
+X-Processed-Time: 0
+Via: 1.1 vegur
+
+HTTP/1.1 302 FOUND
+Connection: keep-alive
+Server: gunicorn/19.7.1
+Date: Wed, 02 May 2018 20:55:20 GMT
+Content-Type: text/html; charset=utf-8
+Content-Length: 0
+Location: /relative-redirect/1
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Credentials: true
+X-Powered-By: Flask
+X-Processed-Time: 0
+Via: 1.1 vegur
+
+HTTP/1.1 302 FOUND
+Connection: keep-alive
+Server: gunicorn/19.7.1
+Date: Wed, 02 May 2018 20:55:20 GMT
+Content-Type: text/html; charset=utf-8
+Content-Length: 0
+Location: /get
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Credentials: true
+X-Powered-By: Flask
+X-Processed-Time: 0
+Via: 1.1 vegur
+
+HTTP/1.1 200 OK
+Connection: keep-alive
+Server: gunicorn/19.7.1
+Date: Wed, 02 May 2018 20:55:20 GMT
+Content-Type: application/json
+Access-Control-Allow-Origin: *
+Access-Control-Allow-Credentials: true
+X-Powered-By: Flask
+X-Processed-Time: 0
+Content-Length: 220
+Via: 1.1 vegur
+

data/test/legacy/eu.httpbin.org/head/status,500

@@ -0,0 +1 @@
+CURLER_ERROR	curl error (22)

data/test/legacy/eu.httpbin.org/redirect,3

@@ -0,0 +1,11 @@
+{
+  "args": {}, 
+  "headers": {
+    "Accept": "*/*", 
+    "Connection": "close", 
+    "Host": "eu.httpbin.org", 
+    "User-Agent": "sinew/1.0.4"
+  }, 
+  "origin": "104.36.46.249", 
+  "url": "http://eu.httpbin.org/get"
+}

data/test/legacy/eu.httpbin.org/status,500

@@ -0,0 +1 @@
+

data/test/legacy/legacy.sinew

@@ -0,0 +1,2 @@
+get 'http://eu.httpbin.org/status/500'
+get 'http://eu.httpbin.org/redirect/3'

data/test/test_legacy.rb

@@ -0,0 +1,21 @@
+require_relative 'test_helper'
+
+class TestLegacy < MiniTest::Test
+  def setup
+    super
+
+    # These are legacy cache files, pulled from an older version of sinew. We
+    # use them to test our legacy head parsing.
+    src = 'legacy/eu.httpbin.org'
+    dst = "#{TMP}/eu.httpbin.org"
+    FileUtils.cp_r(File.expand_path(src, __dir__), dst)
+  end
+
+  def test_legacy
+    sinew.dsl.get('http://eu.httpbin.org/status/500')
+    assert_equal "\n", sinew.dsl.raw
+
+    sinew.dsl.get('http://eu.httpbin.org/redirect/3')
+    assert_equal 'http://eu.httpbin.org/get', sinew.dsl.url
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sinew
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Adam Doppelt
@@ -181,9 +181,15 @@ files:
 - lib/sinew/version.rb
 - sample.sinew
 - sinew.gemspec
+- test/legacy/eu.httpbin.org/head/redirect,3
+- test/legacy/eu.httpbin.org/head/status,500
+- test/legacy/eu.httpbin.org/redirect,3
+- test/legacy/eu.httpbin.org/status,500
+- test/legacy/legacy.sinew
 - test/test.html
 - test/test_cache.rb
 - test/test_helper.rb
+- test/test_legacy.rb
 - test/test_main.rb
 - test/test_nokogiri_ext.rb
 - test/test_output.rb
@@ -214,9 +220,15 @@ signing_key:
 specification_version: 4
 summary: Sinew - structured web crawling using recipes.
 test_files:
+- test/legacy/eu.httpbin.org/head/redirect,3
+- test/legacy/eu.httpbin.org/head/status,500
+- test/legacy/eu.httpbin.org/redirect,3
+- test/legacy/eu.httpbin.org/status,500
+- test/legacy/legacy.sinew
 - test/test.html
 - test/test_cache.rb
 - test/test_helper.rb
+- test/test_legacy.rb
 - test/test_main.rb
 - test/test_nokogiri_ext.rb
 - test/test_output.rb