pupa 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c894758e0375a999ec78a825feb05ff7ec9da66a
-  data.tar.gz: 4f2633cc09888ef1f236cb2c35afdff9681e7924
+  metadata.gz: 37fa9e87e20d4fef24b1f028f046b695a3323b88
+  data.tar.gz: 0b2ae08ac3597955e7175812f05a700c76d91247
 SHA512:
-  metadata.gz: 7e83f57b09cf99dab424032634193aa17055c22721b5ee558362f5fc07face6d6129b71d431e01f0b2b663aa771f752014ce7129e963d44d47beaa154f770c44
-  data.tar.gz: a51e3cee53727013f60623576af75b7c0ce7c047085be577a5f745fed04babc62ffd64a3c59abf234b909ffca47bc73250645f39eaae0364a89ad131f2593426
+  metadata.gz: de98e5ed8f0b145e0ba77401c3e2e983cd47dc2b7a327476a03b6e04eecaeb31f976d76e98a7d51114afd87a95e36c9da491e6a4f4d788f71b02060121cbb94e
+  data.tar.gz: 69fe22ea3079034b34aea5b244d233f66eebefd5010ea923ca9865d475bc81ccaebd6cab94e721ff2944cef3b4842dc297c932e8cbc69d949a593a74b2fc46a3

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
   - 2.0.0
+  - 2.1.0
 env:
   - MODE=default
   - MODE=compat

data/README.md

@@ -5,7 +5,7 @@
 [![Coverage Status](https://coveralls.io/repos/opennorth/pupa-ruby/badge.png?branch=master)](https://coveralls.io/r/opennorth/pupa-ruby)
 [![Code Climate](https://codeclimate.com/github/opennorth/pupa-ruby.png)](https://codeclimate.com/github/opennorth/pupa-ruby)
 
-Pupa.rb is a Ruby 2.0 fork of Sunlight Labs' [Pupa](https://github.com/opencivicdata/pupa). It implements an Extract, Transform and Load (ETL) process to scrape data from online sources, transform it, and write it to a database.
+Pupa.rb is a Ruby 2.x fork of Sunlight Labs' [Pupa](https://github.com/opencivicdata/pupa). It implements an Extract, Transform and Load (ETL) process to scrape data from online sources, transform it, and write it to a database.
 
 ## What it tries to solve
 
@@ -187,6 +187,10 @@ The `json-schema` gem is slow compared to, for example, [JSV](https://github.com
 
 The [pupa-validate](https://npmjs.org/package/pupa-validate) npm package can be used to validate JSON documents using the faster JSV. In an example case, using JSV instead of the `json-schema` gem reduced by half the time to validate 10,000 documents.
 
+### Ruby version
+
+Pupa.rb requires Ruby 2.x. If you have already made all the above optimizations, you may notice a significant improvement by using Ruby 2.1, which has better garbage collection than Ruby 2.0.
+
 ### Profiling
 
 You can profile your code using [perftools.rb](https://github.com/tmm1/perftools.rb). First, install the gem:

data/lib/pupa/processor.rb

@@ -46,18 +46,6 @@ module Pupa
     # @param [String,Hash] params query string parameters
     # @return a parsed document
     def get(url, params = {})
-      # Faraday requires `params` to be a hash.
-      if String === params
-        params = CGI.parse(params)
-
-        # Flatten the parameters for Faraday.
-        params.each do |key,value|
-          if Array === value && value.size == 1
-            params[key] = value.first
-          end
-        end
-      end
-
       client.get(url, params).body
     end
 

data/lib/pupa/processor/client.rb

@@ -2,10 +2,12 @@ require 'active_support/cache'
 require 'faraday_middleware'
 require 'faraday_middleware/response_middleware'
 
+require 'pupa/processor/middleware/gzip'
 require 'pupa/processor/middleware/logger'
 require 'pupa/processor/middleware/parse_html'
 require 'pupa/processor/middleware/parse_json'
 require 'pupa/processor/middleware/raise_error'
+require 'pupa/refinements/faraday'
 require 'pupa/refinements/faraday_middleware'
 
 begin
@@ -49,6 +51,9 @@ module Pupa
             connection.use FaradayMiddleware::ParseXml, preserve_raw: true, content_type: /\bxml$/
           end
 
+          # Must come after the parser middlewares.
+          connection.use Middleware::Gzip
+
           if cache_dir
             connection.response :caching do
               address = cache_dir[%r{\Amemcached://(.+)\z}, 1]

data/lib/pupa/processor/middleware/gzip.rb

@@ -0,0 +1,24 @@
+module Pupa
+  class Processor
+    module Middleware
+      # A Faraday response middleware for parsing gzip responses.
+      #
+      # @see https://gist.github.com/romanbsd/3892387
+      class Gzip < Faraday::Response::Middleware
+        dependency 'zlib'
+
+        def on_complete(env)
+          encoding = env[:response_headers]['content-encoding'].to_s.downcase
+          case encoding
+          when 'gzip'
+            env[:body] = Zlib::GzipReader.new(StringIO.new(env[:body]), encoding: 'ASCII-8BIT').read
+            env[:response_headers].delete('content-encoding')
+          when 'deflate'
+            env[:body] = Zlib::Inflate.inflate(env[:body])
+            env[:response_headers].delete('content-encoding')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pupa/refinements/faraday.rb

@@ -0,0 +1,26 @@
+module Pupa
+  module Refinements
+    # Faraday requires `params` to be a hash.
+    module Connection
+      # @see https://github.com/lostisland/faraday/blob/b8d90a59bafb8dd6e19488fae07945a7700f5664/lib/faraday/connection.rb#L137
+      def get(url=nil, params=nil, headers=nil, &block)
+        if String === params
+          params = CGI.parse(params)
+
+          # Flatten the parameters for Faraday.
+          params.each do |key,value|
+            if Array === value && value.size == 1
+              params[key] = value.first
+            end
+          end
+        end
+
+        super(url, params, headers, &block)
+      end
+    end
+  end
+end
+
+class Faraday::Connection
+  prepend Pupa::Refinements::Connection
+end

data/lib/pupa/version.rb

@@ -1,3 +1,3 @@
 module Pupa
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pupa
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Open North
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-29 00:00:00.000000000 Z
+date: 2014-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -302,12 +302,14 @@ files:
 - lib/pupa/processor/document_store/file_store.rb
 - lib/pupa/processor/document_store/redis_store.rb
 - lib/pupa/processor/helper.rb
+- lib/pupa/processor/middleware/gzip.rb
 - lib/pupa/processor/middleware/logger.rb
 - lib/pupa/processor/middleware/parse_html.rb
 - lib/pupa/processor/middleware/parse_json.rb
 - lib/pupa/processor/middleware/raise_error.rb
 - lib/pupa/processor/persistence.rb
 - lib/pupa/processor/yielder.rb
+- lib/pupa/refinements/faraday.rb
 - lib/pupa/refinements/faraday_middleware.rb
 - lib/pupa/refinements/json-schema.rb
 - lib/pupa/refinements/opencivicdata.rb