pupa 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e225ec9f62bb2c542da3f691b529cea5b6f99c15
-  data.tar.gz: 07726b314755421a2eac3d46756c46cd483ab913
+  metadata.gz: 940e9e4a78d4b1940728bd15b6ee770f07ac213a
+  data.tar.gz: 2cbd50c8ef323d747a37e2a8970253ee3b028bd4
 SHA512:
-  metadata.gz: 5732c1a25dcbcd7aaac28a049905c1fa2c6e4909f18e3b8c75b82e063264a871ec465b79aaa9c5371621b01f23ab4aa73135bb5757de6721e4521ec185d8e27a
-  data.tar.gz: d93fca74f1f8276c44de1ef228b85659875e839ce4a06a54a17f507423186485276c03c0f2be921b7f39e081fb622931f13659b116d76f9a920502f53427d5e6
+  metadata.gz: f5698681774b52440270b76fad66d1916120b2768a588c1ac38bdf55979ffb8f76a3cb1b70900896cfdaa858668633e26b3dab868f9bc875109b1e202deca93e
+  data.tar.gz: 9f4e8a37eea18a0e18584081d25c050057da631e70b66ff1d1afa6f2a73394caf9cc5dfdac86cc4d53138601ee75921d4ee31f00a23fb1dbea5b38fcc77153cc

data/.travis.yml

@@ -10,3 +10,5 @@ services:
   - mongodb
   - postgresql
   - redis
+before_script:
+  - createdb pupa_test

data/lib/pupa/processor/client.rb

@@ -29,9 +29,10 @@ module Pupa
       # @param [String] cache_dir a directory or a Memcached address
       #   (e.g. `memcached://localhost:11211`) in which to cache requests
       # @param [Integer] expires_in the cache's expiration time in seconds
+      # @param [Integer,String] value_max_bytes the maximum Memcached item size
       # @param [String] level the log level
       # @return [Faraday::Connection] a configured Faraday HTTP client
-      def self.new(cache_dir: nil, expires_in: 86400, level: 'INFO') # 1 day
+      def self.new(cache_dir: nil, expires_in: 86400, value_max_bytes: 1048576, level: 'INFO') # 1 day
         Faraday.new do |connection|
           connection.request :url_encoded
           connection.use Middleware::Logger, Logger.new('faraday', level: level)
@@ -58,7 +59,7 @@ module Pupa
             connection.response :caching do
               address = cache_dir[%r{\Amemcached://(.+)\z}, 1]
               if address
-                ActiveSupport::Cache::MemCacheStore.new(address, expires_in: expires_in)
+                ActiveSupport::Cache::MemCacheStore.new(address, expires_in: expires_in, value_max_bytes: 1048576)
               else
                 ActiveSupport::Cache::FileStore.new(cache_dir, expires_in: expires_in)
               end

data/lib/pupa/processor.rb

@@ -24,14 +24,15 @@ module Pupa
     # @param [String] cache_dir the directory or Memcached address
     #   (e.g. `memcached://localhost:11211`) in which to cache HTTP responses
     # @param [Integer] expires_in the cache's expiration time in seconds
+    # @param [Integer,String] value_max_bytes the maximum Memcached item size
     # @param [String] database_url the database URL
     # @param [Boolean] validate whether to validate JSON documents
     # @param [String] level the log level
     # @param [String,IO] logdev the log device
     # @param [Hash] options criteria for selecting the methods to run
-    def initialize(output_dir, pipelined: false, cache_dir: nil, expires_in: 86400, database_url: 'mongodb://localhost:27017/pupa', validate: true, level: 'INFO', logdev: STDOUT, options: {})
+    def initialize(output_dir, pipelined: false, cache_dir: nil, expires_in: 86400, value_max_bytes: 1048576, database_url: 'mongodb://localhost:27017/pupa', validate: true, level: 'INFO', logdev: STDOUT, options: {})
       @store      = DocumentStore.new(output_dir, pipelined: pipelined)
-      @client     = Client.new(cache_dir: cache_dir, expires_in: expires_in, level: level)
+      @client     = Client.new(cache_dir: cache_dir, expires_in: expires_in, value_max_bytes: value_max_bytes, level: level)
       @connection = Connection.new(database_url)
       @logger     = Logger.new('pupa', level: level, logdev: logdev)
       @validate   = validate

data/lib/pupa/refinements/opencivicdata.rb

@@ -32,7 +32,7 @@ end
 #
 # Instead of adding a callback, we can override `to_h` when `persist` is `true`.
 ObjectSpace.each_object(Class) do |base|
-  if base.include?(Pupa::Model)
+  if base != Sequel::Model && base.include?(Pupa::Model) # Sequel::Model will error on #include?
     base.class_eval do
       set_callback(:save, :before) do |object|
         object._type = object._type.camelize.demodulize.underscore

data/lib/pupa/runner.rb

@@ -11,16 +11,17 @@ module Pupa
       @processor_class = processor_class
 
       @options = OpenStruct.new({
-        actions:        [],
-        tasks:          [],
-        output_dir:     File.expand_path('scraped_data', Dir.pwd),
-        pipelined:      false,
-        cache_dir:      File.expand_path('web_cache', Dir.pwd),
-        expires_in:     86400, # 1 day
-        database_url:   'mongodb://localhost:27017/pupa',
-        validate:       true,
-        level:          'INFO',
-        dry_run:        false,
+        actions:         [],
+        tasks:           [],
+        output_dir:      File.expand_path('scraped_data', Dir.pwd),
+        pipelined:       false,
+        cache_dir:       File.expand_path('web_cache', Dir.pwd),
+        expires_in:      86400, # 1 day
+        value_max_bytes: 1048576, # 1 MB
+        database_url:    'mongodb://localhost:27017/pupa',
+        validate:        true,
+        level:           'INFO',
+        dry_run:         false,
       }.merge(defaults))
 
       @actions = {
@@ -82,6 +83,9 @@ module Pupa
         opts.on('-e', '--expires_in SECONDS', "The cache's expiration time in seconds") do |v|
           options.expires_in = v
         end
+        opts.on('-value_max_bytes BYTES', "The maximum Memcached item size") do |v|
+          options.value_max_bytes = v
+        end
         opts.on('-d', '--database_url SCHEME://USERNAME:PASSWORD@HOST:PORT/DATABASE', 'The database URL') do |v|
           options.database_url = v
         end
@@ -142,6 +146,7 @@ module Pupa
         pipelined: options.pipelined,
         cache_dir: options.cache_dir,
         expires_in: options.expires_in,
+        value_max_bytes: options.value_max_bytes,
         database_url: options.database_url,
         validate: options.validate,
         level: options.level,
@@ -160,7 +165,7 @@ module Pupa
       end
 
       if options.level == 'DEBUG'
-        %w(output_dir pipelined cache_dir expires_in database_url validate level).each do |option|
+        %w(output_dir pipelined cache_dir expires_in value_max_bytes database_url validate level).each do |option|
           puts "#{option}: #{options[option]}"
         end
         unless rest.empty?

data/lib/pupa/version.rb

@@ -1,3 +1,3 @@
 module Pupa
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/processor/connection_adapters/postgresql_adapter_spec.rb

@@ -14,7 +14,7 @@ describe Pupa::Processor::Connection::PostgreSQLAdapter do
   end
 
   before :all do
-    connection.raw_connection.drop_table(:people)
+    connection.raw_connection.drop_table?(:people)
     connection.raw_connection.create_table(:people) do
       primary_key :id
       String :_id

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pupa
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Open North
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-11 00:00:00.000000000 Z
+date: 2014-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport