scraperwiki 3.0.1 → 3.0.2

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+# 3.0.1
+* serialise Arrays as json (and deserialize) when saving/getting with save\_var/get\_var
+
+# 3.0.2
+* allow setting of default table name in config file
+* Allow block to be passed to #select

data/lib/scraperwiki.rb

@@ -3,6 +3,8 @@ $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 # require 'scraperwiki/sqlite_save_info.rb'
 require 'scraperwiki/version.rb'
 require 'sqlite_magic'
+require 'json'
+
 
 module ScraperWiki
   extend self
@@ -84,7 +86,8 @@ module ScraperWiki
   # === Example
   # ScraperWiki::save(['id'], {'id'=>1})
   #
-  def save_sqlite(unique_keys, data, table_name="swdata",_verbose=0)
+  def save_sqlite(unique_keys, data, table_name=nil,_verbose=0)
+    table_name ||= default_table_name
     converted_data = convert_data(data)
     sqlite_magic_connection.save_data(unique_keys, converted_data, table_name)
   end
@@ -125,6 +128,8 @@ module ScraperWiki
       result_val.to_f
     when 'NilClass'
       nil
+    when 'Array','Hash'
+      JSON.parse(result_val)
     else
       result_val
     end
@@ -146,11 +151,11 @@ module ScraperWiki
   #
   def save_var(name, value, _verbose=2)
     val_type = value.class.to_s
-    unless ['Fixnum','String','Float','NilClass'].include?(val_type)
+    unless ['Fixnum','String','Float','NilClass', 'Array','Hash'].include?(val_type)
       puts "*** object of type #{val_type} converted to string\n"
     end
-
-    data = { :name => name.to_s, :value_blob => value.to_s, :type => val_type }
+    val = val_type[/Array|Hash/] ? value.to_json : value.to_s
+    data = { :name => name.to_s, :value_blob => val, :type => val_type }
     sqlite_magic_connection.save_data([:name], data, 'swvariables')
   end
 
@@ -161,6 +166,8 @@ module ScraperWiki
   # * _sqlquery_ = A valid select statement, without the select keyword
   # * _data_ = Bind variables provided for ? replacements in the query. See Sqlite3#execute for details
   # * _verbose_ = A verbosity level (not currently implemented, and there just to avoid breaking existing code)
+  # * [optionally] a block can be also be passed and the result rows will be passed
+  # one-by-one to the black rather than loading and returning the whole result set
   #
   # === Returns
   # An array of hashes containing the returned data
@@ -169,7 +176,15 @@ module ScraperWiki
   # ScraperWiki.select('* from swdata')
   #
   def select(sqlquery, data=nil, _verbose=1)
-    sqlite_magic_connection.execute("SELECT "+sqlquery, data)
+    if block_given?
+      sqlite_magic_connection.database.
+                              query("SELECT "+sqlquery, data).
+                              each_hash do |row_hash|
+         yield row_hash
+      end
+    else
+      sqlite_magic_connection.execute("SELECT "+sqlquery, data)
+    end
   end
 
   # Establish an SQLiteMagic::Connection (and remember it)
@@ -178,4 +193,8 @@ module ScraperWiki
     @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
   end
 
+  def default_table_name
+    (@config && @config[:default_table_name]) || 'swdata'
+  end
+
 end

data/lib/scraperwiki/version.rb

@@ -1,3 +1,3 @@
 module ScraperWiki
-  VERSION = '3.0.1'
+VERSION = '3.0.2'
 end

data/spec/scraperwiki_spec.rb

@@ -42,7 +42,8 @@ describe ScraperWiki do
     end
 
     it 'should cache connection' do
-      SqliteMagic::Connection.should_receive(:new).and_return(@dummy_sqlite_magic_connection) # just once
+      SqliteMagic::Connection.should_receive(:new).
+                              and_return(@dummy_sqlite_magic_connection) # just once
       ScraperWiki.sqlite_magic_connection
       ScraperWiki.sqlite_magic_connection
     end
@@ -69,6 +70,34 @@ describe ScraperWiki do
         ScraperWiki.select(sql_snippet)
       end
     end
+
+    context "and block passed" do
+      before do
+        @sql_snippet = 'foo from bar WHERE "baz"=42'
+        @database = mock(:database)
+        @result_enumerator = mock(:result_enumerator, :each_hash => nil)
+        @dummy_sqlite_magic_connection.stub(:database).and_return(@database)
+      end
+
+      it 'should not execute select query on connection' do
+        @database.stub(:query).and_return(@result_enumerator)
+        @dummy_sqlite_magic_connection.should_not_receive(:execute)
+
+        ScraperWiki.select(@sql_snippet, ['foo', 'bar']) { |res| results << res.to_s }
+      end
+
+      it "should make query on connection database and yield results to block" do
+        @database.should_receive(:query).
+                 with("SELECT #{@sql_snippet}", ['foo', 'bar']).
+                 and_return(@result_enumerator)
+        @result_enumerator.should_receive(:each_hash).
+                           and_yield(:result_1).
+                           and_yield(:result_2)
+        results = []
+        ScraperWiki.select(@sql_snippet, ['foo', 'bar']) { |res| results << res.to_s }
+        results.should == ['result_1', 'result_2']
+      end
+    end
   end
 
   describe '#save_sqlite' do
@@ -93,6 +122,18 @@ describe ScraperWiki do
       ScraperWiki.save_sqlite(:unique_keys, :some_data)
     end
 
+    context "and default_table_name set in config" do
+      before do
+        ScraperWiki.config = {:default_table_name => 'my_default_table'}
+      end
+
+      it 'should save data in default_table_name' do
+        @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, 'my_default_table')
+        ScraperWiki.save_sqlite(:unique_keys, :some_data)
+      end
+
+    end
+
     it 'should save data in given table' do
       @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, 'another_table')
       ScraperWiki.save_sqlite(:unique_keys, :some_data, 'another_table')
@@ -138,6 +179,20 @@ describe ScraperWiki do
       @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, "swvariables")
       ScraperWiki.save_var(:foo, 'bar')
     end
+
+    context "and data passed is an Array" do
+      it 'should save data as string with data class as :type' do
+        @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, {:name => 'foo', :value_blob => ['bar',nil,42].to_json, :type => 'Array'}, anything)
+        ScraperWiki.save_var(:foo, ['bar',nil,42])
+      end
+    end
+
+    context "and data passed is a Hash" do
+      it 'should save data as string with data class as :type' do
+        @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, {:name => 'foo', :value_blob => {'bar' => 42}.to_json, :type => 'Hash'}, anything)
+        ScraperWiki.save_var(:foo, {'bar' => 42})
+      end
+    end
   end
 
   describe '#get_var' do
@@ -172,6 +227,20 @@ describe ScraperWiki do
       ScraperWiki.get_var(:foo).should be_nil
     end
 
+    it 'should cast json-serialized Array data to Array' do
+      array = ['a', nil, 123]
+      @dummy_sqlite_magic_connection.stub(:execute).
+                                     and_return([{'value_blob' => array.to_json, 'type' => 'Array'}])
+      ScraperWiki.get_var(:foo).should == array
+    end
+
+    it 'should cast json-serialized Hash data to Hash' do
+      hash = {'a' => 123}
+      @dummy_sqlite_magic_connection.stub(:execute).
+                                     and_return([{'value_blob' => hash.to_json, 'type' => 'Hash'}])
+      ScraperWiki.get_var(:foo).should == hash
+    end
+
     context 'and connection returns empty array' do
       before do
         @dummy_sqlite_magic_connection.stub(:execute).

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraperwiki
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-19 00:00:00.000000000 Z
+date: 2014-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -98,6 +98,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - LICENCE
 - README.md