scraperwiki 3.0.1 → 3.0.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,6 @@
1
+ # 3.0.1
2
+ * serialise Arrays as json (and deserialize) when saving/getting with save\_var/get\_var
3
+
4
+ # 3.0.2
5
+ * allow setting of default table name in config file
6
+ * Allow block to be passed to #select
@@ -3,6 +3,8 @@ $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
3
3
  # require 'scraperwiki/sqlite_save_info.rb'
4
4
  require 'scraperwiki/version.rb'
5
5
  require 'sqlite_magic'
6
+ require 'json'
7
+
6
8
 
7
9
  module ScraperWiki
8
10
  extend self
@@ -84,7 +86,8 @@ module ScraperWiki
84
86
  # === Example
85
87
  # ScraperWiki::save(['id'], {'id'=>1})
86
88
  #
87
- def save_sqlite(unique_keys, data, table_name="swdata",_verbose=0)
89
+ def save_sqlite(unique_keys, data, table_name=nil,_verbose=0)
90
+ table_name ||= default_table_name
88
91
  converted_data = convert_data(data)
89
92
  sqlite_magic_connection.save_data(unique_keys, converted_data, table_name)
90
93
  end
@@ -125,6 +128,8 @@ module ScraperWiki
125
128
  result_val.to_f
126
129
  when 'NilClass'
127
130
  nil
131
+ when 'Array','Hash'
132
+ JSON.parse(result_val)
128
133
  else
129
134
  result_val
130
135
  end
@@ -146,11 +151,11 @@ module ScraperWiki
146
151
  #
147
152
  def save_var(name, value, _verbose=2)
148
153
  val_type = value.class.to_s
149
- unless ['Fixnum','String','Float','NilClass'].include?(val_type)
154
+ unless ['Fixnum','String','Float','NilClass', 'Array','Hash'].include?(val_type)
150
155
  puts "*** object of type #{val_type} converted to string\n"
151
156
  end
152
-
153
- data = { :name => name.to_s, :value_blob => value.to_s, :type => val_type }
157
+ val = val_type[/Array|Hash/] ? value.to_json : value.to_s
158
+ data = { :name => name.to_s, :value_blob => val, :type => val_type }
154
159
  sqlite_magic_connection.save_data([:name], data, 'swvariables')
155
160
  end
156
161
 
@@ -161,6 +166,8 @@ module ScraperWiki
161
166
  # * _sqlquery_ = A valid select statement, without the select keyword
162
167
  # * _data_ = Bind variables provided for ? replacements in the query. See Sqlite3#execute for details
163
168
  # * _verbose_ = A verbosity level (not currently implemented, and there just to avoid breaking existing code)
169
+ # * [optionally] a block can be also be passed and the result rows will be passed
170
+ # one-by-one to the black rather than loading and returning the whole result set
164
171
  #
165
172
  # === Returns
166
173
  # An array of hashes containing the returned data
@@ -169,7 +176,15 @@ module ScraperWiki
169
176
  # ScraperWiki.select('* from swdata')
170
177
  #
171
178
  def select(sqlquery, data=nil, _verbose=1)
172
- sqlite_magic_connection.execute("SELECT "+sqlquery, data)
179
+ if block_given?
180
+ sqlite_magic_connection.database.
181
+ query("SELECT "+sqlquery, data).
182
+ each_hash do |row_hash|
183
+ yield row_hash
184
+ end
185
+ else
186
+ sqlite_magic_connection.execute("SELECT "+sqlquery, data)
187
+ end
173
188
  end
174
189
 
175
190
  # Establish an SQLiteMagic::Connection (and remember it)
@@ -178,4 +193,8 @@ module ScraperWiki
178
193
  @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
179
194
  end
180
195
 
196
+ def default_table_name
197
+ (@config && @config[:default_table_name]) || 'swdata'
198
+ end
199
+
181
200
  end
@@ -1,3 +1,3 @@
1
1
  module ScraperWiki
2
- VERSION = '3.0.1'
2
+ VERSION = '3.0.2'
3
3
  end
@@ -42,7 +42,8 @@ describe ScraperWiki do
42
42
  end
43
43
 
44
44
  it 'should cache connection' do
45
- SqliteMagic::Connection.should_receive(:new).and_return(@dummy_sqlite_magic_connection) # just once
45
+ SqliteMagic::Connection.should_receive(:new).
46
+ and_return(@dummy_sqlite_magic_connection) # just once
46
47
  ScraperWiki.sqlite_magic_connection
47
48
  ScraperWiki.sqlite_magic_connection
48
49
  end
@@ -69,6 +70,34 @@ describe ScraperWiki do
69
70
  ScraperWiki.select(sql_snippet)
70
71
  end
71
72
  end
73
+
74
+ context "and block passed" do
75
+ before do
76
+ @sql_snippet = 'foo from bar WHERE "baz"=42'
77
+ @database = mock(:database)
78
+ @result_enumerator = mock(:result_enumerator, :each_hash => nil)
79
+ @dummy_sqlite_magic_connection.stub(:database).and_return(@database)
80
+ end
81
+
82
+ it 'should not execute select query on connection' do
83
+ @database.stub(:query).and_return(@result_enumerator)
84
+ @dummy_sqlite_magic_connection.should_not_receive(:execute)
85
+
86
+ ScraperWiki.select(@sql_snippet, ['foo', 'bar']) { |res| results << res.to_s }
87
+ end
88
+
89
+ it "should make query on connection database and yield results to block" do
90
+ @database.should_receive(:query).
91
+ with("SELECT #{@sql_snippet}", ['foo', 'bar']).
92
+ and_return(@result_enumerator)
93
+ @result_enumerator.should_receive(:each_hash).
94
+ and_yield(:result_1).
95
+ and_yield(:result_2)
96
+ results = []
97
+ ScraperWiki.select(@sql_snippet, ['foo', 'bar']) { |res| results << res.to_s }
98
+ results.should == ['result_1', 'result_2']
99
+ end
100
+ end
72
101
  end
73
102
 
74
103
  describe '#save_sqlite' do
@@ -93,6 +122,18 @@ describe ScraperWiki do
93
122
  ScraperWiki.save_sqlite(:unique_keys, :some_data)
94
123
  end
95
124
 
125
+ context "and default_table_name set in config" do
126
+ before do
127
+ ScraperWiki.config = {:default_table_name => 'my_default_table'}
128
+ end
129
+
130
+ it 'should save data in default_table_name' do
131
+ @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, 'my_default_table')
132
+ ScraperWiki.save_sqlite(:unique_keys, :some_data)
133
+ end
134
+
135
+ end
136
+
96
137
  it 'should save data in given table' do
97
138
  @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, 'another_table')
98
139
  ScraperWiki.save_sqlite(:unique_keys, :some_data, 'another_table')
@@ -138,6 +179,20 @@ describe ScraperWiki do
138
179
  @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, "swvariables")
139
180
  ScraperWiki.save_var(:foo, 'bar')
140
181
  end
182
+
183
+ context "and data passed is an Array" do
184
+ it 'should save data as string with data class as :type' do
185
+ @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, {:name => 'foo', :value_blob => ['bar',nil,42].to_json, :type => 'Array'}, anything)
186
+ ScraperWiki.save_var(:foo, ['bar',nil,42])
187
+ end
188
+ end
189
+
190
+ context "and data passed is a Hash" do
191
+ it 'should save data as string with data class as :type' do
192
+ @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, {:name => 'foo', :value_blob => {'bar' => 42}.to_json, :type => 'Hash'}, anything)
193
+ ScraperWiki.save_var(:foo, {'bar' => 42})
194
+ end
195
+ end
141
196
  end
142
197
 
143
198
  describe '#get_var' do
@@ -172,6 +227,20 @@ describe ScraperWiki do
172
227
  ScraperWiki.get_var(:foo).should be_nil
173
228
  end
174
229
 
230
+ it 'should cast json-serialized Array data to Array' do
231
+ array = ['a', nil, 123]
232
+ @dummy_sqlite_magic_connection.stub(:execute).
233
+ and_return([{'value_blob' => array.to_json, 'type' => 'Array'}])
234
+ ScraperWiki.get_var(:foo).should == array
235
+ end
236
+
237
+ it 'should cast json-serialized Hash data to Hash' do
238
+ hash = {'a' => 123}
239
+ @dummy_sqlite_magic_connection.stub(:execute).
240
+ and_return([{'value_blob' => hash.to_json, 'type' => 'Hash'}])
241
+ ScraperWiki.get_var(:foo).should == hash
242
+ end
243
+
175
244
  context 'and connection returns empty array' do
176
245
  before do
177
246
  @dummy_sqlite_magic_connection.stub(:execute).
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scraperwiki
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.1
4
+ version: 3.0.2
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-07-19 00:00:00.000000000 Z
12
+ date: 2014-05-21 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: httpclient
@@ -98,6 +98,7 @@ extensions: []
98
98
  extra_rdoc_files: []
99
99
  files:
100
100
  - .gitignore
101
+ - CHANGELOG.md
101
102
  - Gemfile
102
103
  - LICENCE
103
104
  - README.md