scraperwiki 2.0.4 → 2.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,9 @@ require 'sqlite3'
3
3
  $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
4
4
  require 'scraperwiki/sqlite_save_info.rb'
5
5
 
6
+ class SqliteException < RuntimeError
7
+ end
8
+
6
9
  module ScraperWiki
7
10
 
8
11
  # The scrape method fetches the content from a webserver.
@@ -57,7 +60,7 @@ module ScraperWiki
57
60
  # === Example
58
61
  # ScraperWiki::save(['id'], {'id'=>1})
59
62
  #
60
- def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata")
63
+ def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata",verbose=0)
61
64
  raise 'unique_keys must be nil or an array' if unique_keys != nil && !unique_keys.kind_of?(Array)
62
65
  raise 'data must have a non-nil value' if data == nil
63
66
 
@@ -80,6 +83,10 @@ module ScraperWiki
80
83
  SQLiteMagic._do_save_sqlite(unique_keys, rjdata, table_name)
81
84
  end
82
85
 
86
+ def ScraperWiki.sqliteexecute(query,data=nil, verbose=2)
87
+ SQLiteMagic.sqliteexecute(query,data,verbose)
88
+ end
89
+
83
90
  def ScraperWiki.close_sqlite()
84
91
  SQLiteMagic.close
85
92
  end
@@ -126,4 +133,75 @@ module ScraperWiki
126
133
  return jdata
127
134
  end
128
135
 
136
+ # Allows the user to retrieve a previously saved variable
137
+ #
138
+ # === Parameters
139
+ #
140
+ # * _name_ = The variable name to fetch
141
+ # * _default_ = The value to use if the variable name is not found
142
+ # * _verbose_ = Verbosity level
143
+ #
144
+ # === Example
145
+ # ScraperWiki::get_var('current', 0)
146
+ #
147
+ def ScraperWiki.get_var(name, default=nil, verbose=2)
148
+ begin
149
+ result = ScraperWiki.sqliteexecute("select value_blob, type from swvariables where name=?", [name], verbose)
150
+ rescue NoSuchTableSqliteException => e
151
+ return default
152
+ end
153
+
154
+ if !result.has_key?("data")
155
+ return default
156
+ end
157
+
158
+ if result["data"].length == 0
159
+ return default
160
+ end
161
+ # consider casting to type
162
+ svalue = result["data"][0][0]
163
+ vtype = result["data"][0][1]
164
+ if vtype == "Fixnum"
165
+ return svalue.to_i
166
+ end
167
+ if vtype == "Float"
168
+ return svalue.to_f
169
+ end
170
+ if vtype == "NilClass"
171
+ return nil
172
+ end
173
+ return svalue
174
+ end
175
+
176
+ # Allows the user to save a single variable (at a time) to carry state across runs of
177
+ # the scraper.
178
+ #
179
+ # === Parameters
180
+ #
181
+ # * _name_ = The variable name
182
+ # * _value_ = The value of the variable
183
+ # * _verbose_ = Verbosity level
184
+ #
185
+ # === Example
186
+ # ScraperWiki::save_var('current', 100)
187
+ #
188
+ def ScraperWiki.save_var(name, value, verbose=2)
189
+ vtype = String(value.class)
190
+ svalue = value.to_s
191
+ if vtype != "Fixnum" and vtype != "String" and vtype != "Float" and vtype != "NilClass"
192
+ puts "*** object of type "+vtype+" converted to string\n"
193
+ end
194
+ data = { "name" => name, "value_blob" => svalue, "type" => vtype }
195
+ ScraperWiki.save_sqlite(unique_keys=["name"], data=data, table_name="swvariables", verbose=verbose)
196
+ end
197
+
198
+ def ScraperWiki.raisesqliteerror(rerror)
199
+ if /sqlite3.Error: no such table:/.match(rerror) # old dataproxy
200
+ raise NoSuchTableSqliteException.new(rerror)
201
+ end
202
+ if /DB Error: \(OperationalError\) no such table:/.match(rerror)
203
+ raise NoSuchTableSqliteException.new(rerror)
204
+ end
205
+ raise SqliteException.new(rerror)
206
+ end
129
207
  end
@@ -14,10 +14,14 @@ module SQLiteMagic
14
14
  @db = nil
15
15
  @sqlitesaveinfo = {}
16
16
 
17
- def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
17
+ def SQLiteMagic._open_db_if_necessary()
18
18
  if @db.nil?
19
19
  @db = SQLite3::Database.new("scraperwiki.sqlite")
20
20
  end
21
+ end
22
+
23
+ def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
24
+ SQLiteMagic._open_db_if_necessary
21
25
 
22
26
  res = { }
23
27
  if data.class == Hash
@@ -70,6 +74,12 @@ module SQLiteMagic
70
74
  return res
71
75
  end
72
76
 
77
+ def SQLiteMagic.sqliteexecute(query,data=nil, verbose=2)
78
+ SQLiteMagic._open_db_if_necessary
79
+ cols,*rows = (data.nil?)? @db.execute2(query) : @db.execute2(query,data)
80
+ return {"keys"=>cols, "data"=>rows} unless cols.nil? or rows.nil?
81
+ end
82
+
73
83
  def SQLiteMagic.close()
74
84
  @db.close
75
85
  @db = nil
@@ -108,7 +118,6 @@ module SQLiteMagic
108
118
  raise "buildinitialtable: no coldef" unless coldef.length > 0
109
119
  # coldef = coldef[:1] # just put one column in; the rest could be altered -- to prove it's good
110
120
  scoldef = coldef.map { |col| format("`%s` %s", col[0], col[1]) }.join(",")
111
- # used to just add date_scraped in, but without it can't create an empty table
112
121
  @db.execute(format("create table main.`%s` (%s)", @swdatatblname, scoldef))
113
122
  end
114
123
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scraperwiki
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.4
4
+ version: 2.0.5
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors: