scraperwiki 2.0.4 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/scraperwiki.rb +79 -1
- data/lib/scraperwiki/sqlite_save_info.rb +11 -2
- metadata +1 -1
data/lib/scraperwiki.rb
CHANGED
@@ -3,6 +3,9 @@ require 'sqlite3'
|
|
3
3
|
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
|
4
4
|
require 'scraperwiki/sqlite_save_info.rb'
|
5
5
|
|
6
|
+
class SqliteException < RuntimeError
|
7
|
+
end
|
8
|
+
|
6
9
|
module ScraperWiki
|
7
10
|
|
8
11
|
# The scrape method fetches the content from a webserver.
|
@@ -57,7 +60,7 @@ module ScraperWiki
|
|
57
60
|
# === Example
|
58
61
|
# ScraperWiki::save(['id'], {'id'=>1})
|
59
62
|
#
|
60
|
-
def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata")
|
63
|
+
def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata",verbose=0)
|
61
64
|
raise 'unique_keys must be nil or an array' if unique_keys != nil && !unique_keys.kind_of?(Array)
|
62
65
|
raise 'data must have a non-nil value' if data == nil
|
63
66
|
|
@@ -80,6 +83,10 @@ module ScraperWiki
|
|
80
83
|
SQLiteMagic._do_save_sqlite(unique_keys, rjdata, table_name)
|
81
84
|
end
|
82
85
|
|
86
|
+
def ScraperWiki.sqliteexecute(query,data=nil, verbose=2)
|
87
|
+
SQLiteMagic.sqliteexecute(query,data,verbose)
|
88
|
+
end
|
89
|
+
|
83
90
|
def ScraperWiki.close_sqlite()
|
84
91
|
SQLiteMagic.close
|
85
92
|
end
|
@@ -126,4 +133,75 @@ module ScraperWiki
|
|
126
133
|
return jdata
|
127
134
|
end
|
128
135
|
|
136
|
+
# Allows the user to retrieve a previously saved variable
|
137
|
+
#
|
138
|
+
# === Parameters
|
139
|
+
#
|
140
|
+
# * _name_ = The variable name to fetch
|
141
|
+
# * _default_ = The value to use if the variable name is not found
|
142
|
+
# * _verbose_ = Verbosity level
|
143
|
+
#
|
144
|
+
# === Example
|
145
|
+
# ScraperWiki::get_var('current', 0)
|
146
|
+
#
|
147
|
+
def ScraperWiki.get_var(name, default=nil, verbose=2)
|
148
|
+
begin
|
149
|
+
result = ScraperWiki.sqliteexecute("select value_blob, type from swvariables where name=?", [name], verbose)
|
150
|
+
rescue NoSuchTableSqliteException => e
|
151
|
+
return default
|
152
|
+
end
|
153
|
+
|
154
|
+
if !result.has_key?("data")
|
155
|
+
return default
|
156
|
+
end
|
157
|
+
|
158
|
+
if result["data"].length == 0
|
159
|
+
return default
|
160
|
+
end
|
161
|
+
# consider casting to type
|
162
|
+
svalue = result["data"][0][0]
|
163
|
+
vtype = result["data"][0][1]
|
164
|
+
if vtype == "Fixnum"
|
165
|
+
return svalue.to_i
|
166
|
+
end
|
167
|
+
if vtype == "Float"
|
168
|
+
return svalue.to_f
|
169
|
+
end
|
170
|
+
if vtype == "NilClass"
|
171
|
+
return nil
|
172
|
+
end
|
173
|
+
return svalue
|
174
|
+
end
|
175
|
+
|
176
|
+
# Allows the user to save a single variable (at a time) to carry state across runs of
|
177
|
+
# the scraper.
|
178
|
+
#
|
179
|
+
# === Parameters
|
180
|
+
#
|
181
|
+
# * _name_ = The variable name
|
182
|
+
# * _value_ = The value of the variable
|
183
|
+
# * _verbose_ = Verbosity level
|
184
|
+
#
|
185
|
+
# === Example
|
186
|
+
# ScraperWiki::save_var('current', 100)
|
187
|
+
#
|
188
|
+
def ScraperWiki.save_var(name, value, verbose=2)
|
189
|
+
vtype = String(value.class)
|
190
|
+
svalue = value.to_s
|
191
|
+
if vtype != "Fixnum" and vtype != "String" and vtype != "Float" and vtype != "NilClass"
|
192
|
+
puts "*** object of type "+vtype+" converted to string\n"
|
193
|
+
end
|
194
|
+
data = { "name" => name, "value_blob" => svalue, "type" => vtype }
|
195
|
+
ScraperWiki.save_sqlite(unique_keys=["name"], data=data, table_name="swvariables", verbose=verbose)
|
196
|
+
end
|
197
|
+
|
198
|
+
def ScraperWiki.raisesqliteerror(rerror)
|
199
|
+
if /sqlite3.Error: no such table:/.match(rerror) # old dataproxy
|
200
|
+
raise NoSuchTableSqliteException.new(rerror)
|
201
|
+
end
|
202
|
+
if /DB Error: \(OperationalError\) no such table:/.match(rerror)
|
203
|
+
raise NoSuchTableSqliteException.new(rerror)
|
204
|
+
end
|
205
|
+
raise SqliteException.new(rerror)
|
206
|
+
end
|
129
207
|
end
|
@@ -14,10 +14,14 @@ module SQLiteMagic
|
|
14
14
|
@db = nil
|
15
15
|
@sqlitesaveinfo = {}
|
16
16
|
|
17
|
-
def SQLiteMagic.
|
17
|
+
def SQLiteMagic._open_db_if_necessary()
|
18
18
|
if @db.nil?
|
19
19
|
@db = SQLite3::Database.new("scraperwiki.sqlite")
|
20
20
|
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
|
24
|
+
SQLiteMagic._open_db_if_necessary
|
21
25
|
|
22
26
|
res = { }
|
23
27
|
if data.class == Hash
|
@@ -70,6 +74,12 @@ module SQLiteMagic
|
|
70
74
|
return res
|
71
75
|
end
|
72
76
|
|
77
|
+
def SQLiteMagic.sqliteexecute(query,data=nil, verbose=2)
|
78
|
+
SQLiteMagic._open_db_if_necessary
|
79
|
+
cols,*rows = (data.nil?)? @db.execute2(query) : @db.execute2(query,data)
|
80
|
+
return {"keys"=>cols, "data"=>rows} unless cols.nil? or rows.nil?
|
81
|
+
end
|
82
|
+
|
73
83
|
def SQLiteMagic.close()
|
74
84
|
@db.close
|
75
85
|
@db = nil
|
@@ -108,7 +118,6 @@ module SQLiteMagic
|
|
108
118
|
raise "buildinitialtable: no coldef" unless coldef.length > 0
|
109
119
|
# coldef = coldef[:1] # just put one column in; the rest could be altered -- to prove it's good
|
110
120
|
scoldef = coldef.map { |col| format("`%s` %s", col[0], col[1]) }.join(",")
|
111
|
-
# used to just add date_scraped in, but without it can't create an empty table
|
112
121
|
@db.execute(format("create table main.`%s` (%s)", @swdatatblname, scoldef))
|
113
122
|
end
|
114
123
|
|