scraperwiki 2.0.4 → 2.0.5

data/lib/scraperwiki.rb

@@ -3,6 +3,9 @@ require 'sqlite3'
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 require 'scraperwiki/sqlite_save_info.rb'
 
+class SqliteException < RuntimeError
+end
+
 module ScraperWiki
 
     # The scrape method fetches the content from a webserver.
@@ -57,7 +60,7 @@ module ScraperWiki
     # === Example
     # ScraperWiki::save(['id'], {'id'=>1})
     #
-    def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata")
+    def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata",verbose=0)
         raise 'unique_keys must be nil or an array' if unique_keys != nil && !unique_keys.kind_of?(Array)
         raise 'data must have a non-nil value' if data == nil
 
@@ -80,6 +83,10 @@ module ScraperWiki
         SQLiteMagic._do_save_sqlite(unique_keys, rjdata, table_name)
     end 
 
+    def ScraperWiki.sqliteexecute(query,data=nil, verbose=2)
+      SQLiteMagic.sqliteexecute(query,data,verbose)
+    end
+
     def ScraperWiki.close_sqlite()
         SQLiteMagic.close
     end
@@ -126,4 +133,75 @@ module ScraperWiki
         return jdata
     end
 
+    # Allows the user to retrieve a previously saved variable
+    #
+    # === Parameters
+    #
+    # * _name_ = The variable name to fetch
+    # * _default_ = The value to use if the variable name is not found
+    # * _verbose_ = Verbosity level
+    #
+    # === Example
+    # ScraperWiki::get_var('current', 0)
+    #
+    def ScraperWiki.get_var(name, default=nil, verbose=2)
+        begin
+            result = ScraperWiki.sqliteexecute("select value_blob, type from swvariables where name=?", [name], verbose)
+        rescue NoSuchTableSqliteException => e   
+            return default
+        end
+        
+        if !result.has_key?("data") 
+            return default          
+        end 
+        
+        if result["data"].length == 0
+            return default
+        end
+        # consider casting to type
+        svalue = result["data"][0][0]
+        vtype = result["data"][0][1]
+        if vtype == "Fixnum"
+            return svalue.to_i
+        end
+        if vtype == "Float"
+            return svalue.to_f
+        end
+        if vtype == "NilClass"
+            return nil
+        end
+        return svalue
+    end
+    
+    # Allows the user to save a single variable (at a time) to carry state across runs of
+    # the scraper.
+    #
+    # === Parameters
+    #
+    # * _name_ = The variable name
+    # * _value_ = The value of the variable
+    # * _verbose_ = Verbosity level
+    #
+    # === Example
+    # ScraperWiki::save_var('current', 100)
+    #
+    def ScraperWiki.save_var(name, value, verbose=2)
+        vtype = String(value.class)
+        svalue = value.to_s
+        if vtype != "Fixnum" and vtype != "String" and vtype != "Float" and vtype != "NilClass"
+            puts "*** object of type "+vtype+" converted to string\n"
+        end
+        data = { "name" => name, "value_blob" => svalue, "type" => vtype }
+        ScraperWiki.save_sqlite(unique_keys=["name"], data=data, table_name="swvariables", verbose=verbose)
+    end
+
+    def ScraperWiki.raisesqliteerror(rerror)
+        if /sqlite3.Error: no such table:/.match(rerror)  # old dataproxy
+            raise NoSuchTableSqliteException.new(rerror)
+        end
+        if /DB Error: \(OperationalError\) no such table:/.match(rerror)
+            raise NoSuchTableSqliteException.new(rerror)
+        end
+        raise SqliteException.new(rerror)
+    end
 end

data/lib/scraperwiki/sqlite_save_info.rb

@@ -14,10 +14,14 @@ module SQLiteMagic
   @db = nil
   @sqlitesaveinfo = {}
 
-  def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
+  def SQLiteMagic._open_db_if_necessary()
     if @db.nil?
       @db = SQLite3::Database.new("scraperwiki.sqlite")
     end
+  end
+
+  def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
+    SQLiteMagic._open_db_if_necessary
 
     res = { }
     if data.class == Hash
@@ -70,6 +74,12 @@ module SQLiteMagic
     return res
   end
 
+  def SQLiteMagic.sqliteexecute(query,data=nil, verbose=2)
+    SQLiteMagic._open_db_if_necessary
+    cols,*rows = (data.nil?)? @db.execute2(query) : @db.execute2(query,data)
+    return {"keys"=>cols, "data"=>rows} unless cols.nil? or rows.nil?
+  end
+
   def SQLiteMagic.close()
     @db.close
     @db = nil
@@ -108,7 +118,6 @@ module SQLiteMagic
       raise "buildinitialtable: no coldef" unless coldef.length > 0
       # coldef = coldef[:1]  # just put one column in; the rest could be altered -- to prove it's good
       scoldef = coldef.map { |col| format("`%s` %s", col[0], col[1]) }.join(",")
-          # used to just add date_scraped in, but without it can't create an empty table
       @db.execute(format("create table main.`%s` (%s)", @swdatatblname, scoldef))
     end
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraperwiki
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
   prerelease: 
 platform: ruby
 authors: