scraperwiki 2.0.6 → 3.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+*.sw[po]
+.DS_Store
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,6 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+# Specify your gem's spec
+gemspec
+

data/LICENCE

@@ -0,0 +1,8 @@
+Copyright (c) 2013, ScraperWiki Limited
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,90 @@
+# ScraperWiki Ruby library [![Build Status](https://travis-ci.org/openc/scraperwiki-ruby.png)](https://travis-ci.org/openc/scraperwiki-ruby)
+
+This is a Ruby library for scraping web pages and saving data. It is a fork/rewrite of the original [scraperwiki-ruby](https://github.com/scraperwiki/scraperwiki-ruby) gem, extracting the SQLite utility methods into the [sqlite_magic](https://github.com/openc/sqlite_magic) gem.
+
+It is a work in progress (for example, it doesn't yet create indices automatically), but should allow ScraperWiki classic scripts to be run locally. 
+
+## Installing
+
+Add this line to your application's Gemfile:
+
+    gem 'scraperwiki-ruby', :git => 'git@github.com:openc/sqlite_magic.git'
+
+And then execute:
+
+    $ bundle
+
+## Scraping
+
+### ScraperWiki.scrape(url[, params])
+
+Returns the downloaded string from the given *url*. *params* are sent as a POST if set.
+
+### Saving data
+
+Helper functions for saving and querying an SQL database. Updates the schema
+automatically according to the data you save.
+
+Currently only supports SQLite. It will make a local SQLite database.
+
+### ScraperWiki.save\_sqlite(unique\_keys, data[, table\_name = "swdata"],verbose)
+
+Saves a data record into the datastore into the table given
+by *table_name*.
+
+*data* is a hash with field names as keys (can be strings or symbols).
+
+*unique_keys* is a subset of data.keys() which determines when a record is
+overwritten.
+
+For large numbers of records *data* can be an array of hashes.
+
+*verbose*, kept for smooth migration from classic, doesn't do anything yet.
+
+### ScraperWiki.sqliteexecute(query,[params],verbose)
+
+Executes provided query with the parameters against the database and returns the results in key value pairs
+
+*query* is a sql statement
+
+*params*, if prepared statement will contains an array of values
+
+### ScraperWiki.save\_var(name,value,verbose)
+
+Allows the user to save a single variable (at a time) to carry state across runs of the scraper.
+
+*name*, the variable name
+
+*value*, the value of the variable
+
+*verbose*, verbosity level
+
+### ScraperWiki.get\_var(name,default,verbose)
+
+Allows the user to retrieve a previously saved variable
+
+*name*, The variable name to fetch
+
+*value*, The value to use if the variable name is not found
+
+*verbose*, verbosity level
+
+### ScraperWiki.select(partial\_query,[params],verbose)
+
+Allows for a simplified select statement
+
+*partial_query*, A valid select statement, without the select keyword
+
+*params* Any data provided for ? replacements in the query
+
+*verbose*, verbosity level
+
+## Usage
+
+Run your Ruby scraper and any data saved will be put in an SQLite database in the current directory called `scraperwiki.sqlite`.
+
+If you're using scrapers from ScraperWiki Classic, remember to add `require 'scraperwiki'` to your file if it's not already there.
+
+## Development
+
+You need the `sqlite3` program installed to run tests. To install run `sudo apt-get install sqlite3` on Ubuntu.

data/Rakefile

@@ -0,0 +1,8 @@
+# require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+    t.libs << 'test'
+end
+
+task :default => :test

data/lib/scraperwiki.rb

@@ -1,234 +1,181 @@
 require 'httpclient'
-require 'sqlite3'
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
-require 'scraperwiki/sqlite_save_info.rb'
-
-class SqliteException < RuntimeError
-end
+# require 'scraperwiki/sqlite_save_info.rb'
+require 'scraperwiki/version.rb'
+require 'sqlite_magic'
 
 module ScraperWiki
-
-    # The scrape method fetches the content from a webserver.
-    #
-    # === Parameters
-    #
-    # * _url_ = The URL to fetch
-    # * _params_ = The parameters to send with a POST request
-    # * _agent = A manually supplied useragent string
-    #
-    # === Example
-    # ScraperWiki::scrape('http://scraperwiki.com')
-    #
-    def ScraperWiki.scrape(url, params = nil, agent = nil)
-      if agent
-        client = HTTPClient.new(:agent_name => agent)
-      else
-        client = HTTPClient.new
-      end
-      client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      if HTTPClient.respond_to?("client.transparent_gzip_decompression=")
-        client.transparent_gzip_decompression = true
-      end
-
-      if params.nil?
-        html = client.get_content(url)
-      else
-        html = client.post_content(url, params)
-      end
-
-      unless HTTPClient.respond_to?("client.transparent_gzip_decompression=")
-        begin
-          gz = Zlib::GzipReader.new(StringIO.new(html))
-          return gz.read
-        rescue
-          return html
-        end
-      end
+  extend self
+
+  # The scrape method fetches the content from a webserver.
+  #
+  # === Parameters
+  #
+  # * _url_ = The URL to fetch
+  # * _params_ = The parameters to send with a POST request
+  # * _agent = A manually supplied useragent string
+  # NB This method hasn't been refactored or tested, but could
+  # prob do with both
+  #
+  # === Example
+  # ScraperWiki::scrape('http://scraperwiki.com')
+  #
+  def scrape(url, params = nil, agent = nil)
+    if agent
+      client = HTTPClient.new(:agent_name => agent)
+    else
+      client = HTTPClient.new
     end
-
-    # Saves the provided data into a local database for this scraper. Data is upserted
-    # into this table (inserted if it does not exist, updated if the unique keys say it 
-    # does).
-    #
-    # === Parameters
-    #
-    # * _unique_keys_ = A list of column names, that used together should be unique
-    # * _data_ = A hash of the data where the Key is the column name, the Value the row
-    #            value.  If sending lots of data this can be a list of hashes.
-    # * _table_name_ = The name that the newly created table should use.
-    #
-    # === Example
-    # ScraperWiki::save(['id'], {'id'=>1})
-    #
-    def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata",verbose=0)
-        raise 'unique_keys must be nil or an array' if unique_keys != nil && !unique_keys.kind_of?(Array)
-        raise 'data must have a non-nil value' if data == nil
-
-        # convert :symbols to "strings"
-        unique_keys = unique_keys.map { |x| x.kind_of?(Symbol) ? x.to_s : x }
-
-        if data.class == Hash
-            data = [ data ]
-        elsif data.length == 0
-            return
-        end
-
-        rjdata = [ ]
-        for ldata in data
-            ljdata = _convdata(unique_keys, ldata)
-            rjdata.push(ljdata)
-
-        end
-
-        SQLiteMagic._do_save_sqlite(unique_keys, rjdata, table_name)
-    end 
-
-    def ScraperWiki.sqliteexecute(query,data=nil, verbose=2)
-      SQLiteMagic.sqliteexecute(query,data,verbose)
+    client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    if HTTPClient.respond_to?("client.transparent_gzip_decompression=")
+      client.transparent_gzip_decompression = true
     end
 
-    def ScraperWiki.close_sqlite()
-        SQLiteMagic.close
+    if params.nil?
+      html = client.get_content(url)
+    else
+      html = client.post_content(url, params)
     end
 
-    # Internal function to check a row of data, convert to right format
-    def ScraperWiki._convdata(unique_keys, scraper_data)
-        if unique_keys
-            for key in unique_keys
-                if !key.kind_of?(String) and !key.kind_of?(Symbol)
-                    raise 'unique_keys must each be a string or a symbol, this one is not: ' + key
-                end
-                if !scraper_data.include?(key) and !scraper_data.include?(key.to_sym)
-                    raise 'unique_keys must be a subset of data, this one is not: ' + key
-                end
-                if scraper_data[key] == nil and scraper_data[key.to_sym] == nil
-                    raise 'unique_key value should not be nil, this one is nil: ' + key
-                end
-            end
-        end
-
-        jdata = { }
-        scraper_data.each_pair do |key, value|
-            raise 'key must not have blank name' if not key
-
-            key = key.to_s if key.kind_of?(Symbol)
-            raise 'key must be string or symbol type: ' + key if key.class != String
-            raise 'key must be simple text: ' + key if !/[a-zA-Z0-9_\- ]+$/.match(key)
-
-            # convert formats
-            if value.kind_of?(Date)
-                value = value.iso8601
-            end
-            if value.kind_of?(Time)
-                value = value.iso8601
-                raise "internal error, timezone came out as non-UTC while converting to SQLite format" unless value.match(/([+-]00:00|Z)$/)
-                value.gsub!(/([+-]00:00|Z)$/, '')
-            end
-            if ![Fixnum, Float, String, TrueClass, FalseClass, NilClass].include?(value.class)
-                value = value.to_s
-            end
-
-            jdata[key] = value
-        end
-        return jdata
+    unless HTTPClient.respond_to?("client.transparent_gzip_decompression=")
+      begin
+        gz = Zlib::GzipReader.new(StringIO.new(html))
+        return gz.read
+      rescue
+        return html
+      end
     end
-
-    # Allows the user to retrieve a previously saved variable
-    #
-    # === Parameters
-    #
-    # * _name_ = The variable name to fetch
-    # * _default_ = The value to use if the variable name is not found
-    # * _verbose_ = Verbosity level
-    #
-    # === Example
-    # ScraperWiki::get_var('current', 0)
-    #
-    def ScraperWiki.get_var(name, default=nil, verbose=2)
-        begin
-            result = ScraperWiki.sqliteexecute("select value_blob, type from swvariables where name=?", [name], verbose)
-        rescue NoSuchTableSqliteException => e   
-            return default
-        end
-        
-        if !result.has_key?("data") 
-            return default          
-        end 
-        
-        if result["data"].length == 0
-            return default
-        end
-        # consider casting to type
-        svalue = result["data"][0][0]
-        vtype = result["data"][0][1]
-        if vtype == "Fixnum"
-            return svalue.to_i
-        end
-        if vtype == "Float"
-            return svalue.to_f
-        end
-        if vtype == "NilClass"
-            return nil
-        end
-        return svalue
+  end
+
+  def convert_data(value_data)
+    return value_data if value_data.nil? or (value_data.respond_to?(:empty?) and value_data.empty?)
+    [value_data].flatten(1).collect do |datum_hash|
+      datum_hash.inject({}) do |hsh, (k,v)|
+        hsh[k] =
+          case v
+          when Date, DateTime
+            v.iso8601
+          when Time
+            # maintains existing ScraperWiki behaviour
+            v.iso8601.sub(/([+-]00:00|Z)$/, '')
+          else
+            v
+          end
+        hsh
+      end
     end
-    
-    # Allows the user to save a single variable (at a time) to carry state across runs of
-    # the scraper.
-    #
-    # === Parameters
-    #
-    # * _name_ = The variable name
-    # * _value_ = The value of the variable
-    # * _verbose_ = Verbosity level
-    #
-    # === Example
-    # ScraperWiki::save_var('current', 100)
-    #
-    def ScraperWiki.save_var(name, value, verbose=2)
-        vtype = String(value.class)
-        svalue = value.to_s
-        if vtype != "Fixnum" and vtype != "String" and vtype != "Float" and vtype != "NilClass"
-            puts "*** object of type "+vtype+" converted to string\n"
-        end
-        data = { "name" => name, "value_blob" => svalue, "type" => vtype }
-        ScraperWiki.save_sqlite(unique_keys=["name"], data=data, table_name="swvariables", verbose=verbose)
+  end
+
+  def config=(config_hash)
+    @config ||= config_hash
+  end
+  # Saves the provided data into a local database for this scraper. Data is upserted
+  # into this table (inserted if it does not exist, updated if the unique keys say it
+  # does).
+  #
+  # === Parameters
+  #
+  # * _unique_keys_ = A list of column names, that used together should be unique
+  # * _data_ = A hash of the data where the Key is the column name, the Value the row
+  #            value. If sending lots of data this can be a array of hashes.
+  # * _table_name_ = The name that the newly created table should use (default is 'swdata').
+  # * _verbose_ = A verbosity level (not currently implemented, and there just to avoid breaking existing code)
+  #
+  # === Example
+  # ScraperWiki::save(['id'], {'id'=>1})
+  #
+  def save_sqlite(unique_keys, data, table_name="swdata",_verbose=0)
+    converted_data = convert_data(data)
+    sqlite_magic_connection.save_data(unique_keys, converted_data, table_name)
+  end
+
+  # legacy alias for #save_sqlite method, so works with older scrapers
+  def save(*args)
+    save_sqlite(*args)
+  end
+
+  def sqliteexecute(query,data=nil, verbose=2)
+    sqlite_magic_connection.execute(query,data)
+  end
+
+  def close_sqlite
+    sqlite_magic_connection.close
+    @sqlite_magic_connection = nil
+  end
+
+  # Allows the user to retrieve a previously saved variable
+  #
+  # === Parameters
+  #
+  # * _name_ = The variable name to fetch
+  # * _default_ = The value to use if the variable name is not found
+  # * _verbose_ = A verbosity level (not currently implemented, and there just to avoid breaking existing code)
+  #
+  # === Example
+  # ScraperWiki.get_var('current', 0)
+  #
+  def get_var(name, default=nil, _verbose=2)
+    result = sqlite_magic_connection.execute("select value_blob, type from swvariables where name=?", [name])
+    return default if result.empty?
+    result_val = result.first['value_blob']
+    case result.first['type']
+    when 'Fixnum'
+      result_val.to_i
+    when 'Float'
+      result_val.to_f
+    when 'NilClass'
+      nil
+    else
+      result_val
     end
-
-    def ScraperWiki.raisesqliteerror(rerror)
-        if /sqlite3.Error: no such table:/.match(rerror)  # old dataproxy
-            raise NoSuchTableSqliteException.new(rerror)
-        end
-        if /DB Error: \(OperationalError\) no such table:/.match(rerror)
-            raise NoSuchTableSqliteException.new(rerror)
-        end
-        raise SqliteException.new(rerror)
+  rescue SqliteMagic::NoSuchTable
+    return default
+  end
+
+  # Allows the user to save a single variable (at a time) to carry state across runs of
+  # the scraper.
+  #
+  # === Parameters
+  #
+  # * _name_ = The variable name
+  # * _value_ = The value of the variable
+  # * _verbose_ = A verbosity level (not currently implemented, and there just to avoid breaking existing code)
+  #
+  # === Example
+  # ScraperWiki.save_var('current', 100)
+  #
+  def save_var(name, value, _verbose=2)
+    val_type = value.class.to_s
+    unless ['Fixnum','String','Float','NilClass'].include?(val_type)
+      puts "*** object of type #{val_type} converted to string\n"
     end
 
-    # Allows for a simplified select statement
-    #
-    # === Parameters
-    #
-    # * _sqlquery_ = A valid select statement, without the select keyword
-    # * _data_ = Any data provided for ? replacements in the query
-    # * _verbose_ = A verbosity level
-    #
-    # === Returns
-    # A list of hashes containing the returned data
-    #
-    # === Example
-    # ScraperWiki::select('* from swdata')    
-    #    
-    def ScraperWiki.select(sqlquery, data=nil, verbose=1)
-        if data != nil && sqlquery.scan(/\?/).length != 0 && data.class != Array
-            data = [data]
-        end
-        result = ScraperWiki.sqliteexecute("select "+sqlquery, data, verbose)
-        res = [ ]
-        for d in result["data"]
-            #res.push(Hash[result["keys"].zip(d)])           # post-1.8.7
-            res.push(Hash[*result["keys"].zip(d).flatten])   # pre-1.8.7
-        end
-        return res
-    end
+    data = { :name => name.to_s, :value_blob => value.to_s, :type => val_type }
+    sqlite_magic_connection.save_data([:name], data, 'swvariables')
+  end
+
+  # Allows for a simplified select statement
+  #
+  # === Parameters
+  #
+  # * _sqlquery_ = A valid select statement, without the select keyword
+  # * _data_ = Bind variables provided for ? replacements in the query. See Sqlite3#execute for details
+  # * _verbose_ = A verbosity level (not currently implemented, and there just to avoid breaking existing code)
+  #
+  # === Returns
+  # An array of hashes containing the returned data
+  #
+  # === Example
+  # ScraperWiki.select('* from swdata')
+  #
+  def select(sqlquery, data=nil, _verbose=1)
+    sqlite_magic_connection.execute("SELECT "+sqlquery, data)
+  end
+
+  # Establish an SQLiteMagic::Connection (and remember it)
+  def sqlite_magic_connection
+    db = @config ? @config[:db] : 'sqlite.db'
+    @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
+  end
+
 end

data/lib/scraperwiki/version.rb

@@ -0,0 +1,3 @@
+module ScraperWiki
+  VERSION = '3.0.0'
+end

data/scraperwiki.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scraperwiki/version'
+# require File.expand_path('../lib/scraperwiki/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ['Francis Irving']
+  gem.email         = 'francis@scraperwiki.com'
+  gem.description   = 'A library for scraping web pages and saving data easily'
+  gem.summary       = 'ScraperWiki'
+  gem.homepage      = 'http://rubygems.org/gems/scraperwiki'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = 'scraperwiki'
+  gem.require_paths = ['lib']
+  gem.version       = ScraperWiki::VERSION
+
+  gem.add_dependency "httpclient"
+  gem.add_dependency "sqlite_magic"
+  # gem.add_dependency "sqlite3"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "debugger"
+end

data/spec/scraperwiki_spec.rb

@@ -0,0 +1,285 @@
+require 'scraperwiki'
+require 'spec_helper'
+require 'debugger'
+
+describe ScraperWiki do
+  before do
+    @dummy_sqlite_magic_connection = double('sqlite_magic_connection')
+    SqliteMagic::Connection.stub(:new).and_return(@dummy_sqlite_magic_connection)
+  end
+
+  after do
+    # reset cached value
+    ScraperWiki.instance_variable_set(:@sqlite_magic_connection, nil)
+    ScraperWiki.instance_variable_set(:@config, nil)
+  end
+
+  describe "#config=" do
+    it "should set config instance variable" do
+      ScraperWiki.config = :some_config
+      ScraperWiki.instance_variable_get(:@config).should == :some_config
+    end
+
+  end
+
+  describe 'sqlite_magic_connection' do
+    it 'should execute select query and bind variables to connection' do
+      sql_snippet = 'foo from bar WHERE "baz"=42'
+      @dummy_sqlite_magic_connection.should_receive(:execute).with("SELECT #{sql_snippet}", ['foo', 'bar'])
+      ScraperWiki.select(sql_snippet, ['foo', 'bar'])
+    end
+
+    context 'and no config set' do
+      it 'should get an SqliteMagic::Connection with default db name and no path' do
+        SqliteMagic::Connection.should_receive(:new).with('sqlite.db').and_return(@dummy_sqlite_magic_connection)
+        ScraperWiki.sqlite_magic_connection
+      end
+    end
+
+    context 'and config set' do
+      before do
+        ScraperWiki.config = {:db => '/some/location/of/sqlite_file.db'}
+      end
+
+      it 'should get an SqliteMagic::Connection with db set in config' do
+        SqliteMagic::Connection.should_receive(:new).with('/some/location/of/sqlite_file.db').and_return(@dummy_sqlite_magic_connection)
+        ScraperWiki.sqlite_magic_connection
+      end
+    end
+
+    it 'should cache connection' do
+      SqliteMagic::Connection.should_receive(:new).and_return(@dummy_sqlite_magic_connection) # just once
+      ScraperWiki.sqlite_magic_connection
+      ScraperWiki.sqlite_magic_connection
+    end
+
+  end
+
+  describe '#select' do
+    it 'should execute select query with bind variables on connection' do
+      sql_snippet = 'foo from bar WHERE "baz"=42'
+      @dummy_sqlite_magic_connection.should_receive(:execute).with("SELECT #{sql_snippet}", ['foo', 'bar'])
+      ScraperWiki.select(sql_snippet, ['foo', 'bar'])
+    end
+
+    it "should return array of hashes returned by connection" do
+      sqlite_magic_response = [{"animal"=>"fox"}, {"animal"=>"cat"}]
+      @dummy_sqlite_magic_connection.stub(:execute).and_return(sqlite_magic_response)
+      ScraperWiki.select('foo', ['foo', 'bar']).should == sqlite_magic_response
+    end
+
+    context 'and no second argument passed' do
+      it 'should pass nil to connection as second argument' do
+        sql_snippet = 'foo from bar WHERE "baz"=42'
+        @dummy_sqlite_magic_connection.should_receive(:execute).with("SELECT #{sql_snippet}", nil)
+        ScraperWiki.select(sql_snippet)
+      end
+    end
+  end
+
+  describe '#save_sqlite' do
+    before do
+      # don't do anything with raw_data by default
+      ScraperWiki.stub(:convert_data) { |raw_data| raw_data }
+    end
+
+    it 'should save data using :name as unique key' do
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(:unique_keys, anything, anything)
+      ScraperWiki.save_sqlite(:unique_keys, :some_data)
+    end
+
+    it 'should save convert data before saving' do
+      ScraperWiki.should_receive(:convert_data).with(:some_data).and_return(:converted_data)
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, :converted_data, anything)
+      ScraperWiki.save_sqlite(:unique_keys, :some_data)
+    end
+
+    it 'should save data in swdata by default' do
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, 'swdata')
+      ScraperWiki.save_sqlite(:unique_keys, :some_data)
+    end
+
+    it 'should save data in given table' do
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, 'another_table')
+      ScraperWiki.save_sqlite(:unique_keys, :some_data, 'another_table')
+    end
+
+    it 'should return response from connection' do
+      @dummy_sqlite_magic_connection.stub(:save_data).and_return(:save_response)
+      ScraperWiki.save_sqlite(:unique_keys, :some_data).should == :save_response
+    end
+
+    it 'should return response from connection' do
+      @dummy_sqlite_magic_connection.stub(:save_data).and_return(:save_response)
+      ScraperWiki.save_sqlite(:unique_keys, :some_data).should == :save_response
+    end
+  end
+
+  describe '#execute' do
+    it 'execute query on sqlite_magic_connection' do
+      @dummy_sqlite_magic_connection.should_receive(:execute).with(:some_query, :some_data)
+      ScraperWiki.sqliteexecute(:some_query, :some_data)
+    end
+
+    it 'should return result of execute query' do
+      @dummy_sqlite_magic_connection.stub(:execute).and_return(:query_result)
+      ScraperWiki.sqliteexecute(:some_query, :some_data).should == :query_result
+    end
+  end
+
+  describe '#save_var' do
+    it 'should save data using :name as unique key' do
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with([:name], anything, anything)
+      ScraperWiki.save_var(:foo, 'bar')
+    end
+
+    it 'should save data as string with data class as :type' do
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, {:name => 'foo', :value_blob => 'bar', :type => 'String'}, anything)
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, {:name => 'meaning_of_life', :value_blob => '42', :type => 'Fixnum'}, anything)
+      ScraperWiki.save_var(:foo, 'bar')
+      ScraperWiki.save_var(:meaning_of_life, 42)
+    end
+
+    it 'should save data in "swvariables"' do
+      @dummy_sqlite_magic_connection.should_receive(:save_data).with(anything, anything, "swvariables")
+      ScraperWiki.save_var(:foo, 'bar')
+    end
+  end
+
+  describe '#get_var' do
+    it 'should select data using given key' do
+      @dummy_sqlite_magic_connection.should_receive(:execute).
+                                     with("select value_blob, type from swvariables where name=?", [:foo]).
+                                     and_return([{'value_blob' => 'bar', 'type' => 'String'}])
+      ScraperWiki.get_var(:foo)
+    end
+
+    it 'should return data returned by sqlite_magic_connection' do
+      @dummy_sqlite_magic_connection.stub(:execute).
+                                     and_return([{'value_blob' => 'bar', 'type' => 'String'}])
+      ScraperWiki.get_var(:foo).should == 'bar'
+    end
+
+    it 'should cast Fixnum data to integer' do
+      @dummy_sqlite_magic_connection.stub(:execute).
+                                     and_return([{'value_blob' => '42', 'type' => 'Fixnum'}])
+      ScraperWiki.get_var(:foo).should == 42
+    end
+
+    it 'should cast Float data to float' do
+      @dummy_sqlite_magic_connection.stub(:execute).
+                                     and_return([{'value_blob' => '0.234', 'type' => 'Float'}])
+      ScraperWiki.get_var(:foo).should == '0.234'.to_f
+    end
+
+    it 'should cast Nil data to nil' do
+      @dummy_sqlite_magic_connection.stub(:execute).
+                                     and_return([{'value_blob' => 'nil', 'type' => 'NilClass'}])
+      ScraperWiki.get_var(:foo).should be_nil
+    end
+
+    context 'and connection returns empty array' do
+      before do
+        @dummy_sqlite_magic_connection.stub(:execute).
+                                       and_return([])
+      end
+
+      it "should return nil" do
+        ScraperWiki.get_var(:foo).should be_nil
+      end
+
+      it "should return default if default given" do
+        ScraperWiki.get_var(:foo, 'bar').should == 'bar'
+      end
+    end
+
+    context 'and SqliteMagic::NoSuchTable raised' do
+      before do
+        @dummy_sqlite_magic_connection.stub(:execute).
+                                       and_raise(SqliteMagic::NoSuchTable)
+      end
+
+      it "should return nil" do
+        ScraperWiki.get_var(:foo).should be_nil
+      end
+
+      it "should return default if default given" do
+        ScraperWiki.get_var(:foo, 'bar').should == 'bar'
+      end
+    end
+
+    context 'and other error raised' do
+      before do
+        @dummy_sqlite_magic_connection.stub(:execute).
+                                       and_raise
+      end
+
+      it "should raise error" do
+        lambda { ScraperWiki.get_var(:foo)}.should raise_error
+      end
+    end
+  end
+
+  describe '#close_sqlite' do
+    it 'should execute query on sqlite_magic_connection' do
+      @dummy_sqlite_magic_connection.should_receive(:close)
+      ScraperWiki.close_sqlite
+    end
+
+    it 'should lose cached connection' do
+      @dummy_sqlite_magic_connection.stub(:close)
+      ScraperWiki.close_sqlite
+      ScraperWiki.instance_variable_get(:@sqlite_magic_connection).should be_nil
+    end
+  end
+
+  describe "#save" do
+    it "should delegate to #save_sqlite" do
+      ScraperWiki.should_receive(:save_sqlite).with(:foo, :bar).and_return(:result)
+      ScraperWiki.save(:foo, :bar).should == :result
+    end
+  end
+
+  describe "#convert_data" do
+    it "should return nil if passed nil" do
+      ScraperWiki.convert_data(nil).should == nil
+    end
+
+    it "should return empty array if passed empty array" do
+      ScraperWiki.convert_data([]).should == []
+    end
+
+    context 'and passed a hash' do
+      it "should return array containing hash if passed hash" do
+        ScraperWiki.convert_data({:foo => 'bar'}).should == [{:foo => 'bar'}]
+      end
+
+      it "should convert date, time and datetime to iso8601" do
+        date = Date.today
+        time = Time.now
+        datetime = (Time.now - 100).to_datetime
+        values_hash = {:foo => 'bar', :date_val => date, :time_val => time, :datetime_val => datetime}
+        expected_result = [{:foo => 'bar', :date_val => date.iso8601, :time_val => time.utc.iso8601.sub(/([+-]00:00|Z)$/, ''), :datetime_val => datetime.to_s}]
+        ScraperWiki.convert_data(values_hash).should == expected_result
+      end
+    end
+
+    context 'and passed an array of hashes' do
+      it "should return array containing hash if passed hash" do
+        ScraperWiki.convert_data([{:foo => 'bar'}, {:bar => 'baz'}]).should == [{:foo => 'bar'}, {:bar => 'baz'}]
+      end
+
+      it "should convert date, time and datetime to iso8601" do
+        date = Date.today
+        time = Time.now
+        datetime = (Time.now - 100).to_datetime
+        values_array = [{:foo => 'bar', :date_val => date}, {:time_val => time, :datetime_val => datetime}]
+        expected_result = [{:foo => 'bar', :date_val => date.iso8601}, {:time_val => time.utc.iso8601.sub(/([+-]00:00|Z)$/,''), :datetime_val => datetime.to_s}]
+        ScraperWiki.convert_data(values_array).should == expected_result
+      end
+    end
+  end
+
+end
+
+

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rspec/autorun'
+# require 'debugger'
+
+RSpec.configure do |config|
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: scraperwiki
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 3.0.0
   prerelease: 
 platform: ruby
 authors:
-- Francis irving
+- Francis Irving
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-04 00:00:00.000000000 Z
+date: 2013-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -28,7 +28,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: sqlite_magic
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -43,14 +43,70 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A library for scraping web pages and saving data easily
 email: francis@scraperwiki.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
+- Gemfile
+- LICENCE
+- README.md
+- Rakefile
 - lib/scraperwiki.rb
-- lib/scraperwiki/sqlite_save_info.rb
+- lib/scraperwiki/version.rb
+- scraperwiki.gemspec
+- spec/scraperwiki_spec.rb
+- spec/spec_helper.rb
 homepage: http://rubygems.org/gems/scraperwiki
 licenses: []
 post_install_message: 
@@ -75,4 +131,6 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: ScraperWiki
-test_files: []
+test_files:
+- spec/scraperwiki_spec.rb
+- spec/spec_helper.rb

data/lib/scraperwiki/sqlite_save_info.rb

@@ -1,220 +0,0 @@
-# Builds schemas automatically from a hash, for SQLite databases
-# 
-# Ported from ScraperWiki Classic - scraperwiki/services/datastore/datalib.py
-# This will make the code quite unRubyish - it is Julian Todd's Python, ported.
-
-
-# TODO:
-# Sort out 'error' bits
-
-require 'set'
-require 'sqlite3'
-
-module SQLiteMagic
-  @db = nil
-  @sqlitesaveinfo = {}
-
-  def SQLiteMagic._open_db_if_necessary()
-    if @db.nil?
-      @db = SQLite3::Database.new("scraperwiki.sqlite")
-    end
-  end
-
-  def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
-    SQLiteMagic._open_db_if_necessary
-
-    res = { }
-    if data.class == Hash
-      data = [data]
-    end
-
-    if !@sqlitesaveinfo.include?(swdatatblname)
-      ssinfo = SqliteSaveInfo.new(swdatatblname, @db)
-      @sqlitesaveinfo[swdatatblname] = ssinfo
-      if not ssinfo.rebuildinfo() and data.length > 0
-        ssinfo.buildinitialtable(data[0])
-        ssinfo.rebuildinfo()
-        res["tablecreated"] = swdatatblname
-      end
-    else
-      ssinfo = @sqlitesaveinfo[swdatatblname]
-    end
-
-    @db.transaction()
-
-    nrecords = 0
-    data.each do |ldata|
-      newcols = ssinfo.newcolumns(ldata)
-      if newcols.length > 0
-        newcols.each_with_index do |kv, i|
-          ssinfo.addnewcolumn(kv[0], kv[1])
-          res["newcolumn %d" % i] = "%s %s" % kv
-        end
-        ssinfo.rebuildinfo()
-      end
-
-      if nrecords == 0 && unique_keys.length > 0
-        idxname, idxkeys = ssinfo.findclosestindex(unique_keys)
-        # puts "findclosestindex returned name:"+ idxname.to_s + " keys:" + idxkeys.to_s
-        if !idxname || idxkeys != unique_keys.to_set
-          lres = ssinfo.makenewindex(idxname, unique_keys)
-          if lres.include?('error')
-            return lres
-          end
-          res.merge!(lres)
-        end
-      end
-
-      lres = ssinfo.insertdata(ldata)
-      nrecords += 1
-    end
-
-    @db.commit()
-    # log(nrecords + " inserted or replaced")
-    return res
-  end
-
-  def SQLiteMagic.sqliteexecute(query,data=nil, verbose=2)
-    SQLiteMagic._open_db_if_necessary
-    cols,*rows = (data.nil?)? @db.execute2(query) : @db.execute2(query,data)
-    return {"keys"=>cols, "data"=>rows} unless cols.nil? or rows.nil?
-  end
-
-  def SQLiteMagic.close()
-    @db.close
-    @db = nil
-    @sqlitesaveinfo = {}
-  end
-
-
-  class SqliteSaveInfo
-    def initialize(swdatatblname, db)
-      @swdatatblname = swdatatblname
-      @swdatakeys = [ ]
-      @swdatatypes = [  ]
-      @sqdatatemplate = "" 
-      @db = db
-    end
-
-    def rebuildinfo()
-      does_exist = @db.get_first_value("select count(*) from main.sqlite_master where name=?", @swdatatblname)
-      if does_exist == 0
-        return false
-      end
-
-      tblinfo = @db.execute("PRAGMA main.table_info(`%s`)" % @swdatatblname)
-      # puts "tblinfo="+ tblinfo.to_s
-      
-      @swdatakeys = tblinfo.map { |a| a[1] }
-      @swdatatypes = tblinfo.map { |a| a[2] }
-      @sqdatatemplate = format("insert or replace into main.`%s` values (%s)", @swdatatblname, (["?"]*@swdatakeys.length).join(","))
-      return true
-    end
-    
-        
-    def buildinitialtable(data)
-      raise "buildinitialtable: no swdatakeys" unless @swdatakeys.length == 0
-      coldef = self.newcolumns(data)
-      raise "buildinitialtable: no coldef" unless coldef.length > 0
-      # coldef = coldef[:1]  # just put one column in; the rest could be altered -- to prove it's good
-      scoldef = coldef.map { |col| format("`%s` %s", col[0], col[1]) }.join(",")
-      @db.execute(format("create table main.`%s` (%s)", @swdatatblname, scoldef))
-    end
-    
-    def newcolumns(data)
-      newcols = [ ]
-      for k, v in data
-        if !@swdatakeys.include?(k)
-          if v != nil
-            #if k[-5:] == "_blob"
-            #  vt = "blob"  # coerced into affinity none
-            if v.class == Fixnum
-              vt = "integer"
-            elsif v.class == Float
-              vt = "real"
-            else
-              vt = "text"
-            end
-            newcols.push([k, vt])
-          end
-        end
-      end
-      # puts "newcols=" + newcols.to_s
-      return newcols
-    end
-
-    def addnewcolumn(k, vt)
-      @db.execute(format("alter table main.`%s` add column `%s` %s", @swdatatblname, k, vt))
-    end
-
-    def findclosestindex(unique_keys)
-      idxlist = @db.execute(format("PRAGMA main.index_list(`%s`)", @swdatatblname))  # [seq,name,unique]
-      # puts "findclosestindex: idxlist is "+ idxlist.to_s
-      if idxlist.include?('error')
-        return [nil, nil]
-      end
-        
-      uniqueindexes = [ ]
-      for idxel in idxlist
-        if idxel[2]
-          idxname = idxel[1]
-          idxinfo = @db.execute(format("PRAGMA main.index_info(`%s`)", idxname)) # [seqno,cid,name]
-          idxset = idxinfo.map { |a| a[2] }.to_set
-          idxoverlap = idxset.intersection(unique_keys).length
-          uniqueindexes.push([idxoverlap, idxname, idxset])
-        end
-      end
-      
-      if uniqueindexes.length == 0
-        return [nil, nil]
-      end
-      uniqueindexes.sort()
-      # puts "uniqueindexes=" + uniqueindexes.to_s
-      return [uniqueindexes[-1][1], uniqueindexes[-1][2]]
-    end
-
-    # increment to next index number every time there is a change, and add the new index before dropping the old one.
-    def makenewindex(idxname, unique_keys)
-      istart = 0
-      if idxname
-        #mnum = re.search("(\d+)$", idxname)
-        #if mnum
-        #  istart = int(mnum.group(1))
-        #end
-        istart = idxname.match("(\d+)$").first.to_i rescue 0
-      end
-      for i in 0..10000
-        newidxname = format("%s_index%d", @swdatatblname, istart+i)
-        does_exist = @db.get_first_value("select count(*) from main.sqlite_master where name=?", newidxname)
-        if does_exist == 0
-          break
-        end
-      end
-        
-      res = { "newindex" => newidxname }
-      lres = @db.execute(format("create unique index `%s` on `%s` (%s)", newidxname, @swdatatblname, unique_keys.map { |k| format("`%s`", k) }.join(",")))
-      if lres.include?('error')
-        return lres
-      end
-      if idxname
-        lres = @db.execute(format("drop index main.`%s`", idxname))
-        if lres.include?('error')
-          if lres['error'] != 'sqlite3.Error: index associated with UNIQUE or PRIMARY KEY constraint cannot be dropped'
-            return lres
-          end
-        end
-        res["droppedindex"] = idxname
-      end
-      return res
-    end
-
-    def insertdata(data)
-      values = @swdatakeys.map { |k| data[k] } 
-      res = @db.query(@sqdatatemplate, values)
-      res.close
-    end
-  end
-
-end
-
-