scraperwiki 1.0.1 → 2.0.0

data/lib/scraperwiki.rb

@@ -1,480 +1,124 @@
-require 'json'
-require	'uri'
-require	'net/http'
-require 'scraperwiki/datastore'
-require 'httpclient'
-
-class SqliteException < RuntimeError
-end
-
-class NoSuchTableSqliteException < SqliteException
-end
-
-$apiwrapperattacheddata = [ ]
+require 'sqlite3'
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
+require 'scraperwiki/sqlite_save_info.rb'
 
 module ScraperWiki
 
-    $metadatamessagedone = false
-    $attachlist = [ ]
-
-    def ScraperWiki.dumpMessage(hash)
-        msg = JSON.generate(hash)
-        $logfd.write( "JSONRECORD(" + msg.length.to_s() + "):" + msg + "\n")
-        $logfd.flush()
-    end
-
-    def ScraperWiki.httpresponseheader(headerkey, headervalue)
-        ScraperWiki.dumpMessage({'message_type' => 'httpresponseheader', 'headerkey' => headerkey, 'headervalue' => headervalue})
-    end
-
-    def ScraperWiki.scrape(url, params = nil)
-      client = HTTPClient.new
+    # The scrape method fetches the content from a webserver.
+    #
+    # === Parameters
+    #
+    # * _url_ = The URL to fetch
+    # * _params_ = The parameters to send with a POST request
+    # * _agent = A manually supplied useragent string
+    #
+    # === Example
+    # ScraperWiki::scrape('http://scraperwiki.com')
+    #
+    def ScraperWiki.scrape(url, params = nil, agent = nil)
+      if agent
+        client = HTTPClient.new(:agent_name => agent)
+      else
+        client = HTTPClient.new
+      end
       client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      if HTTPClient.respond_to?("client.transparent_gzip_decompression=")
+        client.transparent_gzip_decompression = true
+      end
 
-      if params.nil? 
-        return client.get_content(url)
+      if params.nil?
+        html = client.get_content(url)
       else
-        return client.post_content(url, params)
+        html = client.post_content(url, params)
       end
-    end
 
-    def ScraperWiki.gb_postcode_to_latlng(postcode)
-        uri = URI.parse("http://views.scraperwiki.com/run/uk_postcode_lookup/?postcode="+URI.escape(postcode))
-        sres = Net::HTTP.get(uri)
-        jres = JSON.parse(sres)
-        if jres["lat"] and jres["lng"]
-            return [jres["lat"], jres["lng"]]
+      unless HTTPClient.respond_to?("client.transparent_gzip_decompression=")
+        begin
+          gz = Zlib::GzipReader.new(StringIO.new(html))
+          return gz.read
+        rescue
+          return html
         end
-        return nil
+      end
     end
 
+    # Saves the provided data into a local database for this scraper. Data is upserted
+    # into this table (inserted if it does not exist, updated if the unique keys say it 
+    # does).
+    #
+    # === Parameters
+    #
+    # * _unique_keys_ = A list of column names, that used together should be unique
+    # * _data_ = A hash of the data where the Key is the column name, the Value the row
+    #            value.  If sending lots of data this can be a list of hashes.
+    # * _table_name_ = The name that the newly created table should use.
+    #
+    # === Example
+    # ScraperWiki::save(['id'], {'id'=>1})
+    #
+    def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata")
+        raise 'unique_keys must be nil or an array' if unique_keys != nil && !unique_keys.kind_of?(Array)
+        raise 'data must have a non-nil value' if data == nil
 
+        # convert :symbols to "strings"
+        unique_keys = unique_keys.map { |x| x.kind_of?(Symbol) ? x.to_s : x }
 
-    def ScraperWiki._unicode_truncate(string, size)
-        # Stops 2 byte unicode characters from being chopped in half which kills JSON serializer
-        string.scan(/./u)[0,size].join
-    end
-
-    def ScraperWiki.save(unique_keys, data, date=nil, latlng=nil, table_name="swdata")
-        if unique_keys != nil && !unique_keys.kind_of?(Array)
-            raise 'unique_keys must be nil or an array'
-        end
-        if data == nil
-            raise 'data must have a non-nil value'
-        end
-
-        ds = SW_DataStore.create()
-        ldata = data.dup
-        if date != nil
-            ldata["date"] = date
-        end
-        if latlng != nil
-            ldata["latlng_lat"] = latlng[0]
-            ldata["latlng_lng"] = latlng[1]
+        if data.class == Hash
+            data = [ data ]
+        elsif data.length == 0
+            return
         end
-        return ScraperWiki.save_sqlite(unique_keys, ldata, table_name="swdata", verbose=2)
-    end
-
 
-    def ScraperWiki.sqliteexecute(sqlquery, data=nil, verbose=2)
-        ds = SW_DataStore.create()
-        if ds.m_webstore_port == 0
-            res = ds.request({'maincommand'=>'sqliteexecute', 'sqlquery'=>sqlquery, 'data'=>data, 'attachlist'=>$attachlist})
-        else
-            username = 'resourcedir'  # gets it into the right subdirectory automatically!!!
-            dirscrapername = ds.m_scrapername
-            if ds.m_scrapername == '' or ds.m_scrapername.nil?
-                dirscrapername = 'DRAFT__' + ds.m_runid.gsub(/[\.\-]/, '_')
-            end
-            path = "%s/%s" % [username, dirscrapername]
-            
-            record = {"query"=>sqlquery, "params"=>data, "attach"=>[]}
-            $attachlist.each do |value|
-                record["attach"].push({"user"=>username, "database"=>value["name"], "alias"=>value["asattach"], "securityhash"=>"somthing"})
-            end
-            
-            httpcall = Net::HTTP.new(ds.m_host, ds.m_webstore_port)
-            headers =  { "Accept"=>"application/json+tuples", "X-Scrapername"=>ds.m_scrapername, "X-Runid"=>ds.m_runid, "Content-Type"=>"application/json" }
-            response = httpcall.put(path, JSON.generate(record), headers)
-            res = JSON.parse(response.body)
-            if res["state"] == "error"
-                ScraperWiki.raisesqliteerror(res["message"])
-            end
-            if (res.class == Hash) and (res["keys"].class == Array) and (res["data"].class == Array)
-                if res["keys"].include?("state") and (res["data"].length == 1)
-                    ddata = Hash[*res["keys"].zip(res["data"][0]).flatten]
-                    if ddata["state"] == "error"
-                        ScraperWiki.raisesqliteerror(ddata["message"])
-                    end
-                end
-            end
-        end
+        rjdata = [ ]
+        for ldata in data
+            ljdata = _convdata(unique_keys, ldata)
+            rjdata.push(ljdata)
 
-        if verbose
-            if data.kind_of?(Array) 
-                data.each do |value|
-                    ldata = [ ]
-                    if value == nil
-                        value  = ''
-                    end
-                    ldata.push(ScraperWiki._unicode_truncate(value.to_s, 50))
-                end
-            else
-                ldata = data
-            end
-            ScraperWiki.dumpMessage({'message_type'=>'sqlitecall', 'command'=>"execute", 'val1'=>sqlquery, 'val2'=>ldata})
         end
-        return res
-    end
-
 
+        SQLiteMagic._do_save_sqlite(unique_keys, rjdata, table_name)
+    end 
 
-            # this ought to be a local function
+    # Internal function to check a row of data, convert to right format
     def ScraperWiki._convdata(unique_keys, scraper_data)
         if unique_keys
             for key in unique_keys
                 if !key.kind_of?(String) and !key.kind_of?(Symbol)
-                    return { "error" => 'unique_keys must each be a string or a symbol', "bad_key" => key }
+                    return 'unique_keys must each be a string or a symbol, this one is not: ' + key
                 end
                 if !scraper_data.include?(key) and !scraper_data.include?(key.to_sym)
-                    return { "error" => 'unique_keys must be a subset of data', "bad_key" => key }
+                    return 'unique_keys must be a subset of data, this one is not: ' + key
                 end
                 if scraper_data[key] == nil and scraper_data[key.to_sym] == nil
-                    return { "error" => 'unique_key value should not be nil', "bad_key" => key }
+                    return 'unique_key value should not be nil, this one is nil: ' + key
                 end
             end
         end
 
         jdata = { }
         scraper_data.each_pair do |key, value|
-            if not key
-                return { "error" => 'key must not be blank', "bad_key" => key }
-            end
-            if key.kind_of?(Symbol)
-                key = key.to_s
-            end
-            if key.class != String
-                return { "error" => 'key must be string type', "bad_key" => key }
-            end
+            raise 'key must not have blank name' if not key
 
-            if !/[a-zA-Z0-9_\- ]+$/.match(key)
-                return { "error"=>'key must be simple text', "bad_key"=> key }
-            end
+            key = key.to_s if key.kind_of?(Symbol)
+            raise 'key must be string or symbol type: ' + key if key.class != String
+            raise 'key must be simple text: ' + key if !/[a-zA-Z0-9_\- ]+$/.match(key)
 
-            if value.kind_of?(Date) 
+            # convert formats
+            if value.kind_of?(Date)
                 value = value.iso8601
             end
             if value.kind_of?(Time)
                 value = value.iso8601
-                raise "internal error, timezone came out as non-UTC while converting to SQLite format" unless value.match(/\+00:00$/)
-                value.gsub!(/\+00:00$/, '')
+                raise "internal error, timezone came out as non-UTC while converting to SQLite format" unless value.match(/([+-]00:00|Z)$/)
+                value.gsub!(/([+-]00:00|Z)$/, '')
             end
             if ![Fixnum, Float, String, TrueClass, FalseClass, NilClass].include?(value.class)
                 value = value.to_s
             end
+
             jdata[key] = value
         end
         return jdata
     end
 
-
-    def ScraperWiki.save_sqlite(unique_keys, data, table_name="swdata", verbose=2)
-        if !data
-            ScraperWiki.dumpMessage({'message_type' => 'data', 'content' => "EMPTY SAVE IGNORED"})
-            return
-        end
-
-        # convert :symbols to "strings"
-        unique_keys = unique_keys.map { |x| x.kind_of?(Symbol) ? x.to_s : x }
-
-        if data.class == Hash
-            data = [ data ]
-        end
-            
-        rjdata = [ ]
-        for ldata in data
-            ljdata = _convdata(unique_keys, ldata)
-            if ljdata.include?("error")
-                raise SqliteException.new(ljdata["error"])
-            end
-            rjdata.push(ljdata)
-        end
-
-        ds = SW_DataStore.create()
-        if ds.m_webstore_port == 0
-            res = ds.request({'maincommand'=>'save_sqlite', 'unique_keys'=>unique_keys, 'data'=>rjdata, 'swdatatblname'=>table_name})
-        else
-            username = 'resourcedir'  # gets it into the right subdirectory automatically!!!
-            dirscrapername = ds.m_scrapername
-            if ds.m_scrapername == '' or ds.m_scrapername.nil?
-                dirscrapername = 'DRAFT__' + ds.m_runid.gsub(/[\.\-]/, '_')
-            end
-            
-            # (do something about jargtypes later)
-            qsl = [ ]
-            unique_keys.each do |key|
-                qsl.push("unique="+URI.encode(key))
-            end
-            
-            # quick and dirty provision of column types to the webstore
-            if rjdata.length != 0
-                jargtypes = { }
-                rjdata[0].each_pair do |k, v|
-                    if v != nil
-                        #if k[-5..-1] == "_blob"
-                        #    vt = "blob"  # coerced into affinity none
-                        if v.class == Fixnum
-                            vt = "integer"
-                        elsif v.class == Float
-                            vt = "real"
-                        else
-                            vt = "text"
-                        end
-                        jargtypes[k] = vt
-                    end
-                end
-                qsl.push(("jargtypes="+JSON.generate(jargtypes)))
-            end
-            
-            path = "%s/%s/%s?%s" % [username, dirscrapername, table_name, qsl.join("&")]
-            #puts JSON.generate(rjdata)
-            httpcall = Net::HTTP.new(ds.m_host, ds.m_webstore_port)
-            headers =  { "Accept"=>"application/json", "X-Scrapername"=>ds.m_scrapername, "X-Runid"=>ds.m_runid, "Content-Type"=>"application/json" }
-            response = httpcall.post(path, JSON.generate(rjdata), headers)
-            #puts response.body
-            res = JSON.parse(response.body)
-            if res["state"] == "error"
-                res["error"] = res["message"]
-            end
-        end
-        
-       
-        if res["error"]
-            raise SqliteException.new(res["error"])
-        end
-
-        if verbose >= 2
-            pdata = { }
-            if rjdata.class == Hash
-                sdata = rjdata
-            else
-                sdata = rjdata[0]
-            end
-            sdata.each_pair do |key, value|
-                key = ScraperWiki._unicode_truncate(key.to_s, 50)
-                if value == nil
-                    value  = ''
-                else
-                    value = ScraperWiki._unicode_truncate(value.to_s, 50)
-                end
-                pdata[key] = String(value)
-            end
-            if rjdata.class == Array and rjdata.size > 1
-                pdata["number_records"] = "Number Records: "+String(rjdata.size)
-            end
-            ScraperWiki.dumpMessage({'message_type' => 'data', 'content' => pdata})
-        end
-        return res
-    end
-
-            # also needs to handle the types better (could save json and datetime objects handily
-    def ScraperWiki.save_var(name, value, verbose=2)
-        vtype = String(value.class)
-        svalue = value.to_s
-        if vtype != "Fixnum" and vtype != "String" and vtype != "Float" and vtype != "NilClass"
-            puts "*** object of type "+vtype+" converted to string\n"
-        end
-        data = { "name" => name, "value_blob" => svalue, "type" => vtype }
-        ScraperWiki.save_sqlite(unique_keys=["name"], data=data, table_name="swvariables", verbose=verbose)
-    end
-
-    def ScraperWiki.get_var(name, default=nil, verbose=2)
-        begin
-            result = ScraperWiki.sqliteexecute("select value_blob, type from swvariables where name=?", [name], verbose)
-        rescue NoSuchTableSqliteException => e   
-            return default
-        end
-        
-        if !result.has_key?("data") 
-            return default          
-        end 
-        
-        if result["data"].length == 0
-            return default
-        end
-        # consider casting to type
-        svalue = result["data"][0][0]
-        vtype = result["data"][0][1]
-        if vtype == "Fixnum"
-            return svalue.to_i
-        end
-        if vtype == "Float"
-            return svalue.to_f
-        end
-        if vtype == "NilClass"
-            return nil
-        end
-        return svalue
-    end
-
-    # These are DEPRECATED and just here for compatibility
-    def ScraperWiki.get_metadata(metadata_name, default = nil)
-        if !$metadatamessagedone == nil
-            puts "*** instead of get_metadata('"+metadata_name+"') please use\n    get_var('"+metadata_name+"')"
-            metadatamessagedone = true
-        end
-        result = ScraperWiki.get_var(metadata_name, default)
-        return result
-    end
-
-    # These are DEPRECATED and just here for compatibility
-    def ScraperWiki.save_metadata(metadata_name, value)
-        if !$metadatamessagedone
-            puts "*** instead of save_metadata('"+metadata_name+"') please use\n    save_var('"+metadata_name+"')"
-            $metadatamessagedone = true
-        end
-        return ScraperWiki.save_var(metadata_name, value)
-    end
-
-
-    def ScraperWiki.show_tables(dbname=nil)
-        name = "sqlite_master"
-        if dbname != nil
-            name = "`"+dbname+"`.sqlite_master" 
-        end
-        result = ScraperWiki.sqliteexecute("select tbl_name, sql from "+name+" where type='table'")
-        #return result["data"]
-        return (Hash[*result["data"].flatten])   # pre-1.8.7
-    end
-
-
-    def ScraperWiki.table_info(name)
-        sname = name.split(".")
-        if sname.length == 2
-            result = ScraperWiki.sqliteexecute("PRAGMA %s.table_info(`%s`)" % sname)
-        else
-            result = ScraperWiki.sqliteexecute("PRAGMA table_info(`%s`)" % name)
-        end
-        res = [ ]
-        for d in result["data"]
-            res.push(Hash[*result["keys"].zip(d).flatten])   # pre-1.8.7
-        end
-        return res
-    end
-
-
-    def ScraperWiki.getDataByDate(name, start_date, end_date, limit=-1, offset=0)
-        raise SqliteException.new("getDataByDate has been deprecated")
-    end
-    
-    def ScraperWiki.getDataByLocation(name, lat, lng, limit=-1, offset=0)
-        raise SqliteException.new("getDataByLocation has been deprecated")
-    end
-        
-    def ScraperWiki.search(name, filterdict, limit=-1, offset=0)
-        raise SqliteException.new("SW_APIWrapper.search has been deprecated")
-    end
-
-    def ScraperWiki.raisesqliteerror(rerror)
-        if /sqlite3.Error: no such table:/.match(rerror)  # old dataproxy
-            raise NoSuchTableSqliteException.new(rerror)
-        end
-        if /DB Error: \(OperationalError\) no such table:/.match(rerror)
-            raise NoSuchTableSqliteException.new(rerror)
-        end
-        raise SqliteException.new(rerror)
-    end
-    
-    def ScraperWiki.attach(name, asname=nil, verbose=1)
-        $attachlist.push({"name"=>name, "asname"=>asname})
-
-        ds = SW_DataStore.create()
-
-        if ds.m_webstore_port == 0
-            res = ds.request({'maincommand'=>'sqlitecommand', 'command'=>"attach", 'name'=>name, 'asname'=>asname})
-            if res["error"]
-                ScraperWiki.raisesqliteerror(res)
-            end
-        else
-            res = {'status'=>'ok'}
-        end
-        
-        if verbose
-            ScraperWiki.dumpMessage({'message_type'=>'sqlitecall', 'command'=>"attach", 'val1'=>name, 'val2'=>asname})
-        end
-            
-        return res
-    end
-    
-
-    def ScraperWiki.commit(verbose=1)
-        ds = SW_DataStore.create()
-        if ds.m_webstore_port == 0
-            res = ds.request({'maincommand'=>'sqlitecommand', 'command'=>"commit"})
-        else
-            puts "*** commit() no longer a necessary function call"
-            res = {'status'=>'ok'}
-        end
-    end
-
-    def ScraperWiki.select(sqlquery, data=nil, verbose=1)
-        if data != nil && sqlquery.scan(/\?/).length != 0 && data.class != Array
-            data = [data]
-        end
-        result = ScraperWiki.sqliteexecute("select "+sqlquery, data, verbose)
-        res = [ ]
-        for d in result["data"]
-            #res.push(Hash[result["keys"].zip(d)])           # post-1.8.7
-            res.push(Hash[*result["keys"].zip(d).flatten])   # pre-1.8.7
-        end
-        return res
-    end
-
-    # old functions put back in for regression
-    def ScraperWiki.getData(name, limit=-1, offset=0)
-        if !$apiwrapperattacheddata.include?(name)
-            puts "*** instead of getData('"+name+"') please use\n    ScraperWiki.attach('"+name+"') \n    print ScraperWiki.select('* from `"+name+"`.swdata')"
-            ScraperWiki.attach(name)
-            $apiwrapperattacheddata.push(name)
-        end
-
-        apilimit = 500
-        g = Enumerator.new do |g|
-            count = 0
-            while true
-                if limit == -1
-                    step = apilimit
-                else
-                    step = apilimit < (limit - count) ? apilimit : limit - count
-                end
-                query = "* from `#{name}`.swdata limit #{step} offset #{offset+count}"
-
-                records = ScraperWiki.select(query)
-                for r in records
-                    g.yield r
-                end
-
-                count += records.length
-                if records.length < step
-                    break
-                end
-                if limit != -1 and count >= limit
-                    break
-                end
-            end
-        end
-    end
-
-    def ScraperWiki.getKeys(name)
-        if !$apiwrapperattacheddata.include?(name)
-            puts "*** instead of getKeys('"+name+"') please use\n    ScraperWiki.attach('"+name+"') \n    print ScraperWiki.sqliteexecute('select * from `"+name+"`.swdata limit 0')['keys']"
-            ScraperWiki.attach(name)
-            $apiwrapperattacheddata.push(name)
-        end
-        result = ScraperWiki.sqliteexecute("select * from `"+name+"`.swdata limit 0")
-        if result.include?("error")
-            raise SqliteException.new(result["error"])
-        end
-        return result["keys"]
-    end
 end

metadata

@@ -1,98 +1,45 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: scraperwiki
-version: !ruby/object:Gem::Version 
-  hash: 21
+version: !ruby/object:Gem::Version
+  version: 2.0.0
   prerelease: 
-  segments: 
-  - 1
-  - 0
-  - 1
-  version: 1.0.1
 platform: ruby
-authors: 
-- Francis Irving
+authors:
+- Francis irving
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-10-09 00:00:00 +02:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: json
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: httpclient
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id002
-description: Ruby code used for accessing
-email: 
-- francis@scraperwiki.com
+date: 2013-04-04 00:00:00.000000000 Z
+dependencies: []
+description: A library for scraping web pages and saving data easily
+email: francis@scraperwiki.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- lib/version.rb
+files:
 - lib/scraperwiki.rb
-- lib/scraperwiki/datastore.rb
-- lib/scraperwiki/stacktrace.rb
-has_rdoc: true
-homepage: http://scraperwiki.com
+homepage: http://rubygems.org/gems/scraperwiki
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.5.2
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: ScraperWiki client library for Ruby
+summary: ScraperWiki
 test_files: []
-

data/lib/scraperwiki/datastore.rb

@@ -1,109 +0,0 @@
-require 'json'
-require 'singleton'
-require 'thread'
-require 'cgi'
-
-# the python version of this makes use of a global static copy of the class 
-# so the connection is made only once to the dataproxy
-# I think the Singleton module implements this magically
-
-class SW_DataStore
-    
-    @@lock = Mutex.new
-    
-    include Singleton
-
-    attr_accessor :m_port, :m_host, :m_scrapername, :m_runid, :m_attachables, :m_webstore_port
-
-    def initialize
-      @m_socket = nil
-      @m_host = nil
-      @m_port = nil
-      @m_scrapername = ''
-      @m_runid = ''
-      @m_attachables = []
-      @webstore_port = 0
-    end
-
-
-    def ensure_connected
-        # Connect to the data proxy. The data proxy will need to make an Ident call
-        # back to get the scraperID. Since the data proxy may be on another machine
-        # and the peer address it sees will have been subject to NAT or masquerading,
-        # send the UML name and the socket port number in the request.
-
-        if @m_socket == nil
-            @m_socket = TCPSocket.open(@m_host, @m_port)
-            proto, port, name, ip = @m_socket.addr()
-            if @m_scrapername == '' or @m_scrapername.nil?
-              sname = ''
-            else
-              sname = CGI::escape(@m_scrapername)
-            end
-            if @m_runid == '' or @m_runid.nil?
-              rid = ''
-            else
-              rid = CGI::escape(@m_runid)
-            end
-            
-            getmsg = "GET /?uml=%s&port=%s&vscrapername=%s&vrunid=%s HTTP/1.1\n\n" % ['lxc', port, sname, rid]
-            @m_socket.send(getmsg, 0)
-            @m_socket.flush()
-            
-            buffer = @m_socket.recv(1024)
-            result = JSON.parse(buffer)
-            if result["status"] != "good"
-                raise result["status"]
-            end
-        end
-    end
-
-    def request (req)
-        text = ''
-        @@lock.synchronize {
-          ensure_connected
-          reqmsg  = JSON.generate(req) + "\n"
-
-          bytes_sent = 0
-          while bytes_sent < reqmsg.length
-            bytes_sent += @m_socket.send(reqmsg.slice(bytes_sent, reqmsg.length), 0)
-          end
-          @m_socket.flush()
-
-          while true
-            buffer = @m_socket.recv(1024)
-            if buffer.length == 0
-                break
-            end
-            text += buffer
-            if text[-1] == "\n"[0]
-                break
-            end
-          end
-        }
-        return JSON.parse(text)
-    end
-
-    # function used to both initialize the settings and get an instance! 
-    # this is ridiculous and unnecessary with new webstore.  
-    # we are creating object without the fields merely to access the static variables! 
-    def SW_DataStore.create(host=nil, port = nil, scrapername = '', runid = nil, attachables = nil, webstore_port = nil)
-        instance = SW_DataStore.instance
-        # so, it might be intended that the host and port are
-        # set once, never to be changed, but this is ruby so
-        # there's no way to guarantee that.
-        if host && port && instance.m_port.nil? && instance.m_host.nil?
-          instance.m_host = host
-          instance.m_port = port
-          instance.m_scrapername = scrapername
-          instance.m_runid = runid
-          instance.m_attachables = attachables
-          instance.m_webstore_port = webstore_port
-        elsif host && port
-          raise "Can't change host and port once connection made"
-        elsif !(instance.m_port) || !(instance.m_host)
-          raise "Can't return a datastore without port/host information"
-        end
-        instance
-    end
-end

data/lib/scraperwiki/stacktrace.rb

@@ -1,51 +0,0 @@
-def _get_stackentry(code_filename, code, filename, linenumber, funcname)
-    nlinenumber = linenumber.to_i
-    stackentry = {"file" => filename, "linenumber" => nlinenumber, "duplicates" => 1}
-
-    if filename == "(eval)" or filename == code_filename
-        codelines = code.split("\n")
-        if (nlinenumber >= 1) && (nlinenumber <= codelines.size)
-            stackentry["linetext"] = codelines[nlinenumber-1]
-        elsif (nlinenumber == codelines.size + 1)
-            stackentry["linetext"] = "<end of file>"
-        else
-            stackentry["linetext"] = "getExceptionTraceback: ScraperWiki internal error, line %d out of range in file %s" % [nlinenumber, code_filename]
-        end
-        stackentry["file"] = "<string>"
-    else
-        # XXX bit of a hack to show the line number in third party libraries
-        stackentry["file"] += ":" + linenumber
-    end
-    if funcname
-        stackentry["furtherlinetext"] = funcname
-    end
-    return stackentry
-end
-
-def getExceptionTraceback(e, code, code_filename)
-    lbacktrace = e.backtrace.reverse
-    #File.open("/tmp/fairuby", 'a') {|f| f.write(JSON.generate(lbacktrace)) }
-
-    exceptiondescription = e.to_s
-    
-    stackdump = []
-    for l in lbacktrace
-        (filename, linenumber, funcname) = l.split(":")
-
-        next if filename.match(/\/exec.rb$/) # skip showing stack of wrapper
-
-        stackentry = _get_stackentry(code_filename, code, filename, linenumber, funcname)
-        stackdump.push(stackentry)
-    end
-
-    if e.kind_of?(SyntaxError)
-        (filename, linenumber, message) = exceptiondescription.split(/[:\n]/, 3)
-        exceptiondescription = message
-
-        stackentry = _get_stackentry(code_filename, code, filename, linenumber, nil)
-        stackdump.push(stackentry)
-    end
-
-    return { 'message_type' => 'exception', 'exceptiondescription' => exceptiondescription, "stackdump" => stackdump }
-end
-

data/lib/version.rb

@@ -1,4 +0,0 @@
-
-module ScraperWiki
-  VERSION = "1.0.1"
-end