veritable 0.1.0.0

data/CHANGELOG.txt

@@ -0,0 +1,3 @@
+veritable-ruby 0.1.0 - June 1, 2012
+    * Initial release
+

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Prior Knowledge
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Veritable
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'veritable'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install veritable
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/lib/veritable/api.rb

@@ -0,0 +1,460 @@
+require 'veritable/cursor'
+require 'veritable/datatypes'
+require 'veritable/errors'
+require 'veritable/resource'
+require 'veritable/util'
+
+module Veritable
+  class API
+    include VeritableResource
+
+    def root; get(""); end
+
+    def limits; get("user/limits"); end
+
+    def tables(opts={'start' => nil, 'limit' => nil})
+      Cursor.new({'collection' => "tables",
+        'start' => opts['start'],
+        'limit' => opts['limit']}.update(@opts)) {|x| Table.new(@opts, x)}
+    end
+
+    def table(table_id)
+      Table.new(@opts, get("tables/#{table_id}"))
+    end
+
+    def create_table(table_id=nil, description='', force=false)
+      if table_id.nil?
+        autogen = true
+        table_id = Util.make_table_id
+      else
+        autogen = false
+        Util.check_id table_id
+      end
+
+      if has_table? table_id
+        if autogen
+          return create_table(nil, description, false)
+        end
+        if ! force
+          raise VeritableError.new("Couldn't create table -- table with id #{table_id} already exists.")
+        else
+          delete_table table_id
+        end
+      end
+      doc = post("tables", {:_id => table_id, :description => description})
+      Table.new(@opts, doc)
+    end
+
+    def delete_table(table_id); delete("tables/#{table_id}"); end
+
+    def inspect; to_s; end
+    def to_s; "#<Veritable::API url='#{api_base_url}'>"; end
+
+    def has_table?(table_id)
+      begin
+        table table_id
+      rescue
+        false
+      else
+        true
+      end
+    end
+  end
+
+  class Table
+    include VeritableResource
+
+    alias :rest_delete :delete
+    def delete
+      rest_delete(link('self'))
+    end
+
+    def row(row_id); get("#{link('rows')}/#{row_id}"); end
+
+    def rows(opts={'start' => nil, 'limit' => nil})
+      Cursor.new({'collection' => link('rows'),
+        'start' => opts['start'],
+        'limit' => opts['limit']}.update(@opts))
+    end
+
+    def upload_row(row)
+      Util.check_row row
+      put("#{link('rows')}/#{row['_id']}", row)
+    end
+
+    def batch_upload_rows(rows, per_page=100)
+      batch_modify_rows('put', rows, per_page)
+    end
+
+    def delete_row(row_id)
+      rest_delete("#{link('rows')}/#{row_id}")
+    end
+
+    def batch_delete_rows(rows, per_page=100)
+      begin
+        batch_modify_rows('delete', rows, per_page)
+      rescue VeritableError => e
+        if (not e.respond_to?(:http_code)) or (not (e.http_code == "404 Resource Not Found"))
+          raise e
+        end
+      end
+    end
+
+    def analysis(analysis_id)
+      Analysis.new(@opts, get("#{link('analyses')}/#{analysis_id}"))
+    end
+
+    def analyses(opts={'start' => nil, 'limit' => nil})
+      Cursor.new({'collection' => link('analyses'),
+        'start' => opts['start'],
+        'limit' => opts['limit']}.update(@opts)) {|x| Analysis.new(@opts, x)}
+    end
+
+    def delete_analysis(analysis_id)
+      rest_delete("#{link('analyses')}/#{analysis_id}")
+    end
+
+    def create_analysis(schema, analysis_id=nil, description="", force=false, analysis_type="veritable")
+      if analysis_type != "veritable"
+        if analysis_type.respond_to? :to_s
+          raise VeritableError.new("Invalid analysis type #{analysis_type}.")
+        else
+          raise VeritableError.new("Invalid analysis type.")
+        end
+      end
+
+      if analysis_id.nil?
+        autogen = true
+        analysis_id = Util.make_analysis_id
+      else
+        autogen = false
+        Util.check_id analysis_id
+      end
+
+      if has_analysis? analysis_id
+        if autogen
+          return create_analysis(nil, description, false)
+        end
+        if ! force
+          raise VeritableError.new("Couldn't create table -- table with id #{analysis_id} already exists.")
+        else
+          delete_analysis analysis_id
+        end
+      end
+      doc = post(link('analyses'), {:_id => analysis_id, :description => description, :type => analysis_type, :schema => schema})
+      Analysis.new(@opts, doc)
+    end
+
+    def inspect; to_s; end
+    def to_s; "#<Veritable::Table _id='#{_id}'>"; end
+
+    def _id; @doc['_id']; end
+    def description; @doc['description']; end
+
+    def has_analysis?(analysis_id)
+      begin
+        analysis analysis_id
+      rescue
+        false
+      else
+        true
+      end
+    end
+
+    private
+
+    def batch_modify_rows(action, rows, per_page=100)
+      if not per_page.is_a? Fixnum or not per_page > 0
+        raise VeritableError.new("Batch upload or delete must have integer page size greater than 0.")
+      end
+      rows = rows.collect {|row|
+        Util.check_row(row)
+        row
+      }
+      if (not rows.is_a? Array) and (not rows.is_a? Veritable::Cursor)
+        raise VeritableError.new("Must pass an array of row hashes or a cursor of rows to batch upload or delete.")
+      end
+      ct = (1..per_page).to_a.cycle
+      batch = Array.new()
+      ct.each { |ct|
+        if rows.empty?
+          if batch.size > 0
+            post(link('rows'), {'action' => action, 'rows' => batch})
+          end
+          break
+        end
+        batch.push rows.shift
+        if ct == per_page
+          post(link('rows'), {'action' => action, 'rows' => batch})
+          batch = Array.new()
+        end
+      }
+    end
+  end
+
+  class Analysis
+    include VeritableResource
+
+    def update; @doc = get(link('self')); end
+
+    alias :rest_delete :delete
+    def delete; rest_delete(link('self')); end
+
+    def schema; Schema.new(get(link('schema'))); end
+
+    def wait(max_time=nil, poll=2)
+      elapsed = 0
+      while running?
+        sleep poll
+        if not max_time.nil?
+          elapsed += poll
+          if elapsed > max_time
+            raise VeritableError.new("Wait for analysis -- Maximum time of #{max_time} second exceeded.")
+          end
+        end
+        update
+      end
+    end
+
+    def predict(row, count=100)
+      update if running?
+      if succeeded?
+        if not row.is_a? Hash
+          raise VeritableError.new("Predict -- Must provide a row hash to make predictions.")
+        end
+        res = post(link('predict'), {'data' => row, 'count' => count})
+        if not res.is_a? Array
+          begin
+            res.to_s
+          rescue
+            raise VeritableError.new("Predict -- Error making predictions: #{res}")
+          else
+            raise VeritableError.new("Predict -- Error making predictions.")
+          end
+        end
+        Prediction.new(row, res, schema)
+      elsif running?
+        raise VeritableError.new("Predict -- Analysis with id #{_id} is still running and not yet ready to predict.")
+      elsif failed?
+        raise VeritableError.new("Predict -- Analysis with id #{_id} has failed and cannot predict.")
+      else
+        raise VeritableError.new("Predict -- Shouldn't be here -- please let us know at support@priorknowledge.com.")
+      end
+    end
+
+    def related_to(column_id, opts={'start' => nil, 'limit' => nil})
+      update if running?
+      if succeeded?
+        Cursor.new(
+         {'collection' => "#{link('related')}/#{column_id}",
+          'start' => opts['start'],
+          'limit' => opts['limit']}.update(@opts))
+      elsif running?
+        raise VeritableError.new("Related -- Analysis with id #{_id} is still running and not yet ready to calculate related.")
+      elsif failed?
+        raise VeritableError.new("Related -- Analysis with id #{_id} has failed and cannot calculate related.")
+      else
+        raise VeritableError.new("Related -- Shouldn't be here -- please let us know at support@priorknowledge.com.")
+      end
+    end
+
+    def inspect; to_s; end
+    def to_s; "#<Veritable::Analysis _id='#{_id}'>"; end
+
+    def _id; @doc['_id']; end
+    def created_at; @doc['created_at']; end
+    def finished_at; @doc['finished_at']; end
+    def state; @doc['state']; end
+    def running?; state == 'running'; end
+    def succeeded?; state == 'succeeded'; end
+    def failed?; state == 'failed'; end
+    def error; state == 'failed' ? @doc['error'] : nil; end
+    def progress; state == 'running' ? @doc['progress'] : nil; end
+  end
+
+  class Schema < Hash
+    def initialize(data, subset=nil)
+      begin
+        data.each {|k, v|
+          if subset.is_a? Array
+            self[k] = v if subset.include? k
+          elsif subset.is_a? Hash
+            self[k] = v if subset.has_key? k
+          else
+            self[k] = v
+          end
+        }
+      rescue
+        begin
+          data.to_s
+        rescue
+          raise VeritableError.new("Initialize schema -- invalid schema data.")
+        else
+          raise VeritableError.new("Initialize schema -- invalid schema data #{data}.")
+        end
+      end
+    end
+
+    def type(column)
+      self[column]['type']
+    end
+
+    def validate
+      self.each {|k, v|
+        if not k.is_a? String
+          begin
+            k.to_s
+          rescue
+            raise VeritableError.new("Validate schema -- Invalid schema specification: nonstring column id.")
+          else
+            raise VeritableError.new("Validate schema -- Invalid schema specification: nonstring column id #{k}")
+          end
+        end
+        begin
+          Util.check_id k
+        rescue
+          raise VeritableError.new("Validate schema -- Invalid column name #{k}: must contain only alphanumerics, dashes, and underscores, and may not begin with a dash or underscore.")
+        end
+        if not v.include? 'type'
+          raise VeritableError.new("Validate schema -- Invalid schema specification. Column #{k} must specify a 'type', one of #{DATATYPES}")
+        end
+        if not DATATYPES.include? v['type']
+          raise VeritableError.new("Validate schema -- Invalid schema specification. Column #{k}, type #{v['type']} is not valid. Type must be one of #{DATATYPES}")
+        end
+      }
+    end
+  end
+
+  class Prediction < Hash
+    attr_reader :request
+    attr_reader :distribution
+    attr_reader :schema
+    attr_reader :uncertainty
+
+    def initialize(request, distribution, schema)
+      @request = request
+      @distribution = distribution
+      @schema = Schema.new(schema)
+      @uncertainty = Hash.new()
+
+      request.each { |k,v|
+        if v.nil?
+          self[k] = point_estimate k
+          @uncertainty[k] = calculate_uncertainty k
+        else
+          self[k] = v
+          @uncertainty[k] = 0.0
+        end
+      }
+    end
+    
+    def prob_within(column, range)
+      col_type = schema.type column
+      Veritable::Util.check_datatype(col_type, "Probability within -- ")
+      if col_type == 'boolean' or col_type == 'categorical'
+        count = distribution.inject(0) {|memo, row|
+          if range.include? row[column]
+            memo + 1 
+          else
+            memo
+          end
+        }
+        count.to_f / distribution.size
+      elsif col_type == 'count' or col_type == 'real'
+        mn = range[0]
+        mx = range[1]
+        count = distribution.inject(0) {|memo, row|
+          v = row[column]
+          if (mn.nil? or v >= mn) and (mx.nil? or v <=mx)
+            memo + 1 
+          else
+            memo
+          end
+        }
+        count.to_f / distribution.size
+      end
+    end
+
+    def credible_values(column, p=nil)
+      col_type = schema.type column
+      Veritable::Util.check_datatype(col_type, "Credible values -- ")
+      if col_type == 'boolean' or col_type == 'categorical'
+        p = 0.5 if p.nil?
+        tf = Hash.new
+        ((freqs(counts(column)).sort_by {|k, v| v}).reject {|c, a| a < p}).each {|k, v| tf[k] = v}
+        tf
+      elsif col_type == 'count' or col_type == 'real'
+        p = 0.9 if p.nil?
+        n = distribution.size
+        a = (n * (1.0 - p) / 2.0).round.to_i
+        sv = sorted_values column
+        n = sv.size
+        lo = sv[a]
+        hi = sv[n - 1 - a]
+        [lo, hi]
+      end
+    end
+
+    def inspect; to_s; end
+    def to_s; "<Veritable::Prediction #{super}>"; end
+
+    private
+
+    def sorted_values(column)
+      values = (distribution.collect {|row| row[column]}).reject {|x| x.nil?}
+      values.sort
+    end
+
+    def counts(column)
+      cts = Hash.new
+      distribution.each {|row|
+        if row.has_key? column
+          cat = row[column]
+          if not (cts.has_key? cat)
+            cts[cat] = 0
+          end
+          cts[cat] += 1
+        end
+      }
+      cts
+    end
+
+    def freqs(cts)
+      total = cts.values.inject(0) {|memo, obj| memo + obj}
+      freqs = Hash.new()
+      cts.each {|k, v|
+        freqs[k] = v.to_f / total
+      }
+      freqs
+    end
+
+    def point_estimate(column)
+      col_type = schema.type column
+      Veritable::Util.check_datatype(col_type, "Point estimate -- ")
+      if col_type == 'boolean' or col_type == 'categorical'
+        # use the mode
+        (counts(column).max_by {|k, v| v})[0]
+      elsif col_type == 'real' or col_type == 'count'
+        # use the mean
+        values = distribution.collect {|row| row[column]}
+        mean = (values.inject(0) {|memo, obj| memo + obj}) / values.size.to_f
+        col_type == 'real' ? mean : mean.round.to_i
+      end
+    end
+
+    def calculate_uncertainty(column)
+      values = distribution.collect {|row| row[column]}
+      col_type = schema.type column
+      Veritable::Util.check_datatype(col_type, "Calculate uncertainty -- ")
+      n = values.size
+      if col_type == 'boolean' or col_type == 'categorical'
+        e = ((counts column).max_by {|k, v| v})[0]
+        c = 1.0 - (values.count {|v| v == e} / n.to_f)
+        c.to_f
+      elsif col_type == 'count' or col_type == 'real'
+        r = credible_values column
+        (r[1] - r[0]).to_f
+      end
+    end
+  end
+end

data/lib/veritable/connection.rb

@@ -0,0 +1,84 @@
+require 'veritable/object'
+require 'multi_json'
+
+module Veritable
+  module Connection
+    include VeritableObject
+    def initialize(opts=nil, doc=nil)
+      super(opts, doc)
+      require_opts :api_key, :api_base_url
+      default_opts(:ssl_verify => true, :enable_gzip => true)
+    end
+
+    def get(url, params=nil, headers={})
+      if params and params.count > 0
+        query_string = Util.query_params(params)
+        url += "?#{query_string}"
+      end
+      request(:get, url, nil, headers)
+    end
+
+    def post(url, payload, headers={})
+      payload = MultiJson.encode(payload)
+      headers = headers.merge({:content_type => 'application/json'})
+      request(:post, url, payload, headers)
+    end
+
+    def put(url, payload, headers={})
+      payload = MultiJson.encode(payload)
+      headers = headers.merge({:content_type => 'application/json'})
+      request(:put, url, payload, headers)
+    end
+
+    def delete(url, headers={})
+      begin
+        request(:delete, url, nil, headers)
+      rescue VeritableError => e
+        if not e.respond_to? :http_code or not e.http_code == "404 Resource Not Found"
+          raise e
+        end
+      end
+    end
+
+    def request(verb, url, payload=nil, headers={})
+      url = api_base_url + "/" + url
+
+      headers = {
+        :user_agent => USER_AGENT,
+        :accept => :json,
+        :accept_encoding => enable_gzip ? :gzip : nil
+      }.merge(headers)
+
+      opts = {
+        :method => verb.to_s,
+        :url => url,
+        :user => api_key,
+        :password => "",
+        :headers => headers,
+        :payload => payload,
+        :verify_ssl => ssl_verify,
+      }
+      begin
+        response = RestClient::Request.execute(opts)
+      rescue RestClient::Exception => e
+        begin
+          r = MultiJson.decode(e.response)
+          msg = r['message']
+          code = r['code']
+        rescue
+          raise e
+        end
+        raise VeritableError.new("HTTP Error #{e.message} -- #{code}: #{msg}", {'http_code' => e.message, 'api_code' => code, 'api_message' => msg})
+      end
+      return MultiJson.decode(response)
+    end
+
+    private
+
+    def api_key; @opts[:api_key]; end
+    def api_base_url; @opts[:api_base_url]; end
+    def ssl_verify; @opts[:ssl_verify]; end
+    def enable_gzip; @opts[:enable_gzip]; end
+
+  end
+end

data/lib/veritable/cursor.rb

@@ -0,0 +1,66 @@
+require 'veritable/object'
+require 'veritable/resource'
+
+module Veritable
+  class Cursor
+    include VeritableResource
+    include Enumerable
+    def initialize(opts=nil, doc=nil, &lazymap)
+      super(opts, doc)
+
+      require_opts 'collection'
+      default_opts({'per_page' => 100})
+
+      collection_key = collection.split("/")[-1]
+      @doc = get(collection, params={:count => per_page, :start => start})
+      @doc.has_key?(collection_key) ? @opts['key'] = collection_key : @opts['key'] = 'data'
+      @opts['lazymap'] = lazymap if lazymap
+    end
+
+    def each
+      i = limit if limit
+      loop do
+        if data.length > 0 or refresh > 0
+          if limit
+            raise StopIteration if i == 0
+            i = i - 1
+          end
+          if lazymap
+            yield lazymap.call(data.shift)
+          else
+            yield data.shift
+          end
+        else
+          raise StopIteration
+        end
+      end
+    end
+    def inspect; to_s; end
+    def to_s; "#<Veritable::Cursor collection='" + collection + "'>"; end
+
+    private
+
+    def refresh
+      return data.length if data.length > 0
+      if next_page
+        @doc = get next_page
+      elsif last_page?
+        return 0
+      else
+        @doc = get(collection, params={:count => per_page, :start => start})
+      end
+      return data.length
+    end
+
+    def limit; @opts['limit']; end
+    def limit=(x); @opts['limit'] = x; end
+    def start; @opts['start']; end
+    def per_page; @opts['per_page']; end
+    def collection; @opts['collection'] end
+    def lazymap; @opts['lazymap']; end
+    def key; @opts['key'] end
+    def next_page; link 'next' end
+    def last_page?; ! @doc.has_key? 'next' end
+    def data; @doc[key] end
+  end
+end

data/lib/veritable/datatypes.rb

@@ -0,0 +1,3 @@
+module Veritable
+  DATATYPES = ['boolean', 'categorical', 'real', 'count']
+end

data/lib/veritable/errors.rb

@@ -0,0 +1,17 @@
+class VeritableError < StandardError
+  attr_reader :message
+  def initialize(message, opts=nil)
+    @message = message
+    if opts.is_a? Hash
+      @opts = opts
+      eigenclass = class << self; self; end
+      @opts.keys.each {|k|
+        eigenclass.send(:define_method, k.to_sym) {
+          @opts[k]
+        }
+      }
+    end
+  end
+  def to_s; message; end
+  def inspect; message; end
+end

data/lib/veritable/object.rb

@@ -0,0 +1,20 @@
+require 'veritable/errors'
+
+module Veritable
+  module VeritableObject
+    def initialize(opts=nil, doc=nil)
+      @opts = opts
+      @doc = doc
+    end
+
+    private
+
+    def require_opts(*keys)
+      keys.each {|k| raise VeritableError.new("Error initializing object -- must provide #{k}") unless @opts.has_key?(k)}
+    end
+
+    def default_opts(hash={})
+      hash.each {|k, v| @opts[k] = v unless @opts.has_key?(k)}
+    end
+  end
+end

data/lib/veritable/resource.rb

@@ -0,0 +1,14 @@
+require 'veritable/connection'
+require 'veritable/errors'
+
+module Veritable
+  module VeritableResource
+  	include Connection
+
+    private
+
+    def link(name)
+      @doc['links'][name]
+    end
+  end
+end

data/lib/veritable/util.rb

@@ -0,0 +1,429 @@
+require 'veritable/datatypes'
+require 'veritable/errors'
+require 'uuid'
+require 'uri'
+require 'csv'
+require 'set'
+
+module Veritable
+  module Util
+    class << self
+      def make_table_id; UUID.new.generate :compact ; end
+      def make_analysis_id; UUID.new.generate :compact ; end
+
+      def query_params(params, parent=nil)
+        flatten_params(params).collect {|x|
+          "#{x[0]}=#{x[1]}"
+        }.join("&")
+      end
+
+      def check_id(id)
+        if not id.is_a? String
+          begin
+            id.to_s
+          rescue
+            raise VeritableError.new("Invalid id -- strings only.")
+          else
+            raise VeritableError.new("Invalid id '#{id}' -- strings only.")
+          end
+        elsif not id =~ Regexp.new('\A[a-zA-Z0-9][-_a-zA-Z0-9]*\z')
+          raise VeritableError.new("Invalid id '#{id}' -- must contain only alphanumerics, underscores, and dashes.")
+        elsif id[0] == '_' or id[0] == '-'
+          raise VeritableError.new("Invalid id '#{id}' -- may not begin with a dash or underscore.")
+        end
+      end
+
+      def check_row(row)
+        if not row.is_a? Hash
+          begin
+            row.to_s
+          rescue
+            raise VeritableError.new("Invalid row -- Must provide a hash of column name-value pairs.")
+          else
+            raise VeritableError.new("Invalid row #{row} -- Must provide a hash of column name-value pairs.")
+          end
+        elsif not row.has_key? '_id'
+          raise VeritableError.new("Invalid row #{row} -- rows must contain unique row ids in the '_id' field.")
+        else
+          begin
+            check_id row['_id']
+          rescue VeritableError => e
+            raise VeritableError.new("Invalid row #{row} -- #{e}")
+          end
+        end
+      end
+
+      def check_datatype(datatype, msg=nil)
+        if not DATATYPES.include? datatype
+          begin
+            datatype.to_s
+          rescue
+            raise VeritableError.new("#{msg}Invalid data type.")
+          else
+            raise VeritableError.new("#{msg}Invalid data type '#{datatype}'.")
+          end
+        end
+      end
+
+      def split_rows(rows, frac)
+        rows = rows.to_a
+        n = rows.size
+        inds = (0...n).to_a.shuffle
+        border_ind = (n * frac).floor.to_i
+        train_dataset = (0...border_ind).collect {|i| rows[inds[i]] }
+        test_dataset = (border_ind...n).collect {|i| rows[inds[i]] }
+        return [train_dataset, test_dataset]
+      end
+
+      def validate_schema(schema)
+        schema.is_a? Veritable::Schema ? schema.validate : Veritable::Schema.new(schema).validate
+      end
+
+      def make_schema(schema_rule, opts={})
+		if ((not opts.has_key?('headers')) and (not opts.has_key?('rows')))
+			raise VeritableError.new("Either :headers or :rows must be provided!")
+		end
+		headers = opts.has_key?('headers') ? opts['headers'] : nil
+		if headers.nil?
+			headers = Set.new
+			opts['rows'].each {|row| headers.merge(row.keys)}
+			headers = headers.to_a.sort
+		end
+		schema = {}
+		headers.each do |c|
+			schema_rule.each do |r, t|
+				if r === c
+					schema[c] = t
+					break
+				end
+			end
+		end
+		return Veritable::Schema.new(schema)
+      end
+
+      def write_csv(rows, filename)
+        headers = Set.new
+        rows.each {|row| headers.merge(row.keys)}
+        headers = headers.to_a.sort
+        CSV.open(filename, "w") do |csv|
+          csv << headers
+          rows.each do |row|
+            out_row = headers.collect {|h| row.keys.include?(h) ? row[h] : ''}
+            csv << out_row
+          end
+        end
+      end
+
+      def read_csv(filename, id_col=nil, na_vals=[''])
+        rows = CSV.read(filename)
+        header = rows.shift
+        header = header.collect {|h| (h == id_col ? '_id' : h).strip}
+        if header.include?('_id')
+          id_col = '_id'
+        end
+        rid = 0
+        rows = rows.collect do |raw_row|
+          rid = rid + 1
+          row = {}
+          (0...raw_row.length).each do |i|
+            row[header[i]] = ( na_vals.include?(raw_row[i]) ? nil : raw_row[i] )
+          end
+          if id_col.nil? 
+            row['_id'] = rid.to_s
+          end
+          row
+        end
+        return rows
+      end
+      
+      def clean_data(rows, schema, opts={})
+        validate(rows, schema, {
+          'convert_types' => opts.has_key?('convert_types') ? opts['convert_types'] : true,
+          'allow_nones' => false,
+          'remove_nones' => opts.has_key?('remove_nones') ? opts['remove_nones'] : true,
+          'remove_invalids' => opts.has_key?('remove_invalids') ? opts['remove_invalids'] : true,
+          'reduce_categories' => opts.has_key?('reduce_categories') ? opts['reduce_categories'] : true,
+          'has_ids' => true,
+          'assign_ids' => opts.has_key?('assign_ids') ? opts['assign_ids'] : false,
+          'allow_extra_fields' => true,
+          'remove_extra_fields' => opts.has_key?('remove_extra_fields') ? opts['remove_extra_fields'] : false,
+          'allow_empty_columns' => false})
+      end
+
+      def validate_data(rows, schema)
+        validate(rows, schema, {
+          'convert_types' => false,
+          'allow_nones' => false,
+          'remove_nones' => false,
+          'remove_invalids' => false,
+          'reduce_categories' => false,
+          'has_ids' => true,
+          'assign_ids' => false,
+          'allow_extra_fields' => true,
+          'remove_extra_fields' => false,
+          'allow_empty_columns' => false})
+      end
+
+      def clean_predictions(predictions, schema, opts={})
+        validate(predictions, schema, {
+          'convert_types' => opts.has_key?('convert_types') ? opts['convert_types'] : true,
+          'allow_nones' => true,
+          'remove_nones' => false,
+          'remove_invalids' => opts.has_key?('remove_invalids') ? opts['remove_invalids'] : true,
+          'reduce_categories' => false,
+          'has_ids' => false,
+          'assign_ids' => false,
+          'allow_extra_fields' => false,
+          'remove_extra_fields' => opts.has_key?('remove_extra_fields') ? opts['remove_extra_fields'] : true,
+          'allow_empty_columns' => true})
+      end
+
+      def validate_predictions(predictions, schema)
+        validate(predictions, schema, {
+          'convert_types' => false,
+          'allow_nones' => true,
+          'remove_nones' => false,
+          'remove_invalids' => false,
+          'reduce_categories' => false,
+          'has_ids' => false,
+          'assign_ids' => false,
+          'allow_extra_fields' => false,
+          'remove_extra_fields' => false,
+          'allow_empty_columns' => true})
+      end
+
+      private
+
+      def flatten_params(params, parent=nil)
+        result = []
+        if params.is_a? Hash
+          params.each {|k, v|
+            kk = parent ? "#{parent}[#{urlencode(k)}]" : urlencode(k)
+            if v.is_a?(Hash) or v.is_a?(Array)
+              result += flatten_params(v, kk)
+            else
+              result << [kk, urlencode(v)]
+            end
+          }
+        elsif params.is_a? Array
+          params.each {|v|
+            if v.is_a?(Hash) or v.is_a?(Array)
+              result += flatten_params(v, kk)
+            else
+              result << ["#{parent}[]", urlencode(v)]
+            end
+          }
+        end
+        result
+      end
+
+      def urlencode(k)
+        URI.escape(k.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+      end
+
+      def to_integer(v)
+        return v if v.is_a? Fixnum
+        v.gsub!(/\A([+-]?\d+?)\.0*?\Z/, '\1')
+        Integer(v)
+      end
+	  	  
+      def validate(rows, schema, opts)
+        schema = Veritable::Schema.new(schema) unless schema.is_a? Veritable::Schema
+
+        # ensure the schema is well-formed
+        schema.validate  
+
+        # store the row numbers of each unique id so that we can warn the user
+        unique_ids = Hash.new
+
+        # store the density of fields
+        field_fill = Hash.new
+        schema.keys.each {|c|
+          field_fill[c] = 0 if c != '_id'
+        }
+
+        # store the number of categories in each categorical column
+        category_counts = Hash.new
+
+        # values which will be converted to true and false in boolean cols if convert_types
+        true_strings = ['true', 't', 'yes', 'y']
+        false_strings = ['false', 'f', 'no', 'n']
+
+        max_cats = 256
+        # be careful before changing the order of any of this logic -- the point is to do this all only once
+        (0...rows.size).each {|i|
+          if opts['assign_ids']
+            rows[i]['_id'] = i.to_s  # number the rows sequentially
+          elsif opts['has_ids']
+            raise VeritableError.new("Validate -- row #{i} is missing key '_id'", {'row' => i, 'col' => '_id'}) unless rows[i].include? '_id'
+            
+            if opts['convert_types'] # attempt to convert _id to string
+              begin
+                rows[i]['_id'] = rows[i]['_id'].to_s if not rows[i]['_id'].is_a? String
+              rescue
+                raise VeritableError.new("Validate -- row #{i}, key '_id' cannot be converted to string.", {'row' => i, 'col' => '_id'})
+              end
+            end
+
+            if not rows[i]['_id'].is_a? String # invalid type for _id
+              begin
+                rows[i]['_id'].to_s
+              rescue
+                raise VeritableError.new("Validate -- row #{i}, key '_id' is not a string.", {'row' => i, 'col' => '_id'})
+              else
+                raise VeritableError.new("Validate -- row #{i}, key '_id', value #{rows[i]['_id']} is not a string.", {'row' => i, 'col' => '_id'})
+              end
+            end
+            
+            begin
+              check_id rows[i]['_id'] # make sure _id is alphanumeric
+            rescue
+              raise VeritableError.new("Validate -- row #{i}, key '_id', value #{rows[i]['_id']} contains disallowed characters. Ids must contain only alphanumerics, with underscores and hyphens allowed after the beginning of the id.", {'row' => i, 'col' => '_id'})
+            end
+            
+            if unique_ids.include? rows[i]['_id']
+              raise VeritableError.new("Validate -- row #{i}, key '_id', value #{rows[i]['_id']} is non-unique, conflicts with row #{unique_ids[rows[i]['_id']]}", {'row' => i, 'col' => '_id'})
+            end
+            
+            unique_ids[rows[i]['_id']] = i
+          elsif rows[i].include? '_id' # no ids, no autoid, but _id column
+            if opts['remove_extra_fields'] # just remove it
+              rows[i].delete '_id'
+            else
+              raise VeritableError.new("Validate -- row #{i}, key '_id' should not be included.", {'row' => i, 'col' => '_id'})
+            end
+          end
+          rows[i].keys.each {|c|
+            if c != '_id'
+              if not schema.include? c # keys missing from schema
+                if opts['remove_extra_fields'] # remove it
+                  rows[i].delete c
+                else
+                  if not opts['allow_extra_fields'] # or silently allow
+                    raise VeritableError.new("Row #{i}, key #{c} is not defined in schema", {'row' => i, 'col' => c})
+                  end
+                end
+              elsif rows[i][c].nil? # nil values
+                if opts['remove_nones'] # remove
+                  rows[i].delete c
+                else
+                  if not opts['allow_nones'] # or silently allow
+                    raise VeritableError.new("Row #{i}, key #{c} should be removed because it is nil", {'row' => i, 'col' => c})
+                  end
+                end
+              else # keys present in schema
+                coltype = schema.type c # check the column type
+                if coltype == 'count'
+                  if opts['convert_types'] # try converting to int
+                    begin
+                      rows[i][c] = to_integer(rows[i][c])
+                    rescue
+                      rows[i][c] = opts['remove_invalids'] ? nil : rows[i][c] # flag for removal
+                    end
+                  end
+                  if rows[i][c].nil?
+                    rows[i].delete c  # remove flagged values
+                  elsif opts['remove_invalids'] and (rows[i][c].is_a? Fixnum) and (rows[i][c] < 0)
+                    rows[i].delete c
+                  else
+                    if not (rows[i][c].is_a? Fixnum) or not (rows[i][c] >= 0) # catch invalids
+                      raise VeritableError.new("Validate -- row #{i}, key #{c}, value #{rows[i][c]} is #{rows[i][c].class}, not a non-negative integer.", {'row' => i, 'col' => c})
+                    end
+                  end
+                elsif coltype == 'real'
+                  if opts['convert_types'] # try converting to float
+                    begin
+                      rows[i][c] = Float(rows[i][c]) unless rows[i][c].is_a? Float
+                    rescue
+                      rows[i][c] = opts['remove_invalids'] ? nil : rows[i][c] # flag for removal
+                    end
+                  end
+                  if rows[i][c].nil?
+                    rows[i].delete c
+                  else
+                    if not rows[i][c].is_a? Float
+                      raise VeritableError.new("Validate -- row #{i}, key #{c}, value #{rows[i][c]} is a #{rows[i][c].class}, not a float.", {'row' => i, 'col' => c})
+                    end
+                  end
+                elsif coltype == 'boolean'
+                  if opts['convert_types'] # try converting to bool
+                    lc = (rows[i][c]).to_s.strip.downcase
+                    begin
+                      if true_strings.include? lc
+                        rows[i][c] = true
+                      elsif false_strings.include? lc
+                        rows[i][c] = false
+                      elsif to_integer(rows[i][c]) == 0 # note that this behavior differs from what a rubyist might expect; "0" maps to false
+                        rows[i][c] = false
+                      else
+                        rows[i][c] = true
+                      end 
+                    rescue
+                      rows[i][c] = opts['remove_invalids'] ? nil : rows[i][c] # flag for removal
+                    end
+                  end
+                  if rows[i][c].nil? # remove flagged values
+                    rows[i].delete c
+                  else
+                    if not [true, false].include? rows[i][c]
+                      raise VeritableError.new("Validate -- row #{i}, key #{c}, value #{rows[i][c]} is #{rows[i][c].class}, not a boolean", {'row' => i, 'col' => c})
+                    end
+                  end
+                elsif coltype == 'categorical'
+                  if opts['convert_types'] # try converting to string
+                    begin
+                      rows[i][c] = rows[i][c].to_s unless rows[i][c].is_a? String
+                    rescue
+                      rows[i][c] = opts['remove_invalids'] ? nil : rows[i][c] # flag for removal
+                    end
+                  end
+                  if rows[i][c].nil? # remove flagged values
+                    rows[i].delete c
+                  else
+                    if not rows[i][c].is_a? String # catch invalids
+                      raise VeritableError.new("Validate -- row #{i}, key #{c}, value #{rows[i][c]} is a #{rows[i][c].class}, not a string", {'row' => i, 'col' => c})
+                    end
+                    category_counts[c] = Hash.new if not category_counts.include? c # increment count
+                    category_counts[c][rows[i][c]] = 0 if not category_counts[c].include? rows[i][c]
+                    category_counts[c][rows[i][c]] += 1
+                  end
+                else
+                  raise VeritableError.new("Validate -- didn't recognize column type #{coltype}")
+                end
+              end
+              if not field_fill.include? c and not opts['remove_extra_fields']
+                field_fill[c] = 0
+              end
+              if rows[i].include? c and not rows[i][c].nil?
+                field_fill[c] += 1
+              end
+            end
+          }
+        }
+        category_counts.keys.each {|c|
+          cats = category_counts[c].keys
+          if cats.size > max_cats # too many categories
+            if opts['reduce_categories'] # keep the largest max_cats - 1
+              cats = cats.sort! {|a,b| category_counts[c][b] <=> category_counts[c][a]}
+              category_map = Hash.new
+              (0...cats.size).each {|j|
+                j < max_cats - 1 ? category_map[cats[j]] = cats[j] : category_map[cats[j]] = "Other"
+              }
+              (0...rows.size).each {|i|
+                rows[i][c] = category_map[rows[i][c]] if rows[i].include? c and not rows[i][c].nil?
+              }
+            else
+              raise VeritableError.new("Validate -- categorical column #{c} has #{category_counts[c].keys.size} unique values which exceeds the limits of #{max_cats}.", {'col' => c})
+            end
+          end
+        }
+        if not opts['allow_empty_columns']
+          field_fill.each {|c, fill|
+            raise VeritableError.new("Validate -- column #{c} does not have any values", {'col' => c}) if fill == 0
+          }
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/veritable/version.rb

@@ -0,0 +1,3 @@
+module Veritable
+  VERSION = "0.1.0.0"
+end

data/lib/veritable.rb

@@ -0,0 +1,32 @@
+require 'openssl'
+
+require 'veritable/api'
+require 'veritable/connection'
+require 'veritable/errors'
+require 'veritable/util'
+require 'veritable/version'
+
+require 'rest_client'
+require 'uuid'
+require 'multi_json'
+
+module Veritable
+  USER_AGENT = 'veritable-ruby ' + VERSION
+  BASE_URL = "https://api.priorknowledge.com"
+  
+  def self.connect(opts={})
+    opts[:api_key] = opts[:api_key] || ENV['VERITABLE_KEY']
+    opts[:api_base_url] = opts[:api_base_url] || ENV['VERITABLE_URL'] || BASE_URL
+
+    opts[:ssl_verify] = true unless opts.has_key?(:ssl_verify)
+    opts[:enable_gzip] = true unless opts.has_key?(:enable_gzip)
+
+    api = API.new(opts)
+    connection_test = api.root
+    status = connection_test["status"]
+    entropy = connection_test["entropy"]
+    raise VeritableError.new("No Veritable server responding at #{opts[:api_base_url]}") if status != "SUCCESS"
+    raise VeritableError.new("No Veritable server responding at #{opts[:api_base_url]}") if ! entropy.is_a?(Float)
+    api
+  end
+end

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: veritable
+version: !ruby/object:Gem::Version
+  version: 0.1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Prior Knowledge
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Veritable is the predictive database developed by Prior Knowledge (http://www.priorknowledge.com)
+email:
+- support@priorknowledge.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.txt
+- lib/veritable/api.rb
+- lib/veritable/connection.rb
+- lib/veritable/cursor.rb
+- lib/veritable/datatypes.rb
+- lib/veritable/errors.rb
+- lib/veritable/object.rb
+- lib/veritable/resource.rb
+- lib/veritable/util.rb
+- lib/veritable/version.rb
+- lib/veritable.rb
+- LICENSE
+- README.md
+homepage: https://dev.priorknowledge.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Ruby client for Veritable API
+test_files: []