veritable 0.1.0.80 → 0.1.1.24
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.txt +3 -0
- data/lib/veritable/api.rb +143 -33
- data/lib/veritable/errors.rb +4 -0
- data/lib/veritable/resource.rb +6 -1
- data/lib/veritable/util.rb +67 -29
- data/lib/veritable/version.rb +1 -1
- data/lib/veritable.rb +11 -7
- metadata +86 -118
data/CHANGELOG.txt
CHANGED
data/lib/veritable/api.rb
CHANGED
@@ -23,6 +23,11 @@ module Veritable
|
|
23
23
|
class API
|
24
24
|
include VeritableResource
|
25
25
|
|
26
|
+
def initialize(opts=nil, doc=nil)
|
27
|
+
super
|
28
|
+
@opts['api_limits'] = get("user/limits")
|
29
|
+
end
|
30
|
+
|
26
31
|
# Gets the root of the api
|
27
32
|
#
|
28
33
|
# ==== Returns
|
@@ -34,10 +39,12 @@ module Veritable
|
|
34
39
|
# Gets the user-specific API limits
|
35
40
|
#
|
36
41
|
# ==== Returns
|
37
|
-
# A Hash
|
42
|
+
# A Hash representing the user's current API limits.
|
38
43
|
#
|
39
44
|
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
40
|
-
def limits
|
45
|
+
def limits
|
46
|
+
return @opts['api_limits']
|
47
|
+
end
|
41
48
|
|
42
49
|
# Gets a cursor for the table collection
|
43
50
|
#
|
@@ -156,11 +163,11 @@ module Veritable
|
|
156
163
|
# * +batch_upload_rows+ -- batch uploads multiple rows to the table
|
157
164
|
# * +delete_row+ -- deletes a row from the table by its unique id
|
158
165
|
# * +batch_delete_rows+ -- batch deletes multiple rows from the table
|
159
|
-
# * +analyses+ --
|
160
|
-
# * +analysis+ --
|
161
|
-
# * +create_analysis+ --
|
162
|
-
# * +delete_analysis+ --
|
163
|
-
# * +has_analysis?+ --
|
166
|
+
# * +analyses+ -- gets a Veritable::Cursor over the collection of analyses of the table
|
167
|
+
# * +analysis+ -- gets a single analysis of the table by its unique id
|
168
|
+
# * +create_analysis+ -- creates a new analysis of the table
|
169
|
+
# * +delete_analysis+ -- deletes an analysis of the table by its unique id
|
170
|
+
# * +has_analysis?+ -- checks if the table has an analysis with the given id
|
164
171
|
#
|
165
172
|
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
166
173
|
class Table
|
@@ -469,7 +476,12 @@ module Veritable
|
|
469
476
|
# A new Veritable::Schema object describing the colums contained in the analysis.
|
470
477
|
#
|
471
478
|
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
472
|
-
def schema
|
479
|
+
def schema
|
480
|
+
if @old_schema.nil?
|
481
|
+
@old_schema = Schema.new(get(link('schema')))
|
482
|
+
end
|
483
|
+
return @old_schema
|
484
|
+
end
|
473
485
|
|
474
486
|
# Blocks until the analysis succeeds or fails
|
475
487
|
#
|
@@ -506,31 +518,39 @@ module Veritable
|
|
506
518
|
#
|
507
519
|
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
508
520
|
def predict(row, count=100)
|
509
|
-
|
510
|
-
|
521
|
+
if not row.is_a? Hash
|
522
|
+
raise VeritableError.new("Predict -- Must provide a row hash to make predictions.")
|
523
|
+
end
|
524
|
+
return raw_predict([row], count, api_limits['predictions_max_response_cells'], api_limits['predictions_max_cols'])[0]
|
525
|
+
end
|
526
|
+
|
527
|
+
|
528
|
+
# Makes predictions based on the analysis for multiple rows at a time
|
529
|
+
#
|
530
|
+
# ==== Arguments
|
531
|
+
# * +rows+ -- an Array of Hashes, each of which represents a row whose missing values are to be predicted. Keys must be valid String ids of columns contained in the underlying table, and values must be either fixed (conditioning) values of an appropriate type for each column, or +nil+ for values to be predicted. Each row Hash must also have a '_request_id' key with a unique string value.
|
532
|
+
# * +count+ -- optionally specify the number of samples from the predictive distribution to return. Defaults to +100+.
|
533
|
+
#
|
534
|
+
# ==== Returns
|
535
|
+
# An Array of Veritable::Prediction objects
|
536
|
+
#
|
537
|
+
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
538
|
+
def batch_predict(rows, count=100)
|
539
|
+
if not rows.is_a? Array
|
540
|
+
raise VeritableError.new("Predict -- Must provide an array of row hashes to make predictions.")
|
541
|
+
end
|
542
|
+
rows.each {|row|
|
511
543
|
if not row.is_a? Hash
|
512
|
-
|
544
|
+
raise VeritableError.new("Predict -- Invalid row for predictions: #{row}")
|
513
545
|
end
|
514
|
-
|
515
|
-
|
516
|
-
begin
|
517
|
-
res.to_s
|
518
|
-
rescue
|
519
|
-
raise VeritableError.new("Predict -- Error making predictions: #{res}")
|
520
|
-
else
|
521
|
-
raise VeritableError.new("Predict -- Error making predictions.")
|
522
|
-
end
|
546
|
+
if not row['_request_id'].is_a? String
|
547
|
+
raise VeritableError.new("Predict -- Rows for batch predictions must contain a string '_request_id' field: #{row}")
|
523
548
|
end
|
524
|
-
|
525
|
-
|
526
|
-
raise VeritableError.new("Predict -- Analysis with id #{_id} is still running and not yet ready to predict.")
|
527
|
-
elsif failed?
|
528
|
-
raise VeritableError.new("Predict -- Analysis with id #{_id} has failed and cannot predict.")
|
529
|
-
else
|
530
|
-
raise VeritableError.new("Predict -- Shouldn't be here -- please let us know at support@priorknowledge.com.")
|
531
|
-
end
|
549
|
+
}
|
550
|
+
return raw_predict(rows, count, api_limits['predictions_max_response_cells'], api_limits['predictions_max_cols'])
|
532
551
|
end
|
533
552
|
|
553
|
+
|
534
554
|
# Scores how related columns are to a column of interest
|
535
555
|
#
|
536
556
|
# ==== Arguments
|
@@ -595,7 +615,78 @@ module Veritable
|
|
595
615
|
|
596
616
|
# The String description of the analysis
|
597
617
|
def description; @doc['description']; end
|
598
|
-
|
618
|
+
|
619
|
+
private
|
620
|
+
|
621
|
+
def execute_batch(batch, count, preds)
|
622
|
+
if batch.size == 0
|
623
|
+
return
|
624
|
+
end
|
625
|
+
if batch.size == 1
|
626
|
+
data = batch[0]
|
627
|
+
else
|
628
|
+
data = batch
|
629
|
+
end
|
630
|
+
res = post(link('predict'), {'data' => data, 'count' => count, 'return_fixed' => false})
|
631
|
+
if not res.is_a? Array
|
632
|
+
begin
|
633
|
+
res.to_s
|
634
|
+
rescue
|
635
|
+
raise VeritableError.new("Predict -- Error making predictions.")
|
636
|
+
else
|
637
|
+
raise VeritableError.new("Predict -- Error making predictions: #{res}")
|
638
|
+
end
|
639
|
+
end
|
640
|
+
(0...batch.size).each {|i|
|
641
|
+
request = batch[i].clone
|
642
|
+
request_id = request['_request_id']
|
643
|
+
distribution = res[(i * count)...((i + 1) * count)]
|
644
|
+
preds.push Prediction.new(request, distribution, schema, request_id)
|
645
|
+
}
|
646
|
+
end
|
647
|
+
|
648
|
+
def raw_predict(rows, count, maxcells, maxcols)
|
649
|
+
update if running?
|
650
|
+
if running?
|
651
|
+
raise VeritableError.new("Predict -- Analysis with id #{_id} is still running and not yet ready to predict.")
|
652
|
+
elsif failed?
|
653
|
+
raise VeritableError.new("Predict -- Analysis with id #{_id} has failed and cannot predict.")
|
654
|
+
elsif succeeded?
|
655
|
+
preds = []
|
656
|
+
ncells = 0
|
657
|
+
batch = []
|
658
|
+
rows.each {|row|
|
659
|
+
ncols = (row.values.select {|v| v.nil?}).size
|
660
|
+
tcols = (row.keys.select {|k| k != '_request_id'}).size
|
661
|
+
if tcols > maxcols
|
662
|
+
raise VeritableError.new("Predict -- Cannot predict for row #{row['_request_id']} with more than #{maxcols} combined fixed and predicted values.")
|
663
|
+
end
|
664
|
+
n = ncols * count
|
665
|
+
if n > maxcells
|
666
|
+
raise VeritableError.new("Predict -- Cannot predict for row #{row['_request_id']} with #{ncols} missing values and count #{count}: exceeds predicted cell limit of #{maxcells}.")
|
667
|
+
end
|
668
|
+
}
|
669
|
+
rows.each {|row|
|
670
|
+
ncols = (row.values.select {|v| v.nil?}).size
|
671
|
+
n = ncols * count
|
672
|
+
if (ncells + n) > maxcells
|
673
|
+
execute_batch(batch, count, preds)
|
674
|
+
ncells = n
|
675
|
+
batch = [row]
|
676
|
+
else
|
677
|
+
batch.push row
|
678
|
+
ncells = ncells + n
|
679
|
+
end
|
680
|
+
}
|
681
|
+
execute_batch(batch, count, preds)
|
682
|
+
return preds
|
683
|
+
else
|
684
|
+
raise VeritableError.new("Predict -- Shouldn't be here -- please let us know at support@priorknowledge.com.")
|
685
|
+
end
|
686
|
+
end
|
687
|
+
|
688
|
+
|
689
|
+
end
|
599
690
|
|
600
691
|
# Represents a schema for a Veritable analysis
|
601
692
|
#
|
@@ -712,6 +803,9 @@ module Veritable
|
|
712
803
|
# The original predictions request, as a Hash
|
713
804
|
attr_reader :request
|
714
805
|
|
806
|
+
# The original prediction '_request_id', nil if none was specified
|
807
|
+
attr_reader :request_id
|
808
|
+
|
715
809
|
# The underlying predicted distribution, as an Array of Hashes
|
716
810
|
#
|
717
811
|
# Each Hash represents a single draw from the predictive distribution, and should be regarded as equiprobable with the others.
|
@@ -738,13 +832,27 @@ module Veritable
|
|
738
832
|
# Users should not call directly. Instead, call Veritable::Analysis#predict.
|
739
833
|
#
|
740
834
|
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
741
|
-
def initialize(request, distribution, schema)
|
835
|
+
def initialize(request, distribution, schema, request_id=nil)
|
742
836
|
@request = request
|
743
|
-
@
|
837
|
+
@request.delete '_request_id'
|
838
|
+
|
744
839
|
@schema = Schema.new(schema)
|
745
|
-
@
|
840
|
+
@request_id = request_id
|
746
841
|
|
747
|
-
|
842
|
+
fixed = {}
|
843
|
+
@request.each { |k,v|
|
844
|
+
if not v.nil?
|
845
|
+
fixed[k] = v
|
846
|
+
end
|
847
|
+
}
|
848
|
+
@distribution = distribution
|
849
|
+
@distribution.each {|d|
|
850
|
+
d.delete '_request_id'
|
851
|
+
d.update(fixed)
|
852
|
+
}
|
853
|
+
|
854
|
+
@uncertainty = Hash.new()
|
855
|
+
@request.each { |k,v|
|
748
856
|
if v.nil?
|
749
857
|
self[k] = point_estimate k
|
750
858
|
@uncertainty[k] = calculate_uncertainty k
|
@@ -753,6 +861,8 @@ module Veritable
|
|
753
861
|
@uncertainty[k] = 0.0
|
754
862
|
end
|
755
863
|
}
|
864
|
+
|
865
|
+
|
756
866
|
end
|
757
867
|
|
758
868
|
# Calculates the probability a column's value lies within a range.
|
data/lib/veritable/errors.rb
CHANGED
@@ -20,6 +20,7 @@ module Veritable
|
|
20
20
|
#
|
21
21
|
# See also: https://dev.priorknowledge.com/docs/client/ruby
|
22
22
|
def initialize(message, opts=nil)
|
23
|
+
super(message)
|
23
24
|
@message = message
|
24
25
|
if opts.is_a? Hash
|
25
26
|
@opts = opts
|
@@ -29,6 +30,9 @@ module Veritable
|
|
29
30
|
@opts[k]
|
30
31
|
}
|
31
32
|
}
|
33
|
+
if @opts.include? 'inner_error'
|
34
|
+
set_backtrace(@opts['inner_error'].backtrace)
|
35
|
+
end
|
32
36
|
end
|
33
37
|
end
|
34
38
|
|
data/lib/veritable/resource.rb
CHANGED
@@ -6,7 +6,7 @@ module Veritable
|
|
6
6
|
#
|
7
7
|
# Users should not include this module.
|
8
8
|
module VeritableResource
|
9
|
-
|
9
|
+
include Connection
|
10
10
|
|
11
11
|
private
|
12
12
|
|
@@ -14,5 +14,10 @@ module Veritable
|
|
14
14
|
def link(name)
|
15
15
|
@doc['links'][name]
|
16
16
|
end
|
17
|
+
|
18
|
+
# Private method: retrieves the cached copy of the user's limits
|
19
|
+
def api_limits
|
20
|
+
return @opts['api_limits']
|
21
|
+
end
|
17
22
|
end
|
18
23
|
end
|
data/lib/veritable/util.rb
CHANGED
@@ -238,6 +238,7 @@ module Veritable
|
|
238
238
|
# - +reduce_categories+ -- controls whether clean_data will automatically reduce the number of categories in categorical columns with too many categories (default: +true+) If +true+, the largest categories in a column will be preserved, up to the allowable limit, and the other categories will be binned as <tt>"Other"</tt>.
|
239
239
|
# - +assign_ids+ -- controls whether clean_data will automatically assign new ids to the rows (default: +false=) If +true+, rows will be numbered sequentially. If the rows have an existing <tt>'_id'</tt> column, +remove_extra_fields+ must also be set to +true+ to avoid raising a Veritable::VeritableError.
|
240
240
|
# - +remove_extra_fields+ -- controls whether clean_data will automatically remove columns that are not contained in the schema (default: +false+) If +assign_ids+ is +true+ (default), will also remove the <tt>'_id'</tt> column.
|
241
|
+
# - +rename_columns+ -- an array of two-valued arrays <tt>[[old_col_1, new_col_1],[old_col_2, new_col_2],...]</tt> of column names to rename. If +rename_columns+ is +false+ (default), no columns are renamed.
|
241
242
|
#
|
242
243
|
# ==== Raises
|
243
244
|
# A Veritable::VeritableError containing further details if the data does not validate against the schema.
|
@@ -253,11 +254,12 @@ module Veritable
|
|
253
254
|
'remove_nones' => opts.has_key?('remove_nones') ? opts['remove_nones'] : true,
|
254
255
|
'remove_invalids' => opts.has_key?('remove_invalids') ? opts['remove_invalids'] : true,
|
255
256
|
'reduce_categories' => opts.has_key?('reduce_categories') ? opts['reduce_categories'] : true,
|
256
|
-
'has_ids' =>
|
257
|
+
'has_ids' => '_id',
|
257
258
|
'assign_ids' => opts.has_key?('assign_ids') ? opts['assign_ids'] : false,
|
258
259
|
'allow_extra_fields' => true,
|
259
260
|
'remove_extra_fields' => opts.has_key?('remove_extra_fields') ? opts['remove_extra_fields'] : false,
|
260
|
-
'allow_empty_columns' => false
|
261
|
+
'allow_empty_columns' => false,
|
262
|
+
'rename_columns' => opts.has_key?('rename_columns') ? opts['rename_columns'] : false})
|
261
263
|
end
|
262
264
|
|
263
265
|
# Validates an Array of row Hashes against an analysis schema
|
@@ -280,14 +282,16 @@ module Veritable
|
|
280
282
|
'remove_nones' => false,
|
281
283
|
'remove_invalids' => false,
|
282
284
|
'reduce_categories' => false,
|
283
|
-
'has_ids' =>
|
285
|
+
'has_ids' => '_id',
|
284
286
|
'assign_ids' => false,
|
285
287
|
'allow_extra_fields' => true,
|
286
288
|
'remove_extra_fields' => false,
|
287
|
-
'allow_empty_columns' => false
|
289
|
+
'allow_empty_columns' => false,
|
290
|
+
'rename_columns' => false})
|
288
291
|
end
|
289
292
|
|
290
293
|
# Cleans up a predictions request in accordance with an analysis schema
|
294
|
+
# Automatically renames '_id' to '_request_id', otherwise assigns a new '_request_id' to each row
|
291
295
|
#
|
292
296
|
# This method mutates its +predictions+ argument. If clean_predictions raises an exception, values in some columns may be converted while others are left in their original state.
|
293
297
|
#
|
@@ -298,6 +302,7 @@ module Veritable
|
|
298
302
|
# - +convert_types+ -- controls whether clean_data will attempt to convert cells in a column to be of the correct type (default: +true+)
|
299
303
|
# - +remove_invalids+ -- controls whether clean_data will automatically remove cells that are invalid for a given column (default: +true+)
|
300
304
|
# - +remove_extra_fields+ -- controls whether clean_data will automatically remove columns that are not contained in the schema (default: +true+)
|
305
|
+
# - +rename_columns+ -- an array of two-valued arrays <tt>[[old_col_1, new_col_1],[old_col_2, new_col_2],...]</tt> of column names to rename. If +rename_columns+ is +false+, no columns are renamed. (default: <tt>[['_id','_request_id']]</tt>)
|
301
306
|
#
|
302
307
|
# ==== Raises
|
303
308
|
# A Veritable::VeritableError containing further details if the predictions request does not validate against the schema
|
@@ -313,11 +318,12 @@ module Veritable
|
|
313
318
|
'remove_nones' => false,
|
314
319
|
'remove_invalids' => opts.has_key?('remove_invalids') ? opts['remove_invalids'] : true,
|
315
320
|
'reduce_categories' => false,
|
316
|
-
'has_ids' =>
|
317
|
-
'assign_ids' =>
|
321
|
+
'has_ids' => '_request_id',
|
322
|
+
'assign_ids' => true,
|
318
323
|
'allow_extra_fields' => false,
|
319
324
|
'remove_extra_fields' => opts.has_key?('remove_extra_fields') ? opts['remove_extra_fields'] : true,
|
320
|
-
'allow_empty_columns' => true
|
325
|
+
'allow_empty_columns' => true,
|
326
|
+
'rename_columns' => [['_id','_request_id']]})
|
321
327
|
end
|
322
328
|
|
323
329
|
# Validates a predictions request against an analysis schema
|
@@ -340,11 +346,12 @@ module Veritable
|
|
340
346
|
'remove_nones' => false,
|
341
347
|
'remove_invalids' => false,
|
342
348
|
'reduce_categories' => false,
|
343
|
-
'has_ids' =>
|
349
|
+
'has_ids' => '_request_id',
|
344
350
|
'assign_ids' => false,
|
345
351
|
'allow_extra_fields' => false,
|
346
352
|
'remove_extra_fields' => false,
|
347
|
-
'allow_empty_columns' => true
|
353
|
+
'allow_empty_columns' => true,
|
354
|
+
'rename_columns' => false})
|
348
355
|
end
|
349
356
|
|
350
357
|
private
|
@@ -392,13 +399,29 @@ module Veritable
|
|
392
399
|
# ensure the schema is well-formed
|
393
400
|
schema.validate
|
394
401
|
|
402
|
+
# figure out which column holds the unique id
|
403
|
+
if (opts['has_ids'] or opts['assign_ids'])
|
404
|
+
if (opts['has_ids'].is_a? String and opts['assign_ids'].is_a? String)
|
405
|
+
if (opts['has_ids'] != opts['assign_ids'])
|
406
|
+
raise VeritableError.new("Can't assign new row ids to column '#{opts['assign_ids']}' when ids are expected in column '#{opts['has_ids']}'.",{})
|
407
|
+
end
|
408
|
+
end
|
409
|
+
if opts['assign_ids'].is_a? String
|
410
|
+
id_col = opts['assign_ids']
|
411
|
+
elsif opts['has_ids'].is_a? String
|
412
|
+
id_col = opts['has_ids']
|
413
|
+
else
|
414
|
+
id_col = '_id'
|
415
|
+
end
|
416
|
+
end
|
417
|
+
|
395
418
|
# store the row numbers of each unique id so that we can warn the user
|
396
419
|
unique_ids = Hash.new
|
397
420
|
|
398
421
|
# store the density of fields
|
399
422
|
field_fill = Hash.new
|
400
423
|
schema.keys.each {|c|
|
401
|
-
field_fill[c] = 0 if c !=
|
424
|
+
field_fill[c] = 0 if c != id_col
|
402
425
|
}
|
403
426
|
|
404
427
|
# store the number of categories in each categorical column
|
@@ -411,49 +434,64 @@ module Veritable
|
|
411
434
|
max_cats = 256
|
412
435
|
# be careful before changing the order of any of this logic -- the point is to do this all only once
|
413
436
|
(0...rows.size).each {|i|
|
437
|
+
|
438
|
+
if opts['rename_columns'] # first apply the column renaming rules, if any
|
439
|
+
if not opts['rename_columns'].is_a? Array
|
440
|
+
raise VeritableError.new("Must supply column renaming rules as a list of lists.",{})
|
441
|
+
end
|
442
|
+
opts['rename_columns'].each {|rule|
|
443
|
+
if not rule.is_a? Array
|
444
|
+
raise VeritableError.new("Must supply column renaming rules as a list of lists.",{})
|
445
|
+
end
|
446
|
+
if rows[i].include? rule[0]
|
447
|
+
rows[i][rule[1]] = rows[i][rule[0]]
|
448
|
+
rows[i].delete rule[0]
|
449
|
+
end
|
450
|
+
}
|
451
|
+
end
|
452
|
+
|
414
453
|
if opts['assign_ids']
|
415
|
-
rows[i][
|
454
|
+
rows[i][id_col] = i.to_s # number the rows sequentially
|
416
455
|
elsif opts['has_ids']
|
417
|
-
raise VeritableError.new("Validate -- row #{i} is missing key '
|
456
|
+
raise VeritableError.new("Validate -- row #{i} is missing key '#{id_col}'", {'row' => i, 'col' => id_col}) unless rows[i].include? id_col
|
418
457
|
|
419
458
|
if opts['convert_types'] # attempt to convert _id to string
|
420
459
|
begin
|
421
|
-
rows[i][
|
460
|
+
rows[i][id_col] = rows[i][id_col].to_s if not rows[i][id_col].is_a? String
|
422
461
|
rescue
|
423
|
-
raise VeritableError.new("Validate -- row #{i}, key '
|
462
|
+
raise VeritableError.new("Validate -- row #{i}, key '#{id_col}' cannot be converted to string.", {'row' => i, 'col' => id_col})
|
424
463
|
end
|
425
464
|
end
|
426
465
|
|
427
|
-
if not rows[i][
|
466
|
+
if not rows[i][id_col].is_a? String # invalid type for _id
|
428
467
|
begin
|
429
|
-
rows[i][
|
468
|
+
rows[i][id_col].to_s
|
430
469
|
rescue
|
431
|
-
raise VeritableError.new("Validate -- row #{i}, key '
|
470
|
+
raise VeritableError.new("Validate -- row #{i}, key '#{id_col}' is not a string.", {'row' => i, 'col' => id_col})
|
432
471
|
else
|
433
|
-
raise VeritableError.new("Validate -- row #{i}, key '
|
472
|
+
raise VeritableError.new("Validate -- row #{i}, key '#{id_col}', value #{rows[i][id_col]} is not a string.", {'row' => i, 'col' => id_col})
|
434
473
|
end
|
435
474
|
end
|
436
475
|
|
437
476
|
begin
|
438
|
-
check_id rows[i][
|
477
|
+
check_id rows[i][id_col] # make sure _id is alphanumeric
|
439
478
|
rescue
|
440
|
-
raise VeritableError.new("Validate -- row #{i}, key '
|
441
|
-
end
|
442
|
-
|
443
|
-
if unique_ids.include? rows[i]['_id']
|
444
|
-
raise VeritableError.new("Validate -- row #{i}, key '_id', value #{rows[i]['_id']} is non-unique, conflicts with row #{unique_ids[rows[i]['_id']]}", {'row' => i, 'col' => '_id'})
|
479
|
+
raise VeritableError.new("Validate -- row #{i}, key '#{id_col}', value #{rows[i][id_col]} contains disallowed characters. Ids must contain only alphanumerics, with underscores and hyphens allowed after the beginning of the id.", {'row' => i, 'col' => id_col})
|
445
480
|
end
|
446
481
|
|
447
|
-
unique_ids
|
448
|
-
|
482
|
+
if unique_ids.include? rows[i][id_col]
|
483
|
+
raise VeritableError.new("Validate -- row #{i}, key '#{id_col}', value #{rows[i][id_col]} is non-unique, conflicts with row #{unique_ids[rows[i][id_col]]}", {'row' => i, 'col' => id_col})
|
484
|
+
end
|
485
|
+
unique_ids[rows[i][id_col]] = i
|
486
|
+
elsif rows[i].include? id_col # no ids, no autoid, but _id column
|
449
487
|
if opts['remove_extra_fields'] # just remove it
|
450
|
-
rows[i].delete
|
488
|
+
rows[i].delete id_col
|
451
489
|
else
|
452
|
-
raise VeritableError.new("Validate -- row #{i}, key '
|
490
|
+
raise VeritableError.new("Validate -- row #{i}, key '#{id_col}' should not be included.", {'row' => i, 'col' => id_col})
|
453
491
|
end
|
454
492
|
end
|
455
493
|
rows[i].keys.each {|c|
|
456
|
-
if c !=
|
494
|
+
if c != id_col
|
457
495
|
if not schema.include? c # keys missing from schema
|
458
496
|
if opts['remove_extra_fields'] # remove it
|
459
497
|
rows[i].delete c
|
data/lib/veritable/version.rb
CHANGED
data/lib/veritable.rb
CHANGED
@@ -65,12 +65,16 @@ module Veritable
|
|
65
65
|
opts[:ssl_verify] = true unless opts.has_key?(:ssl_verify)
|
66
66
|
opts[:enable_gzip] = true unless opts.has_key?(:enable_gzip)
|
67
67
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
68
|
+
begin
|
69
|
+
api = API.new(opts)
|
70
|
+
connection_test = api.root
|
71
|
+
status = connection_test["status"]
|
72
|
+
entropy = connection_test["entropy"]
|
73
|
+
raise VeritableError.new("No Veritable server responding at #{opts[:api_base_url]}") if status != "SUCCESS"
|
74
|
+
raise VeritableError.new("No Veritable server responding at #{opts[:api_base_url]}") if ! entropy.is_a?(Float)
|
75
|
+
rescue Exception => e
|
76
|
+
raise VeritableError.new("No Veritable server responding at #{opts[:api_base_url]}", {'inner_error' => e})
|
77
|
+
end
|
78
|
+
return api
|
75
79
|
end
|
76
80
|
end
|
metadata
CHANGED
@@ -1,151 +1,115 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: veritable
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0.80
|
3
|
+
version: !ruby/object:Gem::Version
|
5
4
|
prerelease:
|
5
|
+
version: 0.1.1.24
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Prior Knowledge
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
|
13
|
+
date: 2012-06-14 00:00:00 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
15
16
|
name: rest-client
|
16
|
-
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
17
18
|
none: false
|
18
|
-
requirements:
|
19
|
+
requirements:
|
19
20
|
- - ~>
|
20
|
-
- !ruby/object:Gem::Version
|
21
|
-
version:
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: "1.4"
|
22
23
|
type: :runtime
|
23
24
|
prerelease: false
|
24
|
-
version_requirements:
|
25
|
-
|
26
|
-
requirements:
|
27
|
-
- - ~>
|
28
|
-
- !ruby/object:Gem::Version
|
29
|
-
version: '1.4'
|
30
|
-
- !ruby/object:Gem::Dependency
|
25
|
+
version_requirements: *id001
|
26
|
+
- !ruby/object:Gem::Dependency
|
31
27
|
name: uuid
|
32
|
-
requirement: !ruby/object:Gem::Requirement
|
28
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
33
29
|
none: false
|
34
|
-
requirements:
|
35
|
-
- -
|
36
|
-
- !ruby/object:Gem::Version
|
37
|
-
version:
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: "0"
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
|
-
version_requirements:
|
41
|
-
|
42
|
-
requirements:
|
43
|
-
- - ! '>='
|
44
|
-
- !ruby/object:Gem::Version
|
45
|
-
version: '0'
|
46
|
-
- !ruby/object:Gem::Dependency
|
36
|
+
version_requirements: *id002
|
37
|
+
- !ruby/object:Gem::Dependency
|
47
38
|
name: multi_json
|
48
|
-
requirement: !ruby/object:Gem::Requirement
|
39
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
49
40
|
none: false
|
50
|
-
requirements:
|
51
|
-
- -
|
52
|
-
- !ruby/object:Gem::Version
|
53
|
-
version:
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: "0"
|
54
45
|
type: :runtime
|
55
46
|
prerelease: false
|
56
|
-
version_requirements:
|
57
|
-
|
58
|
-
requirements:
|
59
|
-
- - ! '>='
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
62
|
-
- !ruby/object:Gem::Dependency
|
47
|
+
version_requirements: *id003
|
48
|
+
- !ruby/object:Gem::Dependency
|
63
49
|
name: test-unit
|
64
|
-
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
65
51
|
none: false
|
66
|
-
requirements:
|
67
|
-
- -
|
68
|
-
- !ruby/object:Gem::Version
|
69
|
-
version:
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: "0"
|
70
56
|
type: :development
|
71
57
|
prerelease: false
|
72
|
-
version_requirements:
|
73
|
-
|
74
|
-
requirements:
|
75
|
-
- - ! '>='
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: '0'
|
78
|
-
- !ruby/object:Gem::Dependency
|
58
|
+
version_requirements: *id004
|
59
|
+
- !ruby/object:Gem::Dependency
|
79
60
|
name: rake
|
80
|
-
requirement: !ruby/object:Gem::Requirement
|
61
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
81
62
|
none: false
|
82
|
-
requirements:
|
83
|
-
- -
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
version:
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: "0"
|
86
67
|
type: :development
|
87
68
|
prerelease: false
|
88
|
-
version_requirements:
|
89
|
-
|
90
|
-
requirements:
|
91
|
-
- - ! '>='
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: '0'
|
94
|
-
- !ruby/object:Gem::Dependency
|
69
|
+
version_requirements: *id005
|
70
|
+
- !ruby/object:Gem::Dependency
|
95
71
|
name: rdoc
|
96
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
97
73
|
none: false
|
98
|
-
requirements:
|
99
|
-
- -
|
100
|
-
- !ruby/object:Gem::Version
|
101
|
-
version:
|
74
|
+
requirements:
|
75
|
+
- - ">="
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: "0"
|
102
78
|
type: :development
|
103
79
|
prerelease: false
|
104
|
-
version_requirements:
|
105
|
-
|
106
|
-
requirements:
|
107
|
-
- - ! '>='
|
108
|
-
- !ruby/object:Gem::Version
|
109
|
-
version: '0'
|
110
|
-
- !ruby/object:Gem::Dependency
|
80
|
+
version_requirements: *id006
|
81
|
+
- !ruby/object:Gem::Dependency
|
111
82
|
name: simplecov
|
112
|
-
requirement: !ruby/object:Gem::Requirement
|
83
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
113
84
|
none: false
|
114
|
-
requirements:
|
115
|
-
- -
|
116
|
-
- !ruby/object:Gem::Version
|
117
|
-
version:
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
118
89
|
type: :development
|
119
90
|
prerelease: false
|
120
|
-
version_requirements:
|
121
|
-
|
122
|
-
requirements:
|
123
|
-
- - ! '>='
|
124
|
-
- !ruby/object:Gem::Version
|
125
|
-
version: '0'
|
126
|
-
- !ruby/object:Gem::Dependency
|
91
|
+
version_requirements: *id007
|
92
|
+
- !ruby/object:Gem::Dependency
|
127
93
|
name: json
|
128
|
-
requirement: !ruby/object:Gem::Requirement
|
94
|
+
requirement: &id008 !ruby/object:Gem::Requirement
|
129
95
|
none: false
|
130
|
-
requirements:
|
131
|
-
- -
|
132
|
-
- !ruby/object:Gem::Version
|
133
|
-
version:
|
96
|
+
requirements:
|
97
|
+
- - ">="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
version: "0"
|
134
100
|
type: :development
|
135
101
|
prerelease: false
|
136
|
-
version_requirements:
|
137
|
-
none: false
|
138
|
-
requirements:
|
139
|
-
- - ! '>='
|
140
|
-
- !ruby/object:Gem::Version
|
141
|
-
version: '0'
|
102
|
+
version_requirements: *id008
|
142
103
|
description: Veritable is the predictive database developed by Prior Knowledge (http://www.priorknowledge.com)
|
143
|
-
email:
|
104
|
+
email:
|
144
105
|
- support@priorknowledge.com
|
145
106
|
executables: []
|
107
|
+
|
146
108
|
extensions: []
|
109
|
+
|
147
110
|
extra_rdoc_files: []
|
148
|
-
|
111
|
+
|
112
|
+
files:
|
149
113
|
- lib/veritable.rb
|
150
114
|
- lib/veritable/util.rb
|
151
115
|
- lib/veritable/errors.rb
|
@@ -161,32 +125,36 @@ files:
|
|
161
125
|
- README.md
|
162
126
|
homepage: https://dev.priorknowledge.com
|
163
127
|
licenses: []
|
128
|
+
|
164
129
|
post_install_message:
|
165
130
|
rdoc_options: []
|
166
|
-
|
131
|
+
|
132
|
+
require_paths:
|
167
133
|
- lib
|
168
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
134
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
169
135
|
none: false
|
170
|
-
requirements:
|
171
|
-
- -
|
172
|
-
- !ruby/object:Gem::Version
|
173
|
-
|
174
|
-
segments:
|
136
|
+
requirements:
|
137
|
+
- - ">="
|
138
|
+
- !ruby/object:Gem::Version
|
139
|
+
hash: -4025254634787494658
|
140
|
+
segments:
|
175
141
|
- 0
|
176
|
-
|
177
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
142
|
+
version: "0"
|
143
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
178
144
|
none: false
|
179
|
-
requirements:
|
180
|
-
- -
|
181
|
-
- !ruby/object:Gem::Version
|
182
|
-
|
183
|
-
segments:
|
145
|
+
requirements:
|
146
|
+
- - ">="
|
147
|
+
- !ruby/object:Gem::Version
|
148
|
+
hash: -4025254634787494658
|
149
|
+
segments:
|
184
150
|
- 0
|
185
|
-
|
151
|
+
version: "0"
|
186
152
|
requirements: []
|
153
|
+
|
187
154
|
rubyforge_project:
|
188
155
|
rubygems_version: 1.8.24
|
189
156
|
signing_key:
|
190
157
|
specification_version: 3
|
191
158
|
summary: Ruby client for Veritable API
|
192
159
|
test_files: []
|
160
|
+
|