dbd 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b21e77e8316f18a011e2356b79a28a35e5dde7d
-  data.tar.gz: d8a0adeebbc7311a512ef08bfbc63902e16f2159
+  metadata.gz: f0ecabc694d7fb96ca29461e84ec65065d15e3ec
+  data.tar.gz: b31a6c1b351ddcfde4e90d10589526fb7e15bc78
 SHA512:
-  metadata.gz: c8e9c42c082ad4bdc8c6af283c9a73fad3c35a75e5ebace47aefbbe127cb0ee001ef066d17ef6e9b253dcd2a6311ccbb520810bdb061ab233c77950be7a99542
-  data.tar.gz: a63bb4462ebdc8fa4a6c2e0f3a1513146f7aac1e85a9d748c3126f5352464a613ee9bf956e27e0fad06edc4816e7226e78a37a1c5218196e6ea0f4ccf13db5e5
+  metadata.gz: 9b5aac83bb9c2ae36d95121aec6570513821daf56372af99713930ab44e11e19dc585a2d730d35710c3614ca3c62e5f18fd1c3a4cfd4bff01e1e21bd74f2a7cb
+  data.tar.gz: 161310b4ba873d17a52707da9ae58f6b1c40a5c4163739f512410ae0c1445ecca818d88a0f7362008d8eb392fa103ae7ff0dbf883265dd838913801f315fe767

data/Guardfile

@@ -1,6 +1,6 @@
-guard 'rspec' do
+guard 'rspec', :all_after_pass => true, :all_on_start => true do
   watch(%r{^spec/.+_spec\.rb$})
-  watch(%r{^spec/factories})    { "spec" }
+  watch(%r{^spec/test_factories})    { "spec" }
   watch(%r{^lib/dbd/helpers})   { "spec" }
 #  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
   watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}/" }

data/HISTORY.txt

@@ -50,3 +50,14 @@
 * new function graph#to_CSV_file
 * bin/test_5.rb was used to write 10M facts using ruby-2.0.0, 1.9.3 and jruby-1.7.4
 * jruby is 3 time faster, but 10% more memory comsumption
+
+0.0.10 (7 July 2013)
+======
+
+* Input validation for Graph#from_CSV (pairing with @marksim)
+* Convert Graph.from_CSV => Graph#from_CSV (can apply multiple
+  from_CSV's on same graph)
+* moved factories out of Fact to Fact::Factory
+  (removing the dependency injection violations from the Fact class)
+* work-arounds for JRuby nanosecond rounding issues (#1)
+* renamed Factories to TestFactories

data/README.md

@@ -48,7 +48,7 @@ Open Source [MIT]
 
 ## Examples
 
-Also see the file `docs/test.rb` to execute the script below.
+Running `ruby docs/test.rb` will execute the script below.
 
 ```
 require 'dbd'
@@ -103,7 +103,7 @@ puts csv
 # "a3da9295-b43a-4c3a-8e8c-97c3f04c1fa3","2013-06-19 22:02:20.490036790 UTC","5eb1ea27-6691-4a57-ab13-8a59021968e1","3767c493-79d3-4a97-a832-79e6361ddc4c","todo:story","A long period of peace,
 #  that is a ""bliss""."
 
-imported_graph = Dbd::Graph.from_CSV(csv)
+imported_graph = Dbd::Graph.new.from_CSV(csv)
 
 puts imported_graph.map(&:short)
 
@@ -121,21 +121,40 @@ puts imported_graph.map(&:short)
 ## Performance tests on 10 M facts
 
 In version 0.0.9 a number of test programs where added (e.g. ../bin/test_5.rb)
-that where used to populated in memory and write to disk a data set with 10 M facts.
+that where used to populate in memory and write to disk a data set with 10 M facts.
 
 This function was tested on ruby-2.0.0, ruby-1.9.3 and jruby-1.7.4. The facts
 had an approximate size of 250 Bytes each (80 Bytes object).
 
 The time needed and memory size (RSS) for populating the in-memory dataset was:
 
-10 M facts (of 250 Bytes; 2.5 GB netto data):
+Generate in memory 10 M facts (of 250 Bytes; 2.5 GB netto data):
 
-| ruby	     | time        | memory (RSS} |
+| ruby	     | time        | memory (RSS) |
 |------------|-------------| ------------:|
 | ruby-1.9.3 | 863 seconds |       8.1 GB |
 | ruby-2.0.0 | 862 seconds |       9.0 GB |
 |jruby-1.7.4 | 345 seconds |      10.8 GB |
 
+In version 0.0.10 a test for reading a fact stream from a CSV file was added
+(e.g. ../bin/test_6.rb). Reading back a CSV file that was written earlier with
+10 M facts (with test_5.rb) was tested on jruby-1.7.4. and ruby-2.0.0.
+
+This version also has input validation on the strings in the CSV. The time needed
+and memory size (RSS) for reading the file (and populating the in-memory dataset
+was):
+
+Read from CSV (to_CSV) 10 M facts (of 250 Bytes; 2.5 GB netto data):
+
+| ruby	     | time          |  memory (RSS) |
+|------------|---------------|--------------:|
+| ruby-1.9.3 | 4434  seconds | approx. 10 GB |
+| ruby-2.0.0 | 5163  seconds | approx. 15 GB |
+|jruby-1.7.4 | 1513  seconds | approx. 14 GB |
+
+The significantly larger times to read from_CSV versus writing to_CSV are _not_
+significantly caused by input validation (a test in JRuby without validation on
+reading 1M facts was only 6% faster with the input validation turned off).
 
 [RDF]:              http://www.w3.org/RDF/
 [Rationale]:        http://github.com/petervandenabeele/dbd/blob/master/docs/rationale.md

data/bin/test_4.rb

@@ -19,7 +19,7 @@ row_data = [
   "test",
   "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB 0"]
 
-puts "starting CSV.open"
+puts "starting CSV.generate"
 
 start_time = Time.now
 
@@ -29,4 +29,4 @@ csv_string = CSV.generate(force_quotes: true) do |csv|
   end
 end
 
-puts "CSV.open took #{Time.now - start_time} seconds"
+puts "CSV.generate took #{Time.now - start_time} seconds"

data/bin/test_6.rb

@@ -13,9 +13,9 @@ require 'dbd'
 start = Time.now
 
 graph = File.open(filename) do |f|
-  Dbd::Graph.from_CSV(f)
+  Dbd::Graph.new.from_CSV(f)
 end
 
-puts "Graph is ready (took #{Time.now - start}s), now starting the write to disk"
+puts "Graph is ready (took #{Time.now - start}s)."
 
 puts "graph.size is #{graph.size}"

data/docs/test.rb

@@ -50,7 +50,7 @@ puts csv
 # "a3da9295-b43a-4c3a-8e8c-97c3f04c1fa3","2013-06-19 22:02:20.490036790 UTC","5eb1ea27-6691-4a57-ab13-8a59021968e1","3767c493-79d3-4a97-a832-79e6361ddc4c","todo:story","A long period of peace,
 #  that is a ""bliss""."
 
-imported_graph = Dbd::Graph.from_CSV(csv)
+imported_graph = Dbd::Graph.new.from_CSV(csv)
 
 puts imported_graph.map(&:short)
 

data/lib/dbd/fact.rb

@@ -1,3 +1,4 @@
+require 'dbd/fact/factory'
 require 'dbd/fact/collection'
 require 'dbd/fact/subject'
 require 'dbd/fact/id'
@@ -57,6 +58,12 @@ module Dbd
   #   in RDF. Probably more detailed modeling using RDF object will follow.
   class Fact
 
+    ##
+    # @return [Module] The module that has the factories for Fact
+    def self.factory
+      self::Factory
+    end
+
     ##
     # @return [Array] The 6 attributes of a Fact.
     def self.attributes
@@ -73,13 +80,15 @@ module Dbd
     end
 
     ##
-    # A set_once setter for time_stamp.
+    # These "set once" setters implement a form of immutable behavior.
+    # The value can be set once (after initial creation the object),
+    # but can never be changed after that.
     #
-    # This implements a "form" of immutable behavior. The value can
-    # be set once (possibly after creation the object), but can
-    # never be changed after that.
+    # A set_once setter for time_stamp.
     #
     # The input class is validated (easy confusion with String or Time).
+    #
+    # @param [TimeStamp] :time_stamp a time_stamp (not a Time or a String)
     def time_stamp=(time_stamp)
       validate_time_stamp_class(time_stamp)
       set_once(:time_stamp, time_stamp)
@@ -88,9 +97,7 @@ module Dbd
     ##
     # A set_once setter for provenance_subject.
     #
-    # This implements a "form" of immutable behavior. The value can
-    # be set once (possibly after creation the object), but can
-    # never be changed after that.
+    # @param [String] :provenance_subject a string representation of the uuid
     def provenance_subject=(provenance_subject)
       set_once(:provenance_subject, provenance_subject)
     end
@@ -98,29 +105,15 @@ module Dbd
     ##
     # A set_once setter for subject.
     #
-    # This implements a "form" of immutable behavior. The value can
-    # be set once (possibly after creation the object), but can
-    # never be changed after that.
+    # @param [String] :subject a string representation of the uuid
     def subject=(subject)
       set_once(:subject, subject)
     end
 
-    ##
-    # @return [String] A new subject string.
-    def self.new_subject
-      Subject.new_subject
-    end
-
-    ##
-    # @return [String] A new id string.
-    def self.new_id
-      ID.new_id
-    end
-
     ##
     # Builds a new Fact.
     #
-    # @param [Hash{Symbol => Object}] options
+    # @param [Hash{Symbol => Object}] :options
     # @option options [#to_s] :predicate Required : the predicate for this Fact
     # @option options [#to_s] :object Required :  the object for this Fact (required)
     # @option options [String (uuid)] :provenance_subject (nil) Optional: the subject of the provenance(resource|fact)
@@ -128,13 +121,13 @@ module Dbd
     # @option options [TimeStamp] :time_stamp (nil) Optional: the time_stamp for this Fact
     # @option options [String (uuid)] :id Optional : set the id
     def initialize(options)
-      @id = options[:id] || self.class.new_id
+      @id = options[:id] || self.class.factory.new_id
       @time_stamp = options[:time_stamp]
-      validate_time_stamp_class(@time_stamp)
       @provenance_subject = options[:provenance_subject]
       @subject = options[:subject]
       @predicate = options[:predicate]
       @object = options[:object]
+      validate_time_stamp_class(@time_stamp)
       raise PredicateError, "predicate cannot be nil" if predicate.nil?
       raise ObjectError, "object cannot be nil" if object.nil?
     end
@@ -153,18 +146,7 @@ module Dbd
     end
 
     ##
-    # Constructs a Fact or ProvenanceFact from a string values array
-    # (e.g. pulled from a CSV row).
-    #
-    # @param [Array] string_values Required : the array with values, organized as in attributes
-    # @return [Fact, ProvenanceFact] the constructed fact
-    def self.from_string_values(string_values)
-      string_hash = hash_from_values(string_values)
-      fact_from_hash(values_hash(string_hash))
-    end
-
-    ##
-    # Equivalent facts (have all same values, except time_stamp).
+    # Equivalent facts (have all same values, except time_stamp which is near?).
     #
     # For "equality" only a test on the id is used. If the id
     # (which is a uuid) is the same, we assume that is the "same"
@@ -174,6 +156,9 @@ module Dbd
     # The time_stamp may be slightly different (because shifts
     # of a few nanoseconds will be required to resolve collisions
     # on merge).
+    #
+    # @param [Fact] :other the other fact to compare with
+    # @return [trueish]
     def equivalent?(other)
       (self.class.attributes - [:time_stamp]).
         all?{ |attribute| self.send(attribute) == other.send(attribute) } &&
@@ -198,6 +183,8 @@ module Dbd
     # ProvenanceResource with this provenance_subject.
     #
     # This is overridden in the ProvenanceFact, since only relevant for a Fact.
+    #
+    # @param[Hash] :h the hash that contains the provenance_subject index
     def update_used_provenance_subjects(h)
       # using a provenance_subject sets the key
       h[provenance_subject] = true
@@ -210,22 +197,22 @@ module Dbd
     def errors
       # * id not validated, is set automatically upon creation
       # * time_stamp not validated, is set automatically later
-      # * predicate not validated, is validated in initialize
-      # * object not validated, is validated in initialize
-      [].tap do |a|
-        a << provenance_subject_error(provenance_subject)
-        a << "Subject is missing" unless subject
-      end.compact
+      # * predicate not validated, is validated upon creation
+      # * object not validated, is validated upon creation
+      [provenance_subject_error(provenance_subject),
+       subject ? nil : "Subject is missing"].compact
     end
 
     ##
     # Validates the presence or absence of provenance_subject.
     #
-    # Here, in (base) Fact, provenance_subject must be present
-    # In the derived ProvenanceFact it must not be present.
+    # Here, in (base) Fact, provenance_subject must be present.
+    #
+    # In the derived ProvenanceFact it must NOT be present.
     # This is how the difference is encoded between Fact and
     # ProvenanceFact in the fact stream.
-    # @param [#nil?] provenance_subject
+    #
+    # @param [Object] provenance_subject
     # Return [nil, String] nil or an error message
     def provenance_subject_error(provenance_subject)
       "Provenance subject is missing" unless provenance_subject
@@ -236,6 +223,8 @@ module Dbd
     #
     # Needed for validations that depend on different behavior for
     # a provenance_fact (mainly, no provenance_subject).
+    #
+    # @return [trueish] false in the Fact implementation
     def provenance_fact?
       false
     end
@@ -246,27 +235,6 @@ module Dbd
       "#{provenance_subject.to_s[0...8]}"
     end
 
-    # FIXME This has to move to a Fact::Factory
-    def self.hash_from_values(values)
-      # Do not keep "empty" values (e.g. the provenance_subject for a ProvenanceFact).
-      attributes_values_array = [attributes, values].transpose.delete_if{|a,v| v.nil? || v == ''}
-      Hash[attributes_values_array]
-    end
-
-    def self.values_hash(string_hash)
-      string_hash.dup.tap do |h|
-        h[:time_stamp] = TimeStamp.new(time: h[:time_stamp])
-      end
-    end
-
-    def self.fact_from_hash(hash)
-      if hash[:provenance_subject]
-        Fact.new(hash)
-      else
-        ProvenanceFact.new(hash)
-      end
-    end
-
     def validate_time_stamp_class(time_stamp)
       unless time_stamp.nil? || time_stamp.is_a?(TimeStamp)
         raise ArgumentError, "time_stamp is of class #{time_stamp.class}, should be TimeStamp"

data/lib/dbd/fact/factory.rb

@@ -0,0 +1,90 @@
+module Dbd
+  class Fact
+    module Factory
+
+      class << self
+
+        ##
+        # @return [Class] the top class for which instances are created here.
+        def top_class
+          Fact
+        end
+
+        ##
+        # @return [String] A new subject string.
+        def new_subject
+          top_class::Subject.new_subject
+        end
+
+        ##
+        # @return [String] A new id string.
+        def new_id
+          top_class::ID.new_id
+        end
+
+        ##
+        # Constructs a Fact or ProvenanceFact from a string values array
+        # (e.g. pulled from a CSV row).
+        #
+        # @param [Array] string_values Required : the array with values, organized as in attributes
+        # @return [Fact, ProvenanceFact] the constructed fact
+        def from_string_values(string_values, options={})
+          string_hash = string_hash_from_values(string_values)
+          validate_string_hash(string_hash) if options[:validate]
+          fact_from_values_hash(values_hash(string_hash))
+        end
+
+        def attribute_formats
+          # TODO clean this up
+          {
+            id: [true, Fact::ID.valid_regexp],
+            time_stamp: [true, TimeStamp.valid_regexp],
+            provenance_subject: [false, Fact::Subject.valid_regexp],
+            subject: [true, Fact::Subject.valid_regexp],
+            predicate: [true, /./],
+            object: [true, /./]
+          }
+        end
+
+      private
+
+        def string_hash_from_values(string_values)
+          attributes_strings_array = [top_class.attributes, string_values].transpose
+          # Remove empty values (e.g. the provenance_subject for a ProvenanceFact).
+          attributes_strings_array.delete_if{|a,v| v.nil? || v == ''}
+          Hash[attributes_strings_array]
+        end
+
+        def values_hash(string_hash)
+          string_hash.tap do |h|
+            h[:time_stamp] = TimeStamp.new(time: h[:time_stamp])
+          end
+        end
+
+        def fact_from_values_hash(values_hash)
+          if values_hash[:provenance_subject]
+            Fact.new(values_hash)
+          else
+            ProvenanceFact.new(values_hash)
+          end
+        end
+
+        def validate_string_hash(string_hash)
+          attribute_formats.each do |attr, validation|
+            string = string_hash[attr]
+            mandatory, format = validation
+            validate_string(mandatory, string, format)
+          end
+        end
+
+        def validate_string(mandatory, string, format)
+          if (mandatory || string) && (string !~ format)
+            raise FactError, "invalid entry found : #{string}"
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/dbd/fact/id.rb

@@ -2,8 +2,8 @@ module Dbd
   class Fact
     module ID
 
-      def self.regexp
-        Helpers::UUID.regexp
+      def self.valid_regexp
+        Helpers::UUID.valid_regexp
       end
 
       def self.new_id

data/lib/dbd/fact/subject.rb

@@ -4,8 +4,8 @@ module Dbd
   class Fact
     module Subject
 
-      def self.regexp
-        Helpers::UUID.regexp
+      def self.valid_regexp
+        Helpers::UUID.valid_regexp
       end
 
       def self.new_subject