xamplr 1.2.0 → 1.3.15

data/examples/tokyo-cabinet-experimental/xampl-gen.rb

@@ -8,6 +8,8 @@ if $0 == __FILE__ then
     end
   end
 
+  $LOAD_PATH.unshift('../../lib/')
+  
   require 'xamplr-generator'
 
   include XamplGenerator

data/examples/write-testing/README

@@ -0,0 +1,4 @@
+
+-- raconteur.info is about 114 times faster writing unsynced
+-- raconteur.info is about 12 times faster writing synced
+

data/examples/write-testing/RESULTS.home

@@ -0,0 +1,33 @@
+time puts: 0.004565
+time puts: 0.001416
+time puts: 0.001879
+time puts: 0.000844
+time puts: 0.000763
+time puts: 0.000973
+time puts: 0.000936
+time puts: 0.000788
+time puts: 0.000849
+time puts: 0.000917
+TOTAL: 0.033575
+time sync write: 0.045199
+time sync write: 0.149448
+time sync write: 0.110496
+time sync write: 0.121815
+time sync write: 0.133079
+time sync write: 0.122049
+time sync write: 0.122168
+time sync write: 0.132572
+time sync write: 0.110322
+time sync write: 0.133309
+TOTAL: 1.188656
+time write: 0.004363
+time write: 0.000237
+time write: 0.000394
+time write: 0.000243
+time write: 0.000378
+time write: 0.000506
+time write: 0.000242
+time write: 0.000377
+time write: 0.000743
+time write: 0.000599
+TOTAL: 0.009168

data/examples/write-testing/RESULTS.raconteur

@@ -0,0 +1,33 @@
+time print: 9.2e-05
+time print: 6.0e-05
+time print: 5.3e-05
+time print: 5.4e-05
+time print: 5.8e-05
+time print: 5.4e-05
+time print: 5.3e-05
+time print: 6.1e-05
+time print: 5.4e-05
+time print: 5.2e-05
+TOTAL: 0.00077
+time sync write: 0.002917
+time sync write: 0.00076
+time sync write: 0.00077
+time sync write: 0.001231
+time sync write: 0.000767
+time sync write: 0.000771
+time sync write: 0.000768
+time sync write: 0.000773
+time sync write: 0.000782
+time sync write: 0.000761
+TOTAL: 0.010476
+time write: 7.9e-05
+time write: 6.9e-05
+time write: 7.5e-05
+time write: 7.6e-05
+time write: 5.2e-05
+time write: 5.2e-05
+time write: 5.2e-05
+time write: 5.2e-05
+time write: 5.5e-05
+time write: 5.3e-05
+TOTAL: 0.000767

data/examples/write-testing/write-speed-test.rb

@@ -0,0 +1,40 @@
+
+is_darwin = RUBY_PLATFORM.include? 'darwin'
+
+#output = (1..1000000).to_a.inspect
+output = (1..1000).to_a.inspect
+
+file_name = "./junk.out"
+
+ss = Time.now
+10.times do | i |
+  s = Time.now
+  File.open("./junk-a-#{ i }.out", 'w') { | f | f.print output; f.fsync }
+  puts "time puts: #{ Time.now - s }"
+end
+
+puts "TOTAL: #{ Time.now - ss}\n"
+
+ss = Time.now
+10.times do | i |
+  s = Time.now
+  File.open("./junk-b-#{ i }.out", 'w') do | f |
+    f.write output
+    f.fsync
+    if is_darwin then
+      f.fcntl(51, 0) # Attempt an F_FULLFSYNC fcntl to commit data to disk
+    end
+  end
+  puts "time sync write: #{ Time.now - s }"
+end
+
+puts "TOTAL: #{ Time.now - ss}\n"
+
+ss = Time.now
+10.times do | i |
+  s = Time.now
+  File.open("./junk-b-#{ i }.out", 'w') { | f | f.write output }
+  puts "time write: #{ Time.now - s }"
+end
+
+puts "TOTAL: #{ Time.now - ss}\n"

data/lib/xamplr/.cvsignore

@@ -0,0 +1 @@
+.cvs.log

data/lib/xamplr/TODO

@@ -1 +1,2 @@
--- escaped content is being repeatedly escaped (confirm, just a suspiscion from a long time ago)
+-- use the libxml form of escape attrs (look for apos)
+

data/lib/xamplr/exceptions.rb

@@ -1,4 +1,3 @@
-
 module Xampl
 
   class XamplIsInvalid < Exception
@@ -58,6 +57,18 @@ module Xampl
     end
   end
 
+  class ReturnOrThrowInTransaction < Exception
+    attr_reader :xampl
+
+    def initialize(xampl=nil)
+      @xampl = xampl
+    end
+
+    def message
+      "attempt to change #{@xampl}, pid: #{@xampl.get_the_index}, oid: #{@xampl.object_id} when changes are blocked"
+    end
+  end
+
   class UnmanagedChange < Exception
     attr_reader :xampl
 
@@ -82,6 +93,18 @@ module Xampl
     end
   end
 
+  class IncompatiblePersisterConfiguration < Exception
+    attr_reader :msg
+
+    def initialize(persister_kind, feature_name)
+      @msg = "persister kind #{persister_kind}:: requested feature: #{feature_name}"
+    end
+
+    def message
+      @msg
+    end
+  end
+
   class MixedPersisters < Exception
     attr_reader :msg
 

data/lib/xamplr/from-xml.rb

@@ -4,9 +4,9 @@ module Xampl
 
   class FromXML
 
-    attr :checkWellFormed, false
-    attr :is_realising, false
-    attr :tokenise_content, false
+    attr :checkWellFormed #1.9.1 , false
+    attr :is_realising #1.9.1 , false
+    attr :tokenise_content #1.9.1 , false
 
     @reader = nil
 
@@ -65,7 +65,8 @@ module Xampl
                                                  LibXML::XML::Parser::Options::NONET |
                                                  LibXML::XML::Parser::Options::NOCDATA |
                                                  LibXML::XML::Parser::Options::DTDATTR |
-                                                 LibXML::XML::Parser::Options::COMPACT)
+                                                 # LibXML::XML::Parser::Options::COMPACT |
+                                                 0)
       #TODO CLOSE THIS THING!!
     end
 
@@ -81,7 +82,8 @@ module Xampl
                                                    LibXML::XML::Parser::Options::NONET |
                                                    LibXML::XML::Parser::Options::NOCDATA |
                                                    LibXML::XML::Parser::Options::DTDATTR |
-                                                   LibXML::XML::Parser::Options::COMPACT)
+                                                   # LibXML::XML::Parser::Options::COMPACT) |
+                                                   0)
       #TODO CLOSE THIS THING!!
     end
 
@@ -355,7 +357,13 @@ TODO -- can these ever happen?
 
       #describe_current_element_type
 
+#TODO -- get rid of this, it is for debugging only
+begin
       okay = @reader.read
+rescue => e
+  puts "WHAT???? #{ e }"
+  raise e
+end
 
       @just_opened_an_element = start_element?
       @insert_end_element = (@just_opened_an_element and @reader.empty_element?)

data/lib/xamplr/graphml-out.rb

@@ -9,13 +9,6 @@ module XamplGenerator
       @elements_map = elements_map
     end
 
-    def devise_filename(filename)
-      bn = File.basename(filename)
-      ext = File.extname(bn)
-      bn = bn[0..(bn.size - ext.size - 1)]
-      fn = "#{bn}.graphml"
-    end
-
     def generate_class_nodes(element, include_mixins)
 
       node = @element_to_node_map[element.nstag]
@@ -83,20 +76,8 @@ module XamplGenerator
       return false
     end
 
-#    def ignore_package(package)
-#      puts "#{File.basename(__FILE__)}:#{__LINE__} pkg: [#{ package }], exclude: #{ @excluded_packages.inspect }, include: #{ @included_packages.inspect }"
-#      r = ignore_package_work(package)
-#      puts "#{File.basename(__FILE__)}:#{__LINE__}      ignore? #{ r }"
-#      r
-#    end
-
-    #def write_graph_ml(filename, excluded_packages=[ 'C2', 'NavRelationships', 'App' ], included_packages=nil, include_mixins=false)
-    def write_graph_ml(filename, excluded_packages=[ ], included_packages=nil, include_mixins=true)
-
-      #puts "#{File.basename(__FILE__)}:#{__LINE__} no graphml yet"
-      #return
-
-      filename = devise_filename(filename)
+    def write_graph_ml(base_filename, excluded_packages=[ ], included_packages=nil, include_mixins=true)
+      filename = "#{base_filename}.graphml"
 
       @excluded_packages = Set.new(excluded_packages)
       @included_packages = included_packages ? Set.new(included_packages) : nil
@@ -386,8 +367,8 @@ EOS
     def write_mixin_ref_edge(edge, class_node, mixin_node)
       @out << <<EOS
         <edge id="e#{ edge }"
-              source="n#{ class_node }"
-              target="n#{ mixin_node }">
+              source="n#{ mixin_node }"
+              target="n#{ class_node }">
             <data key="d2">
                 <y:PolyLineEdge>
                     <y:Path sx="0.0"
@@ -397,8 +378,8 @@ EOS
                     <y:LineStyle color="#000000"
                                  type="line"
                                  width="1.0"/>
-                    <y:Arrows source="none"
-                              target="white_delta"/>
+                    <y:Arrows source="white_delta"
+                              target="none"/>
                     <y:EdgeLabel alignment="center"
                                  distance="2.0"
                                  fontFamily="Dialog"

data/lib/xamplr/indexed-array.rb

@@ -48,14 +48,22 @@ class IndexedArray
     if String === index or Symbol === index then
       pos = @map.delete(index)
       if pos then
-        @map.each{ | k, v| @map[k] = (v - 1) if pos < v }
+#        @map.each{ | k, v| @map[k] = (v - 1) if pos < v }
+        @map.keys.each do |k|
+          v = @map[k]
+          @map[k] = (v - 1) if pos < v
+        end
         return @store.delete_at(pos)
       end
     else
       key = @map.index(index)
       pos = @map.delete(key) if key
       if pos then
-        @map.each{ | k, v | @map[k] = (v - 1) if pos < v }
+#        @map.each{ | k, v | @map[k] = (v - 1) if pos < v }
+        @map.keys.each do |k|
+          v = @map[k]
+          @map[k] = (v - 1) if pos < v
+        end
         return @store.delete_at(pos)
       end
     end

data/lib/xamplr/persist-to-xml.rb

@@ -43,7 +43,9 @@ module Xampl
       return prefix
     end
 
-    def attr_esc(s)
+    def attr_esc_fast(s)
+      #NOTE -- there are known issues with using Ruby 1.9.1 and libxml-ruby, which this is using. Seems to mostly
+      #        be related to DOM and XPATH but...
       unless defined?(@@doc) then
         @@doc = LibXML::XML::Document.new()
         @@doc.root = LibXML::XML::Node.new('r')
@@ -54,10 +56,7 @@ module Xampl
       (@@doc.root.to_s)[6..-4]
     end
 
-
-=begin
-
-    def attr_esc_old(s)
+    def attr_esc_slow(s)
       if (s.kind_of? XamplObject)
         return attr_esc(s.to_xml)
       end
@@ -75,8 +74,7 @@ module Xampl
       return result
     end
 
-=end
-
+    alias attr_esc attr_esc_fast
 
 #    def content_esc(s)
 #      #NO! the attribute has the right to compact white space
@@ -98,6 +96,7 @@ module Xampl
 
       result.gsub!("&", "&")
       result.gsub!("<", "&lt;")
+      result.gsub!(">", "&gt;")
 
       return result
     end

data/lib/xamplr/persistence.rb

@@ -1,4 +1,3 @@
-
 require 'sync'
 
 module Xampl
@@ -24,6 +23,7 @@ module Xampl
   end
 
   def Xampl.register_persister_kind(klass)
+    #puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] REGISTER: #{ klass.kind } --> #{ klass }"
     @@persister_kinds[klass.kind] = klass
   end
 
@@ -37,28 +37,49 @@ module Xampl
     @@persister = nil
   end
 
-  @@default_persister_kind = :simple
-  @@default_persister_format = :xml_format
+  $is_darwin = RUBY_PLATFORM.include? 'darwin'
+
+  @@factory_default_persister_options = {
+          :kind => :simple,
+          :format => :xml_format
+  }
+  @@default_persister_options = {}.merge(@@factory_default_persister_options)
+
+  def Xampl.default_persister_options
+    {}.merge(@@default_persister_options)
+  end
+
+  def Xampl.raw_persister_options
+    @@default_persister_options
+  end
+
+  def Xampl.set_default_persister_properties(options)
+    if options[:reset] then
+      @@default_persister_options = @@factory_default_persister_options.merge(options)
+    else
+      @@default_persister_options = @@default_persister_options.merge(options)
+    end
+  end
 
   def Xampl.default_persister_kind
-    @@default_persister_kind
+    @@default_persister_options[:kind]
   end
+
   def Xampl.set_default_persister_kind(kind)
-    @@default_persister_kind = kind
-    #puts "SET KIND format: #{@@default_persister_format}, kind: #{@@default_persister_kind}"
+    @@default_persister_options[:kind] = kind
   end
 
   def Xampl.default_persister_format
-    @@default_persister_format
+    @@default_persister_options[:format]
   end
+
   def Xampl.set_default_persister_format(format)
-    @@default_persister_format = format
-    #puts "SET FORMAT format: #{@@default_persister_format}, kind: #{@@default_persister_kind}"
+    @@default_persister_options[:format] = format
   end
 
   def Xampl.enable_persister(name, kind=nil, format=nil)
-    kind = kind || @@default_persister_kind
-    format = format || @@default_persister_format
+    kind = kind || Xampl.default_persister_kind
+    format = format || Xampl.default_persister_format
     @@persister = @@known_persisters[name]
 
     if @@persister then
@@ -68,7 +89,6 @@ module Xampl
       kind = @@persister.kind || kind
       format = @@persister.format || format
 
-      #raise XamplException.new(:live_across_rollback) if @@persister.rolled_back
       if kind and kind != @@persister.kind then
         raise IncompatiblePersisterRequest.new(@@persister, "kind", kind, @@persister.kind)
       end
@@ -78,7 +98,8 @@ module Xampl
     end
 
     unless @@persister then
-      # puts "CREATE PERSISTER #{name}, format: #{format}, kind: #{kind}"
+#      puts "CREATE PERSISTER #{name}, format: #{format}, kind: #{kind}"
+#      puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] kinds available: #{ @@persister_kinds.keys.inspect }"
       @@persister = @@persister_kinds[kind].new(name, format)
       if (nil != name) then
         @@known_persisters[name] = @@persister
@@ -90,7 +111,7 @@ module Xampl
 
   def Xampl.print_known_persisters
     puts "Known Persisters:: --------------------------"
-    @@known_persisters.each { | n, k | puts "    #{n} #{k}" }
+    @@known_persisters.each { | n, k | puts " #{n} #{k}" }
     puts "---------------------------------------------"
   end
 
@@ -104,7 +125,7 @@ module Xampl
   def Xampl.drop_all_persisters(verbose=false)
     puts "Drop All Persisters:: --------------------------" if verbose
     @@known_persisters.each do |name, persister|
-      puts "    #{ name } #{ persister.class.name }" if verbose
+      puts " #{ name } #{ persister.class.name }" if verbose
       next if persister == @@persister
       persister.close
       persister.clear_cache
@@ -123,9 +144,28 @@ module Xampl
     Xampl.print_known_persisters
   end
 
+  def Xampl.add_lexical_indexs(indexes)
+    case Xampl.default_persister_kind
+      when :tokyo_cabinet then
+        Xampl::TokyoCabinetPersister.add_lexical_indexs(indexes)
+      else
+        raise IncompatiblePersisterConfiguration.new(Xampl.default_persister_kind, "lexical_indexes")
+    end
+  end
+
+  def Xampl.add_numerical_indexs(indexes)
+    case Xampl.default_persister_kind
+      when :tokyo_cabinet then
+        Xampl::TokyoCabinetPersister.add_numerical_indexs(indexes)
+      else
+        raise IncompatiblePersisterConfiguration.new(Xampl.default_persister_kind, "numerical_indexs")
+    end
+  end
+
   @@xampl_lock = Sync.new
 
   @@verbose_transactions = true
+
   def Xampl.verboseTransactions(v)
     @@verbose_transactions = v
   end
@@ -141,40 +181,80 @@ module Xampl
 
     if block_given? then
       @@xampl_lock.synchronize(:EX) do
-        initial_persister = @@persister
-        Xampl.enable_persister(name, kind, format)
-
-        rollback = true
-        exception = nil
-        original_automatic = @@persister.automatic
-
+#      if true then
         begin
-          #TODO -- impose some rules on nested transactions/enable_persisters??
-
-          Xampl.auto_persistence(automatic)
-          result = yield
-          Xampl.block_future_changes(true)
-          Xampl.sync
-          rollback = false
-          return result
-        rescue => e
-          exception = e
-        ensure
-          Xampl.block_future_changes(false)
-          Xampl.auto_persistence(original_automatic)
-          if rollback then
-            if exception then
-              puts "ROLLBACK(#{__LINE__}):: #{exception}" if rollback and @@verbose_transactions
-              #print exception.backtrace.join("\n") if rollback
-              raise exception
-            else
-              if @@verbose_transactions and rollback then
-                puts "ROLLBACK(#{__LINE__}):: UNKNOWN CAUSE" if rollback
+#          @@xampl_lock.sync_lock(:EX)
+          initial_persister = @@persister
+          Xampl.enable_persister(name, kind, format)
+
+          rollback = true
+          exception = nil
+          original_automatic = @@persister.automatic
+
+          begin
+            #TODO -- impose some rules on nested transactions/enable_persisters??
+
+            Xampl.auto_persistence(automatic)
+            result = yield
+            Xampl.block_future_changes(true)
+            Xampl.sync
+            rollback = false
+            return result
+          rescue => e
+            exception = e
+          ensure
+            Xampl.block_future_changes(false)
+            Xampl.auto_persistence(original_automatic)
+            if rollback then
+              # we get here if the transaction block finishes early
+              if exception then
+                # the early finish was caused by an exception
+                puts "ROLLBACK(#{__LINE__}):: #{exception}" if rollback and @@verbose_transactions
+                raise exception
+              else
+                # How could we have arrived at this point???
+                # Well, I don't know all the reasons, but the ones I do know are:
+                #  - return was used in the block passed into the transaction
+                #  - a throw was made
+                # There's no way that I know of to distinguish, so, assume that the transaction worked
+                #    20091202 -- well maybe not...
+
+#                Xampl.rollback # this is bad
+                rollback = true
+#                raise  ReturnOrThrowInTransaction
+                STDERR.puts "---------"
+                STDERR.puts "Either a return or a throw from a transaction. The DB is possibly not synced."
+                caller(0).each do |trace|
+                  STDERR.puts(trace)
+                end
+                STDERR.puts "---------"
+
+=begin
+
+                begin
+                  puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] "
+                  Xampl.block_future_changes(true)
+                  puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] "
+                  Xampl.sync
+                  rollback = false
+                rescue => e
+                  puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] "
+                  # so we know the persister had a problem
+                  puts "PERSISTER ERROR(#{__LINE__}) #{ e }"
+                ensure
+                  puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] "
+                  Xampl.block_future_changes(false)
+                end
+
+=end
               end
             end
+            Xampl.rollback if rollback
+            @@persister = initial_persister
           end
-          Xampl.rollback if rollback
-          @@persister = initial_persister
+          raise exception if exception
+#        ensure
+#          @@xampl_lock.sync_unlock
         end
       end
     end
@@ -216,11 +296,15 @@ module Xampl
           if 0 == @changed.size then
             @changed = original_changed
 
-            if rollback then
+            if exception then
               puts "ROLLBACK(#{__LINE__}):: #{exception}"
-              print exception.backtrace.join("\n")
-              Xampl.rollback
+              print exception.backtrace.join("\n") if exception
             end
+
+            #no change so don't bother with rollback
+#            if rollback then
+#              Xampl.rollback
+#            end
             @@persister = initial_persister
           else
             puts "CHANGED COUNT: #{@changed.size}"