hutch-xamplr 1.0.5 → 1.1.0

data/lib/xamplr/persistence.rb.more_thread_safe

@@ -569,8 +569,6 @@ module Xampl
           return xampl.persist
         when :ruby_format then
           return xampl.to_ruby
-        when :yaml_format then
-          return xampl.as_yaml
       end
     end
 
@@ -580,20 +578,9 @@ module Xampl
       
       if representation =~ /^</ then
         return XamplObject.realise_from_xml_string(representation, target)
-      elsif representation =~ /^-/ then
-        return XamplObject.from_yaml(representation, target)
       else
         XamplObject.from_ruby(representation, target)
       end
-      
-      # case @format
-      #   when nil, :xml_format then
-      #     return XamplObject.realise_from_xml_string(representation, target)
-      #   when :ruby_format then
-      #     XamplObject.from_ruby(representation, target)
-      #   when :yaml_format then
-      #     return XamplObject.from_yaml(representation, target)
-      # end
     end
 
     def write(xampl)

data/lib/xamplr/persistence.rb.partially_thread_safe

@@ -561,8 +561,6 @@ module Xampl
           return xampl.persist
         when :ruby_format then
           return xampl.to_ruby
-        when :yaml_format then
-          return xampl.as_yaml
       end
     end
 
@@ -572,20 +570,9 @@ module Xampl
       
       if representation =~ /^</ then
         return XamplObject.realise_from_xml_string(representation, target)
-      elsif representation =~ /^-/ then
-        return XamplObject.from_yaml(representation, target)
       else
         XamplObject.from_ruby(representation, target)
       end
-      
-      # case @format
-      #   when nil, :xml_format then
-      #     return XamplObject.realise_from_xml_string(representation, target)
-      #   when :ruby_format then
-      #     XamplObject.from_ruby(representation, target)
-      #   when :yaml_format then
-      #     return XamplObject.from_yaml(representation, target)
-      # end
     end
 
     def write(xampl)

data/lib/xamplr/persister.rb

@@ -0,0 +1,298 @@
+
+require "xamplr/persistence"
+
+module Xampl
+  class Persister
+    attr_accessor :name,
+                  :automatic,
+                  :block_changes,
+                  :read_count, :total_read_count,
+                  :write_count, :total_write_count,
+                  :total_sync_count, :total_rollback_count,
+                  :cache_hits, :total_cache_hits,
+                  :last_write_count,
+                  :rolled_back
+    attr_reader :syncing, :format
+
+    def initialize(name=nil, format=nil)
+      @name = name
+      @format = format
+      @automatic = false
+      @changed = {}
+      @cache_hits = 0
+      @total_cache_hits = 0
+      @read_count = 0
+      @total_read_count = 0
+      @write_count = 0
+      @total_write_count = 0
+      @last_write_count = 0
+      @total_sync_count = 0
+      @total_rollback_count = 0
+      @rolled_back = false
+      @syncing = false
+
+      @busy_count = 0
+    end
+
+    def optimise(opts)
+    end
+
+    def close
+      self.sync
+    end
+
+    def busy(yes)
+      if yes then
+        @busy_count += 1
+      elsif 0 < @busy_count then
+        @busy_count -= 1
+      end
+    end
+
+    def is_busy
+      return 0 < @busy_count
+    end
+
+    def introduce(xampl)
+      if xampl.introduce_persister(self) then
+        cache(xampl)
+      end
+      has_changed(xampl) if xampl.is_changed
+    end
+
+    def has_changed(xampl)
+      #raise XamplException.new(:live_across_rollback) if @rolled_back
+      # puts "!!!! has_changed #{xampl} #{xampl.get_the_index} -- persist required: #{xampl.persist_required}"
+      if xampl.persist_required && xampl.is_changed then
+        unless self == xampl.persister
+          raise MixedPersisters.new(xampl.persister, self)
+        end
+        @changed[xampl] = xampl
+#         puts "!!!! change recorded ==> #{@changed.size}/#{count_changed} #{@changed.object_id} !!!!"
+        #         @changed.each{ | thing, ignore |
+        #           puts "             changed: #{thing}, index: #{thing.get_the_index},  changed: #{thing.is_changed}"
+        #         }
+      end
+    end
+
+    def has_not_changed(xampl)
+      #       puts "!!!! has_not_changed #{xampl} #{xampl.get_the_index} -- in @changed: #{nil != @changed[xampl]}"
+      @changed.delete(xampl) if xampl
+    end
+
+    def count_changed
+#      @changed.each{ | thing, ignore |
+      #        puts "changed: #{thing}, index: #{thing.get_the_index}"
+      #      }
+      return @changed.size
+    end
+
+    def cache(xampl)
+      raise XamplException.new(:unimplemented)
+    end
+
+    def uncache(xampl)
+      raise XamplException.new(:unimplemented)
+    end
+
+    def clear_cache
+      raise XamplException.new(:unimplemented)
+    end
+
+    def Persister.replace(old_xampl, new_xampl)
+      pid = old_xampl.get_the_index
+      if old_xampl.persister != @@persister then
+        raise MixedPersisters.new(@@persister, old_xampl.persister)
+      end
+      if new_xampl.persister != @@persister then
+        raise MixedPersisters.new(@@persister, new_xampl.persister)
+      end
+
+      new_xampl.note_replacing(old_xampl)
+
+      unless old_xampl.load_needed then
+        Xampl.log.warn("Replacing live xampl: #{old_xampl} pid: #{pid}")
+        @@persister.uncache(old_xampl)
+        old_xampl.invalidate
+      end
+      new_xampl.pid = nil
+      new_xampl.pid = pid
+      @@persister.introduce(new_xampl)
+    end
+
+    def represent(xampl, mentions=[])
+      #puts "REPRESENT #{xampl} load needed: #{xampl.load_needed}"
+      #      return nil if xampl.load_needed
+      case xampl.default_persister_format || @format
+        when nil, :xml_format then
+          return xampl.persist("", mentions)
+        when :ruby_format then
+          return xampl.to_ruby(mentions)
+      end
+    end
+
+    def realise(representation, target=nil)
+      # Normally we'd expect to see the representation in the @format format, but
+      # that isn't necessarily the case. Try to work out what the format might be...
+
+      if representation =~ /^</ then
+        return XamplObject.realise_from_xml_string(representation, target)
+      else
+        XamplObject.from_ruby(representation, target)
+      end
+    end
+
+    def version(stream)
+      raise XamplException.new(:unimplemented)
+      # catch(:refuse_to_version) do
+      # end
+    end
+
+    def write(xampl)
+      raise XamplException.new(:unimplemented)
+    end
+
+    def read(klass, pid, target=nil)
+      raise XamplException.new(:unimplemented)
+    end
+
+    def lookup(klass, pid)
+      #raise XamplException.new(:live_across_rollback) if @rolled_back
+      #puts "#{File.basename(__FILE__)} #{__LINE__} LOOKUP:: klass: #{klass} pid: #{pid}"
+
+      begin
+        busy(true)
+        xampl = read(klass, pid)
+      ensure
+        busy(false)
+      end
+
+      #puts "#{File.basename(__FILE__)} #{__LINE__}       ---> #{ xampl }"
+
+      return xampl
+    end
+
+    def find_known(klass, pid)
+      #raise XamplException.new(:live_across_rollback) if @rolled_back
+
+      xampl = read_from_cache(klass, pid, nil)
+
+      return xampl
+    end
+
+    def lazy_load(target, klass, pid)
+      #      puts "#{File.basename(__FILE__)} #{__LINE__} LAZY_LOAD:: klass: #{klass} pid: #{pid} target: #{target}"
+
+      xampl = read(klass, pid, target)
+
+      # puts "   LAZY_LOAD --> #{xampl}"
+
+      return xampl
+    end
+
+    def put_changed(msg="")
+      puts "Changed::#{msg}:"
+      @changed.each { | xampl, ignore | puts "   #{xampl.tag} #{xampl.get_the_index}" }
+    end
+
+    def do_sync_write
+      unchanged_in_changed_list = 0
+      @changed.each do |xampl, ignore|
+        unchanged_in_changed_list += 1 unless xampl.is_changed
+        unless xampl.kind_of?(InvalidXampl) then
+          write(xampl)
+        end
+      end
+    end
+
+    def sync
+      #raise XamplException.new(:live_across_rollback) if @rolled_back
+      begin
+        #puts "SYNC"
+        #puts "SYNC"
+        #puts "SYNC changed: #{@changed.size}" if 0 < @changed.size
+        #@changed.each do | key, value |
+        ##puts "   #{key.class.name}"
+        ##puts "key: #{key.class.name}, value: #{value.class.name}"
+        #puts key.to_xml
+        #end
+        #puts "SYNC"
+        #puts "SYNC"
+
+        #if 0 < @changed.size then
+        #puts "SYNC changed: #{@changed.size}"
+        ##caller(0).each do | trace |
+        ##  next if /xamplr/ =~ trace
+        ##  puts "  #{trace}"
+        ##  break if /actionpack/ =~ trace
+        ##end
+        #end
+        busy(true)
+        @syncing = true
+
+        do_sync_write
+
+        @changed = {}
+
+        @total_read_count = @total_read_count + @read_count
+        @total_write_count = @total_write_count + @write_count
+        @total_cache_hits = @total_cache_hits + @cache_hits
+        @total_sync_count = @total_sync_count + 1
+
+        @read_count = 0
+        @last_write_count = @write_count
+        @write_count = 0
+
+        self.sync_done()
+
+        return @last_write_count
+      ensure
+        busy(false)
+        @syncing = false
+      end
+    end
+
+    def sync_done
+      raise XamplException.new(:unimplemented)
+    end
+
+    def rollback
+      begin
+        busy(true)
+
+        return Xampl.rollback(self)
+      ensure
+        busy(false)
+      end
+    end
+
+    def rollback_cleanup
+      @changed = {}
+    end
+
+    def print_stats
+      printf("SYNC:: TOTAL cache_hits: %d, reads: %d, writes: %d\n",
+             @total_cache_hits, @total_read_count, @total_write_count)
+      printf("             cache_hits: %d, reads: %d, last writes: %d\n",
+             @cache_hits, @read_count, @last_write_count)
+      printf("             syncs: %d\n", @total_sync_count)
+      printf("             changed count: %d (%d)\n", count_changed, @changed.size)
+      @changed.each do |thing, ignore|
+        if thing.is_changed then
+          puts "             changed: #{thing}, index: #{thing.get_the_index}"
+        else
+          puts "             UNCHANGED: #{thing}, index: #{thing.get_the_index} <<<<<<<<<<<<<<<<<<< BAD!"
+        end
+      end
+    end
+  end
+
+  require "xamplr/persisters/simple"
+  require "xamplr/persisters/in-memory"
+  require "xamplr/persisters/filesystem"
+
+  if require 'tokyocabinet' then
+    require "xamplr/persisters/tokyo-cabinet"
+  end
+
+end

data/lib/xamplr/{persister → persisters}/caching.rb

@@ -1,7 +1,7 @@
 module Xampl
 
   require "fileutils"
-  require "xamplr/persister/caches"
+  require "xamplr/persisters/caches"
 
   class AbstractCachingPersister < Persister
 

data/lib/xamplr/{persister → persisters}/filesystem.rb

@@ -3,7 +3,7 @@ module Xampl
   require 'xamplr/version'
 
   require "fileutils"
-  require "xamplr/persister/caching"
+  require "xamplr/persisters/caching"
 
   class FilesystemPersister < AbstractCachingPersister
 
@@ -39,9 +39,7 @@ module Xampl
 
       representation = represent(xampl)
       if representation then
-        File.open(place, "w"){ | out |
-          out.puts representation
-        }
+        File.open(place, "w"){ | out | out.puts representation }
         @write_count = @write_count + 1
       end
       xampl.changes_accepted
@@ -52,7 +50,8 @@ module Xampl
       place = File.join(@root_dir, klass.name.split("::"), pid)
 
       return nil unless File.exist?(place)
-      return File.read(place)
+      representation = File.read(place)
+      return representation
     end
   end
 

data/lib/xamplr/{persister → persisters}/in-memory.rb

@@ -1,6 +1,6 @@
 module Xampl
 
-  require "xamplr/persister/caches"
+  require "xamplr/persisters/caches"
 
   class InMemoryPersister < Persister
 

data/lib/xamplr/{persister → persisters}/tokyo-cabinet.rb

@@ -2,9 +2,11 @@ module Xampl
 
   require 'fileutils'
   require 'tokyocabinet'
-  require 'xamplr/persister/caching'
+  require 'xamplr/persisters/caching'
   require 'set'
 
+  require 'ruby-prof'
+
   class TokyoCabinetPersister < AbstractCachingPersister
     include TokyoCabinet
 
@@ -43,29 +45,32 @@ module Xampl
 
       open_tc_db()
 
-      #      note_errors("TC:: optimisation error: %s\n") do
+      #      note_errors("TC[[#{ @filename }]]:: optimisation error: %s\n") do
       #        @tc_db.optimize(-1, -1, -1, TDB::TDEFLATE)
       #      end
-      #      note_errors("TC:: close error: %s\n") do
+      #      note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
       #        @tc_db.close
       #      end
     end
 
     def open_tc_db
       return if @tc_db
+#      puts "#{File.basename(__FILE__)}:#{__LINE__} open tc db: #{ @filename }"
+#puts "#{File.basename(__FILE__)}:#{__LINE__} callers..."
+      #caller(0).each { | trace | puts "   #{trace}"}
       @tc_db = TDB.new
-      note_errors("TC:: tuning error: %s\n") do
+      note_errors("TC[[#{ @filename }]]:: tuning error: %s\n") do
         @tc_db.tune(-1, -1, -1, TDB::TDEFLATE)
       end
 
-      note_errors("TC:: open error: %s\n") do
+      note_errors("TC[[#{ @filename }]]:: open [#{ @filename }] error: %s\n") do
         @tc_db.open(@filename, TDB::OWRITER | TDB::OCREAT | TDB::OLCKNB ) #TDB::OTSYNC slows it down by almost 50 times
       end
 
       # Don't care if there are errors (in fact, if the index exists a failure is the expected thing)
 
       $lexical_indexes.each do | index_name |
-        @tc_db.setindex(index_name, TDB::ITLEXICAL | TDB::ITKEEP)
+        r = @tc_db.setindex(index_name, TDB::ITLEXICAL | TDB::ITKEEP)
       end
       $numeric_indexes.each do | index_name |
         @tc_db.setindex(index_name, TDB::ITDECIMAL | TDB::ITKEEP)
@@ -84,7 +89,7 @@ module Xampl
           @tc_db.setindex(index_name, 9998)
         end
       else
-        note_errors("TC:: optimisation error: %s\n") do
+        note_errors("TC[[#{ @filename }]]:: optimisation error: %s\n") do
           @tc_db.optimize(-1, -1, -1, 0xff)
         end
       end
@@ -93,10 +98,11 @@ module Xampl
     def close
       if @tc_db then
         self.sync
-        note_errors("TC:: close error: %s\n") do
+        note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
           @tc_db.close
         end
         @tc_db = nil
+#        puts "#{File.basename(__FILE__)}:#{__LINE__} close tc db: #{ @filename }"
       end
     end
 
@@ -274,6 +280,17 @@ module Xampl
     end
 
     def do_sync_write
+#      RubyProf.start
+      #
+      #      do_sync_write_work
+      #
+      #      result = RubyProf.stop
+      #      printer = RubyProf::FlatPrinter.new(result)
+      #      printer.print(STDOUT, 0)
+      #      puts "#{File.basename(__FILE__)}:#{__LINE__} stop this profiler"
+      #    end
+      #
+      #    def do_sync_write_work
       open_tc_db
       @time_stamp = Time.now.to_f.to_s
 
@@ -283,7 +300,7 @@ module Xampl
       #      end
 
       begin
-        note_errors("TC:: tranbegin error: %s\n") do
+        note_errors("TC[[#{ @filename }]]:: tranbegin error: %s\n") do
           @tc_db.tranbegin
         end
 
@@ -292,16 +309,16 @@ module Xampl
         end
       rescue => e
         msg = "no TC.abort attempted"
-        msg = note_errors("TC:: trancommit error: %s\n") do
+        msg = note_errors("TC[[#{ @filename }]]:: trancommit error: %s\n") do
           @tc_db.tranabort
         end
         raise "TokyoCabinetPersister Error:: #{ msg }/#{ e }"
       else
-        note_errors("TC:: trancommit error: %s\n") do
+        note_errors("TC[[#{ @filename }]]:: trancommit error: %s\n") do
           @tc_db.trancommit
         end
       ensure
-        #        note_errors("TC:: close error: %s\n") do
+        #        note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
         #          @tc_db.close()
         #        end
       end
@@ -317,7 +334,7 @@ module Xampl
 
       query = TableQuery.new(@tc_db)
       query.add_condition('xampl_from', :equals, place)
-      note_errors("TC:: failed to remove from mentions, error: %s\n") do
+      note_errors("TC[[#{ @filename }]]:: failed to remove from mentions, error: %s\n") do
         query.searchout
       end
 
@@ -333,7 +350,7 @@ module Xampl
                 'xampl_to' => mention_place
         }
 
-        note_errors("TC:: write error: %s\n") do
+        note_errors("TC[[#{ @filename }]]:: write error: %s\n") do
           @tc_db.put(pk, mention_hash)
         end
       end
@@ -350,7 +367,7 @@ module Xampl
         xampl_hash = hash.merge(xampl_hash)
       end
 
-      note_errors("TC:: write error: %s\n") do
+      note_errors("TC[[#{ @filename }]]:: write error: %s\n") do
         @tc_db.put(place, xampl_hash)
       end
 
@@ -359,7 +376,15 @@ module Xampl
       return true
     end
 
+    $TC_COUNT = 0
+    $FS_COUNT = 0
+    $NF_COUNT = 0
+
     def read_representation(klass, pid)
+      #TODO -- is this being called too often, e.g. by new_xxx???
+      #      puts "#{File.basename(__FILE__)}:#{__LINE__} READ #{ klass }/#{ pid }"
+      #      caller(0).each { | trace | puts "  #{trace}"}
+
       open_tc_db
       place = File.join(klass.name.split("::"), pid)
       representation = nil
@@ -367,9 +392,22 @@ module Xampl
       meta = @tc_db[place]
       representation = meta['xampl'] if meta
 
+      #      puts "#{File.basename(__FILE__)}:#{__LINE__} TC: #{ klass }/#{ pid }" if representation
+      $TC_COUNT += 1 if representation
+
       # puts "read: #{ place }, size: #{ representation.size }"
       # puts representation[0..100]
 
+      unless representation then
+        # try the filesystem if it is not in the TC repository
+        place = File.join(@root_dir, klass.name.split("::"), pid)
+        representation = File.read(place) if File.exist?(place)
+        $FS_COUNT += 1 if representation
+#        puts "#{File.basename(__FILE__)}:#{__LINE__} FS: #{ klass }/#{ pid } (FS: #{ $FS_COUNT}, TC: #{ $TC_COUNT }, NF: #{ $NF_COUNT }" if representation
+      end
+#      puts "#{File.basename(__FILE__)}:#{__LINE__} ??: #{ klass }/#{ pid }" unless representation
+      $NF_COUNT += 1
+
       return representation
     end
   end