xamplr 1.2.0 → 1.3.15

data/lib/xamplr/persister.rb

@@ -1,4 +1,3 @@
-
 require "xamplr/persistence"
 
 module Xampl
@@ -41,6 +40,11 @@ module Xampl
       self.sync
     end
 
+    def shutdown
+#      self.sync
+#      self.close
+    end
+
     def busy(yes)
       if yes then
         @busy_count += 1
@@ -204,7 +208,15 @@ module Xampl
 
     def put_changed(msg="")
       puts "Changed::#{msg}:"
-      @changed.each { | xampl, ignore | puts "   #{xampl.tag} #{xampl.get_the_index}" }
+      @changed.each { | xampl, ignore | puts " #{xampl.tag} #{xampl.get_the_index}" }
+    end
+
+    def start_sync_write
+      #only if needed
+    end
+
+    def done_sync_write
+      # only if needed
     end
 
     def do_sync_write
@@ -220,29 +232,39 @@ module Xampl
     def sync
       #raise XamplException.new(:live_across_rollback) if @rolled_back
       begin
-        #puts "SYNC"
-        #puts "SYNC"
-        #puts "SYNC changed: #{@changed.size}" if 0 < @changed.size
-        #@changed.each do | key, value |
-        ##puts "   #{key.class.name}"
-        ##puts "key: #{key.class.name}, value: #{value.class.name}"
-        #puts key.to_xml
-        #end
-        #puts "SYNC"
-        #puts "SYNC"
-
-        #if 0 < @changed.size then
-        #puts "SYNC changed: #{@changed.size}"
-        ##caller(0).each do | trace |
-        ##  next if /xamplr/ =~ trace
-        ##  puts "  #{trace}"
-        ##  break if /actionpack/ =~ trace
-        ##end
-        #end
-        busy(true)
-        @syncing = true
-
-        do_sync_write
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] SYNC changed: #{@changed.size}" if 0 < @changed.size
+#        @changed.each do | key, value |
+#          puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] key: #{key.class.name}, pid: #{key.get_the_index}"
+#        end
+#
+#        if 0 < @changed.size then
+#          puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] SYNC changed: #{@changed.size}"
+#          caller(0).each do | trace |
+#            next if /xamplr/ =~ trace
+#            puts " #{trace}"
+#            break if /actionpack/ =~ trace
+#          end
+#        end
+#
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] SYNC changed: #{@changed.size}"
+        if 0 < @changed.size then
+          begin
+#            puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] SYNC changed: #{@changed.size}"
+            busy(true)
+#            if @syncing then
+#              puts "\n\n\n\n#{ __FILE__ }:#{ __LINE__ } [#{__method__}] SYNCING IS ALREADY TRUE!!!!!!\n\n\n"
+#            end
+            @syncing = true
+
+            start_sync_write
+            do_sync_write
+          ensure
+            done_sync_write
+            @syncing = false
+          end
+        else
+#          puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] SYNC, noting changed"
+        end
 
         @changed = {}
 
@@ -260,6 +282,7 @@ module Xampl
         return @last_write_count
       ensure
         busy(false)
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] **** SYNCING IS FALSE"
         @syncing = false
       end
     end

data/lib/xamplr/persisters/filesystem.rb

@@ -25,7 +25,6 @@ module Xampl
       end
     end
 
-
     def write(xampl)
       #raise XamplException.new(:no_index_so_no_persist) unless xampl.get_the_index or xampl.ignore_when_no_index
       raise XamplException.new("no_index_so_no_persist [#{xampl.class.name}]") unless xampl.get_the_index or xampl.ignore_when_no_index
@@ -39,7 +38,14 @@ module Xampl
 
       representation = represent(xampl)
       if representation then
-        File.open(place, "w"){ | out | out.puts representation }
+        File.open(place, "w")do |out|
+          out.puts representation
+          out.fsync
+          if $is_darwin then
+            out.fcntl(51, 0) # Attempt an F_FULLFSYNC fcntl to commit data to disk
+          end
+
+        end
         @write_count = @write_count + 1
       end
       xampl.changes_accepted

data/lib/xamplr/persisters/tokyo-cabinet.rb

@@ -5,29 +5,49 @@ module Xampl
   require 'xamplr/persisters/caching'
   require 'set'
 
-#  require 'ruby-prof'
+  #  require 'ruby-prof'
 
   class TokyoCabinetPersister < AbstractCachingPersister
     include TokyoCabinet
 
     def note_errors(msg="TokyoCabinet Error:: %s\n")
-      result = yield
+#      puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] START"
+#      puts "--------------------------------------------------"
+#      caller(0).each { |trace| puts " #{trace}" }
+#      puts "--------------------------------------------------\n\n"
+      exception = nil
+      begin
+        result = yield
+      rescue => e
+        exception = e
+      end
 
       rmsg = nil
       unless result then
         rmsg = sprintf(msg, @tc_db.errmsg(@tc_db.ecode))
+        STDERR.puts "NOTE: TokyoCabinet Error!"
         STDERR.printf(rmsg)
         STDERR.puts "---------"
         caller(0).each do |trace|
           STDERR.puts(trace)
         end
         STDERR.puts "---------"
+
+#        STDOUT.puts "NOTE: TokyoCabinet Error!"
+#        STDOUT.printf(rmsg)
+#        STDOUT.puts "---------"
+#        caller(0).each do |trace|
+#          STDOUT.puts(trace)
+#        end
+#        STDOUT.puts "---------"
       end
+      raise exception if exception
       return rmsg
     end
 
-    $lexical_indexes = Set.new(%w{ class pid time-stamp xampl_from xampl_to }) unless defined?($lexical_indexes)
-    $numeric_indexes = Set.new unless defined?($numeric_indexes)
+    $lexical_indexes = Set.new(%w{ class pid time-stamp xampl-from xampl-to xampl-place }) unless defined?($lexical_indexes)
+
+    $numeric_indexes = Set.new(%w{ scheduled-delete-at }) unless defined?($numeric_indexes)
 
     def TokyoCabinetPersister.add_lexical_indexs(indexes)
       $lexical_indexes.merge(indexes)
@@ -40,31 +60,54 @@ module Xampl
     def initialize(name=nil, format=nil, root=File.join(".", "repo"))
       super(root, name, format)
 
-      FileUtils.mkdir_p(@root_dir) unless File.exist?(@root_dir)
+      @files_dir = "#{ @root_dir }/files"
+#      FileUtils.mkdir_p(@root_dir) unless File.exist?(@root_dir)
+      FileUtils.mkdir_p(@files_dir) unless File.exist?(@files_dir)
       @filename = "#{@root_dir}/repo.tct"
+      @tc_db = nil
+#      puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] file: #{ @filename }, db: #{ @tc_db.class.name }"
 
       open_tc_db()
 
-      #      note_errors("TC[[#{ @filename }]]:: optimisation error: %s\n") do
-      #        @tc_db.optimize(-1, -1, -1, TDB::TDEFLATE)
-      #      end
-      #      note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
-      #        @tc_db.close
-      #      end
+#      note_errors("TC[[#{ @filename }]]:: optimisation error: %s\n") do
+#        @tc_db.optimize(-1, -1, -1, TDB::TDEFLATE)
+#      end
+#      note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
+#        @tc_db.close
+#      end
+
+      begin
+        note_errors("TC[[#{ @filename }]]:: close error in initialize: %s\n") do
+          @tc_db.close
+        end
+        @tc_db = nil
+      rescue => e
+        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] OH CRAP!!! #{ e }"
+      end
     end
 
     def open_tc_db
-      return if @tc_db
-#      puts "#{File.basename(__FILE__)}:#{__LINE__} open tc db: #{ @filename }"
-#puts "#{File.basename(__FILE__)}:#{__LINE__} callers..."
-      #caller(0).each { | trace | puts "   #{trace}"}
+#      if @tcdb then
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] ALREADY OPEN #{ @filename }"
+#      else
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] OPEN #{ @filename }"
+##        callers = caller(0)
+##        puts "   0 #{ callers[0] }"
+##        puts "   1 #{ callers[1] }"
+##        puts "   2 #{ callers[2] }"
+#        #puts "#{File.basename(__FILE__)}:#{__LINE__} callers..."
+#        #caller(0).each { | trace | puts "   #{trace}"}
+#      end
+
+      return if @tc_db # if there is a tc_db then it is already open
+
       @tc_db = TDB.new
       note_errors("TC[[#{ @filename }]]:: tuning error: %s\n") do
         @tc_db.tune(-1, -1, -1, TDB::TDEFLATE)
       end
 
       note_errors("TC[[#{ @filename }]]:: open [#{ @filename }] error: %s\n") do
-        @tc_db.open(@filename, TDB::OWRITER | TDB::OCREAT | TDB::OLCKNB ) #TDB::OTSYNC slows it down by almost 50 times
+        @tc_db.open(@filename, TDB::OWRITER | TDB::OCREAT | TDB::OLCKNB | TDB::OTSYNC ) #TDB::OTSYNC slows it down by almost 50 times
       end
 
       # Don't care if there are errors (in fact, if the index exists a failure is the expected thing)
@@ -96,13 +139,24 @@ module Xampl
     end
 
     def close
+#      puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] CLOSE #{ @filename }"
+#      callers = caller(0)
+#      puts "   0 #{ callers[0] }"
+#      puts "   1 #{ callers[1] }"
+#      puts "   2 #{ callers[2] }"
+
       if @tc_db then
-        self.sync
-        note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
-          @tc_db.close
+        begin
+#          puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] NO SELF SYNC?? [#{ @currently_syncing }] --> db: #{ @tc_db.class.name }"
+          self.sync unless @currently_syncing
+        rescue => e
+          puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] OH CRAP!!! #{ e }"
+        ensure
+          note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
+            @tc_db.close
+          end
+          @tc_db = nil
         end
-        @tc_db = nil
-#        puts "#{File.basename(__FILE__)}:#{__LINE__} close tc db: #{ @filename }"
       end
     end
 
@@ -155,9 +209,9 @@ module Xampl
       end
 
       if hint then
-        return results, the_hint
+        return results.uniq, the_hint
       else
-        return results
+        return results.uniq
       end
     end
 
@@ -199,9 +253,9 @@ module Xampl
       end
 
       if hint then
-        return results, the_hint
+        return results.uniq, the_hint
       else
-        return results
+        return results.uniq
       end
     end
 
@@ -219,10 +273,15 @@ module Xampl
         result_keys = query.search
       end
 
+      results = result_keys.collect do |key|
+        meta = @tc_db[ key ]
+        meta['xampl-place'] || meta['place']
+      end
+
       if hint then
-        return result_keys, the_hint
+        return results.uniq, the_hint
       else
-        return result_keys
+        return results.uniq
       end
     end
 
@@ -255,7 +314,7 @@ module Xampl
       place = File.join(xampl.class.name.split("::"), xampl.get_the_index)
 
       query = TableQuery.new(@tc_db)
-      query.add_condition('xampl_to', :equals, place)
+      query.add_condition('xampl-to', :equals, place)
       result_keys = query.search
 
       class_cache = {}
@@ -263,8 +322,8 @@ module Xampl
         result = @tc_db[ key ]
         next unless result
 
-        mentioner = result['xampl_from']
-        class_name = result['class']
+        mentioner = result['xampl-from']
+        class_name = result['mentioned_class']
         result_class = class_cache[class_name]
         unless result_class then
           class_name.split("::").each do | chunk |
@@ -280,30 +339,43 @@ module Xampl
 
         self.lookup(result_class, result['pid'])
       end
-      return results
+      return results.uniq
     end
 
-    def do_sync_write
-#      RubyProf.start
-      #
-      #      do_sync_write_work
-      #
-      #      result = RubyProf.stop
-      #      printer = RubyProf::FlatPrinter.new(result)
-      #      printer.print(STDOUT, 0)
-      #      puts "#{File.basename(__FILE__)}:#{__LINE__} stop this profiler"
-      #    end
-      #
-      #    def do_sync_write_work
+    def start_sync_write
+#      puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] #{ @filename }"
+#      callers = caller(0)
+#      puts "   0 #{ callers[0] }"
+#      puts "   1 #{ callers[1] }"
+#      puts "   2 #{ callers[2] }"
+
+      @currently_syncing = true
       open_tc_db
-      @time_stamp = Time.now.to_f.to_s
+    end
+
+    def done_sync_write
+      begin
+        note_errors("TC[[#{ @filename }]]:: sync error in done_sync_write: %s\n") do
+          @tc_db.sync
+        end
+#        close
+      ensure
+        @currently_syncing = false
+      end
+    end
 
-      #      puts "DO SYNC WRITE: #{ @changed.size } to be written (#{ @filename })"
-      #      note_errors("TC:: open error: %s\n") do
-      #        @tc_db.open(@filename, TDB::OWRITER | TDB::OCREAT | TDB::OLCKNB ) #TDB::OTSYNC slows it down by almost 50 times
-      #      end
 
+    def do_sync_write
       begin
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] **************************"
+#        callers = caller(0)
+#        puts "   0 #{ callers[0] }"
+#        puts "   1 #{ callers[1] }"
+#        puts "   2 #{ callers[2] }"
+
+#        open_tc_db
+        @time_stamp = Time.now.to_f.to_s
+
         note_errors("TC[[#{ @filename }]]:: tranbegin error: %s\n") do
           @tc_db.tranbegin
         end
@@ -313,45 +385,97 @@ module Xampl
         end
       rescue => e
         msg = "no TC.abort attempted"
-        msg = note_errors("TC[[#{ @filename }]]:: trancommit error: %s\n") do
+        msg = note_errors("TC[[#{ @filename }]]:: tranabort error: %s\n") do
           @tc_db.tranabort
         end
+        puts "------------------------------------------------------------------------"
+        puts "TokyoCabinetPersister Error:: #{ msg }/#{ e }"
+        puts e.backtrace.join("\n")
+        puts "------------------------------------------------------------------------"
         raise "TokyoCabinetPersister Error:: #{ msg }/#{ e }"
       else
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] COMMIT"
         note_errors("TC[[#{ @filename }]]:: trancommit error: %s\n") do
           @tc_db.trancommit
         end
       ensure
-        #        note_errors("TC[[#{ @filename }]]:: close error: %s\n") do
-        #          @tc_db.close()
-        #        end
-      end
 #      puts "               num records: #{ @tc_db.rnum() }"
+        #      puts "#{ __FILE__ }:#{ __LINE__ } keys..."
+        #      @tc_db.keys.each do | key |
+        #        meta = @tc_db[key]
+        #        meta['xampl'] = (meta['xampl'] || "no rep")[0..25]
+        #        puts "         key: [#{ key }] -- #{ meta.inspect }"
+        #      end
+
+#        close
+      end
+    end
+
+    def how_indexed(xampl)
+      raise XamplException.new(:no_index_so_no_persist) unless xampl.get_the_index
+      place = File.join(xampl.class.name.split("::"), xampl.get_the_index)
+
+      open_tc_db
+
+      result_keys = Set.new
+
+      query = TableQuery.new(@tc_db)
+      query.add_condition('xampl-place', :equals, place)
+      search_results = query.search
+      result_keys.merge( search_results)
+
+      primary = @tc_db[ place ]
+      if primary then
+        primary.delete('xampl')
+      end
+
+      results = primary ? [ primary ] : []
+      result_keys.each do | key |
+        result = @tc_db[ key ]
+        next unless result
+
+        result.delete('xampl')
+
+        results << result
+      end
+
+      results
     end
 
     def write(xampl)
       raise XamplException.new(:no_index_so_no_persist) unless xampl.get_the_index
 
-      place = File.join(xampl.class.name.split("::"), xampl.get_the_index)
+      place_dir = xampl.class.name.split("::")
+      place = File.join( place_dir, xampl.get_the_index)
+      place_dir = File.join( @files_dir, place_dir )
       mentions = Set.new
       data = represent(xampl, mentions)
 
+      #get rid of any supplimentary indexes associated with this xampl object
+      # TODO -- This can be slow
       query = TableQuery.new(@tc_db)
-      query.add_condition('xampl_from', :equals, place)
+      query.add_condition('xampl-from', :equals, place)
       note_errors("TC[[#{ @filename }]]:: failed to remove from mentions, error: %s\n") do
         query.searchout
       end
 
+      query = TableQuery.new(@tc_db)
+      query.add_condition('xampl-place', :equals, place)
+      note_errors("TC[[#{ @filename }]]:: failed to remove from mentions, error: %s\n") do
+        query.searchout
+      end
+
+      # TODO -- This can be slow
       mentions.each do | mention |
         mention_place = File.join(mention.class.name.split("::"), mention.get_the_index)
         #TODO -- will repeadedly changing a persisted xampl object fragment the TC db?
 
         pk = @tc_db.genuid
         mention_hash = {
-                'xampl_from' => place,
-                'class' => xampl.class.name,
+                'xampl-from' => place,
+                'mentioned_class' => xampl.class.name,
                 'pid' => xampl.get_the_index,
-                'xampl_to' => mention_place
+                'xampl-to' => mention_place
         }
 
         note_errors("TC[[#{ @filename }]]:: write error: %s\n") do
@@ -366,15 +490,56 @@ module Xampl
               'xampl' => data
       }
 
-      hash = xampl.describe_yourself
-      if hash then
-        xampl_hash = hash.merge(xampl_hash)
+#      puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] #{ xampl.class.name } ... describe"
+      primary_description, secondary_descriptions = xampl.describe_yourself
+      if primary_description then
+        xampl_hash = primary_description.merge(xampl_hash)
       end
 
       note_errors("TC[[#{ @filename }]]:: write error: %s\n") do
+        if Xampl.raw_persister_options[:write_through] then
+          FileUtils.mkdir_p(place_dir) unless File.exist?(place_dir)
+          file_place = "#{ @files_dir }/#{ place }"
+          File.open(file_place, "w")do |out|
+            out.write xampl_hash['xampl']
+            if :sync == Xampl.raw_persister_options[:write_through] then
+              out.fsync
+              if $is_darwin then
+                out.fcntl(51, 0) # Attempt an F_FULLFSYNC fcntl to commit data to disk (darwin *ONLY*)
+              end
+            end
+          end
+
+        end
         @tc_db.put(place, xampl_hash)
       end
 
+      #TODO -- smarter regarding when to delete (e.g. mentions)
+      if xampl.should_schedule_delete? and xampl.scheduled_for_deletion_at then
+        secondary_descriptions = [] unless secondary_descriptions
+        secondary_descriptions << { 'scheduled-delete-at' => xampl.scheduled_for_deletion_at }
+      elsif xampl.scheduled_for_deletion_at then
+        puts "#{ __FILE__ }:#{ __LINE__ } HOW TO DO THIS without violating xampl's change rules????? "
+        #xampl.scheduled_for_deletion_at = nil
+      end
+
+      if secondary_descriptions then
+        xampl_hash = {
+                'class' => xampl.class.name,
+                'pid' => xampl.get_the_index,
+                'xampl-place' => place
+        }
+
+        secondary_descriptions.each do | secondary_description |
+          description = secondary_description.merge(xampl_hash)
+
+          note_errors("TC[[#{ @filename }]]:: write error: %s\n") do
+            pk = @tc_db.genuid
+            @tc_db.put(pk, description)
+          end
+        end
+      end
+
       @write_count = @write_count + 1
       xampl.changes_accepted
       return true
@@ -389,10 +554,14 @@ module Xampl
       #      puts "#{File.basename(__FILE__)}:#{__LINE__} READ #{ klass }/#{ pid }"
       #      caller(0).each { | trace | puts "  #{trace}"}
 
-      open_tc_db
-      place = File.join(klass.name.split("::"), pid)
       representation = nil
 
+      unless @tc_db then
+#        puts "#{ __FILE__ }:#{ __LINE__ } [#{__method__}] READ REP"
+        open_tc_db
+      end
+      place = File.join(klass.name.split("::"), pid)
+
       meta = @tc_db[place]
       representation = meta['xampl'] if meta
 
@@ -508,6 +677,7 @@ module Xampl
 
     def initialize (table)
       @query = TDBQRY::new(table)
+      @text_form = []
       @opts = {}
     end
 
@@ -602,6 +772,8 @@ module Xampl
       op = op | TDBQRY::QCNEGATE unless affirmative
       op = op | TDBQRY::QCNOIDX if no_index
 
+      @text_form << "operator: #{ operator }#{ affirmative ? '' : ' NEGATED'}#{ no_index ? ' NO INDEX' : ''} -- col: '#{ colname }', val: '#{ val }'"
+
       @query.addcond(colname, op, val)
     end
 
@@ -634,6 +806,14 @@ module Xampl
     def order_by (colname, direction=:strasc)
       @query.setorder(colname, DIRECTIONS[direction])
     end
+
+    def inspect
+      "TableQuery:\n#{ @text_form.join("\n") }"
+    end
+
+    def to_s
+      inspect
+    end
   end
 
   Xampl.register_persister_kind(TokyoCabinetPersister)