dbmlite3 1.0.0 → 2.0.0.pre.alpha.4

data/lib/internal_lite3/handle.rb

@@ -0,0 +1,284 @@
+
+require 'sequel'
+
+require 'weakref'
+
+module Lite3
+
+  # Wrapper around a Sequel::Database object.
+  #
+  # We do this instead of using them directly because transactions
+  # happen at the handle level rather than the file level and this
+  # lets us share the transaction across multiple tables in the same
+  # file.
+  #
+  # In addition, we can use this to transparently close and reopen the
+  # underlying database file when (e.g.) forking the process.
+  #
+  # Instances contain references to DBM objects using them.  When the
+  # set becomes empty, the handle is closed; adding a reference will
+  # ensure the handle is open.
+  class Handle
+    attr_reader :path
+    def initialize(path)
+      @path = path
+      @db = open_db(path)
+      @refs = {}
+    end
+
+    private
+
+    def open_db(path)
+      return IS_JRUBY                           ?
+        Sequel.connect("jdbc:sqlite:#{path}")   :
+        Sequel.sqlite(@path)
+    end
+
+    public
+
+    def to_s
+      "<#{self.class}:0x#{object_id.to_s(16)} path=#{@path}>"
+    end
+    alias inspect to_s
+
+    
+    #
+    # References to the DBM object(s) using this handle.
+    #
+    # References are weak. scrub_refs! will remove all reclaimed refs
+    # and close the handle if there are none left.  (Note that this
+    # doesn't preclude us from reopening the handle later, though.  We
+    # could keep Handles around longer if we want and reuse them, but we
+    # don't.)
+    #
+
+    def addref(parent)
+      @refs[parent.object_id] = WeakRef.new(parent)
+    end
+
+    def delref(parent)
+      @refs.delete(parent.object_id)
+      scrub_refs!
+    end
+
+    def scrub_refs!
+      @refs.delete_if{|k,v| ! v.weakref_alive? }
+      disconnect! if @refs.empty?
+    end
+
+    def live_refs
+      scrub_refs!
+      return @refs.size
+    end
+
+
+    #
+    # Opening and closing
+    #
+
+    # Disconnect the underlying database handle.
+    def disconnect!
+      @db.disconnect
+    end
+
+
+    #
+    # Transactions
+    #
+    
+    # Perform &block in a transaction.  See DBM.transaction.
+    def transaction(&block)
+      @db.transaction({}, &block)
+    end
+
+    # Test if there is currently a transaction in progress
+    def transaction_active?
+      return @db.in_transaction?
+    end
+
+
+
+    #
+    # Table access; the common SQL idioms we care about.  These all
+    # deal with tables of key/value pairs.
+    #
+
+    # Create a table of key-value pairs if it does not already exist.
+    def create_key_value_table(name)
+      @db.create_table?(name) do
+        String :key, primary_key: true
+        String :value
+      end
+    end
+
+    # Perform an upsert for the row with field 'key'
+    def upsert(table, key, value)
+      transaction {
+        recs = @db[table].where(key: key)
+        if recs.count == 0
+          @db[table].insert(key: key, value: value)
+        elsif recs.count == 1
+          recs.update(value: value)
+        else
+          raise InternalError.new("Duplicate entry for key '#{key}'")
+        end
+      }
+
+      return value
+    end
+
+    # Retrieve the 'value' field of the row with value 'key' in the given table.
+    def lookup(table, key)
+      row = @db[table].where(key:key).first
+      return nil unless row
+
+      return row[:value]
+    end
+
+    def clear_table(table)
+      @db[table].delete
+    end
+
+    def delete(table, key)
+      @db[table].where(key: key).delete
+    end
+
+    def get_size(table)
+      return @db[table].count
+    end
+
+
+    # Backend for `each`; evaluates `block` on each row in `table`
+    # with the undecoded key and value as arguments.  It is *not* a
+    # single transaction.
+    #
+    # We do this instead of using `Dataset.each` because the latter is
+    # not guaranteed to be re-entrant.
+    #
+    # Each key/value pair is retrieved via a separate query so that it
+    # is safe to access the database from inside the block.  Items are
+    # retrieved by rowid in increasing order. Since we preserve those,
+    # modifications done in the block (probably) won't break things.
+    #
+    # This is (probably) not very fast but it's (probably) good enough
+    # for most things.
+    def tbl_each(table, &block)
+      return if @db[table].count == 0
+
+      curr = -1
+      while true
+        row = @db[table].where{rowid > curr}
+          .limit(1)
+          .select(:rowid, :key, :value)
+          .first
+
+        return unless row
+        curr, key, value = *row.values
+
+        block.call(key, value)
+      end
+    end
+
+    # Wrapper around Dataset.each, with all the ensuing limitations.
+    def tbl_each_fast(table, &block)
+      @db[table].each(&block)
+    end
+
+
+  end
+
+
+
+  #
+  # Private classes
+  #
+
+  # Dummy `Handle` that throws an `Error` exception whenever something
+  # tries to treat it as an open handle.  This replaces a `DBM`'s
+  # `Handle` object when `DBM.close` is called so that the error
+  # message will be useful if something tries to access a closed
+  # handle.
+  class ClosedHandle
+    def initialize(filename, table)
+      @filename, @table = [filename, table]
+    end
+
+    # We clone the rest of Handle's interface with methods that throw
+    # an Error.
+    Handle.instance_methods(false).each { |name|
+      next if method_defined? name
+      define_method(name) { |*args|
+        raise Error.new("Use of closed database at #{@filename}/#{@table}")
+      }
+    }
+  end
+
+
+  # Module to manage the collection of active Handle objects.  See the
+  # docs for `Lite3::SQL` for an overview; this module hold the actual
+  # code and data.
+  module HandlePool
+    @@handles = {}      # The hash of `Handle` objects keyed by filename
+
+    # Retrieve the `Handle` associated with `filename`, creating it
+    # first if necessary.  `filename` is normalized with
+    # `File.realpath` before using as a key and so is as good or bad
+    # as that for detecting an existing file.
+    def self.get(filename)
+
+      # Scrub @@handles of all inactive Handles
+      self.gc
+
+      # We need to convert the filename to a canonical
+      # form. `File.realpath` does this for us but only if the file
+      # exists.  If not, we use it on the parent directory instead and
+      # use `File.join` to create the full path.
+      if File.exist?(filename)
+        File.file?(filename) or
+          raise Error.new("Filename '#{filename}' exists but is not a file.")
+
+        filename = File.realpath(filename)
+      else
+        dn = File.dirname(filename)
+        File.directory?(dn) or
+          raise Error.new("Parent directory '#{dn}' nonexistant or " +
+                          "not a directory.")
+
+        filename = File.join(File.realpath(dn), File.basename(filename))
+      end
+
+      @@handles[filename] = Handle.new(filename) unless
+        @@handles.has_key?(filename)
+
+      return @@handles[filename]
+    end
+
+    # Close all underlying database connections.
+    def self.close_all
+      Sequel::DATABASES.each(&:disconnect)
+    end
+
+    # Close and remove all Handle objects with no refs and return a
+    # hash mapping the filename for each live Handle to the number of
+    # DBM objects that currently reference it.  Does **NOT** perform a
+    # Ruby GC.
+    def self.gc
+      results = {}
+      @@handles.select!{|path, handle|
+        handle.scrub_refs!
+
+        if handle.live_refs == 0
+          @@handles.delete(path)
+          next false
+        end
+
+        results[path] = handle.live_refs
+        true
+      }
+
+      return results
+    end
+  end
+
+  private_constant :Handle, :ClosedHandle, :HandlePool
+end

data/lib/internal_lite3/sql.rb

@@ -0,0 +1,87 @@
+
+module Lite3
+
+  # This module provides some basic access to the underlying
+  # `Sequel::Database` objects used by `Lite3::DBM` to actually store
+  # and retrieve data.
+  #
+  # The only thing you need to care about is that if your process
+  # forks, you *must* invoke `Lite3::SQL.close_all` before forking the
+  # process.  Otherwise, it will clone the connection and could lead
+  # to database corruption.
+  #
+  # More details:
+  #
+  # `Lite3` maintains a pool of private handle objects (private class
+  # `Lite3::Handle`) which in turn manage the `Sequel::Database`
+  # objects that actually do the work.  There is one handle per
+  # SQLite3 database file; since each `DBM` represents one table in a
+  # SQLite3 file, multiple `DBM` objects will use the same handle.
+  #
+  # Handle objects can themselves close and replace their
+  # `Sequel::Database` objects transparently.
+  #
+  # The underlying system keeps track of which `DBM` objects reference
+  # which files and will close a file's `Sequel::Database` when all
+  # of the `DBM`s using it have been closed.  (It does **not** handle
+  # the case where a `DBM` object remains open and goes out of scope;
+  # that object will be kept around for the life of the process.)
+  #
+  # Mostly, you don't need to care about this.  However, it affects
+  # you in two ways:
+  #
+  # 1. Transactions are done at the file level and not the table level.
+  #    This means that you can access separate tables in the same
+  #    transaction, which is a Very Good Thing.
+  #
+  # 2. You can safely fork the current process and keep using existing
+  #    `DBM` objects in both processes, provided you've invoked
+  #    `close_all` before the fork.  This will have closed the actual
+  #    database handles (which can't tolerate being carried across a
+  #    fork) and opens new ones the next time they're needed.
+  #
+  # If you find yourself needing to be sure that you don't have any
+  # unexpected open file handles (e.g. before forking or if you need
+  # Windows to unlock it), you should call `close_all`.
+  #
+  # Otherwise, it's safe to ignore this stuff.
+
+
+  # This module provides some basic, consistent access to the
+  # underlying database library(es) (currently `sequel`).
+  module SQL
+
+    # Tests if the underlying database libraries are threadsafe.
+    #
+    # (Currently, it always returns true, since Sequel does that for
+    # us.)
+    def self.threadsafe?
+      return true
+    end
+
+    # Disconnect and delete all database handles and associated
+    # metadata that are no longer needed (i.e. because their
+    # corresponding `DBM`s have been closed or reclaimed).
+    #
+    # Returns a hash mapping the path to each open database file to
+    # the number of live DBM objects referencing it.
+    #
+    # You normally won't need to explicitly call this, but it's
+    # useful for testing and debugging.
+    def self.gc()           return HandlePool.gc;           end
+
+    # Close and remove the underlying database connections.  This does
+    # not invalidate existing `Lite3::DBM` objects; they will recreate
+    # the connections when needed.
+    #
+    # The main use for this is for safely forking the current process.
+    # You should call this just before each `fork` to avoid potential
+    # corruption from duplicated database handles.
+    #
+    # This **should not** be called while a database operation is in
+    # progress.  (E.g. do **not** call this from the block of
+    # `DBM.each`.)
+    def self.close_all()    return HandlePool.close_all     end
+  end
+
+end

data/spec/dbmlite3_spec.rb

@@ -212,9 +212,11 @@ Serializations = Set.new
     it "implements each_*" do
       db = newdb.call(Tmp.file, "floop")
 
+      # Empty DBMs don't evaluate their bloc
       count = 0
       db.each {|k,v| count += 1}
       db.each_pair {|k,v| count += 1}
+      expect( count ) .to eq 0
 
 
       db["foo"] = 42
@@ -236,6 +238,64 @@ Serializations = Set.new
       db.close
     end
 
+    it "allows database modification in `each`" do
+      db = newdb.call(Tmp.file, "floop")
+
+      db["foo"] = 42
+      db["bar"] = 99
+      db["quux"] = 123
+      db["baz"] = 999
+
+      count = 0
+      db.each do|key, value|
+        count += 1
+
+        case count
+        when 1
+          expect(key) .to eq "foo"
+          expect(value) .to eq 42
+          db['foo'] = 'new_foo'
+
+        when 2
+          expect(key) .to eq "bar"
+          expect(value) .to eq 99
+
+          db['baz'] = "new_baz"
+
+          expect(db['foo']) .to eq "new_foo"
+
+          db.delete("quux")
+
+        when 3
+          expect(key) .to eq "baz"
+          expect(value) .to eq "new_baz"
+
+        when 4
+          fail "there should not be 4 items!"
+        end
+      end
+
+      expect(count) .to eq 3
+
+      db.close
+    end
+
+    it "implements fast_each" do
+      db = newdb.call(Tmp.file, "floop")
+
+      db["foo"] = 42
+      db["bar"] = 99
+      db["quux"] = 123
+
+      expected = []
+      db.fast_each{|key, value| expected.push [key, value]}
+
+      expect(expected) .to eq [ ["foo", 42], ["bar", 99], ["quux", 123]]
+
+      db.close
+    end
+
+
     it "deletes items from the table" do
       db = newdb.call(Tmp.file, "floop")
 
@@ -792,7 +852,7 @@ describe Lite3::DBM do
   end
 
   it "keeps most of its names private" do
-    expect( Lite3.constants.to_set ) .to eq %i{SQL DBM Error}.to_set
+    expect( Lite3.constants.to_set ) .to eq %i{SQL DBM Error InternalError}.to_set
   end
 end
 
@@ -816,15 +876,6 @@ describe Lite3::SQL do
     db
   }
 
-  # it "manages a pool of DB handles that should now all be closed." do
-  #   # If this fails, it (probably) means the previous tests didn't
-  #   # clean up after themselves.
-  #   GC.start
-  #   expect( Lite3::SQL.gc.empty? ) .to be true
-
-  #   Lite3::SQL.close_all        # smoketest
-  # end
-
   it "lets you close the actual handle without impeding database use" do
     expect( Lite3::SQL.gc.size ) .to eq 0
 
@@ -840,72 +891,19 @@ describe Lite3::SQL do
 
     # Referencing DBM objects should be db1 and db2
     path, refs = stats.to_a[0]
+    expect( path ) .to eq file
+    expect( refs ) .to eq 2
 
-    expect( refs.size ) .to eq 2
-    expect( refs.include?(db1) ) .to be true
-    expect( refs.include?(db2) ) .to be true
-
-    # Underlying handles should be open
-    expect( db1.handle_closed? ) .to be false
-    expect( db2.handle_closed? ) .to be false
+    # We can no longer test if the underlying file handles are still
+    # open, so we don't.
 
-    # Test closing it
+    # Test closing it and forcing a re-open
     Lite3::SQL.close_all
-    expect( db1.handle_closed? ) .to be true
-    expect( db2.handle_closed? ) .to be true
-
-    # Test auto-opening them.
     expect( db1["foo"] ) .to eq vv["foo"]
-    expect( db1.handle_closed? ) .to be false
-    expect( db2.handle_closed? ) .to be false
 
-    db1.close
-    db2.close
-
-    expect( Lite3::SQL.gc.keys.size ) .to eq 0
-  end
-
-  # it "(eventually) closes handles that have gone out of scope" do
-  #   expect( Lite3::SQL.gc.keys.size ) .to eq 0
-
-  #   file = Tmp.file
-  #   db1 = newbasic.call(file, "first")
-
-  #   expect( db1.handle_closed? ) .to be false
-  #   expect( Lite3::SQL.gc.keys.size ) .to eq 1
-
-  #   db1 = nil
-  #   GC.start
-  #   expect( Lite3::SQL.gc.keys.size ) .to eq 0
-  # end
-
-  it "does close_all with multiple files" do
-    db1 = newbasic.call(Tmp.file, "first")
-    db2 = newbasic.call(Tmp.file, "second")
-
-    # The above should be using the same handle, which is currently
-    # open.
-
-    stats = Lite3::SQL.gc
-    expect( stats.keys.size ) .to eq 2
-
-    all_refs = stats.values.flatten
-    expect( all_refs.include?(db1) ) .to be true
-    expect( all_refs.include?(db2) ) .to be true
-
-    # Underlying handles should be open
-    expect( db1.handle_closed? ) .to be false
-    expect( db2.handle_closed? ) .to be false
-
-    # Test closing it
-    Lite3::SQL.close_all
-    expect( db1.handle_closed? ) .to be true
-    expect( db2.handle_closed? ) .to be true
-
-    # Test auto-opening them.
+    # Repeat, but this time use the underlying lib
+    Sequel::DATABASES.each(&:disconnect)
     expect( db1["foo"] ) .to eq vv["foo"]
-    expect( db1.handle_closed? ) .to be false
-    expect( db2.handle_closed? ) .to be true
 
     db1.close
     db2.close
@@ -913,8 +911,7 @@ describe Lite3::SQL do
     expect( Lite3::SQL.gc.keys.size ) .to eq 0
   end
 
-
-  it "allows multipe table accesses in the same transaction" do
+  it "allows multiple table accesses in the same transaction" do
     file = Tmp.file
     db1 = newbasic.call(file, "first")
     db2 = Lite3::DBM.new(file, "second")
@@ -964,6 +961,50 @@ describe Lite3::SQL do
     expect{ db1.size } .to raise_error Lite3::Error
     expect{ db1.to_a } .to raise_error Lite3::Error
   end
+
+  it "finalizes DBM objects that have gone out of scope." do
+
+    # This is really difficult to test because there's no reliable way
+    # to get the garbage collector to clean up when we want it to.  As
+    # such, we make the attempt and skip with a warning if db2 hasn't
+    # been reclaimed.
+    #
+    # (Dropping into the debugger after GC.start seems to help.)
+
+
+    file = Tmp.file
+    db1 = newbasic.call(file, "first")
+    db2 = Lite3::DBM.new(file, "second")
+
+    # Two DBMs are currently open.
+    stats = Lite3::SQL.gc
+    expect( stats.size ) .to be 1
+    expect( stats.values[0] ) .to be 2
+
+    # Make db2 a weak reference so it goes out of scope after a GC.
+    # It's possible that a GC redesign will this.
+    db2 = WeakRef.new(db2)
+    GC.start
+
+    # Uncommenting the next line and then resuming seems to work:
+    #byebug
+
+    if db2.weakref_alive?
+      db2.close
+      db1.close
+      skip "GC has't reclaimed the handle; bailing."
+    end
+
+    # There should now be exactly 1 open DBM
+    stats = Lite3::SQL.gc
+    expect( stats.size ) .to be 1
+    expect( stats.values[0] ) .to be 1
+
+    db1.close
+  end
+
+
+
 end