amalgalite 0.2.4 → 0.4.0

data/HISTORY

@@ -1,21 +1,46 @@
 = Changelog
-== Version 0.2.4
+== Version 0.4.0 2008-09-14
+
+* Major Enhancements
+  * update to SQLite3 version 3.6.2 and enable the RTree option by default
+  * Amalgalite::Requires module allowing ruby code to be 'required' from columns
+    in an SQLite database
+  * Amagalite::Requires::Bootstrap extension module enabling low level boot
+    strapping of the pure ruby Amalgalite code from an sqlite database
+
+* Minor Enhancements
+  * more indepth information about indexes is available via the Index class
+  * add support for sqlite3_status and sqlite3_db_status information
+
+* Bugfixes
+  * fix nil exception when using a declared_data_type on primary key column that
+    has no declared_data_type
+  * when Database#transaction is passed a block, the return value is the return
+    value of the block
+  * nested transactions are 'faked'.  Calling Database#transaction while
+    Databased#in_transaction? is true does not result in an exception, but
+    continues on in the current transaction.  
+  * raise LoadError if required in the same program as sqlite3-ruby.  These
+    libraries conflict with each other.
+
+== Version 0.2.4 2008-07-13
 
 * Bugfixes
   * fix compilation when ruby is compiled without pthreads using 
 
-== Version 0.2.3
+== Version 0.2.3 2008-07-12
 
 * Bugfixes
   * make sure file permissions are all read before shipping gem
 
-== Version 0.2.2
+== Version 0.2.2 2008-07-12
 
 * Bugfixes
   * Database#pragma should accept a block just like Database#execute does
 
 * compatibility fix
-  * convert to using extconf.rb instead of mkrf for muster integration
+  * convert to using extconf.rb instead of mkrf to enable compilation as a
+    direct ruby extension in the ruby source tree
 
 == Version 0.2.1 - 2008-07-05
 
@@ -32,7 +57,7 @@
   * added examples/gem_db.rb script demonstrating taps and prepared statements
   * added examples/schema-info.rb script demonstrating meta information
   * added examples/blob.rb demonstrating incremental blob IO
-  * added acces to the SQLite3 errcode and errmsg api
+  * added access to the SQLite3 errcode and errmsg api
   
 * Bugfixes
   * added taps.rb for requiring

data/bin/amalgalite-pack-into-db

@@ -0,0 +1,155 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+#
+# A commandline utility to pack all of amalgalite into a database so that it can
+# be loaded with the C extension only.
+#
+
+#
+# given a file, see if it can be found in the ruby load path, if so, return that
+# full path
+#
+def full_path_of( rb_file )
+  $:.each do |load_path|
+    guess = File.join( load_path, rb_file )
+    return guess if File.exist?( guess )
+  end
+  return nil
+end
+
+OPTIONS = { :force => false }
+
+parser = OptionParser.new do |op|
+  op.banner  = "Usage: #{op.program_name} [options] <dbfile>"
+  op.separator ""
+  
+  op.on("-f", "--force", "Force overwriting of an existing database") do |f|
+    OPTIONS[:force]= true
+  end
+
+end
+
+begin
+  parser.parse!
+rescue OptionParser::ParseError => pe
+  STDERR.puts "ERROR : #{pe}"
+  STDERR.puts parser
+  exit 1
+end
+
+#
+# capture before and after snapshots of the loaded features to see what changed
+# when we required amalgalite
+#
+require 'rubygems'
+$: << "lib"
+$: << "ext"
+loaded_features_before = $".dup
+require 'amalgalite/requires'
+loaded_features_after = $".dup
+
+#
+# reorder the items that should be required, putting the system level .rb files
+# first in the list
+#
+amalgalite_needs = loaded_features_after - loaded_features_before
+core_libs = (amalgalite_needs - Amalgalite::Requires.require_order).delete_if { |l| l.index(".rb").nil? }
+
+#
+# check and make sure nothing is missed
+#
+core_libs.each do |l| 
+  if l.index("malgalite") then
+    STDERR.puts "ERROR!  require_order needs an update #{l}" 
+    STDERR.puts "run rake test:check_requries and fix"
+    exit 2
+  end
+end
+amalgalite_needs = core_libs.concat( Amalgalite::Requires.require_order )
+
+
+# width value for tidy output
+max_width = amalgalite_needs.sort_by { |l| l.length }.last.length
+
+#
+# Get the filename, and do not overwrite if the file already exists, unless of
+# course, --force is used
+#
+dbfile = ARGV.shift || Amalgalite::Requires::Bootstrap::DEFAULT_DB
+if OPTIONS[:force] then
+  File.unlink( dbfile ) if File.exist?( dbfile )
+end
+
+if File.exist?( dbfile ) then
+  STDERR.puts "ERROR: #{dbfile} already exists, erase manually or use '--force' option"
+  STDERR.puts parser
+  exit 1
+end
+
+
+#
+# Create the datbase
+#
+puts "Creating database #{dbfile}"
+db = Amalgalite::Database.new( dbfile )
+table_name   = Amalgalite::Requires::Bootstrap::DEFAULT_TABLE
+rowid_col    = Amalgalite::Requires::Bootstrap::DEFAULT_ROWID_COLUMN
+filename_col = Amalgalite::Requires::Bootstrap::DEFAULT_FILENAME_COLUMN
+contents_col = Amalgalite::Requires::Bootstrap::DEFAULT_CONTENTS_COLUMN
+db.execute(<<-create)
+    CREATE TABLE #{table_name} (
+      #{rowid_col}    INTEGER PRIMARY KEY AUTOINCREMENT,
+      #{filename_col} TEXT UNIQUE,
+      #{contents_col} BLOB
+    )
+    create
+db.reload_schema!
+
+# 
+# for every file in the list, insert it into the database
+#
+db.transaction do |dbt|
+  db.prepare( "INSERT INTO #{table_name}(#{filename_col}, #{contents_col}) VALUES ( $filename, $contents )" ) do |stmt|
+    amalgalite_needs.each do |am_requires|
+
+      msg = " skipped, probably a binary extension"
+
+      if File.extname( am_requires ) == ".rb" then
+        full_path = File.expand_path( full_path_of( am_requires ) )
+
+        if full_path and File.readable?( full_path ) then
+          contents = IO.readlines( full_path )
+          contents.each { |l| l.gsub!( /^(\s*require .*)$/m, "# commented out by #{parser.program_name} \\1") }
+          # strip off the .rb
+          rq = am_requires[ /\A(.*)\.rb\Z/, 1]
+          stmt.execute( { "$filename" => rq,
+                          "$contents" => Amalgalite::Blob.new( :string => contents.join,
+                                                               :column => dbt.schema.tables[table_name].columns[contents_col] )  } )
+          msg = "stored #{full_path}" 
+        end
+
+        puts "  -> #{am_requires.ljust( max_width )} : #{msg}"
+      else
+        STDERR.puts "ERROR : #{am_requires} is an invalid file to pack"
+      end
+    end
+  end
+end
+db.close
+
+puts <<-text
+
+Packing complete.  To utilize the bootstrapping in #{dbfile} you must do
+one of the following:
+
+  * statically compile the amalgalite C extension into your application
+  * require 'amalgalite3'
+
+Once one of those is working you can boostrap the Amalgalite library with
+this line in your code:
+ 
+  Amalgalite::Requries::Boostrap.lift( 'dbfile' => '#{dbfile}' )
+
+text

data/examples/a.rb

@@ -0,0 +1,9 @@
+class A
+  def initialize 
+    puts "Initialized A"
+  end
+
+  def a
+    puts "called a"
+  end
+end

data/examples/blob.rb

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+#
+# An Amalgalite example showing how Blob's can be utilized
+#
+# We'll make a database with one table, that we store files in.  We'll use the
+# Blob incremental IO to store the files and retrieve them from the database
+#
+# This little program will store 1 or more files in the sqlite3 database when
+# the 'store' action is given, and cat a file to stdout on 'retrieve'
+#
+# e.g.
+#
+#   ruby blob.rb store a.rb b.rb c.rb # => stores a.rb b.rb and c.rb in the db
+#
+#   ruby blob.rb retrieve a.rb        # => dumps a.rb to stdout
+#
+
+require 'rubygems'
+require 'amalgalite'
+VALID_ACTIONS = %w[ list retrieve store ]
+def usage 
+  STDERR.puts "Usage: #{File.basename($0)} ( #{VALID_ACTIONS.join(' | ')} )  file(s)"
+  exit 1
+end
+
+#
+# This does the basic command line parsing
+#
+usage if ARGV.size < 1
+action    = ARGV.shift
+usage unless VALID_ACTIONS.include? action 
+file_list = ARGV
+
+#
+# create the database if it doesn't exist
+#
+db = Amalgalite::Database.new( "filestore.db" )
+unless db.schema.tables['files']
+  STDERR.puts "Creating files table"
+  db.execute(<<-create)
+  CREATE TABLE files(
+    id     INTEGER PRIMARY KEY AUTOINCREMENT,
+    path   VARCHAR UNIQUE,
+    data   BLOB
+  )
+  create
+  db.reload_schema!
+end
+
+
+case action
+  #
+  # list all the files that are stored in the database
+  #
+when 'list'
+  db.execute("SELECT path FROM files") do |row|
+    puts row['path']
+  end
+
+  #
+  # if we are doing the store action, then loop over the files and store them in
+  # the database.  This will use incremental IO to store the files directly from
+  # the file names. 
+  #
+  # It is slightly strange in that you have to tell the Blob object what column
+  # it is going to, but that is necessary at this point to be able to hook
+  # automatically into the lower level incremental blob IO api.
+  #
+  # This also shows using the $var syntax for binding name sql values in a
+  # prepared statement.
+  #
+when 'store'
+  usage if file_list.empty?
+  db.transaction do |db_in_trans|
+    db_in_trans.prepare("INSERT INTO files(path, data) VALUES( $path, $data )") do |stmt|
+      file_list.each do |file_path|
+        begin        
+          if File.exist?( file_path ) then
+            stmt.execute( "$path" => file_path, 
+                          "$data" => Amalgalite::Blob.new( :file => file_path, :column => db_in_trans.schema.tables['files'].columns['data'] ) )
+            STDERR.puts "inserted #{file_path} with id #{db.last_insert_rowid}"
+          else
+            STDERR.puts "#{file_path} does not exist"
+          end
+        rescue => e
+          STDERR.puts e
+        end
+      end
+    end
+  end
+  STDERR.puts "inserted a total of #{db.total_changes} changes"
+
+  #
+  # dump the file that matches the right path to stdout.  This also shows
+  # positional sql varible binding using the '?' syntax.
+  #
+when 'retrieve'
+  usage if file_list.empty?
+  db.execute("SELECT id, path, data FROM files WHERE path = ?", file_list.first) do |row|
+  STDERR.puts "Dumping #{row['path']} to stdout"
+  row['data'].write_to_io( STDOUT )
+  end
+end
+db.close

data/examples/bootstrap.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+# An example of requiring all the files in a table via the Bootstrap::lift
+# method.
+#
+# First use the blob.rb example in this same directory to load the a.rb file
+# into an example database:
+#
+#   ruby blob.rb store a.rb
+#
+# Then run this example.
+#
+
+# Require "ONLY" then binary extension, do not +require+ the ruby based code
+$: << "../ext"
+require 'amalgalite3'
+
+# See what the load path is, notice that it is very small
+puts "Before $\" : #{$".inspect}"
+
+# tell the binary extension to "require" every file in the filestore.db in the
+# table 'files' orderd by column 'id'.  The 'path' column is added to $" and the
+# code in 'data' is evaled.
+Amalgalite::Requires::Bootstrap.lift( "dbfile"          => "filestore.db", 
+                                      "table_name"      => "files", 
+                                      "rowid_column"    => "id", 
+                                      "filename_column" => "path", 
+                                      "contents_column" => "data" )
+
+# Notice that a.rb is in the list of files that has been required
+puts "After $\" : #{$".inspect}"
+
+# and look we prove that the code was eval'd appropriately
+a = A.new
+a.a
+

data/examples/gem-db.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+
+#
+# Basic amalgalite example creating a table, inserting rows and doing various
+# selects and prepared statements
+#
+require 'rubygems'
+require 'amalgalite'
+
+#
+# Create a database, this will create the DB if it doesn't exist
+#
+puts "Opening database (version #{Amalgalite::Version})"
+db = Amalgalite::Database.new("gems.db")
+
+#
+# Setup taps into profile and trace information of sqlite, the profile tap will
+# goto the profile_tap.log file and the trace information will go to the
+# trace_tap.log file
+#
+puts "Establishing taps"
+db.trace_tap   = Amalgalite::Taps::IO.new( trace_tap_file = File.open("trace_tap.log", "w+") )
+db.profile_tap = Amalgalite::Taps::IO.new( profile_tap_file = File.open("profile_tap.log", "w+") )
+
+#
+# Create the schema unless it already exists in the table.  The meta information
+# about the database schema is available as the result of the db.schema method
+#
+schema = db.schema
+unless schema.tables['gems'] 
+  puts "Create schema"
+  db.execute <<-SQL
+  CREATE TABLE gems (
+    id      INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    name    VARCHAR(128),
+    version VARCHAR(32),
+    author  VARCHAR(128)
+  );
+  SQL
+  db.reload_schema!
+end
+
+#
+# Get some data from the system to insert into the database.  Since everyone
+# probably has gems installed, that's a ready known piece of information.  We'll
+# just pull in the latest version of each installed gem and dump some meta
+# information into a db for testing.
+#
+latest_specs = Gem.source_index.latest_specs
+
+puts "Inserting #{latest_specs.size} rows of gem information..."
+before = Time.now
+
+# Inserting bulk rows as a transaction is good practice with SQLite, it is
+# MUCH faster.  
+db.transaction do |db_in_transaction|
+  db_in_transaction.prepare("INSERT INTO gems(name, version, author) VALUES( :name, :version, :author );") do |stmt|
+    latest_specs.each do |spec|
+      insert_data = {}
+      insert_data[':name']    = spec.name.to_s
+      insert_data[':version'] = spec.version.to_s
+      insert_data[':author']  = spec.authors.join(' ')
+      #puts "Inserting #{insert_data.inspect}"
+      stmt.execute( insert_data )
+    end
+  end
+end
+puts "Took #{Time.now - before} seconds"
+puts "Done Inserting"
+
+authors_by_number = db.execute("SELECT author, count( name ) as num_gems FROM gems GROUP BY author ORDER BY num_gems DESC")
+favorite_author   = authors_by_number.first
+puts "Your favorite gem author is <#{favorite_author['author']}>, with #{favorite_author['num_gems']} gems installed."
+
+#
+# Now we'll look at the profile sampler and see what information it traced about
+# our behavoir.
+#
+db.profile_tap.samplers.each do |stat_name, stat_values|
+  puts "-" * 20
+  puts stat_values.to_s
+end
+
+#
+# Clear out the taps (not really necessary, just cleaning up)
+#
+db.trace_tap = profile_tap = nil
+
+#
+# close things down
+#
+db.close
+trace_tap_file.close
+profile_tap_file.close