amalgalite 1.8.0-x64-mingw-ucrt

data/TODO.md

@@ -0,0 +1,57 @@
+# Future Release possibilties:
+- rebuild statement constants
+- look at all pragma statements
+
+## SQLite API:
+- authorizers
+- loading of extensions -- readfile / writefile
+- utf-16 integration
+- create_collation 
+- encryption key support
+- expose sqlite3_strnicmp
+- table name and column name in a type map?
+- type conversion for manifest typing? how to allow it through?
+- explicit pragma handler
+- application_id pragma setter
+
+## Non backwards compatible changes:
+- change the schema objects to be more consistent
+- change taps to use to_proc protocol
+- convert type dependency to just use 'call'
+- integrate transaction and savepoint under the same api
+
+## SQLite Features:
+- activate SQLITE_ENABLE_ICU extension
+- activate SQLITE_ENABLE_LOCKING_STYLE
+- activate SQLITE_ENABLE_UNLOCK_NOTIFY
+- expose PRAGMA foreign_keys
+- virtual file system
+- full text search (FTS3)
+- expose the sqlite mutex lib
+- statement status ( sqlite3_stmt_status )
+- db status ( sqlite3_db_status )
+- library status ( sqlite3_status )
+- sqlite3_index_info
+- sqlite3_create_function has 4th parameter SQLITE_DETERMINISTIC 
+- sqlite3_rtree_query_callback()
+
+## Drivers:
+- data mapper driver
+- sequel driver optimization
+
+## Features:
+- Think about moving from arrayfields to ordered hash?
+- add to command line which directory to pack into a rubylibs table
+- amalgalite command line tool
+- use ruby's ALLOC_N and hook into sqlite3_mem_methods
+
+## Functions to possibly expose:
+- sqlite3_backup_remaining, sqlite3_backup_pagecount
+- sqlite3_compileoption_used, sqlite3_compileoption_get
+- sqlite3_config
+- sqlite3_data_count - returns number of colums in the result set of a
+  prepared statement
+- sqlite_sourceid, sqlite_source_id
+- sqlite3_strnicmp
+-
+

data/bin/amalgalite-pack

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'pathname'
+
+#
+# add relative paths to the load path if we are not a gem and calculate what the
+# strip path will be if we decide to pack --self
+#
+this_path  = Pathname.new( File.expand_path( __FILE__ ) )
+gem_path   = Pathname.new( Gem.dir )
+rel_path   = this_path.relative_path_from( gem_path )
+if  ".." == rel_path.to_s.split( File::SEPARATOR ).first then
+  lib_path = File.join( File.dirname( __FILE__ ), "../lib" )
+  $:.unshift lib_path
+  $:.unshift File.join( File.dirname( __FILE__ ), "../ext" )
+end
+
+#
+# snapshot of what is needed for amalgalite requires, this info may only be used
+# when packing amalgalite itself
+#
+loaded_features_before = $LOADED_FEATURES.dup
+require 'amalgalite/packer'
+loaded_features_after  = $LOADED_FEATURES.dup
+load_diff = loaded_features_after - loaded_features_before
+
+#
+# strip off any LOAD_PATH elements from the front of load_diff since that
+# will conflict with Amalgalite::Packer.amalgalite_require_order.  Also
+# strip out any 'rubygems' items since those are not used by Amalgalite
+# and show as a side effect fo the "require 'amalgalite/packer'"
+#
+strip_paths = $LOAD_PATH.sort.reverse
+amalgalite_needs = []
+load_diff.each do |f|
+  next if f.split( File::SEPARATOR ).include?( "rubygems" )
+  appended = false
+  strip_paths.each do |path|
+    if 0 == f.index(path ) then
+      rel_path = f.sub( path, '' ).sub(%r{\A#{File::SEPARATOR}},'')
+      amalgalite_needs <<  rel_path
+      appended = true
+      break
+    end
+  end
+  amalgalite_needs << f unless appended
+end
+
+#
+# Commandline parser
+#
+options = {}
+begin
+  parser = OptionParser.new do |op|
+    op.banner  = "Usage: #{op.program_name} [options] <files>"
+    op.separator ""
+
+    op.on("--dbfile DB", "The Database file in which to pack files") do |d|
+      options[:dbfile] = d
+    end
+
+    op.on("--drop-table", "Drop the table before inserting rows") do |t|
+      options[:drop_table] = t
+    end
+
+    op.on("-m", "--merge", "Merge these files into the existing table overwriting rows that conflict") do |m|
+      options[:merge] = true
+    end
+
+    op.on("--require-order", "Dump Amalgalite's require order" ) do |m|
+      puts amalgalite_needs
+      exit 0
+    end
+
+    op.on("--self", "pack amalgalite itself into the database") do |d|
+      options[:self] = true
+    end
+
+    op.on("--strip-prefix PREFIX", "strip this path prefix off the front of each file") do |p|
+      options[:strip_prefix] = File.expand_path( p )
+    end
+
+    op.on("-t", "--table TABLE", "the table name to pack into") do |t|
+      options[:table_name] = t
+    end
+
+    op.on("--verbose", "Be verbose about output") do |v|
+      options[:verbose] = v
+    end
+
+    op.on("-z", "--compressed", "compress the file contents on storage") do |z|
+      options[:compressed] = true
+    end
+
+  end
+
+  parser.parse!
+  require 'amalgalite/packer'
+  file_list = ARGV.dup
+
+
+  if options[:self] then
+    options[:table_name] = Amalgalite::Requires::Bootstrap::DEFAULT_BOOTSTRAP_TABLE
+    core_libs = (amalgalite_needs - Amalgalite::Packer.amalgalite_require_order).delete_if { |l| l.index(".rb").nil? }
+
+    #
+    # check and make sure nothing is missed
+    #
+    core_libs.each do |l|
+      if l.index("amalgalite") then
+        STDERR.puts "ERROR!  require_order needs an update #{l}"
+        exit 2
+      end
+    end
+    file_list = core_libs.concat( Amalgalite::Packer.amalgalite_require_order )
+    if options[:compressed] then
+      STDERR.puts "Compressing --self is not allowed, reverting to uncompressed"
+      options[:compressed] = false
+    end
+  end
+  STDERR.puts parser if file_list.empty?
+
+  packer = Amalgalite::Packer.new( options )
+  packer.pack( file_list )
+
+rescue OptionParser::ParseError => pe
+  STDERR.puts "ERROR : #{pe}"
+  STDERR.puts parser
+  exit 1
+end
+
+__END__
+
+puts <<-text
+
+Packing complete.  To utilize the bootstrapping in #{dbfile} you must do
+one of the following:
+
+  * statically compile the amalgalite C extension into your application
+  * require 'amalgalite/#{RUBY_VERSION.sub(/\.\d$/,'')}/amalgalite3'
+
+Once one of those is working you can boostrap the Amalgalite library with
+this line in your code:
+
+  Amalgalite::Requries::Boostrap.lift( 'dbfile' => '#{dbfile}' )
+
+text

data/examples/a.rb

@@ -0,0 +1,9 @@
+class A
+  def initialize 
+    puts "Initialized A"
+  end
+
+  def a
+    puts "called a"
+  end
+end

data/examples/blob.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+#
+# An Amalgalite example showing how Blob's can be utilized
+#
+# We'll make a database with one table, that we store files in.  We'll use the
+# Blob incremental IO to store the files and retrieve them from the database
+#
+# This little program will store 1 or more files in the sqlite3 database when
+# the 'store' action is given, and cat a file to stdout on 'retrieve'
+#
+# e.g.
+#
+#   ruby blob.rb store a.rb b.rb c.rb # => stores a.rb b.rb and c.rb in the db
+#
+#   ruby blob.rb retrieve a.rb        # => dumps a.rb to stdout
+#
+
+require 'rubygems'
+$: << "../lib"
+$: << "../ext"
+require 'amalgalite'
+require 'amalgalite/packer'
+VALID_ACTIONS = %w[ list retrieve store ]
+def usage 
+  STDERR.puts "Usage: #{File.basename($0)} ( #{VALID_ACTIONS.join(' | ')} )  file(s)"
+  exit 1
+end
+
+#
+# This does the basic command line parsing
+#
+usage if ARGV.size < 1
+action    = ARGV.shift
+usage unless VALID_ACTIONS.include? action 
+file_list = ARGV
+
+#
+# create the database if it doesn't exist
+#
+db = Amalgalite::Database.new( "filestore.db" )
+
+case action
+  #
+  # list all the files that are stored in the database
+  #
+when 'list'
+  db.execute("SELECT filename FROM rubylibs") do |row|
+    puts row['filename']
+  end
+
+  #
+  # if we are doing the store action, then loop over the files and store them in
+  # the database.  This will use incremental IO to store the files directly from
+  # the file names. 
+  #
+  # It is slightly strange in that you have to tell the Blob object what column
+  # it is going to, but that is necessary at this point to be able to hook
+  # automatically into the lower level incremental blob IO api.
+  #
+  # This also shows using the $var syntax for binding name sql values in a
+  # prepared statement.
+  #
+when 'store'
+  usage if file_list.empty?
+  require 'amalgalite/packer'
+
+  packer = Amalgalite::Packer.new( :dbfile => 'filestore.db',
+                                   :compressed => false )
+  packer.pack( file_list )
+
+  #
+  # dump the file that matches the right path to stdout.  This also shows
+  # positional sql varible binding using the '?' syntax.
+  #
+when 'retrieve'
+  usage if file_list.empty?
+  db.execute("SELECT id, compressed, filename, contents FROM rubylibs WHERE filename = ?", file_list.first) do |row|
+    STDERR.puts "Dumping #{row['filename']} to stdout"
+    if row['compressed'] then
+      s = row['contents'].to_s
+      STDOUT.puts Amalgalite::Packer.gunzip( s )
+    else
+      row['contents'].write_to_io( STDOUT )
+    end
+  end
+end
+db.close

data/examples/bootstrap.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+# An example of requiring all the files in a table via the Bootstrap::lift
+# method.
+#
+# First use the blob.rb example in this same directory to load the a.rb file
+# into an example database:
+#
+#   ruby blob.rb store a.rb
+#
+# Then run this example.
+#
+
+# Require "ONLY" then binary extension, do not +require+ the ruby based code
+$: << "../ext"
+require 'amalgalite/amalgalite3'
+
+# See what the load path is, notice that it is very small
+puts "Before $\" : #{$LOADED_FEATURES.inspect}"
+
+# tell the binary extension to "require" every file in the filestore.db in the
+# table 'files' orderd by column 'id'.  The 'path' column is added to $LOADED_FEATURES and the
+# code in 'data' is evaled.
+Amalgalite::Requires::Bootstrap.lift( "dbfile"          => "filestore.db", 
+                                      "table_name"      => "rubylibs", 
+                                      "rowid_column"    => "id", 
+                                      "filename_column" => "filename", 
+                                      "contents_column" => "contents" )
+
+# Notice that a.rb is in the list of files that has been required
+puts "After $\" : #{$LOADED_FEATURES.inspect}"
+
+# and look we prove that the code was eval'd appropriately
+a = A.new
+a.a
+

data/examples/define_aggregate.rb

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$: << "../lib"
+$: << "../ext"
+require 'amalgalite'
+
+#--
+# Create a database and a table to put some results from the functions in
+#--
+db = Amalgalite::Database.new( ":memory:" )
+db.execute( "CREATE TABLE atest( words )" )
+
+#------------------------------------------------------------------------------
+# Create unique word count aggregate 
+#------------------------------------------------------------------------------
+class UniqueWordCount < ::Amalgalite::Aggregate
+  attr_accessor :words
+
+  def initialize
+    @name = 'unique_word_count'
+    @arity = 1
+    @words = Hash.new { |h,k| h[k] = 0 }
+  end
+
+  def step( str )
+    str.split(/\W+/).each do |word|
+      words[ word.downcase ] += 1
+    end
+    return nil
+  end
+
+  def finalize
+    return words.size
+  end
+end
+
+db.define_aggregate( 'unique_word_count', UniqueWordCount )
+
+#------------------------------------------------------------------------------
+# Now we have a new aggregate function, lets insert some rows into the database
+# and see what we can find.
+#------------------------------------------------------------------------------
+sql = "INSERT INTO atest( words ) VALUES( ? )"
+verify = {}
+db.prepare( sql ) do |stmt|
+  DATA.each do |words|
+    words.strip!
+    puts "Inserting #{words}"
+    stmt.execute( words )
+    words.split(/\W+/).each { |w| verify[w] = true }
+  end
+end
+
+#------------------------------------------------------------------------------
+# And show the results
+#------------------------------------------------------------------------------
+puts
+puts "Getting results..."
+puts
+all_rows = db.execute("SELECT unique_word_count( words ) AS uwc FROM atest")
+puts "#{all_rows.first['uwc']} unique words found"
+puts "#{verify.size} unique words to verify"
+
+__END__
+some random
+words with
+which
+to play
+and there should
+be a couple of different
+words that appear
+more than once and
+some that appear only
+once

data/examples/define_function.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$: << "../lib"
+$: << "../ext"
+require 'amalgalite'
+
+#--
+# Create a database and a table to put some results from the functions in
+#--
+db = Amalgalite::Database.new( ":memory:" )
+db.execute( "CREATE TABLE ftest( data, md5, sha1, sha2_bits, sha2)" )
+
+#------------------------------------------------------------------------------
+# Create an MD5 method using the block format of defining an sql fuction
+#------------------------------------------------------------------------------
+require 'digest/md5'
+db.define_function( 'md5' ) do |x|
+  Digest::MD5.hexdigest( x.to_s )
+end
+
+#------------------------------------------------------------------------------
+# Create a SHA1 method using the lambda format of defining an sql function
+#------------------------------------------------------------------------------
+require 'digest/sha1'
+sha1 = lambda do |y|
+  Digest::SHA1.hexdigest( y.to_s )
+end
+db.define_function( "sha1", sha1 )
+
+#------------------------------------------------------------------------------
+# Create a SHA2 method using the class format for defining an sql function
+# In this one we will allow any number of parameters, but we will only use the
+# first two.
+#------------------------------------------------------------------------------
+require 'digest/sha2'
+class SQLSha2
+  # track the number of invocations
+  attr_reader :call_count
+
+  def initialize
+    @call_count = 0
+  end
+
+  # the protocol that is used for sql function definition
+  def to_proc() self ; end
+
+  # say we take any number of parameters
+  def arity
+    -1
+  end
+
+  # The method that is called by SQLite, must be named 'call'
+  def call( *args )
+    text = args.shift.to_s
+    bitlength = (args.shift || 256).to_i
+    Digest::SHA2.new( bitlength ).hexdigest( text )
+  end
+end
+db.define_function('sha2', SQLSha2.new)
+
+
+#------------------------------------------------------------------------------
+# Now we have 3 new sql functions, each defined in one of the available methods
+# to define sql functions in amalgalite.  Lets insert some rows and look at the
+# results
+#------------------------------------------------------------------------------
+possible_bits = [ 256, 384, 512 ]
+sql = "INSERT INTO ftest( data, md5, sha1, sha2_bits, sha2 ) VALUES( @word , md5( @word ), sha1( @word ), @bits, sha2(@word,@bits) )" 
+db.prepare( sql ) do |stmt|
+  DATA.each do |word|
+    word.strip!
+    bits = possible_bits[ rand(3) ]
+    puts "Inserting #{word}, #{bits}"
+    stmt.execute( { '@word' => word, '@bits' => bits } )
+  end
+end
+
+#------------------------------------------------------------------------------
+# And show the results
+#------------------------------------------------------------------------------
+puts
+puts "Getting results..."
+puts
+columns = db.schema.tables['ftest'].columns.keys.sort
+i = 0
+db.execute("SELECT #{columns.join(',')} FROM ftest") do |row|
+  i += 1
+  puts "-----[ row #{i} ]-" + "-" * 42
+  columns.each do |col|
+    puts "#{col.ljust(10)} : #{row[col]}"
+  end
+  puts
+end
+
+
+__END__
+some
+random
+words
+with
+which
+to
+play

data/examples/fts5.rb

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'amalgalite'
+require 'benchmark'
+require 'pathname'
+
+begin
+  require 'json'
+rescue LoadError 
+  abort "'gem install json' to run this example"
+end
+
+
+README = <<_
+ This example program assumes that you have available the 'fortune' BSD command dataset.
+
+ execute `fortune -f` to see if you have this command available.
+_
+
+fortune_dir = %x[ fortune -f 2>&1 ].split("\n").first.split.last
+abort README unless fortune_dir and File.directory?( fortune_dir )
+
+fortune_path = Pathname.new(fortune_dir)
+
+#
+# Lets create a database that utilizes FTS5 http://www.sqlite.org/fts5.html
+#
+#
+
+#
+# Create a database, this will create the DB if it doesn't exist
+#
+puts "Opening database (version #{Amalgalite::VERSION})"
+db = Amalgalite::Database.new("fts5.db")
+
+#
+# Create the schema unless it already exists in the table.  The meta information
+# about the database schema is available as the result of the db.schema method
+#
+schema = db.schema
+unless schema.tables['search'] 
+  puts "Create schema"
+  db.execute_batch <<-SQL
+  CREATE VIRTUAL TABLE search USING fts5(
+    filename,
+    content
+  );
+
+  CREATE TABLE plain (
+    filename VARCHAR(128),
+    content  TEXT 
+  );
+  SQL
+  db.reload_schema!
+end
+
+def each_fortune(path,&block)
+  fortune = []
+  path.each_line do |line|
+    line.strip!
+    if line == "%" then
+      yield fortune.join("\n")
+      fortune.clear
+    else 
+      fortune << line
+    end
+  end
+end
+
+#
+# Only load the data if the db is empty
+#
+if db.first_value_from( "SELECT count(*) from search" ) == 0 then
+  before = Time.now
+  idx = 0
+
+  # Inserting bulk rows as a transaction is good practice with SQLite, it is
+  # MUCH faster.  
+  db.transaction do |db_in_transaction|
+    # Iterate over the files in the fortunes dir and split on the fortunes, then 
+
+    fortune_path.each_child do |fortune_file|
+      next if fortune_file.directory?
+      next if fortune_file.extname == ".dat"
+      $stdout.puts "Loading #{fortune_file}"
+
+      each_fortune(fortune_file) do |fortune|
+        insert_data  = {
+          ':fname'   => fortune_file.to_s,
+          ':content' => fortune
+        }
+
+        # insert into the FTS5 table
+        db_in_transaction.prepare("INSERT INTO search( filename, content ) VALUES( :fname, :content );") do |stmt|
+          stmt.execute( insert_data )
+        end
+
+        # insert into the normal table for comparison
+        db_in_transaction.prepare("INSERT INTO plain( filename, content ) VALUES( :fname, :content );") do |stmt|
+          stmt.execute( insert_data )
+        end
+
+        idx += 1
+        print "Processed #{idx}\r"
+        $stdout.flush
+      end
+      puts "\nFinalizing..."
+    end
+  end
+  puts "Took #{Time.now - before} seconds to insert #{idx} fortunes"
+  puts "Done Inserting"
+end
+
+doc_count = db.first_value_from( "SELECT count(*) from search" ) 
+
+#
+# Now lets do some searching for some various words
+#
+
+%w[ president salmon thanks ].each do |word|
+
+  count = 100
+  puts
+  puts "Searching for '#{word}' #{count} times across #{doc_count} fortunes"
+  puts "=" * 60
+
+  Benchmark.bm( 15 ) do |x|
+
+    #
+    # search using the fts search to get the cont of tweets with the given word
+    #
+    x.report('fts5: ') do 
+      db.prepare( "SELECT count(filename) FROM search WHERE search MATCH 'content:#{word}'" ) do |stmt|
+        count.times do
+          stmt.execute
+        end
+      end
+    end
+
+    #
+    # search using basic string matching in sqlite.  
+    #
+    x.report('plain: ') do 
+      db.prepare( "SELECT count(filename) FROM plain WHERE content LIKE '% #{word} %'" ) do |stmt|
+        count.times do
+          stmt.execute
+        end
+      end
+    end
+  end
+end
+