libsql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4a4257a3382ce29836c8946b14640967d51dc53c58cc6a61374d5c3522d75249
+  data.tar.gz: ed5aa55d6e3a20d93c946fc3cef64943bbdc30c734eed6862e7b2c220b309445
+SHA512:
+  metadata.gz: 559bc6235c96ba9e27eac6ad22c235c697dd6212ea70767fae3071c78b6279bb9b5b7a27cd9744309394b62a933e21238df706e7a6a3a7754de5aa8c53cc9446
+  data.tar.gz: 9abf46e3166aa0ee653dd87be63b2fdcbd29f4c54c64cbdf24526e08ae6babb9499f8d73d7736d9517e9b4e13465be06aaaafd153f7a4b1e1056b8b43d73ef5e

data/CONTRIBUTING.md

@@ -0,0 +1,60 @@
+# Hi there!
+
+I see you are interested in contributing. That is wonderful. I love
+contributions.
+
+I guarantee that there are bugs in this software. And I guarantee that there is
+a feature you want that is not in here yet. As such, any and all bugs reports
+are gratefully accepted, bugfixes even more so. Helping out with bugs is the
+easiest way to contribute.
+
+
+## The Quick Version
+
+* Have a [GitHub Account][].
+* Search the [GitHub Issues][] and see if your issue already present. If so
+  add your comments, :thumbsup:, etc.
+* Issue not there? Not a problem, open up a [new issue][].
+    * **Bug reports** please be as detailed as possible. Include:
+        * full ruby engine and version: `ruby -e 'puts RUBY_DESCRIPTION'`
+        * operating system and version
+        * version of libsql `ruby -rubygems -e "require 'libsql'; puts Libsql::VERSION"`
+        * as much detail about the bug as possible so I can replicate it. Feel free
+          to link in a [gist][]
+    * **New Feature**
+        * What the new feature should do.
+        * What benefit the new feature brings to the project.
+* Fork the [repo][].
+* Create a new branch for your issue: `git checkout -b issue/my-issue`
+* Lovingly craft your contribution:
+    * `rake develop` to get started, or if you prefer bundler `rake develop:using_bundler && bundle`.
+    * `rake test` to run tests
+* Make sure that `rake test` passes. It's important, I said it twice.
+* Add yourself to the contributors section below.
+* Submit your [pull request][].
+
+## Building Windows Binaries
+
+This is done using https://github.com/rake-compiler/rake-compiler-dock
+
+1. have VirtualBox installed
+2. have Docker Machine installed (https://docs.docker.com/engine/installation/)
+3. `gem install rake-compiler-dock`
+4. `rake-compiler-dock` (this could take a while)
+5. `bundle`
+6. `rake cross native gem`
+
+# Contributors
+
+* [Jeremy Hinegardner](https://github.com/copiousfreetime)
+* [Jos Backus](https://github.com/josb)
+* [Alex Young](https://github.com/regularfry)
+* [Michael Granger](https://github.com/ged)
+* [Alex Young](https://github.com/bmalex)
+
+[GitHub Account]: https://github.com/signup/free "GitHub Signup"
+[GitHub Issues]:  https://github.com/copiousfreetime/libsql-ruby/issues "libsql-ruby Issues"
+[new issue]:      https://github.com/copiousfreetime/libsql-ruby/issues/new "New libsql-ruby Issue"
+[gist]:           https://gist.github.com/ "New Gist"
+[repo]:           https://github.com/copiousfreetime/libsql-ruby "libsql-ruby Repo"
+[pull request]:   https://help.github.com/articles/using-pull-requests "Using Pull Requests"

data/HISTORY.md

@@ -0,0 +1,6 @@
+# Libsql Changelog
+
+## Version 0.1.0 - 2023-05-03
+* Use the the [amalgalite](https://github.com/copiousfreetime/amalgalite) codebase to bootstrap libsql.
+* Initial public release
+* embedds [libSQL-v0.2.1](https://github.com/libsql/libsql/releases/tag/libsql-0.2.1)

data/LICENSE

@@ -0,0 +1,31 @@
+BSD License - https://opensource.org/licenses/BSD-3-Clause
+
+Copyright (c) 2023 Jeremy Hinegardner
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.  
+
+* Neither the name of Jeremy Hinegardner nor the names of its contributors 
+  may be used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/Manifest.txt

@@ -0,0 +1,96 @@
+CONTRIBUTING.md
+HISTORY.md
+LICENSE
+Manifest.txt
+README.md
+Rakefile
+TODO.md
+examples/a.rb
+examples/blob.rb
+examples/define_aggregate.rb
+examples/define_function.rb
+examples/fts5.rb
+examples/gem-db.rb
+examples/schema-info.rb
+ext/libsql/c/extconf.rb
+ext/libsql/c/gen_constants.rb
+ext/libsql/c/libsql_blob.c
+ext/libsql/c/libsql_constants.c
+ext/libsql/c/libsql_database.c
+ext/libsql/c/libsql_ext.c
+ext/libsql/c/libsql_ext.h
+ext/libsql/c/libsql_statement.c
+ext/libsql/c/notes.txt
+ext/libsql/c/sqlite3.c
+ext/libsql/c/sqlite3.h
+lib/libsql-ruby.rb
+lib/libsql.rb
+lib/libsql/aggregate.rb
+lib/libsql/blob.rb
+lib/libsql/boolean.rb
+lib/libsql/busy_timeout.rb
+lib/libsql/column.rb
+lib/libsql/csv_table_importer.rb
+lib/libsql/database.rb
+lib/libsql/function.rb
+lib/libsql/index.rb
+lib/libsql/memory_database.rb
+lib/libsql/paths.rb
+lib/libsql/profile_tap.rb
+lib/libsql/progress_handler.rb
+lib/libsql/schema.rb
+lib/libsql/sqlite3.rb
+lib/libsql/sqlite3/constants.rb
+lib/libsql/sqlite3/database/function.rb
+lib/libsql/sqlite3/database/status.rb
+lib/libsql/sqlite3/libsql_version.rb
+lib/libsql/sqlite3/status.rb
+lib/libsql/sqlite3/version.rb
+lib/libsql/statement.rb
+lib/libsql/table.rb
+lib/libsql/taps.rb
+lib/libsql/taps/console.rb
+lib/libsql/taps/io.rb
+lib/libsql/trace_tap.rb
+lib/libsql/type_map.rb
+lib/libsql/type_maps/default_map.rb
+lib/libsql/type_maps/storage_map.rb
+lib/libsql/type_maps/text_map.rb
+lib/libsql/version.rb
+lib/libsql/view.rb
+spec/aggregate_spec.rb
+spec/blob_spec.rb
+spec/boolean_spec.rb
+spec/busy_handler.rb
+spec/data/iso-3166-country.txt
+spec/data/iso-3166-schema.sql
+spec/data/iso-3166-subcountry.txt
+spec/data/make-iso-db.sh
+spec/database_spec.rb
+spec/default_map_spec.rb
+spec/function_spec.rb
+spec/integeration_spec.rb
+spec/iso_3166_database.rb
+spec/json_spec.rb
+spec/libsql_spec.rb
+spec/paths_spec.rb
+spec/progress_handler_spec.rb
+spec/rtree_spec.rb
+spec/schema_spec.rb
+spec/spec_helper.rb
+spec/sqlite3/constants_spec.rb
+spec/sqlite3/database_status_spec.rb
+spec/sqlite3/libsql_version_spec.rb
+spec/sqlite3/status_spec.rb
+spec/sqlite3/version_spec.rb
+spec/sqlite3_spec.rb
+spec/statement_spec.rb
+spec/storage_map_spec.rb
+spec/tap_spec.rb
+spec/text_map_spec.rb
+spec/type_map_spec.rb
+spec/version_spec.rb
+tasks/custom.rake
+tasks/default.rake
+tasks/extension.rake
+tasks/this.rb

data/README.md

@@ -0,0 +1,59 @@
+## libsql
+
+[![Build Status](https://copiousfreetime.semaphoreci.com/badges/libsql-ruby/branches/main.svg)](https://copiousfreetime.semaphoreci.com/projects/libsql-ruby)
+
+* [Homepage](http://github.com/copiousfreetime/libsql-ruby)
+* `git clone git://github.com/copiousfreetime/libsql-ruby.git`
+* [Github](http://github.com/copiousfreetime/libsql-ruby/)
+* [Bug Tracking](http://github.com/copiousfreetime/libsql-ruby/issues)
+
+## INSTALL
+
+* `gem install libsql`
+* `bundle add libsql`
+
+## DESCRIPTION
+
+libsql embeds the libsql fork of the SQLite database engine as a ruby extension.
+There is no need to install libsql separately.
+
+Look in the examples/ directory to see
+
+* general usage
+* blob io
+* schema information
+* custom functions
+* custom aggregates
+* requiring ruby code from a database
+* full text search
+
+Also scroll through Libsql::Database for a quick example, and a general
+overview of the API.
+
+libsql adds in the following additional non-default SQLite extensions:
+
+* [R*Tree index extension](http://sqlite.org/rtree.html)
+* [Full Text Search](http://sqlite.org/fts5.html) - both fts3 and fts5
+* [Geopoly Interface to R*Tree](https://www.sqlite.org/geopoly.html)
+* [JSON Extension](https://www.sqlite.org/json1.html)
+
+Other extensions are add that might not be usable/visible by users of the gem.
+The full list of extensions added is in
+[extconf.rb](ext/libsql/c/extconf.rb). And those may be cross referenced
+against the [compile options from SQLite](https://www.sqlite.org/compile.html)
+
+## CREDITS
+
+* This is a straight port of the [amalgalite](https://github.com/copiousfreetime/amalgalite) gem, also written by me.
+
+## CHANGES
+
+Read the [HISTORY.md](HISTORY.md) file.
+
+## LICENSE
+
+Copyright (c) 2023 Jeremy Hinegardner
+
+All rights reserved.
+
+See LICENSE and/or COPYING for details.

data/Rakefile

@@ -0,0 +1,28 @@
+# vim: syntax=ruby
+load 'tasks/this.rb'
+
+This.name     = "libsql"
+This.author   = "Jeremy Hinegardner"
+This.email    = "jeremy@copiousfreetime.org"
+This.homepage = "http://github.com/copiousfreetime/#{ This.name }-ruby"
+
+This.ruby_gemspec do |spec|
+  spec.add_dependency( 'arrayfields', '~> 4.9' )
+
+  spec.add_development_dependency( 'debug',              '~> 1.0' )
+  spec.add_development_dependency( 'rspec',              '~> 3.12' )
+  spec.add_development_dependency( 'rspec_junit_formatter','~> 0.6' )
+  spec.add_development_dependency( 'rake',               '~> 13.0' )
+  spec.add_development_dependency( 'rake-compiler',      '~> 1.2'  )
+  spec.add_development_dependency( 'rake-compiler-dock', '~> 1.3'  )
+  spec.add_development_dependency( 'rdoc',               '~> 6.5'  )
+  spec.add_development_dependency( 'simplecov',          '~> 0.21' )
+  spec.add_development_dependency( 'archive-zip',        '~> 0.12' )
+
+  spec.extensions.concat This.extension_conf_files
+  spec.license = "BSD-3-Clause"
+end
+
+load 'tasks/default.rake'
+load 'tasks/extension.rake'
+load 'tasks/custom.rake'

data/TODO.md

@@ -0,0 +1,57 @@
+# Future Release possibilties:
+- rebuild statement constants
+- look at all pragma statements
+
+## SQLite API:
+- authorizers
+- loading of extensions -- readfile / writefile
+- utf-16 integration
+- create_collation 
+- encryption key support
+- expose sqlite3_strnicmp
+- table name and column name in a type map?
+- type conversion for manifest typing? how to allow it through?
+- explicit pragma handler
+- application_id pragma setter
+
+## Non backwards compatible changes:
+- change the schema objects to be more consistent
+- change taps to use to_proc protocol
+- convert type dependency to just use 'call'
+- integrate transaction and savepoint under the same api
+
+## SQLite Features:
+- activate SQLITE_ENABLE_ICU extension
+- activate SQLITE_ENABLE_LOCKING_STYLE
+- activate SQLITE_ENABLE_UNLOCK_NOTIFY
+- expose PRAGMA foreign_keys
+- virtual file system
+- full text search (FTS3)
+- expose the sqlite mutex lib
+- statement status ( sqlite3_stmt_status )
+- db status ( sqlite3_db_status )
+- library status ( sqlite3_status )
+- sqlite3_index_info
+- sqlite3_create_function has 4th parameter SQLITE_DETERMINISTIC 
+- sqlite3_rtree_query_callback()
+
+## Drivers:
+- data mapper driver
+- sequel driver optimization
+
+## Features:
+- Think about moving from arrayfields to ordered hash?
+- add to command line which directory to pack into a rubylibs table
+- command line tool
+- use ruby's ALLOC_N and hook into sqlite3_mem_methods
+
+## Functions to possibly expose:
+- sqlite3_backup_remaining, sqlite3_backup_pagecount
+- sqlite3_compileoption_used, sqlite3_compileoption_get
+- sqlite3_config
+- sqlite3_data_count - returns number of colums in the result set of a
+  prepared statement
+- sqlite_sourceid, sqlite_source_id
+- sqlite3_strnicmp
+-
+

data/examples/a.rb

@@ -0,0 +1,9 @@
+class A
+  def initialize 
+    puts "Initialized A"
+  end
+
+  def a
+    puts "called a"
+  end
+end

data/examples/blob.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+#
+# A Libsql example showing how Blob's can be utilized
+#
+# We'll make a database with one table, that we store files in.  We'll use the
+# Blob incremental IO to store the files and retrieve them from the database
+#
+# This little program will store 1 or more files in the sqlite3 database when
+# the 'store' action is given, and cat a file to stdout on 'retrieve'
+#
+# e.g.
+#
+#   ruby blob.rb store a.rb b.rb c.rb # => stores a.rb b.rb and c.rb in the db
+#
+#   ruby blob.rb retrieve a.rb        # => dumps a.rb to stdout
+#
+
+require 'rubygems'
+$: << "../lib"
+$: << "../ext"
+require 'libsql'
+VALID_ACTIONS = %w[ list retrieve store ]
+def usage 
+  STDERR.puts "Usage: #{File.basename($0)} ( #{VALID_ACTIONS.join(' | ')} )  file(s)"
+  exit 1
+end
+
+#
+# This does the basic command line parsing
+#
+usage if ARGV.size < 1
+action    = ARGV.shift
+usage unless VALID_ACTIONS.include? action 
+file_list = ARGV
+
+#
+# create the database if it doesn't exist
+#
+db = ::Libsql::Database.new( "filestore.db" )
+schema = db.schema
+unless schema.tables['files']
+
+  schema = <<~SQL
+  CREATE TABLE files (
+    id integer primary key autoincrement,
+    filename text unique,
+    contents blob
+  )
+  SQL
+
+  db.execute(schema)
+  db.reload_schema!
+end
+
+
+case action
+  #
+  # list all the files that are stored in the database
+  #
+when 'list'
+  db.execute("SELECT filename FROM files") do |row|
+    puts row['filename']
+  end
+
+  #
+  # if we are doing the store action, then loop over the files and store them in
+  # the database.  This will use incremental IO to store the files directly from
+  # the file names. 
+  #
+  # It is slightly strange in that you have to tell the Blob object what column
+  # it is going to, but that is necessary at this point to be able to hook
+  # automatically into the lower level incremental blob IO api.
+  #
+  # This also shows using the $var syntax for binding name sql values in a
+  # prepared statement.
+  #
+when 'store'
+  usage if file_list.empty?
+
+  contents_column = db.schema.tables['files'].columns['contents']
+  db.transaction do |trans|
+    trans.prepare("INSERT INTO files (filename, contents) VALUES ($filename, $contents)") do |stmt|
+      file_list.each do |file_path|
+        contents = IO.read(file_path)
+        content_io = StringIO.new(contents)
+        stmt.execute("$filename" => file_path,
+                     "$contents" => ::Libsql::Blob.new(io: content_io,
+                                                       column: contents_column)
+                    )
+      end
+    end
+  end
+
+    #
+    # dump the file that matches the right path to stdout.  This also shows
+    # positional sql varible binding using the '?' syntax.
+    #
+  when 'retrieve'
+    usage if file_list.empty?
+    db.execute("SELECT id, filename, contents FROM files WHERE filename = ?", file_list.first) do |row|
+      STDERR.puts "Dumping #{row['filename']} to stdout"
+      row['contents'].write_to_io( STDOUT )
+    end
+  end
+  db.close

data/examples/define_aggregate.rb

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$: << "../lib"
+$: << "../ext"
+require 'libsql'
+
+#--
+# Create a database and a table to put some results from the functions in
+#--
+db = ::Libsql::Database.new( ":memory:" )
+db.execute( "CREATE TABLE atest( words )" )
+
+#------------------------------------------------------------------------------
+# Create unique word count aggregate 
+#------------------------------------------------------------------------------
+class UniqueWordCount < ::Libsql::Aggregate
+  attr_accessor :words
+
+  def initialize
+    @name = 'unique_word_count'
+    @arity = 1
+    @words = Hash.new { |h,k| h[k] = 0 }
+  end
+
+  def step( str )
+    str.split(/\W+/).each do |word|
+      words[ word.downcase ] += 1
+    end
+    return nil
+  end
+
+  def finalize
+    return words.size
+  end
+end
+
+db.define_aggregate( 'unique_word_count', UniqueWordCount )
+
+#------------------------------------------------------------------------------
+# Now we have a new aggregate function, lets insert some rows into the database
+# and see what we can find.
+#------------------------------------------------------------------------------
+sql = "INSERT INTO atest( words ) VALUES( ? )"
+verify = {}
+db.prepare( sql ) do |stmt|
+  DATA.each do |words|
+    words.strip!
+    puts "Inserting #{words}"
+    stmt.execute( words )
+    words.split(/\W+/).each { |w| verify[w] = true }
+  end
+end
+
+#------------------------------------------------------------------------------
+# And show the results
+#------------------------------------------------------------------------------
+puts
+puts "Getting results..."
+puts
+all_rows = db.execute("SELECT unique_word_count( words ) AS uwc FROM atest")
+puts "#{all_rows.first['uwc']} unique words found"
+puts "#{verify.size} unique words to verify"
+
+__END__
+some random
+words with
+which
+to play
+and there should
+be a couple of different
+words that appear
+more than once and
+some that appear only
+once

data/examples/define_function.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+$: << "../lib"
+$: << "../ext"
+require 'libsql'
+
+#--
+# Create a database and a table to put some results from the functions in
+#--
+db = ::Libsql::Database.new( ":memory:" )
+db.execute( "CREATE TABLE ftest( data, md5, sha1, sha2_bits, sha2)" )
+
+#------------------------------------------------------------------------------
+# Create an MD5 method using the block format of defining an sql fuction
+#------------------------------------------------------------------------------
+require 'digest/md5'
+db.define_function( 'md5' ) do |x|
+  Digest::MD5.hexdigest( x.to_s )
+end
+
+#------------------------------------------------------------------------------
+# Create a SHA1 method using the lambda format of defining an sql function
+#------------------------------------------------------------------------------
+require 'digest/sha1'
+sha1 = lambda do |y|
+  Digest::SHA1.hexdigest( y.to_s )
+end
+db.define_function( "sha1", sha1 )
+
+#------------------------------------------------------------------------------
+# Create a SHA2 method using the class format for defining an sql function
+# In this one we will allow any number of parameters, but we will only use the
+# first two.
+#------------------------------------------------------------------------------
+require 'digest/sha2'
+class SQLSha2
+  # track the number of invocations
+  attr_reader :call_count
+
+  def initialize
+    @call_count = 0
+  end
+
+  # the protocol that is used for sql function definition
+  def to_proc() self ; end
+
+  # say we take any number of parameters
+  def arity
+    -1
+  end
+
+  # The method that is called by SQLite, must be named 'call'
+  def call( *args )
+    text = args.shift.to_s
+    bitlength = (args.shift || 256).to_i
+    Digest::SHA2.new( bitlength ).hexdigest( text )
+  end
+end
+db.define_function('sha2', SQLSha2.new)
+
+
+#------------------------------------------------------------------------------
+# Now we have 3 new sql functions, each defined in one of the available methods
+# to define sql functions in libsql.  Lets insert some rows and look at the
+# results
+#------------------------------------------------------------------------------
+possible_bits = [ 256, 384, 512 ]
+sql = "INSERT INTO ftest( data, md5, sha1, sha2_bits, sha2 ) VALUES( @word , md5( @word ), sha1( @word ), @bits, sha2(@word,@bits) )" 
+db.prepare( sql ) do |stmt|
+  DATA.each do |word|
+    word.strip!
+    bits = possible_bits[ rand(3) ]
+    puts "Inserting #{word}, #{bits}"
+    stmt.execute( { '@word' => word, '@bits' => bits } )
+  end
+end
+
+#------------------------------------------------------------------------------
+# And show the results
+#------------------------------------------------------------------------------
+puts
+puts "Getting results..."
+puts
+columns = db.schema.tables['ftest'].columns.keys.sort
+i = 0
+db.execute("SELECT #{columns.join(',')} FROM ftest") do |row|
+  i += 1
+  puts "-----[ row #{i} ]-" + "-" * 42
+  columns.each do |col|
+    puts "#{col.ljust(10)} : #{row[col]}"
+  end
+  puts
+end
+
+
+__END__
+some
+random
+words
+with
+which
+to
+play