extralite-bundle 2.4 → 2.6

data/ext/sqlite3/sqlite3.h

@@ -146,9 +146,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.44.2"
-#define SQLITE_VERSION_NUMBER 3044002
-#define SQLITE_SOURCE_ID      "2023-11-24 11:41:44 ebead0e7230cd33bcec9f95d2183069565b9e709bf745c9b5db65cc0cbf92c0f"
+#define SQLITE_VERSION        "3.45.0"
+#define SQLITE_VERSION_NUMBER 3045000
+#define SQLITE_SOURCE_ID      "2024-01-15 17:01:13 1066602b2b1976fe58b5150777cced894af17c803e068f5918390d6915b46e1d"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -3954,15 +3954,17 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
-** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** text that describes the error, as either UTF-8 or UTF-16 respectively,
+** or NULL if no error message is available.
 ** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
 ** subsequent calls to other SQLite interface functions.)^
 **
-** ^The sqlite3_errstr() interface returns the English-language text
-** that describes the [result code], as UTF-8.
+** ^The sqlite3_errstr(E) interface returns the English-language text
+** that describes the [result code] E, as UTF-8, or NULL if E is not an
+** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
@@ -8037,9 +8039,11 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*);
 **
 ** ^(Some systems (for example, Windows 95) do not support the operation
 ** implemented by sqlite3_mutex_try().  On those systems, sqlite3_mutex_try()
-** will always return SQLITE_BUSY. The SQLite core only ever uses
-** sqlite3_mutex_try() as an optimization so this is acceptable
-** behavior.)^
+** will always return SQLITE_BUSY. In most cases the SQLite core only uses
+** sqlite3_mutex_try() as an optimization, so this is acceptable
+** behavior. The exceptions are unix builds that set the
+** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working
+** sqlite3_mutex_try() is required.)^
 **
 ** ^The sqlite3_mutex_leave() routine exits a mutex that was
 ** previously entered by the same thread.   The behavior
@@ -8298,6 +8302,7 @@ SQLITE_API int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
@@ -12811,8 +12816,11 @@ struct Fts5PhraseIter {
 **   created with the "columnsize=0" option.
 **
 ** xColumnText:
-**   This function attempts to retrieve the text of column iCol of the
-**   current document. If successful, (*pz) is set to point to a buffer
+**   If parameter iCol is less than zero, or greater than or equal to the
+**   number of columns in the table, SQLITE_RANGE is returned.
+**
+**   Otherwise, this function attempts to retrieve the text of column iCol of
+**   the current document. If successful, (*pz) is set to point to a buffer
 **   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
 **   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
 **   if an error occurs, an SQLite error code is returned and the final values
@@ -12822,8 +12830,10 @@ struct Fts5PhraseIter {
 **   Returns the number of phrases in the current query expression.
 **
 ** xPhraseSize:
-**   Returns the number of tokens in phrase iPhrase of the query. Phrases
-**   are numbered starting from zero.
+**   If parameter iCol is less than zero, or greater than or equal to the
+**   number of phrases in the current query, as returned by xPhraseCount,
+**   0 is returned. Otherwise, this function returns the number of tokens in
+**   phrase iPhrase of the query. Phrases are numbered starting from zero.
 **
 ** xInstCount:
 **   Set *pnInst to the total number of occurrences of all phrases within
@@ -12839,12 +12849,13 @@ struct Fts5PhraseIter {
 **   Query for the details of phrase match iIdx within the current row.
 **   Phrase matches are numbered starting from zero, so the iIdx argument
 **   should be greater than or equal to zero and smaller than the value
-**   output by xInstCount().
+**   output by xInstCount(). If iIdx is less than zero or greater than
+**   or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
 **
-**   Usually, output parameter *piPhrase is set to the phrase number, *piCol
+**   Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
 **   to the column in which it occurs and *piOff the token offset of the
-**   first token of the phrase. Returns SQLITE_OK if successful, or an error
-**   code (i.e. SQLITE_NOMEM) if an error occurs.
+**   first token of the phrase. SQLITE_OK is returned if successful, or an
+**   error code (i.e. SQLITE_NOMEM) if an error occurs.
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option.
@@ -12870,6 +12881,10 @@ struct Fts5PhraseIter {
 **   Invoking Api.xUserData() returns a copy of the pointer passed as
 **   the third argument to pUserData.
 **
+**   If parameter iPhrase is less than zero, or greater than or equal to
+**   the number of phrases in the query, as returned by xPhraseCount(),
+**   this function returns SQLITE_RANGE.
+**
 **   If the callback function returns any value other than SQLITE_OK, the
 **   query is abandoned and the xQueryPhrase function returns immediately.
 **   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
@@ -12984,9 +12999,42 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   If iPhrase or iToken are less than zero, or if iPhrase is greater than
+**   or equal to the number of phrases in the query as reported by
+**   xPhraseCount(), or if iToken is equal to or greater than the number of
+**   tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
+     are both zeroed.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row. If iIdx is less than zero or greater than or equal to the
+**   value returned by xInstCount(), SQLITE_RANGE is returned.  Otherwise,
+**   output variable (*ppToken) is set to point to a buffer containing the
+**   matching document token, and (*pnToken) to the size of that buffer in
+**   bytes. This API is not available if the specified token matches a
+**   prefix query term. In that case both output variables are always set
+**   to 0.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */
 
   void *(*xUserData)(Fts5Context*);
 
@@ -13021,6 +13069,13 @@ struct Fts5ExtensionApi {
 
   int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
   void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
+
+  /* Below this point are iVersion>=3 only */
+  int (*xQueryToken)(Fts5Context*,
+      int iPhrase, int iToken,
+      const char **ppToken, int *pnToken
+  );
+  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };
 
 /*

data/gemspec.rb

@@ -16,7 +16,7 @@ def common_spec(s)
   s.rdoc_options = ["--title", "extralite", "--main", "README.md"]
   s.extra_rdoc_files = ["README.md"]
   s.require_paths = ["lib"]
-  s.required_ruby_version = '>= 2.7'
+  s.required_ruby_version = '>= 3.0'
 
   s.add_development_dependency  'rake-compiler',        '1.1.6'
   s.add_development_dependency  'minitest',             '5.15.0'

data/lib/extralite/version.rb

@@ -1,4 +1,4 @@
 module Extralite
   # Extralite version
-  VERSION = '2.4'
+  VERSION = '2.6'
 end

data/lib/extralite.rb

@@ -33,16 +33,20 @@ module Extralite
   class Database
     # @!visibility private
     TABLES_SQL = <<~SQL
-      SELECT name FROM sqlite_master
+      SELECT name FROM %<db>s.sqlite_master
       WHERE type ='table'
-        AND name NOT LIKE 'sqlite_%';
+        AND name NOT LIKE 'sqlite_%%';
     SQL
 
-    # Returns the list of currently defined tables.
+    alias_method :execute_multi, :batch_execute
+
+    # Returns the list of currently defined tables. If a database name is given,
+    # returns the list of tables for the relevant attached database.
     #
+    # @param db [String] name of attached database
     # @return [Array] list of tables
-    def tables
-      query_single_column(TABLES_SQL)
+    def tables(db = 'main')
+      query_single_column(format(TABLES_SQL, db: db))
     end
 
     # Gets or sets one or more pragmas:
@@ -56,6 +60,11 @@ module Extralite
       value.is_a?(Hash) ? pragma_set(value) : pragma_get(value)
     end
 
+    # Error class used to roll back a transaction without propagating an
+    # exception.
+    class Rollback < Error
+    end
+
     # Starts a transaction and runs the given block. If an exception is raised
     # in the block, the transaction is rolled back. Otherwise, the transaction
     # is commited after running the block.
@@ -72,13 +81,56 @@ module Extralite
     
       abort = false
       yield self
-    rescue
+    rescue => e
       abort = true
-      raise
+      raise unless e.is_a?(Rollback)
     ensure
       execute(abort ? 'rollback' : 'commit')
     end
 
+    # Creates a savepoint with the given name.
+    #
+    # @param name [String, Symbol] savepoint name
+    # @return [Extralite::Database] database
+    def savepoint(name)
+      execute "savepoint #{name}"
+      self
+    end
+
+    # Release a savepoint with the given name.
+    #
+    # @param name [String, Symbol] savepoint name
+    # @return [Extralite::Database] database
+    def release(name)
+      execute "release #{name}"
+      self
+    end
+
+    # Rolls back changes to a savepoint with the given name.
+    #
+    # @param name [String, Symbol] savepoint name
+    # @return [Extralite::Database] database
+    def rollback_to(name)
+      execute "rollback to #{name}"
+      self
+    end
+
+    # Rolls back the currently active transaction. This method should only be
+    # called from within a block passed to Database#transaction. This method
+    # raises a Extralite::Rollback exception, which will stop execution of the
+    # transaction block without propagating the exception.
+    #
+    #   db.transaction do
+    #     db.execute('insert into foo (42)')
+    #     db.rollback!
+    #   end
+    #
+    # @param name [String, Symbol] savepoint name
+    # @return [Extralite::Database] database
+    def rollback!
+      raise Rollback
+    end
+
     private
 
     def pragma_set(values)
@@ -87,7 +139,11 @@ module Extralite
     end
 
     def pragma_get(key)
-      query("pragma #{key}")
+      query_single_value("pragma #{key}")
     end
   end
+
+  class Query
+    alias_method :execute_multi, :batch_execute
+  end
 end

data/test/helper.rb

@@ -7,3 +7,11 @@ require 'minitest/autorun'
 puts "sqlite3 version: #{Extralite.sqlite3_version}"
 
 IS_LINUX = RUBY_PLATFORM =~ /linux/
+SKIP_RACTOR_TESTS = !IS_LINUX || (RUBY_VERSION =~ /^3\.[01]/)
+
+module Minitest::Assertions
+  def assert_in_range exp_range, act
+    msg = message(msg) { "Expected #{mu_pp(act)} to be in range #{mu_pp(exp_range)}" }
+    assert exp_range.include?(act), msg
+  end
+end

data/test/issue-54.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "./lib/extralite"
+
+puts 'Connecting to database...'
+
+connection_1 = Extralite::Database.new("test.sqlite3")
+puts "#{connection_1} connected"
+connection_2 = Extralite::Database.new("test.sqlite3")
+connection_2.busy_timeout = 0
+puts "#{connection_2} connected"
+
+[connection_1, connection_2].each do |connection|
+  puts "#{connection} beginning transaction..."
+  connection.execute "begin immediate transaction"
+end
+
+[connection_1, connection_2].each do |connection|
+  puts "#{connection} rolling back transaction..."
+  connection.execute "rollback transaction"
+end

data/test/issue-59.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "./lib/extralite"
+require "benchmark"
+require "tempfile"
+require "fileutils"
+
+p sqlite_version: Extralite.sqlite3_version
+
+N = (ENV['N'] || 1000).to_i
+p N: N
+
+fn1 = '/tmp/db1'
+fn2 = '/tmp/db2'
+
+FileUtils.rm(fn1) rescue nil
+FileUtils.rm(fn2) rescue nil
+
+p fn1: fn1
+p fn2: fn2
+
+db1 = Extralite::Database.new fn1
+db1.execute "pragma journal_mode = wal;"
+db1.transaction do
+  db1.execute "create table t1 ( a integer primary key, b text );"
+  values = N.times.map { |i| "#{i}-#{rand(1000)}" }
+  db1.execute_multi "insert into t1 ( b ) values ( ? );", values
+
+  p count: db1.query_single_value("select count(*) from t1")
+  p some_rows: db1.query("select * from t1 limit 5")
+end
+
+db2 = Extralite::Database.new fn2
+db2.execute "pragma journal_mode = wal;"
+db2.execute "attach '#{fn1}' as db1;"
+db2.execute "create table t2 ( a integer primary key, b text );"
+
+p main_tables: db2.tables
+p db1_tables: db2.tables('db1')
+
+overall = Benchmark.realtime do
+  t1 = Thread.new do
+    time1 = Benchmark.realtime do
+      db2.execute "create unique index db1.t1_b_unique on t1 (b);"
+    end
+    p({ indexing: time1 })
+  end
+
+  t2 = Thread.new do
+    time2 = Benchmark.realtime do
+      (N / 10000).times do |i|
+        values = 10000.times.map { |i| "#{i}-#{rand(1000)}" }
+        db2.transaction do
+          db2.execute_multi "insert into main.t2 ( b ) values ( ? );", values
+        end
+      end
+    end
+    p({ inserting: time2 })
+    p count_t2: db2.query_single_value("select count(*) from main.t2")
+    p some_rows_t2: db2.query("select * from main.t2 limit 5")
+    end
+
+  t1.join
+  t2.join
+end
+
+p({ overall: overall })
+
+db1.close
+db2.close

data/test/perf_ary.rb

@@ -15,26 +15,25 @@ require 'fileutils'
 DB_PATH = "/tmp/extralite_sqlite3_perf-#{Time.now.to_i}-#{rand(10000)}.db"
 puts "DB_PATH = #{DB_PATH.inspect}"
 
+$sqlite3_db = SQLite3::Database.new(DB_PATH)
+$extralite_db = Extralite::Database.new(DB_PATH, gvl_release_threshold: -1)
 
 def prepare_database(count)
   db = Extralite::Database.new(DB_PATH)
-  db.query('create table if not exists foo ( a integer primary key, b text )')
-  db.query('delete from foo')
-  db.query('begin')
-  count.times { db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
-  db.query('commit')
-  db.close
+  $extralite_db.query('create table if not exists foo ( a integer primary key, b text )')
+  $extralite_db.query('delete from foo')
+  $extralite_db.query('begin')
+  count.times { $extralite_db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
+  $extralite_db.query('commit')
 end
 
 def sqlite3_run(count)
-  db = SQLite3::Database.new(DB_PATH)
-  results = db.execute('select * from foo')
+  results = $sqlite3_db.execute('select * from foo')
   raise unless results.size == count
 end
 
 def extralite_run(count)
-  db = Extralite::Database.new(DB_PATH)
-  results = db.query_ary('select * from foo')
+  results = $extralite_db.query('select * from foo')
   raise unless results.size == count
 end
 
@@ -43,12 +42,15 @@ end
 
   prepare_database(c)
 
-  Benchmark.ips do |x|
-    x.config(:time => 3, :warmup => 1)
+  bm = Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 2)
 
     x.report("sqlite3") { sqlite3_run(c) }
     x.report("extralite") { extralite_run(c) }
 
     x.compare!
   end
+  puts;
+  bm.entries.each { |e| puts "#{e.label}: #{(e.ips * c).round.to_i} rows/s" }
+  puts;
 end

data/test/perf_hash.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# Run on Ruby 3.3 with YJIT enabled
+
 require 'bundler/inline'
 
 gemfile do
@@ -15,40 +17,40 @@ require 'fileutils'
 DB_PATH = "/tmp/extralite_sqlite3_perf-#{Time.now.to_i}-#{rand(10000)}.db"
 puts "DB_PATH = #{DB_PATH.inspect}"
 
+$sqlite3_db = SQLite3::Database.new(DB_PATH, results_as_hash: true)
+$extralite_db = Extralite::Database.new(DB_PATH, gvl_release_threshold: -1)
+
 def prepare_database(count)
-  db = Extralite::Database.new(DB_PATH)
-  db.query('create table if not exists foo ( a integer primary key, b text )')
-  db.query('delete from foo')
-  db.query('begin')
-  count.times { db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
-  db.query('commit')
-  db.close
+  $extralite_db.query('create table if not exists foo ( a integer primary key, b text )')
+  $extralite_db.query('delete from foo')
+  $extralite_db.query('begin')
+  count.times { $extralite_db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
+  $extralite_db.query('commit')
 end
 
 def sqlite3_run(count)
-  db = SQLite3::Database.new(DB_PATH, :results_as_hash => true)
-  results = db.execute('select * from foo')
+  results = $sqlite3_db.execute('select * from foo')
   raise unless results.size == count
 end
 
 def extralite_run(count)
-  db = Extralite::Database.new(DB_PATH)
-  results = db.query('select * from foo')
+  results = $extralite_db.query('select * from foo')
   raise unless results.size == count
 end
 
 [10, 1000, 100000].each do |c|
   puts "Record count: #{c}"
-
   prepare_database(c)
 
-  Benchmark.ips do |x|
-    x.config(:time => 3, :warmup => 1)
+  bm = Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 2)
 
     x.report("sqlite3") { sqlite3_run(c) }
     x.report("extralite") { extralite_run(c) }
 
     x.compare!
   end
-  puts; puts; 
+  puts;
+  bm.entries.each { |e| puts "#{e.label}: #{(e.ips * c).round.to_i} rows/s" }
+  puts;
 end

data/test/perf_hash_prepared.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'bundler/inline'
+
+gemfile do
+  source 'https://rubygems.org'
+  gem 'extralite', path: '..'
+  gem 'sqlite3'
+  gem 'benchmark-ips'
+end
+
+require 'benchmark/ips'
+require 'fileutils'
+
+DB_PATH = "/tmp/extralite_sqlite3_perf-#{Time.now.to_i}-#{rand(10000)}.db"
+puts "DB_PATH = #{DB_PATH.inspect}"
+
+def prepare_database(count)
+  $sqlite3_db = SQLite3::Database.new(DB_PATH, results_as_hash: true)
+  $extralite_db = Extralite::Database.new(DB_PATH, gvl_release_threshold: -1)
+    
+  $extralite_db.query('create table if not exists foo ( a integer primary key, b text )')
+  $extralite_db.query('delete from foo')
+  $extralite_db.query('begin')
+  count.times { $extralite_db.query('insert into foo (b) values (?)', "hello#{rand(1000)}" )}
+  $extralite_db.query('commit')
+
+  $sqlite3_stmt = $sqlite3_db.prepare('select * from foo')
+  $extralite_q = $extralite_db.prepare('select * from foo')
+end
+
+def sqlite3_run(count)
+  results = $sqlite3_stmt.execute
+  raise unless results.to_a.size == count
+end
+
+def extralite_run(count)
+  results = $extralite_q.to_a
+  raise unless results.size == count
+end
+
+[10, 1000, 100000].each do |c|
+  puts "Record count: #{c}"
+
+  prepare_database(c)
+
+  bm = Benchmark.ips do |x|
+    x.config(:time => 5, :warmup => 2)
+
+    x.report("sqlite3") { sqlite3_run(c) }
+    x.report("extralite") { extralite_run(c) }
+
+    x.compare!
+  end
+  puts;
+  bm.entries.each { |e| puts "#{e.label}: #{(e.ips * c).round.to_i} rows/s" }
+  puts;
+end