extralite 1.12 → 1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed6cb05e12c36bbbc92701af98639a9a92b91d9ba0fca0362b35fb12f3af676e
-  data.tar.gz: 6d7976fc5b39aa414a5a3431a9c5a1a6c517166d4e82de621b22da3be50fe38b
+  metadata.gz: 5d4d7fe119c7197de4a8cf2a7c54a98845a6bf3cbb5e5140c6b272e9b5d1e458
+  data.tar.gz: bfd3010a039dd6445cb0a0fe0c5dca12a1a7aa19d7c44ec3f89c5b8768f464f7
 SHA512:
-  metadata.gz: 9baa147f13fa4498264e6e942e9c154bcdcd458f503944be9385f5ea61b82a8e6d381f77a75ad7a4c7643aa124cc2a4d5aa8ffb58400efa641dcb62fad810437
-  data.tar.gz: 789d203582c86f54a8f234b67d7b54b03150b900d25a5c76e81a57d779d1735fa7a8dc9822d121df7d08af4513481bf4c0a1a6d2fb1d6dfc54d80022b2b6e559
+  metadata.gz: bc80083972aafcca792665edaeaed05dedcc8d22f3f1c277d7211d99a807344ba6ec203098aa6b4038f77318fc40aa44511d46c6d586e7ed8c75862596c7a092
+  data.tar.gz: 0a4f2d8779ba19732e1ec2f5a2367b39d0889de83fc6cf03a90cd49fae2f378ffcf7238842ca17aff4e8826c66d98158a497e200361cff6caac0ff70cde59ad9

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    extralite (1.12)
+    extralite (1.13)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -23,22 +23,24 @@
 
 ## What is Extralite?
 
-Extralite is a fast, extra-lightweight (less than 460 lines of C-code) SQLite3
-wrapper for Ruby. It provides a single class with a minimal set of methods for
-interacting with an SQLite3 database.
+Extralite is a fast, extra-lightweight (about 600 lines of C-code) SQLite3
+wrapper for Ruby. It provides a minimal set of methods for interacting with an
+SQLite3 database, as well as prepared statements. Extralite bundles the latest
+version of SQLite, offering access to the latest features and enhancements.
 
 ## Features
 
-- Zero dependencies: Extralite bundles SQLite3 version 3.37.2 - no need to
-  install any `libsqlite3` packages.
+- Zero dependencies: Extralite bundles SQLite3 version
+  [3.38.0](https://sqlite.org/releaselog/3_38_0.html) - no need to install any
+  `libsqlite3` packages.
 - A variety of methods for different data access patterns: rows as hashes, rows
   as arrays, single row, single column, single value.
+- Prepared statements.
 - Super fast - [up to 12.5x faster](#performance) than the
   [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) gem (see also
   [comparison](#why-not-just-use-the-sqlite3-gem).)
-- Improved [concurrency](#what-about-concurrency) for multithreaded apps: the
-  Ruby GVL is released while preparing SQL statements and while iterating over
-  results.
+- Improved [concurrency](#concurrency) for multithreaded apps: the Ruby GVL is
+  released while preparing SQL statements and while iterating over results.
 - Iterate over records with a block, or collect records into an array.
 - Parameter binding.
 - Automatically execute SQL strings containing multiple semicolon-separated
@@ -49,13 +51,27 @@ interacting with an SQLite3 database.
   some useful extensions here: https://github.com/nalgeon/sqlean.)
 - Includes a [Sequel adapter](#usage-with-sequel).
 
+## Installation
+
+To use Extralite in your Ruby app, add the following to your `Gemfile`:
+
+```ruby
+gem 'extralite'
+```
+
+You can also run `gem install extralite` if you just want to check it out.
+
+> **Important note**: Extralite will take a while to install (on my modest
+> machine it takes about a minute). This is owing to the fact that Extralite
+> bundles the sqlite3 code, which is compiled upon installation.
+
 ## Usage
 
 ```ruby
 require 'extralite'
 
 # get sqlite3 version
-Extralite.sqlite3_version #=> "3.37.2"
+Extralite.sqlite3_version #=> "3.38.0"
 
 # open a database
 db = Extralite::Database.new('/tmp/my.db')
@@ -94,6 +110,12 @@ db.query('select * from foo where bar = :bar', bar: 42)
 db.query('select * from foo where bar = :bar', 'bar' => 42)
 db.query('select * from foo where bar = :bar', ':bar' => 42)
 
+# prepared statements
+stmt = db.prepare('select ? as foo, ? as bar') #=> Extralite::PreparedStatement
+stmt.query(1, 2) #=> [{ :foo => 1, :bar => 2 }]
+# PreparedStatement offers the same data access methods as the Database class,
+# but without the sql parameter.
+
 # get last insert rowid
 rowid = db.last_insert_rowid
 
@@ -134,9 +156,9 @@ simpler API that gives me query results in a variety of ways. Thus extralite was
 born.
 
 Extralite is quite a bit [faster](#performance) than sqlite3-ruby and is also
-[thread-friendly](#what-about-concurrency). On the other hand, Extralite does
-not have support for defining custom functions, aggregates and collations. If
-you're using those features, you'll need to stick with sqlite3-ruby.
+[thread-friendly](#concurrency). On the other hand, Extralite does not have
+support for defining custom functions, aggregates and collations. If you're
+using any of those features, you'll have to stick to sqlite3-ruby.
 
 Here's a table summarizing the differences between the two gems:
 
@@ -145,15 +167,16 @@ Here's a table summarizing the differences between the two gems:
 |SQLite3 dependency|depends on OS-installed libsqlite3|bundles latest version of SQLite3|
 |API design|multiple classes|single class|
 |Query results|row as hash, row as array, single row, single value|row as hash, row as array, __single column__, single row, single value|
-|execute multiple statements|separate API (#execute_batch)|integrated|
+|Execute multiple statements|separate API (#execute_batch)|integrated|
+|Prepared statements|yes|yes|
 |custom functions in Ruby|yes|no|
 |custom collations|yes|no|
 |custom aggregate functions|yes|no|
-|Multithread friendly|no|[yes](#what-about-concurrency)|
-|Code size|~2650LoC|~530LoC|
+|Multithread friendly|no|[yes](#concurrency)|
+|Code size|~2650LoC|~600LoC|
 |Performance|1x|1.5x to 12.5x (see [below](#performance))|
 
-## What about concurrency?
+## Concurrency
 
 Extralite releases the GVL while making blocking calls to the sqlite3 library,
 that is while preparing SQL statements and fetching rows. Releasing the GVL
@@ -165,29 +188,42 @@ performance:
 
 A benchmark script is included, creating a table of various row counts, then
 fetching the entire table using either `sqlite3` or `extralite`. This benchmark
-shows Extralite to be up to 12.5 times faster than `sqlite3` when fetching a
-large number of rows. Here are the [results for fetching rows as hashes](https://github.com/digital-fabric/extralite/blob/main/test/perf_hash.rb):
+shows Extralite to be up to ~12 times faster than `sqlite3` when fetching a
+large number of rows.
+
+### Rows as hashes
+
+ [Benchmark source code](https://github.com/digital-fabric/extralite/blob/main/test/perf_hash.rb)
 
 |Row count|sqlite3-ruby|Extralite|Advantage|
 |-:|-:|-:|-:|
-|10|57620 rows/s|95340 rows/s|__1.65x__|
-|1K|286.8K rows/s|2106.4 rows/s|__7.35x__|
-|100K|181K rows/s|2275.3K rows/s|__12.53x__|
+|10|75.3K rows/s|134.2K rows/s|__1.78x__|
+|1K|286.8K rows/s|2106.4K rows/s|__7.35x__|
+|100K|181.0K rows/s|2275.3K rows/s|__12.53x__|
 
-When [fetching rows as arrays](https://github.com/digital-fabric/extralite/blob/main/test/perf_ary.rb) Extralite also significantly outperforms sqlite3-ruby:
+### Rows as arrays
+
+[Benchmark source code](https://github.com/digital-fabric/extralite/blob/main/test/perf_ary.rb)
 
 |Row count|sqlite3-ruby|Extralite|Advantage|
 |-:|-:|-:|-:|
-|10|64365 rows/s|94031 rows/s|__1.46x__|
+|10|64.3K rows/s|94.0K rows/s|__1.46x__|
 |1K|498.9K rows/s|2478.2K rows/s|__4.97x__|
 |100K|441.1K rows/s|3023.4K rows/s|__6.85x__|
 
-(If you're interested in checking this yourself, just run the script and let me
-know if your results are better/worse.)
+### Prepared statements
+
+[Benchmark source code](https://github.com/digital-fabric/extralite/blob/main/test/perf_prepared.rb)
+
+|Row count|sqlite3-ruby|Extralite|Advantage|
+|-:|-:|-:|-:|
+|10|241.8K rows/s|888K rows/s|__3.67x__|
+|1K|298.6K rows/s|2606K rows/s|__8.73x__|
+|100K|201.6K rows/s|1934K rows/s|__9.6x__|
 
-As those benchmarks show, Extralite is capabale of reading more than 3M
-rows/second (when fetching rows as arrays), and more than 2.2M rows/second (when
-fetching rows as hashes.)
+As those benchmarks show, Extralite is capabale of reading up to 3M rows/second
+when fetching rows as arrays, and up to 2.2M rows/second when fetching
+rows as hashes.
 
 ## Contributing
 

data/Rakefile

@@ -16,7 +16,7 @@ task :test do
   exec 'ruby test/run.rb'
 end
 
-CLEAN.include '**/*.o', '**/*.so', '**/*.so.*', '**/*.a', '**/*.bundle', '**/*.jar', 'pkg', 'tmp'
+CLEAN.include 'lib/*.o', 'lib/*.so', 'lib/*.so.*', 'lib/*.a', 'lib/*.bundle', 'lib/*.jar', 'pkg', 'tmp'
 
 require 'yard'
 YARD_FILES = FileList['ext/extralite/extralite.c', 'lib/extralite.rb', 'lib/sequel/adapters/extralite.rb']

data/bin/update_sqlite_source

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+version = ARGV[0]
+raise "Please specify version" unless version
+
+require 'fileutils'
+require 'date'
+
+FileUtils.cd '/tmp'
+
+version_id = version.gsub('.', '')
+version_id += '0' * (7 - version_id.length)
+url = "https://sqlite.org/#{Date.today.year}/sqlite-amalgamation-#{version_id}.zip"
+dest = File.expand_path('../ext/extralite', __dir__)
+
+puts "Downloading from #{url}..."
+`curl #{url} > #{version_id}.zip`
+
+puts "Unzipping zip file..."
+`unzip -o #{version_id}.zip`
+
+puts "Copying source files"
+`cp sqlite-amalgamation-#{version_id}/sqlite3.* #{dest}/`
+
+puts 'Done updating source files'

data/ext/extralite/common.c

@@ -0,0 +1,347 @@
+#include <stdio.h>
+#include "extralite.h"
+
+static inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
+  switch (type) {
+    case SQLITE_NULL:
+      return Qnil;
+    case SQLITE_INTEGER:
+      return LL2NUM(sqlite3_column_int64(stmt, col));
+    case SQLITE_FLOAT:
+      return DBL2NUM(sqlite3_column_double(stmt, col));
+    case SQLITE_TEXT:
+      return rb_str_new_cstr((char *)sqlite3_column_text(stmt, col));
+    case SQLITE_BLOB:
+      return rb_str_new((const char *)sqlite3_column_blob(stmt, col), (long)sqlite3_column_bytes(stmt, col));
+    default:
+      rb_raise(cError, "Unknown column type: %d", type);
+  }
+
+  return Qnil;
+}
+
+void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value);
+
+void bind_hash_parameter_values(sqlite3_stmt *stmt, VALUE hash) {
+  VALUE keys = rb_funcall(hash, ID_KEYS, 0);
+  long len = RARRAY_LEN(keys);
+  for (long i = 0; i < len; i++) {
+    VALUE k = RARRAY_AREF(keys, i);
+    VALUE v = rb_hash_aref(hash, k);
+
+    switch (TYPE(k)) {
+      case T_FIXNUM:
+        bind_parameter_value(stmt, NUM2INT(k), v);
+        break;
+      case T_SYMBOL:
+        k = rb_funcall(k, ID_TO_S, 0);
+      case T_STRING:
+        if(RSTRING_PTR(k)[0] != ':') k = rb_str_plus(rb_str_new2(":"), k);
+        int pos = sqlite3_bind_parameter_index(stmt, StringValuePtr(k));
+        bind_parameter_value(stmt, pos, v);
+        break;
+      default:
+        rb_raise(cError, "Cannot bind hash key value idx %ld", i);
+    }
+  }
+  RB_GC_GUARD(keys);
+}
+
+inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
+  switch (TYPE(value)) {
+    case T_NIL:
+      sqlite3_bind_null(stmt, pos);
+      return;
+    case T_FIXNUM:
+      sqlite3_bind_int64(stmt, pos, NUM2LL(value));
+      return;
+    case T_FLOAT:
+      sqlite3_bind_double(stmt, pos, NUM2DBL(value));
+      return;
+    case T_TRUE:
+      sqlite3_bind_int(stmt, pos, 1);
+      return;
+    case T_FALSE:
+      sqlite3_bind_int(stmt, pos, 0);
+      return;
+    case T_STRING:
+      sqlite3_bind_text(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
+      return;
+    case T_HASH:
+      bind_hash_parameter_values(stmt, value);
+      return;
+    default:
+      rb_raise(cError, "Cannot bind parameter at position %d", pos);
+  }
+}
+
+void bind_all_parameters(sqlite3_stmt *stmt, int argc, VALUE *argv) {
+  for (int i = 0; i < argc; i++) {
+    bind_parameter_value(stmt, i + 1, argv[i]);
+  }
+}
+
+static inline VALUE get_column_names(sqlite3_stmt *stmt, int column_count) {
+  VALUE arr = rb_ary_new2(column_count);
+  for (int i = 0; i < column_count; i++) {
+    VALUE name = ID2SYM(rb_intern(sqlite3_column_name(stmt, i)));
+    rb_ary_push(arr, name);
+  }
+  return arr;
+}
+
+static inline VALUE row_to_hash(sqlite3_stmt *stmt, int column_count, VALUE column_names) {
+  VALUE row = rb_hash_new();
+  for (int i = 0; i < column_count; i++) {
+    VALUE value = get_column_value(stmt, i, sqlite3_column_type(stmt, i));
+    rb_hash_aset(row, RARRAY_AREF(column_names, i), value);
+  }
+  return row;
+}
+
+static inline VALUE row_to_ary(sqlite3_stmt *stmt, int column_count) {
+  VALUE row = rb_ary_new2(column_count);
+  for (int i = 0; i < column_count; i++) {
+    VALUE value = get_column_value(stmt, i, sqlite3_column_type(stmt, i));
+    rb_ary_push(row, value);
+  }
+  return row;
+}
+
+typedef struct {
+  sqlite3 *db;
+  sqlite3_stmt **stmt;
+  const char *str;
+  long len;
+  int rc;
+} prepare_stmt_ctx;
+
+void *prepare_multi_stmt_without_gvl(void *ptr) {
+  prepare_stmt_ctx *ctx = (prepare_stmt_ctx *)ptr;
+  const char *rest = NULL;
+  const char *str = ctx->str;
+  const char *end = ctx->str + ctx->len;
+  while (1) {
+    ctx->rc = sqlite3_prepare_v2(ctx->db, str, end - str, ctx->stmt, &rest);
+    if (ctx->rc) {
+      // error
+      sqlite3_finalize(*ctx->stmt);
+      return NULL;
+    }
+
+    if (rest == end) return NULL;
+
+    // perform current query, but discard its results
+    ctx->rc = sqlite3_step(*ctx->stmt);
+    sqlite3_finalize(*ctx->stmt);
+    switch (ctx->rc) {
+    case SQLITE_BUSY:
+    case SQLITE_ERROR:
+    case SQLITE_MISUSE:
+      return NULL;
+    }
+    str = rest;
+  }
+  return NULL;
+}
+
+/*
+This function prepares a statement from an SQL string containing one or more SQL
+statements. It will release the GVL while the statements are being prepared and
+executed. All statements excluding the last one are executed. The last statement
+is not executed, but instead handed back to the caller for looping over results.
+*/
+void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
+  prepare_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
+  rb_thread_call_without_gvl(prepare_multi_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  RB_GC_GUARD(sql);
+
+  switch (ctx.rc) {
+  case 0:
+    return;
+  case SQLITE_BUSY:
+    rb_raise(cBusyError, "Database is busy");
+  case SQLITE_ERROR:
+    rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+  default:
+    rb_raise(cError, "Invalid return code for prepare_multi_stmt_without_gvl: %d (please open an issue on https://github.com/digital-fabric/extralite)", ctx.rc);
+  }
+}
+
+#define SQLITE_MULTI_STMT -1
+
+void *prepare_single_stmt_without_gvl(void *ptr) {
+  prepare_stmt_ctx *ctx = (prepare_stmt_ctx *)ptr;
+  const char *rest = NULL;
+  const char *str = ctx->str;
+  const char *end = ctx->str + ctx->len;
+
+  ctx->rc = sqlite3_prepare_v2(ctx->db, str, end - str, ctx->stmt, &rest);
+  if (ctx->rc)
+    goto discard_stmt;
+  else if (rest != end) {
+    ctx->rc = SQLITE_MULTI_STMT;
+    goto discard_stmt;
+  }
+  goto end;
+discard_stmt:
+  if (*ctx->stmt) {
+    sqlite3_finalize(*ctx->stmt);
+    *ctx->stmt = NULL;
+  }
+end:
+  return NULL;
+}
+
+void prepare_single_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
+  prepare_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
+  rb_thread_call_without_gvl(prepare_single_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  RB_GC_GUARD(sql);
+
+  switch (ctx.rc) {
+  case 0:
+    return;
+  case SQLITE_BUSY:
+    rb_raise(cBusyError, "Database is busy");
+  case SQLITE_ERROR:
+    rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+  case SQLITE_MULTI_STMT:
+    rb_raise(cSQLError, "A prepared statement does not accept SQL strings with multiple queries");
+  default:
+    rb_raise(cError, "Invalid return code for prepare_multi_stmt_without_gvl: %d (please open an issue on https://github.com/digital-fabric/extralite)", ctx.rc);
+  }
+}
+
+struct step_ctx {
+  sqlite3_stmt *stmt;
+  int rc;
+};
+
+void *stmt_iterate_without_gvl(void *ptr) {
+  struct step_ctx *ctx = (struct step_ctx *)ptr;
+  ctx->rc = sqlite3_step(ctx->stmt);
+  return NULL;
+}
+
+inline int stmt_iterate(sqlite3_stmt *stmt, sqlite3 *db) {
+  struct step_ctx ctx = {stmt, 0};
+  rb_thread_call_without_gvl(stmt_iterate_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
+  switch (ctx.rc) {
+    case SQLITE_ROW:
+      return 1;
+    case SQLITE_DONE:
+      return 0;
+    case SQLITE_BUSY:
+      rb_raise(cBusyError, "Database is busy");
+    case SQLITE_ERROR:
+      rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+    default:
+      rb_raise(cError, "Invalid return code for sqlite3_step: %d (please open an issue on https://github.com/digital-fabric/extralite)", ctx.rc);
+  }
+
+  return 0;
+}
+
+VALUE cleanup_stmt(query_ctx *ctx) {
+  if (ctx->stmt) sqlite3_finalize(ctx->stmt);
+  return Qnil;
+}
+
+VALUE safe_query_hash(query_ctx *ctx) {
+  VALUE result = ctx->self;
+  int yield_to_block = rb_block_given_p();
+  VALUE row;
+  int column_count;
+  VALUE column_names;
+
+  column_count = sqlite3_column_count(ctx->stmt);
+  column_names = get_column_names(ctx->stmt, column_count);
+
+  // block not given, so prepare the array of records to be returned
+  if (!yield_to_block) result = rb_ary_new();
+
+  while (stmt_iterate(ctx->stmt, ctx->sqlite3_db)) {
+    row = row_to_hash(ctx->stmt, column_count, column_names);
+    if (yield_to_block) rb_yield(row); else rb_ary_push(result, row);
+  }
+
+  RB_GC_GUARD(column_names);
+  RB_GC_GUARD(row);
+  RB_GC_GUARD(result);
+  return result;
+}
+
+VALUE safe_query_ary(query_ctx *ctx) {
+  int column_count;
+  VALUE result = ctx->self;
+  int yield_to_block = rb_block_given_p();
+  VALUE row;
+
+  column_count = sqlite3_column_count(ctx->stmt);
+
+  // block not given, so prepare the array of records to be returned
+  if (!yield_to_block) result = rb_ary_new();
+
+  while (stmt_iterate(ctx->stmt, ctx->sqlite3_db)) {
+    row = row_to_ary(ctx->stmt, column_count);
+    if (yield_to_block) rb_yield(row); else rb_ary_push(result, row);
+  }
+
+  RB_GC_GUARD(row);
+  RB_GC_GUARD(result);
+  return result;
+}
+
+VALUE safe_query_single_row(query_ctx *ctx) {
+  int column_count;
+  VALUE row = Qnil;
+  VALUE column_names;
+
+  column_count = sqlite3_column_count(ctx->stmt);
+  column_names = get_column_names(ctx->stmt, column_count);
+
+  if (stmt_iterate(ctx->stmt, ctx->sqlite3_db))
+    row = row_to_hash(ctx->stmt, column_count, column_names);
+
+  RB_GC_GUARD(row);
+  RB_GC_GUARD(column_names);
+  return row;
+}
+
+VALUE safe_query_single_column(query_ctx *ctx) {
+  int column_count;
+  VALUE result = ctx->self;
+  int yield_to_block = rb_block_given_p();
+  VALUE value;
+
+  column_count = sqlite3_column_count(ctx->stmt);
+  if (column_count != 1)
+    rb_raise(cError, "Expected query result to have 1 column");
+
+  // block not given, so prepare the array of records to be returned
+  if (!yield_to_block) result = rb_ary_new();
+
+  while (stmt_iterate(ctx->stmt, ctx->sqlite3_db)) {
+    value = get_column_value(ctx->stmt, 0, sqlite3_column_type(ctx->stmt, 0));
+    if (yield_to_block) rb_yield(value); else rb_ary_push(result, value);
+  }
+
+  RB_GC_GUARD(value);
+  RB_GC_GUARD(result);
+  return result;
+}
+
+VALUE safe_query_single_value(query_ctx *ctx) {
+  int column_count;
+  VALUE value = Qnil;
+
+  column_count = sqlite3_column_count(ctx->stmt);
+  if (column_count != 1)
+    rb_raise(cError, "Expected query result to have 1 column");
+
+  if (stmt_iterate(ctx->stmt, ctx->sqlite3_db))
+    value = get_column_value(ctx->stmt, 0, sqlite3_column_type(ctx->stmt, 0));
+
+  RB_GC_GUARD(value);
+  return value;
+}