extralite 1.27 → 2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a027328ec4af0d01bc81706f4f8e1ea08770a1341d8a36953882244b2d20ed88
-  data.tar.gz: 916fabb4a5328e93bcdcb8121b54aa3e958b87915e9b1c9027cccb44941837b9
+  metadata.gz: bd8cf56eff701ddf57586d6f7510d5959cdf81ff15e6a64c6ed8ff698f93b43b
+  data.tar.gz: 42000f8c83849577fd1672996969961357c5bbadb82d89d2168c74be7acfa303
 SHA512:
-  metadata.gz: fc4cd60e38d6e229d5d28ab6e28db3a16f7ddf1ffde53e3f5a97cacebe607158604421016bab34fa5bf96659ab4d0074fffb0f51712e4429d5529f59bbcdec0d
-  data.tar.gz: db9905c1c839a4135fe4c1156b3991f8cad5ac438b5c70091be34e8afee7563371e19afe45cd835cec2f83dd26fc92e5855bf7c53c6ee473728b80caabb3e5e1
+  metadata.gz: 1159c965d8ce9bfade2c81de97b6ef311dcfdb84ce65eb85238aa6a23b29b786d31de2bb8f30f87a24a800b88d01475a7cb400b97245401d587bef365b140087
+  data.tar.gz: 74f975529a9efb7291524eaa4c285fadc6976377a607cd28e72cbd3010cbc9fd061264aabb9a5df1ce864f097ca330c9acd9dded62b8de7cc63c1217be88412a

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+# 2.1 2023-07-11
+
+- Implement `Database#execute`, `Query#execute` for data-manipulation queries
+- Add option for opening databases for read only access
+
+# 2.0 2023-07-08
+
+- Fix Sequel migrations (#8)
+- Redesign prepared statement functionality (#24)
+  - Rewrite `Extralite::PreparedStatement` into `Extralite::Query` with breaking API changes
+  - Add `Extralite::Iterator` class for external iteration
+  - Add `Query#each_xxx`, `Query#to_a_xxx` method
+  - Add `Query#eof?` method
+
 # 1.27 2023-06-12
 
 - Fix execution of prepared statements in Sequel adapter (#23 @gschlager)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    extralite (1.27)
+    extralite (2.1)
 
 GEM
   remote: https://rubygems.org/

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Sharon Rosner
+Copyright (c) 2023 Sharon Rosner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -9,7 +9,7 @@
 
 Extralite is a super fast, extra-lightweight (about 1300 lines of C-code)
 SQLite3 wrapper for Ruby. It provides a minimal set of methods for interacting
-with an SQLite3 database, as well as prepared statements.
+with an SQLite3 database, as well as prepared queries (prepared statements).
 
 Extralite comes in two flavors: the `extralite` gem which uses the
 system-installed sqlite3 library, and the `extralite-bundle` gem which bundles
@@ -26,11 +26,11 @@ latest features and enhancements.
   as arrays, single row, single column, single value.
 - Prepared statements.
 - Parameter binding.
+- External iteration - get single records or batches of records.
 - Use system-installed sqlite3, or the [bundled latest version of
   SQLite3](#installing-the-extralite-sqlite3-bundle).
 - Improved [concurrency](#concurrency) for multithreaded apps: the Ruby GVL is
   released while preparing SQL statements and while iterating over results.
-- Iterate over records with a block, or collect records into an array.
 - Automatically execute SQL strings containing multiple semicolon-separated
   queries (handy for creating/modifying schemas).
 - Execute the same query with multiple parameter lists (useful for inserting records).
@@ -111,11 +111,37 @@ db.query('select * from foo where bar = :bar', ':bar' => 42)
 db.execute_multi('insert into foo values (?)', ['bar', 'baz'])
 db.execute_multi('insert into foo values (?, ?)', [[1, 2], [3, 4]])
 
-# prepared statements
-stmt = db.prepare('select ? as foo, ? as bar') #=> Extralite::PreparedStatement
-stmt.query(1, 2) #=> [{ :foo => 1, :bar => 2 }]
-# PreparedStatement offers the same data access methods as the Database class,
-# but without the sql parameter.
+# prepared queries
+query = db.prepare('select ? as foo, ? as bar') #=> Extralite::Query
+query.bind(1, 2) #=> [{ :foo => 1, :bar => 2 }]
+
+query.next #=> next row in result_set (as hash)
+query.next_hash #=> next row in result_set (as hash)
+query.next_ary #=> next row in result_set (as array)
+query.next_single_column #=> next row in result_set (as single value)
+
+query.next(10) #=> next 10 rows in result_set (as hash)
+query.next_hash(10) #=> next 10 rows in result_set (as hash)
+query.next_ary(10) #=> next 10 rows in result_set (as array)
+query.next_single_column(10) #=> next 10 rows in result_set (as single value)
+
+query.to_a #=> all rows as array of hashes
+query.to_a_hash #=> all rows as array of hashes
+query.to_a_ary #=> all rows as array of arrays
+query.to_a_single_column #=> all rows as array of single values
+
+query.each { |r| ... } #=> iterate over all rows as hashes
+query.each_hash { |r| ... } #=> iterate over all rows as hashes
+query.each_ary { |r| ... } #=> iterate over all rows as arrays
+query.each_single_column { |r| ... } #=> iterate over all rows as single columns
+
+iterator = query.each #=> create enumerable iterator
+iterator.next #=> next row
+iterator.each { |r| ... } #=> iterate over all rows
+values = iterator.map { |r| r[:foo] * 10 } #=> map all rows
+
+iterator = query.each_ary #=> create enumerable iterator with rows as arrays
+iterator = query.each_single_column #=> create enumerable iterator with single values
 
 # get last insert rowid
 rowid = db.last_insert_rowid
@@ -195,10 +221,10 @@ end
 
 Extralite provides methods for retrieving status information about the sqlite
 runtime, database-specific status and prepared statement-specific status,
-`Extralite.runtime_status`, `Database#status` and `PreparedStatement#status`
-respectively. You can also reset the high water mark for the specific status
-code by providing true as the reset argument. The status codes mirror those
-defined by the SQLite API. Some examples:
+`Extralite.runtime_status`, `Database#status` and `Query#status` respectively.
+You can also reset the high water mark for the specific status code by providing
+true as the reset argument. The status codes mirror those defined by the SQLite
+API. Some examples:
 
 ```ruby
 # The Extralite.runtime_status returns a tuple consisting of the current value
@@ -212,9 +238,9 @@ current, high_watermark = Extralite.runtime_status(Extralite::SQLITE_STATUS_MEMO
 # argument in order to reset the high watermark):
 current, high_watermark = db.status(Extralite::SQLITE_DBSTATUS_CACHE_USED)
 
-# The PreparedStatement#status method returns a single value (pass true as a
+# The Query#status method returns a single value (pass true as a
 # second argument in order to reset the high watermark):
-value = stmt.status(Extralite::SQLITE_STMTSTATUS_RUN)
+value = query.status(Extralite::SQLITE_STMTSTATUS_RUN)
 ```
 
 ### Working with Database Limits
@@ -302,7 +328,7 @@ large number of rows.
 |1K|502.1K rows/s|2.065M rows/s|__4.11x__|
 |100K|455.7K rows/s|2.511M rows/s|__5.51x__|
 
-### Prepared Statements
+### Prepared Queries (Prepared Statements)
 
 [Benchmark source code](https://github.com/digital-fabric/extralite/blob/main/test/perf_prepared.rb)
 

data/TODO.md

@@ -0,0 +1,21 @@
+- Improve tracing
+- Transactions and savepoints:
+
+  - `DB#transaction {}` - does a `BEGIN..COMMIT` - non-reentrant!
+  - `DB#savepoint(name)` - creates a savepoint
+  - `DB#release(name)` - releases a savepoint
+  - `DB#rollback` - raises `Extralite::Rollback`, which is rescued by `DB#transaction`
+  - `DB#rollback_to(name)` - rolls back to a savepoint
+
+- More database methods:
+
+  - `Database#quote`
+  - `Database#busy_timeout=` https://sqlite.org/c3ref/busy_timeout.html
+  - `Database#cache_flush` https://sqlite.org/c3ref/db_cacheflush.html
+  - `Database#release_memory` https://sqlite.org/c3ref/db_release_memory.html
+
+- Security
+
+  - Enable extension loading by using
+    [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION](https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigenableloadextension)
+    in order to prevent usage of `load_extension()` SQL function.

data/ext/extralite/common.c

@@ -75,13 +75,11 @@ inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
   }
 }
 
-void bind_all_parameters(sqlite3_stmt *stmt, int argc, VALUE *argv) {
-  for (int i = 0; i < argc; i++) {
-    bind_parameter_value(stmt, i + 1, argv[i]);
-  }
+inline void bind_all_parameters(sqlite3_stmt *stmt, int argc, VALUE *argv) {
+  for (int i = 0; i < argc; i++) bind_parameter_value(stmt, i + 1, argv[i]);
 }
 
-void bind_all_parameters_from_object(sqlite3_stmt *stmt, VALUE obj) {
+inline void bind_all_parameters_from_object(sqlite3_stmt *stmt, VALUE obj) {
   if (TYPE(obj) == T_ARRAY) {
     int count = RARRAY_LEN(obj);
     for (int i = 0; i < count; i++)
@@ -233,22 +231,23 @@ void *stmt_iterate_without_gvl(void *ptr) {
   return NULL;
 }
 
-int stmt_iterate(sqlite3_stmt *stmt, sqlite3 *db) {
-  struct step_ctx ctx = {stmt, 0};
-  rb_thread_call_without_gvl(stmt_iterate_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
-  switch (ctx.rc) {
+inline int stmt_iterate(query_ctx *ctx) {
+  struct step_ctx step_ctx = {ctx->stmt, 0};
+  rb_thread_call_without_gvl(stmt_iterate_without_gvl, (void *)&step_ctx, RUBY_UBF_IO, 0);
+  switch (step_ctx.rc) {
     case SQLITE_ROW:
       return 1;
     case SQLITE_DONE:
+      ctx->eof = 1;
       return 0;
     case SQLITE_BUSY:
       rb_raise(cBusyError, "Database is busy");
     case SQLITE_INTERRUPT:
       rb_raise(cInterruptError, "Query was interrupted");
     case SQLITE_ERROR:
-      rb_raise(cSQLError, "%s", sqlite3_errmsg(db));
+      rb_raise(cSQLError, "%s", sqlite3_errmsg(ctx->sqlite3_db));
     default:
-      rb_raise(cError, "%s", sqlite3_errmsg(db));
+      rb_raise(cError, "%s", sqlite3_errmsg(ctx->sqlite3_db));
   }
 
   return 0;
@@ -260,50 +259,61 @@ VALUE cleanup_stmt(query_ctx *ctx) {
 }
 
 VALUE safe_query_hash(query_ctx *ctx) {
-  VALUE result = ctx->self;
-  int yield_to_block = rb_block_given_p();
-  VALUE row;
-  int column_count;
-  VALUE column_names;
-
-  column_count = sqlite3_column_count(ctx->stmt);
-  column_names = get_column_names(ctx->stmt, column_count);
-
-  // block not given, so prepare the array of records to be returned
-  if (!yield_to_block) result = rb_ary_new();
+  VALUE array = MULTI_ROW_P(ctx->mode) ? rb_ary_new() : Qnil;
+  VALUE row = Qnil;
+  int column_count = sqlite3_column_count(ctx->stmt);
+  VALUE column_names = get_column_names(ctx->stmt, column_count);
+  int row_count = 0;
 
-  while (stmt_iterate(ctx->stmt, ctx->sqlite3_db)) {
+  while (stmt_iterate(ctx)) {
     row = row_to_hash(ctx->stmt, column_count, column_names);
-    if (yield_to_block) rb_yield(row);
-    else                rb_ary_push(result, row);
+    row_count++;
+    switch (ctx->mode) {
+      case QUERY_YIELD:
+        rb_yield(row);
+        break;
+      case QUERY_MULTI_ROW:
+        rb_ary_push(array, row);
+        break;
+      case QUERY_SINGLE_ROW:
+        return row;
+    }
+    if (ctx->max_rows != ALL_ROWS && row_count >= ctx->max_rows)
+      return MULTI_ROW_P(ctx->mode) ? array : ctx->self;
   }
 
   RB_GC_GUARD(column_names);
   RB_GC_GUARD(row);
-  RB_GC_GUARD(result);
-  return result;
+  RB_GC_GUARD(array);
+  return MULTI_ROW_P(ctx->mode) ? array : Qnil;
 }
 
 VALUE safe_query_ary(query_ctx *ctx) {
-  int column_count;
-  VALUE result = ctx->self;
-  int yield_to_block = rb_block_given_p();
-  VALUE row;
-
-  column_count = sqlite3_column_count(ctx->stmt);
-
-  // block not given, so prepare the array of records to be returned
-  if (!yield_to_block) result = rb_ary_new();
+  VALUE array = MULTI_ROW_P(ctx->mode) ? rb_ary_new() : Qnil;
+  VALUE row = Qnil;
+  int column_count = sqlite3_column_count(ctx->stmt);
+  int row_count = 0;
 
-  while (stmt_iterate(ctx->stmt, ctx->sqlite3_db)) {
+  while (stmt_iterate(ctx)) {
     row = row_to_ary(ctx->stmt, column_count);
-    if (yield_to_block) rb_yield(row);
-    else                rb_ary_push(result, row);
+    row_count++;
+    switch (ctx->mode) {
+      case QUERY_YIELD:
+        rb_yield(row);
+        break;
+      case QUERY_MULTI_ROW:
+        rb_ary_push(array, row);
+        break;
+      case QUERY_SINGLE_ROW:
+        return row;
+    }
+    if (ctx->max_rows != ALL_ROWS && row_count >= ctx->max_rows)
+      return MULTI_ROW_P(ctx->mode) ? array : ctx->self;
   }
 
   RB_GC_GUARD(row);
-  RB_GC_GUARD(result);
-  return result;
+  RB_GC_GUARD(array);
+  return MULTI_ROW_P(ctx->mode) ? array : Qnil;
 }
 
 VALUE safe_query_single_row(query_ctx *ctx) {
@@ -314,7 +324,7 @@ VALUE safe_query_single_row(query_ctx *ctx) {
   column_count = sqlite3_column_count(ctx->stmt);
   column_names = get_column_names(ctx->stmt, column_count);
 
-  if (stmt_iterate(ctx->stmt, ctx->sqlite3_db))
+  if (stmt_iterate(ctx))
     row = row_to_hash(ctx->stmt, column_count, column_names);
 
   RB_GC_GUARD(row);
@@ -323,26 +333,33 @@ VALUE safe_query_single_row(query_ctx *ctx) {
 }
 
 VALUE safe_query_single_column(query_ctx *ctx) {
-  int column_count;
-  VALUE result = ctx->self;
-  int yield_to_block = rb_block_given_p();
-  VALUE value;
-
-  column_count = sqlite3_column_count(ctx->stmt);
-  if (column_count != 1)
-    rb_raise(cError, "Expected query result to have 1 column");
+  VALUE array = MULTI_ROW_P(ctx->mode) ? rb_ary_new() : Qnil;
+  VALUE value = Qnil;
+  int column_count = sqlite3_column_count(ctx->stmt);
+  int row_count = 0;
 
-  // block not given, so prepare the array of records to be returned
-  if (!yield_to_block) result = rb_ary_new();
+  if (column_count != 1) rb_raise(cError, "Expected query result to have 1 column");
 
-  while (stmt_iterate(ctx->stmt, ctx->sqlite3_db)) {
+  while (stmt_iterate(ctx)) {
     value = get_column_value(ctx->stmt, 0, sqlite3_column_type(ctx->stmt, 0));
-    if (yield_to_block) rb_yield(value); else rb_ary_push(result, value);
+    row_count++;
+    switch (ctx->mode) {
+      case QUERY_YIELD:
+        rb_yield(value);
+        break;
+      case QUERY_MULTI_ROW:
+        rb_ary_push(array, value);
+        break;
+      case QUERY_SINGLE_ROW:
+        return value;
+    }
+    if (ctx->max_rows != ALL_ROWS && row_count >= ctx->max_rows)
+      return MULTI_ROW_P(ctx->mode) ? array : ctx->self;
   }
 
   RB_GC_GUARD(value);
-  RB_GC_GUARD(result);
-  return result;
+  RB_GC_GUARD(array);
+  return MULTI_ROW_P(ctx->mode) ? array : Qnil;
 }
 
 VALUE safe_query_single_value(query_ctx *ctx) {
@@ -353,7 +370,7 @@ VALUE safe_query_single_value(query_ctx *ctx) {
   if (column_count != 1)
     rb_raise(cError, "Expected query result to have 1 column");
 
-  if (stmt_iterate(ctx->stmt, ctx->sqlite3_db))
+  if (stmt_iterate(ctx))
     value = get_column_value(ctx->stmt, 0, sqlite3_column_type(ctx->stmt, 0));
 
   RB_GC_GUARD(value);
@@ -369,7 +386,7 @@ VALUE safe_execute_multi(query_ctx *ctx) {
     sqlite3_clear_bindings(ctx->stmt);
     bind_all_parameters_from_object(ctx->stmt, RARRAY_AREF(ctx->params, i));
 
-    while (stmt_iterate(ctx->stmt, ctx->sqlite3_db));
+    while (stmt_iterate(ctx));
     changes += sqlite3_changes(ctx->sqlite3_db);
   }
 
@@ -379,3 +396,8 @@ VALUE safe_execute_multi(query_ctx *ctx) {
 VALUE safe_query_columns(query_ctx *ctx) {
   return get_column_names(ctx->stmt, sqlite3_column_count(ctx->stmt));
 }
+
+VALUE safe_query_changes(query_ctx *ctx) {
+  while (stmt_iterate(ctx));
+  return INT2FIX(sqlite3_changes(ctx->sqlite3_db));
+}