extralite 2.3 → 2.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 130809d0ec92b7ae91e31f05e24c072500ef3096bda542a293469f1e206ff8eb
4
- data.tar.gz: b7236d13577003ed2ccd806eb31b3a6703d3126196147f78220504b7fd0264b8
3
+ metadata.gz: d8ccbfbf37744a5b90d30038c3ba4e73f7d2a094a0f18f21cd8b58d66ec32b42
4
+ data.tar.gz: 5c6d3f9c967f7056f8eeb7ba471e2db50a26be334bdeec7ddf80dbd521efff77
5
5
  SHA512:
6
- metadata.gz: 44b5ab1374077c97418b14581947aaa2dd20010590e21beaa755be9aaa6228b1408e5593c69f9f901f7935b0ba58494772cd9c0062b7632dc8c48d4ea54845bd
7
- data.tar.gz: 218b35977b8ce307b9e738dfb280bfcfa4e67910542d03cf11aaec36acbaf61ba74db5c10f5b2d21364384c9cfbc830f91f945d5417dd00b6b090895dadfe276
6
+ metadata.gz: 19f10ce33c25e3b0837ec452b8eff0c4c433a729c79162d8b73e65411142f6df61703e0d7c254ac269fbf02ae39aa1c0cde5cb8f9fe3f262ab9ad0af503ceb99
7
+ data.tar.gz: 921aeeb63e23bfc145af2c978fb3ce5a47b643cf23e265aaf581eb067e8a76ef5c63effde30c567d7565a19fa8f844868456f996d5413a70ef2ba774a927d638
data/.editorconfig ADDED
@@ -0,0 +1,6 @@
1
+ [*]
2
+ charset = utf-8
3
+ trim_trailing_whitespace = true
4
+ insert_final_newline = true
5
+ indent_style = space
6
+ indent_size = 2
@@ -2,6 +2,10 @@ name: Tests
2
2
 
3
3
  on: [push, pull_request]
4
4
 
5
+ concurrency:
6
+ group: tests-${{ format('{0}-{1}', github.head_ref || github.run_number, github.job) }}
7
+ cancel-in-progress: true
8
+
5
9
  jobs:
6
10
  build:
7
11
  strategy:
@@ -15,7 +19,7 @@ jobs:
15
19
 
16
20
  runs-on: ${{matrix.os}}
17
21
  steps:
18
- - uses: actions/checkout@v3
22
+ - uses: actions/checkout@v4
19
23
  - uses: ruby/setup-ruby@v1
20
24
  with:
21
25
  ruby-version: ${{matrix.ruby}}
@@ -24,3 +28,13 @@ jobs:
24
28
  run: bundle exec rake compile
25
29
  - name: Run tests
26
30
  run: bundle exec rake test
31
+ - name: Build bundled gem
32
+ run: bundle exec rake build_bundled
33
+ - name: Run tests with bundled gem
34
+ run: |
35
+ echo Installing extralite-bundle...
36
+ bundle config --local frozen false
37
+ mv Gemfile-bundle Gemfile
38
+ bundle install
39
+ echo Running tests...
40
+ bundle exec rake test
data/.gitignore CHANGED
@@ -10,6 +10,9 @@
10
10
  /test/version_tmp/
11
11
  /tmp/
12
12
  lib/extralite_ext.*
13
+ Gemfile-bundle.lock
14
+ /cmake-build-debug/
15
+ CMakeLists.txt
13
16
 
14
17
  # Used by dotenv library to load environment variables.
15
18
  # .env
data/CHANGELOG.md CHANGED
@@ -1,3 +1,15 @@
1
+ # 2.4 2023-12-24
2
+
3
+ - Implement GVL release threshold. [#46](https://github.com/digital-fabric/extralite/pull/46)
4
+ - Implement write barriers for better GC performance.
5
+ - Add support for binding large numbers and symbols. [#43](https://github.com/digital-fabric/extralite/pull/43)
6
+ - Implement Database#transaction. [#42](https://github.com/digital-fabric/extralite/pull/42)
7
+ - Add support for binding BLOBs. [#40](https://github.com/digital-fabric/extralite/pull/40)
8
+ - Minor fixes and cleanup of C-code.
9
+ - Fix `Database#inspect` for a closed database instance [#37](https://github.com/digital-fabric/extralite/issues/37)
10
+ - Add support for binding named parameters from Struct and Data classes [#30](https://github.com/digital-fabric/extralite/pull/30)
11
+ - Update bundled SQLite code to version 3.44.2 [#32](https://github.com/digital-fabric/extralite/pull/32)
12
+
1
13
  # 2.3 2023-11-12
2
14
 
3
15
  - Update bundled SQLite to version 3.44.0 (#29)
data/Gemfile-bundle ADDED
@@ -0,0 +1,5 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec name: 'extralite'
4
+
5
+ gem "extralite-bundle", git: "file://#{File.expand_path(__dir__)}", ref: `git rev-parse --abbrev-ref HEAD`.strip
data/Gemfile.lock CHANGED
@@ -1,13 +1,13 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- extralite (2.3)
4
+ extralite (2.4)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
8
8
  specs:
9
9
  docile (1.4.0)
10
- json (2.6.3)
10
+ json (2.7.1)
11
11
  minitest (5.15.0)
12
12
  rake (13.1.0)
13
13
  rake-compiler (1.1.6)
@@ -34,4 +34,4 @@ DEPENDENCIES
34
34
  yard (= 0.9.27)
35
35
 
36
36
  BUNDLED WITH
37
- 2.3.3
37
+ 2.4.22
data/README.md CHANGED
@@ -14,14 +14,13 @@ with an SQLite3 database, as well as prepared queries (prepared statements).
14
14
  Extralite comes in two flavors: the `extralite` gem which uses the
15
15
  system-installed sqlite3 library, and the `extralite-bundle` gem which bundles
16
16
  the latest version of SQLite
17
- ([3.44.0](https://sqlite.org/releaselog/3_44_0.html)), offering access to the
17
+ ([3.44.2](https://sqlite.org/releaselog/3_44_2.html)), offering access to the
18
18
  latest features and enhancements.
19
19
 
20
20
  ## Features
21
21
 
22
22
  - Super fast - [up to 11x faster](#performance) than the
23
- [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) gem (see also
24
- [comparison](#why-not-just-use-the-sqlite3-gem).)
23
+ [sqlite3](https://github.com/sparklemotion/sqlite3-ruby) gem.
25
24
  - A variety of methods for different data access patterns: rows as hashes, rows
26
25
  as arrays, single row, single column, single value.
27
26
  - Prepared statements.
@@ -30,10 +29,12 @@ latest features and enhancements.
30
29
  - Use system-installed sqlite3, or the [bundled latest version of
31
30
  SQLite3](#installing-the-extralite-sqlite3-bundle).
32
31
  - Improved [concurrency](#concurrency) for multithreaded apps: the Ruby GVL is
33
- released while preparing SQL statements and while iterating over results.
32
+ released peridically while preparing SQL statements and while iterating over
33
+ results.
34
34
  - Automatically execute SQL strings containing multiple semicolon-separated
35
35
  queries (handy for creating/modifying schemas).
36
- - Execute the same query with multiple parameter lists (useful for inserting records).
36
+ - Execute the same query with multiple parameter lists (useful for inserting
37
+ records).
37
38
  - Load extensions (loading of extensions is autmatically enabled. You can find
38
39
  some useful extensions here: https://github.com/nalgeon/sqlean.)
39
40
  - Includes a [Sequel adapter](#usage-with-sequel).
@@ -101,12 +102,24 @@ db.query_single_value("select 'foo'") #=> "foo"
101
102
 
102
103
  # parameter binding (works for all query_xxx methods)
103
104
  db.query_hash('select ? as foo, ? as bar', 1, 2) #=> [{ :foo => 1, :bar => 2 }]
105
+ db.query_hash('select ?2 as foo, ?1 as bar, ?1 * ?2 as baz', 6, 7) #=> [{ :foo => 7, :bar => 6, :baz => 42 }]
104
106
 
105
107
  # parameter binding of named parameters
106
108
  db.query('select * from foo where bar = :bar', bar: 42)
107
109
  db.query('select * from foo where bar = :bar', 'bar' => 42)
108
110
  db.query('select * from foo where bar = :bar', ':bar' => 42)
109
111
 
112
+ # parameter binding of named parameters from Struct and Data
113
+ SomeStruct = Struct.new(:foo, :bar)
114
+ db.query_single_column('select :bar', SomeStruct.new(41, 42)) #=> [42]
115
+ SomeData = Data.define(:foo, :bar)
116
+ db.query_single_column('select :bar', SomeData.new(foo: 41, bar: 42)) #=> [42]
117
+
118
+ # parameter binding for binary data (BLOBs)
119
+ db.execute('insert into foo values (?)', File.binread('/path/to/file'))
120
+ db.execute('insert into foo values (?)', Extralite::Blob.new('Hello, 世界!'))
121
+ db.execute('insert into foo values (?)', 'Hello, 世界!'.force_encoding(Encoding::ASCII_8BIT))
122
+
110
123
  # insert multiple rows
111
124
  db.execute_multi('insert into foo values (?)', ['bar', 'baz'])
112
125
  db.execute_multi('insert into foo values (?, ?)', [[1, 2], [3, 4]])
@@ -166,6 +179,13 @@ db.pragma(:journal_mode) #=> 'wal'
166
179
  # load an extension
167
180
  db.load_extension('/path/to/extension.so')
168
181
 
182
+ # run queries in a transaction
183
+ db.transaction do
184
+ db.execute('insert into foo values (?)', 1)
185
+ db.execute('insert into foo values (?)', 2)
186
+ db.execute('insert into foo values (?)', 3)
187
+ end
188
+
169
189
  # close database
170
190
  db.close
171
191
  db.closed? #=> true
@@ -194,6 +214,23 @@ ensure
194
214
  end
195
215
  ```
196
216
 
217
+ ### Running transactions
218
+
219
+ In order to run multiple queries in a single transaction, use
220
+ `Database#transaction`, passing a block that runs the queries. You can specify
221
+ the [transaction
222
+ mode](https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions).
223
+ The default mode is `:immediate`:
224
+
225
+ ```ruby
226
+ db.transaction { ... } # Run an immediate transaction
227
+ db.transaction(:deferred) { ... } # Run a deferred transaction
228
+ db.transaction(:exclusive) { ... } # Run an exclusive transaction
229
+ ```
230
+
231
+ If an exception is raised in the given block, the transaction will be rolled
232
+ back. Otherwise, it is committed.
233
+
197
234
  ### Creating Backups
198
235
 
199
236
  You can use `Database#backup` to create backup copies of a database. The
@@ -295,11 +332,33 @@ p articles.to_a
295
332
 
296
333
  ## Concurrency
297
334
 
298
- Extralite releases the GVL while making blocking calls to the sqlite3 library,
299
- that is while preparing SQL statements and fetching rows. Releasing the GVL
300
- allows other threads to run while the sqlite3 library is busy compiling SQL into
301
- bytecode, or fetching the next row. This *does not* hurt Extralite's
302
- performance, as you can see:
335
+ Extralite releases the GVL while making calls to the sqlite3 library that might
336
+ block, such as when backing up a database, or when preparing a query. Extralite
337
+ also releases the GVL periodically when iterating over records. By default, the
338
+ GVL is released every 1000 records iterated. The GVL release threshold can be
339
+ set separately for each database:
340
+
341
+ ```ruby
342
+ db.gvl_release_threshold = 10 # release GVL every 10 records
343
+
344
+ db.gvl_release_threshold = nil # use default value (currently 1000)
345
+ ```
346
+
347
+ For most applications, there's no need to tune the GVL threshold value, as it
348
+ provides [excellent](#performance) performance characteristics for both single-threaded and
349
+ multi-threaded applications.
350
+
351
+ In a heavily multi-threaded application, releasing the GVL more often (lower
352
+ threshold value) will lead to less latency (for threads not running a query),
353
+ but will also hurt the throughput (for the thread running the query). Releasing
354
+ the GVL less often (higher threshold value) will lead to better throughput for
355
+ queries, while increasing latency for threads not running a query. The following
356
+ diagram demonstrates the relationship between the GVL release threshold value,
357
+ latency and throughput:
358
+
359
+ ```
360
+ less latency & throughput <<< GVL release threshold >>> more latency & throughput
361
+ ```
303
362
 
304
363
  ## Performance
305
364
 
@@ -9,10 +9,10 @@ require 'date'
9
9
 
10
10
  FileUtils.cd '/tmp'
11
11
 
12
- version_id = version.gsub('.', '')
12
+ version_id = version.split('.').each_with_index.map { |v, i| i == 0 ? v : v.rjust(2, '0') }.join
13
13
  version_id += '0' * (7 - version_id.length)
14
14
  url = "https://sqlite.org/#{Date.today.year}/sqlite-amalgamation-#{version_id}.zip"
15
- dest = File.expand_path('../ext/extralite', __dir__)
15
+ dest = File.expand_path('../ext/sqlite3', __dir__)
16
16
 
17
17
  puts "Downloading from #{url}..."
18
18
  `curl #{url} > #{version_id}.zip`
@@ -23,4 +23,11 @@ puts "Unzipping zip file..."
23
23
  puts "Copying source files"
24
24
  `cp sqlite-amalgamation-#{version_id}/sqlite3.* #{dest}/`
25
25
 
26
- puts 'Done updating source files'
26
+ puts "Updating README"
27
+ readme_path = File.expand_path('../README.md', __dir__)
28
+ readme = File.read(readme_path)
29
+ readme.gsub!(/\[\d+\.\d+\.\d+\]/, "[#{version}]")
30
+ readme.gsub!(/\d+_\d+_\d+\.html/, "#{version.gsub('.', '_')}.html")
31
+ File.write(readme_path, readme)
32
+
33
+ puts 'Done updating source files'
@@ -3,6 +3,15 @@
3
3
 
4
4
  rb_encoding *UTF8_ENCODING;
5
5
 
6
+ inline void *gvl_call(enum gvl_mode mode, void *(*fn)(void *), void *data) {
7
+ switch (mode) {
8
+ case GVL_RELEASE:
9
+ return rb_thread_call_without_gvl(fn, data, RUBY_UBF_IO, 0);
10
+ default:
11
+ return fn(data);
12
+ }
13
+ }
14
+
6
15
  static inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
7
16
  switch (type) {
8
17
  case SQLITE_NULL:
@@ -24,37 +33,52 @@ static inline VALUE get_column_value(sqlite3_stmt *stmt, int col, int type) {
24
33
 
25
34
  void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value);
26
35
 
36
+ static inline void bind_key_value(sqlite3_stmt *stmt, VALUE k, VALUE v) {
37
+ switch (TYPE(k)) {
38
+ case T_FIXNUM:
39
+ bind_parameter_value(stmt, FIX2INT(k), v);
40
+ break;
41
+ case T_SYMBOL:
42
+ k = rb_sym2str(k);
43
+ case T_STRING:
44
+ if (RSTRING_PTR(k)[0] != ':') k = rb_str_plus(rb_str_new2(":"), k);
45
+ int pos = sqlite3_bind_parameter_index(stmt, StringValuePtr(k));
46
+ bind_parameter_value(stmt, pos, v);
47
+ break;
48
+ default:
49
+ rb_raise(cParameterError, "Cannot bind parameter with a key of type %"PRIsVALUE"",
50
+ rb_class_name(rb_obj_class(k)));
51
+ }
52
+ }
53
+
27
54
  void bind_hash_parameter_values(sqlite3_stmt *stmt, VALUE hash) {
28
55
  VALUE keys = rb_funcall(hash, ID_keys, 0);
29
56
  long len = RARRAY_LEN(keys);
30
57
  for (long i = 0; i < len; i++) {
31
58
  VALUE k = RARRAY_AREF(keys, i);
32
59
  VALUE v = rb_hash_aref(hash, k);
33
-
34
- switch (TYPE(k)) {
35
- case T_FIXNUM:
36
- bind_parameter_value(stmt, FIX2INT(k), v);
37
- break;
38
- case T_SYMBOL:
39
- k = rb_funcall(k, ID_to_s, 0);
40
- case T_STRING:
41
- if(RSTRING_PTR(k)[0] != ':') k = rb_str_plus(rb_str_new2(":"), k);
42
- int pos = sqlite3_bind_parameter_index(stmt, StringValuePtr(k));
43
- bind_parameter_value(stmt, pos, v);
44
- break;
45
- default:
46
- rb_raise(cError, "Cannot bind hash key value idx %ld", i);
47
- }
60
+ bind_key_value(stmt, k, v);
48
61
  }
49
62
  RB_GC_GUARD(keys);
50
63
  }
51
64
 
65
+ void bind_struct_parameter_values(sqlite3_stmt *stmt, VALUE struct_obj) {
66
+ VALUE members = rb_struct_members(struct_obj);
67
+ for (long i = 0; i < RSTRUCT_LEN(struct_obj); i++) {
68
+ VALUE k = rb_ary_entry(members, i);
69
+ VALUE v = RSTRUCT_GET(struct_obj, i);
70
+ bind_key_value(stmt, k, v);
71
+ }
72
+ RB_GC_GUARD(members);
73
+ }
74
+
52
75
  inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
53
76
  switch (TYPE(value)) {
54
77
  case T_NIL:
55
78
  sqlite3_bind_null(stmt, pos);
56
79
  return;
57
80
  case T_FIXNUM:
81
+ case T_BIGNUM:
58
82
  sqlite3_bind_int64(stmt, pos, NUM2LL(value));
59
83
  return;
60
84
  case T_FLOAT:
@@ -66,14 +90,23 @@ inline void bind_parameter_value(sqlite3_stmt *stmt, int pos, VALUE value) {
66
90
  case T_FALSE:
67
91
  sqlite3_bind_int(stmt, pos, 0);
68
92
  return;
93
+ case T_SYMBOL:
94
+ value = rb_sym2str(value);
69
95
  case T_STRING:
70
- sqlite3_bind_text(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
96
+ if (rb_enc_get_index(value) == rb_ascii8bit_encindex() || CLASS_OF(value) == cBlob)
97
+ sqlite3_bind_blob(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
98
+ else
99
+ sqlite3_bind_text(stmt, pos, RSTRING_PTR(value), RSTRING_LEN(value), SQLITE_TRANSIENT);
71
100
  return;
72
101
  case T_HASH:
73
102
  bind_hash_parameter_values(stmt, value);
74
103
  return;
104
+ case T_STRUCT:
105
+ bind_struct_parameter_values(stmt, value);
106
+ return;
75
107
  default:
76
- rb_raise(cError, "Cannot bind parameter at position %d", pos);
108
+ rb_raise(cParameterError, "Cannot bind parameter at position %d of type %"PRIsVALUE"",
109
+ pos, rb_class_name(rb_obj_class(value)));
77
110
  }
78
111
  }
79
112
 
@@ -126,7 +159,7 @@ typedef struct {
126
159
  int rc;
127
160
  } prepare_stmt_ctx;
128
161
 
129
- void *prepare_multi_stmt_without_gvl(void *ptr) {
162
+ void *prepare_multi_stmt_impl(void *ptr) {
130
163
  prepare_stmt_ctx *ctx = (prepare_stmt_ctx *)ptr;
131
164
  const char *rest = NULL;
132
165
  const char *str = ctx->str;
@@ -163,7 +196,7 @@ is not executed, but instead handed back to the caller for looping over results.
163
196
  */
164
197
  void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
165
198
  prepare_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
166
- rb_thread_call_without_gvl(prepare_multi_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
199
+ gvl_call(GVL_RELEASE, prepare_multi_stmt_impl, (void *)&ctx);
167
200
  RB_GC_GUARD(sql);
168
201
 
169
202
  switch (ctx.rc) {
@@ -180,7 +213,7 @@ void prepare_multi_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
180
213
 
181
214
  #define SQLITE_MULTI_STMT -1
182
215
 
183
- void *prepare_single_stmt_without_gvl(void *ptr) {
216
+ void *prepare_single_stmt_impl(void *ptr) {
184
217
  prepare_stmt_ctx *ctx = (prepare_stmt_ctx *)ptr;
185
218
  const char *rest = NULL;
186
219
  const char *str = ctx->str;
@@ -205,7 +238,7 @@ end:
205
238
 
206
239
  void prepare_single_stmt(sqlite3 *db, sqlite3_stmt **stmt, VALUE sql) {
207
240
  prepare_stmt_ctx ctx = {db, stmt, RSTRING_PTR(sql), RSTRING_LEN(sql), 0};
208
- rb_thread_call_without_gvl(prepare_single_stmt_without_gvl, (void *)&ctx, RUBY_UBF_IO, 0);
241
+ gvl_call(GVL_RELEASE, prepare_single_stmt_impl, (void *)&ctx);
209
242
  RB_GC_GUARD(sql);
210
243
 
211
244
  switch (ctx.rc) {
@@ -227,15 +260,26 @@ struct step_ctx {
227
260
  int rc;
228
261
  };
229
262
 
230
- void *stmt_iterate_without_gvl(void *ptr) {
263
+ void *stmt_iterate_step(void *ptr) {
231
264
  struct step_ctx *ctx = (struct step_ctx *)ptr;
232
265
  ctx->rc = sqlite3_step(ctx->stmt);
233
266
  return NULL;
234
267
  }
235
268
 
269
+ inline enum gvl_mode stepwise_gvl_mode(query_ctx *ctx) {
270
+ // a negative or zero threshold means the GVL is always held during iteration.
271
+ if (ctx->gvl_release_threshold <= 0) return GVL_HOLD;
272
+
273
+ if (!sqlite3_stmt_busy(ctx->stmt)) return GVL_RELEASE;
274
+
275
+ // if positive, the GVL is normally held, and release every <threshold> steps.
276
+ return (ctx->step_count % ctx->gvl_release_threshold) ? GVL_HOLD : GVL_RELEASE;
277
+ }
278
+
236
279
  inline int stmt_iterate(query_ctx *ctx) {
237
280
  struct step_ctx step_ctx = {ctx->stmt, 0};
238
- rb_thread_call_without_gvl(stmt_iterate_without_gvl, (void *)&step_ctx, RUBY_UBF_IO, 0);
281
+ ctx->step_count += 1;
282
+ gvl_call(stepwise_gvl_mode(ctx), stmt_iterate_step, (void *)&step_ctx);
239
283
  switch (step_ctx.rc) {
240
284
  case SQLITE_ROW:
241
285
  return 1;