extralite 2.6 → 2.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -48,4 +48,22 @@ end
48
48
  task :build_bundled do
49
49
  puts 'Building extralite-bundle...'
50
50
  `gem build extralite-bundle.gemspec`
51
+ end
52
+
53
+ test_config = lambda do |t|
54
+ t.libs << "test"
55
+ t.libs << "lib"
56
+ t.test_files = FileList["test/**/test_*.rb"]
57
+ end
58
+
59
+ # Rake::TestTask.new(:test, &test_config)
60
+
61
+ begin
62
+ require "ruby_memcheck"
63
+
64
+ namespace :test do
65
+ RubyMemcheck::TestTask.new(:valgrind, &test_config)
66
+ end
67
+ rescue LoadError => e
68
+ warn("NOTE: ruby_memcheck is not available in this environment: #{e}")
51
69
  end
data/TODO.md CHANGED
@@ -1,12 +1,3 @@
1
- - Transactions and savepoints:
2
-
3
- - https://www.sqlite.org/lang_savepoint.html
4
-
5
- - `DB#savepoint(name)` - creates a savepoint
6
- - `DB#release(name)` - releases a savepoint
7
- - `DB#rollback` - raises `Extralite::Rollback`, which is rescued by `DB#transaction`
8
- - `DB#rollback_to(name)` - rolls back to a savepoint
9
-
10
1
  - More database methods:
11
2
 
12
3
  - `Database#quote`
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/inline'
4
+
5
+ gemfile do
6
+ gem 'polyphony'
7
+ gem 'extralite', path: '.'
8
+ end
9
+
10
+ class KVStore
11
+ def initialize(db)
12
+ @db = db
13
+ setup_kv_tables
14
+ setup_queries
15
+ end
16
+
17
+ SETUP_SQL = <<~SQL
18
+ create table if not exists kv (key text primary key, value, expires float);
19
+ create index if not exists idx_kv_expires on kv (expires) where expires is not null;
20
+ SQL
21
+
22
+ def setup_kv_tables
23
+ @db.execute SETUP_SQL
24
+ end
25
+
26
+ def setup_queries
27
+ @q_get = @db.prepare_argv('select value from kv where key = ?')
28
+ @q_set = @db.prepare('insert into kv (key, value) values($1, $2) on conflict (key) do update set value = $2')
29
+ end
30
+
31
+ def get(key)
32
+ @q_get.bind(key).next
33
+ end
34
+
35
+ def set(key, value)
36
+ @q_set.bind(key, value).execute
37
+ end
38
+ end
39
+
40
+ db = Extralite::Database.new(':memory:')
41
+ kv = KVStore.new(db)
42
+
43
+ p get: kv.get('foo')
44
+ p set: kv.set('foo', 42)
45
+ p get: kv.get('foo')
46
+ p set: kv.set('foo', 43)
47
+ p get: kv.get('foo')
48
+
49
+ p db.query('select * from kv order by key')
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require './lib/extralite'
4
+ require 'fiber'
5
+
6
+ f1 = Fiber.new do |f2|
7
+ while true
8
+ STDOUT << '.'
9
+ f2 = f2.transfer
10
+ end
11
+ end
12
+
13
+ db = Extralite::Database.new(':memory:')
14
+ db.on_progress(1) { f1.transfer(Fiber.current) }
15
+
16
+ p db.query('select 1, 2, 3')
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require './lib/extralite'
4
+
5
+ db = Extralite::Database.new(':memory:')
6
+ count = 0
7
+ db.on_progress(10) { count += 1 }
8
+ 10000.times { db.query('select 1') }
9
+ p count: count
@@ -0,0 +1,194 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/inline'
4
+
5
+ gemfile do
6
+ gem 'polyphony'
7
+ gem 'extralite', path: '.'
8
+ gem 'benchmark-ips'
9
+ end
10
+
11
+ require 'tempfile'
12
+
13
+ class PubSub
14
+ def initialize(db)
15
+ @db = db
16
+ end
17
+
18
+ def attach
19
+ @db.execute('insert into subscribers (stamp) values (?)', Time.now.to_i)
20
+ @id = @db.last_insert_rowid
21
+ end
22
+
23
+ def detach
24
+ @db.execute('delete from subscribers where id = ?', @id)
25
+ end
26
+
27
+ def subscribe(*topics)
28
+ @db.transaction do
29
+ topics.each do |t|
30
+ @db.execute('insert into subscriber_topics (subscriber_id, topic) values (?, ?)', @id, t)
31
+ end
32
+ end
33
+ end
34
+
35
+ def unsubscribe(*topics)
36
+ @db.transaction do
37
+ topics.each do |t|
38
+ @db.execute('delete from subscriber_topics where subscriber_id = ? and topic = ?', @id, t)
39
+ end
40
+ end
41
+ end
42
+
43
+ def get_messages(&block)
44
+ # @db.execute('update subscribers set stamp = ? where id = ?', Time.now.to_i, @id)
45
+ @db.query_argv('delete from messages where subscriber_id = ? returning topic, message', @id, &block)
46
+ end
47
+
48
+ SCHEMA = <<~SQL
49
+ PRAGMA foreign_keys = ON;
50
+
51
+ create table if not exists subscribers (
52
+ id integer primary key,
53
+ stamp float
54
+ );
55
+ create index if not exists idx_subscribers_stamp on subscribers (stamp);
56
+
57
+ create table if not exists subscriber_topics (
58
+ subscriber_id integer references subscribers(id) on delete cascade,
59
+ topic text
60
+ );
61
+
62
+ create table if not exists messages(
63
+ subscriber_id integer,
64
+ topic text,
65
+ message text,
66
+ foreign key (subscriber_id, topic)
67
+ references subscriber_topics(subscriber_id, topic)
68
+ on delete cascade
69
+ );
70
+ create index if not exists idx_messages_subscriber_id_topic on messages (subscriber_id, topic);
71
+ SQL
72
+
73
+ def setup
74
+ @db.transaction do
75
+ @db.execute(SCHEMA)
76
+ end
77
+ end
78
+
79
+ def publish(topic, message)
80
+ @db.execute("
81
+ with subscribed as (
82
+ select subscriber_id
83
+ from subscriber_topics
84
+ where topic = $1
85
+ )
86
+ insert into messages
87
+ select subscriber_id, $1, $2
88
+ from subscribed
89
+ ", topic, message)
90
+ end
91
+
92
+ def prune_subscribers
93
+ @db.execute('delete from subscribers where stamp < ?', Time.now.to_i - 3600)
94
+ end
95
+ end
96
+
97
+ fn = Tempfile.new('pubsub_store').path
98
+ p fn: fn
99
+ db1 = Extralite::Database.new(fn)
100
+ db1.pragma(journal_mode: :wal, synchronous: 1)
101
+ db2 = Extralite::Database.new(fn)
102
+ db2.pragma(journal_mode: :wal, synchronous: 1)
103
+ db3 = Extralite::Database.new(fn)
104
+ db3.pragma(journal_mode: :wal, synchronous: 1)
105
+
106
+ db1.on_progress(1000) { |b| b ? sleep(0.0001) : snooze }
107
+ db2.on_progress(1000) { |b| b ? sleep(0.0001) : snooze }
108
+ db3.on_progress(1000) { |b| b ? sleep(0.0001) : snooze }
109
+
110
+ producer = PubSub.new(db1)
111
+ producer.setup
112
+
113
+ consumer1 = PubSub.new(db2)
114
+ consumer1.setup
115
+ consumer1.attach
116
+ consumer1.subscribe('foo', 'bar')
117
+
118
+ consumer2 = PubSub.new(db3)
119
+ consumer2.setup
120
+ consumer2.attach
121
+ consumer2.subscribe('foo', 'baz')
122
+
123
+
124
+ # producer.publish('foo', 'foo1')
125
+ # producer.publish('bar', 'bar1')
126
+ # producer.publish('baz', 'baz1')
127
+
128
+ # puts "- get messages"
129
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
130
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
131
+
132
+ # puts "- get messages again"
133
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
134
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
135
+
136
+ topics = %w{bar baz}
137
+
138
+ publish_count = 0
139
+ f1 = spin do
140
+ while true
141
+ message = "message#{rand(1000)}"
142
+ producer.publish(topics.sample, message)
143
+
144
+ publish_count += 1
145
+ snooze
146
+ end
147
+ end
148
+
149
+ receive_count = 0
150
+ f2 = spin do
151
+ while true
152
+ consumer1.get_messages { |t, m| receive_count += 1 }
153
+ sleep 0.1
154
+ end
155
+ end
156
+
157
+ f3 = spin do
158
+ while true
159
+ consumer2.get_messages { |t, m| receive_count += 1 }
160
+ sleep 0.1
161
+ end
162
+ end
163
+
164
+ db4 = Extralite::Database.new(fn)
165
+ db4.pragma(journal_mode: :wal, synchronous: 1)
166
+ db4.on_progress(1000) { |busy| busy ? sleep(0.05) : snooze }
167
+
168
+ last_t = Time.now
169
+ last_publish_count = 0
170
+ last_receive_count = 0
171
+ while true
172
+ sleep 1
173
+ now = Time.now
174
+ elapsed = now - last_t
175
+ d_publish = publish_count - last_publish_count
176
+ d_receive = receive_count - last_receive_count
177
+ pending = db4.query_single_argv('select count(*) from messages')
178
+ puts "#{Time.now} publish: #{d_publish/elapsed}/s receive: #{d_receive/elapsed}/s pending: #{pending}"
179
+ last_t = now
180
+ last_publish_count = publish_count
181
+ last_receive_count = receive_count
182
+ end
183
+
184
+ # bm = Benchmark.ips do |x|
185
+ # x.config(:time => 10, :warmup => 2)
186
+
187
+ # x.report("pubsub") do
188
+ # db.transaction { 10.times { |i| producer.publish('foo', "foo#{i}") } }
189
+ # consumer1.get_messages
190
+ # consumer2.get_messages
191
+ # end
192
+
193
+ # x.compare!
194
+ # end
@@ -0,0 +1,204 @@
1
+ # frozen_string_literal: true
2
+
3
+ require './lib/extralite'
4
+ require 'tempfile'
5
+
6
+ class PubSub
7
+ def initialize(db)
8
+ @db = db
9
+ end
10
+
11
+ def attach
12
+ @db.execute('insert into subscribers (stamp) values (?)', Time.now.to_i)
13
+ @id = @db.last_insert_rowid
14
+ end
15
+
16
+ def detach
17
+ @db.execute('delete from subscribers where id = ?', @id)
18
+ end
19
+
20
+ def subscribe(*topics)
21
+ @db.transaction do
22
+ topics.each do |t|
23
+ @db.execute('insert into subscriber_topics (subscriber_id, topic) values (?, ?)', @id, t)
24
+ end
25
+ end
26
+ end
27
+
28
+ def unsubscribe(*topics)
29
+ @db.transaction do
30
+ topics.each do |t|
31
+ @db.execute('delete from subscriber_topics where subscriber_id = ? and topic = ?', @id, t)
32
+ end
33
+ end
34
+ end
35
+
36
+ def get_messages(&block)
37
+ @db.transaction(:deferred) do
38
+ results = @db.query_ary('select topic, message from messages where subscriber_id = ?', @id)
39
+ return [] if results.empty?
40
+
41
+ @db.execute('delete from messages where subscriber_id = ?', @id)
42
+ results
43
+ end
44
+
45
+ # messages = @db.query_ary('delete from messages where subscriber_id = ? returning topic, message', @id)
46
+ # if block
47
+ # messages.each(&block)
48
+ # nil
49
+ # else
50
+ # messages
51
+ # end
52
+ rescue Extralite::BusyError
53
+ p busy: :get_message
54
+ block_given? ? nil : []
55
+ end
56
+
57
+ SCHEMA = <<~SQL
58
+ PRAGMA foreign_keys = ON;
59
+
60
+ create table if not exists subscribers (
61
+ id integer primary key,
62
+ stamp float
63
+ );
64
+ create index if not exists idx_subscribers_stamp on subscribers (stamp);
65
+
66
+ create table if not exists subscriber_topics (
67
+ subscriber_id integer references subscribers(id) on delete cascade,
68
+ topic text
69
+ );
70
+
71
+ create table if not exists messages(
72
+ subscriber_id integer,
73
+ topic text,
74
+ message text,
75
+ foreign key (subscriber_id, topic)
76
+ references subscriber_topics(subscriber_id, topic)
77
+ on delete cascade
78
+ );
79
+ create index if not exists idx_messages_subscriber_id_topic on messages (subscriber_id, topic);
80
+ SQL
81
+
82
+ def setup
83
+ @db.transaction do
84
+ @db.execute(SCHEMA)
85
+ end
86
+ end
87
+
88
+ def publish(topic, message)
89
+ # @db.transaction do
90
+ @db.execute("
91
+ with subscribed as (
92
+ select subscriber_id
93
+ from subscriber_topics
94
+ where topic = $1
95
+ )
96
+ insert into messages
97
+ select subscriber_id, $1, $2
98
+ from subscribed
99
+ ", topic, message)
100
+ # end
101
+ rescue Extralite::BusyError
102
+ p busy: :publish
103
+ retry
104
+ end
105
+
106
+ def prune_subscribers
107
+ @db.execute('delete from subscribers where stamp < ?', Time.now.to_i - 3600)
108
+ end
109
+ end
110
+
111
+ fn = Tempfile.new('pubsub_store').path
112
+ p fn
113
+ db1 = Extralite::Database.new(fn)
114
+ db1.busy_timeout = 0.1
115
+ db1.pragma(journal_mode: :wal, synchronous: 1)
116
+ db1.gvl_release_threshold = -1
117
+ db2 = Extralite::Database.new(fn)
118
+ db2.pragma(journal_mode: :wal, synchronous: 1)
119
+ db2.busy_timeout = 0.1
120
+ db2.gvl_release_threshold = -1
121
+ db3 = Extralite::Database.new(fn)
122
+ db3.pragma(journal_mode: :wal, synchronous: 1)
123
+ db3.busy_timeout = 0.1
124
+ db3.gvl_release_threshold = -1
125
+
126
+ # db1.on_progress(100) { Thread.pass }
127
+ # db2.on_progress(100) { Thread.pass }
128
+ # db3.on_progress(100) { Thread.pass }
129
+
130
+ producer = PubSub.new(db1)
131
+ producer.setup
132
+
133
+ consumer1 = PubSub.new(db2)
134
+ consumer1.setup
135
+ consumer1.attach
136
+ consumer1.subscribe('foo', 'bar')
137
+
138
+ consumer2 = PubSub.new(db3)
139
+ consumer2.setup
140
+ consumer2.attach
141
+ consumer2.subscribe('foo', 'baz')
142
+
143
+ # producer.publish('foo', 'foo1')
144
+ # producer.publish('bar', 'bar1')
145
+ # producer.publish('baz', 'baz1')
146
+
147
+ # puts "- get messages"
148
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
149
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
150
+
151
+ # puts "- get messages again"
152
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
153
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
154
+
155
+ topics = %w{bar baz}
156
+
157
+ publish_count = 0
158
+ t1 = Thread.new do
159
+ while true
160
+ message = "message#{rand(1000)}"
161
+ producer.publish(topics.sample, message)
162
+
163
+ publish_count += 1
164
+ Thread.pass
165
+ end
166
+ end
167
+
168
+ receive_count = 0
169
+
170
+ t2 = Thread.new do
171
+ while true
172
+ consumer1.get_messages { |t, m| receive_count += 1 }
173
+ sleep(0.1)
174
+ end
175
+ end
176
+
177
+ t3 = Thread.new do
178
+ while true
179
+ consumer2.get_messages { |t, m| receive_count += 1 }
180
+ sleep(0.1)
181
+ end
182
+ end
183
+
184
+ db4 = Extralite::Database.new(fn)
185
+ db4.pragma(journal_mode: :wal, synchronous: 1)
186
+ db4.gvl_release_threshold = -1
187
+ db4.busy_timeout = 3
188
+
189
+ last_t = Time.now
190
+ last_publish_count = 0
191
+ last_receive_count = 0
192
+ while true
193
+ sleep 1
194
+ now = Time.now
195
+ elapsed = now - last_t
196
+ d_publish = publish_count - last_publish_count
197
+ d_receive = receive_count - last_receive_count
198
+
199
+ count = db4.query_single_argv('select count(*) from messages')
200
+ puts "#{Time.now} publish: #{d_publish/elapsed}/s receive: #{d_receive/elapsed}/s pending: #{count}"
201
+ last_t = now
202
+ last_publish_count = publish_count
203
+ last_receive_count = receive_count
204
+ end
@@ -186,13 +186,13 @@ static inline VALUE convert_value(sqlite3_value *value) {
186
186
  case SQLITE_BLOB:
187
187
  {
188
188
  int len = sqlite3_value_bytes(value);
189
- void *blob = sqlite3_value_blob(value);
189
+ const void *blob = sqlite3_value_blob(value);
190
190
  return rb_str_new(blob, len);
191
191
  }
192
192
  case SQLITE_TEXT:
193
193
  {
194
194
  int len = sqlite3_value_bytes(value);
195
- void *text = sqlite3_value_text(value);
195
+ const void *text = sqlite3_value_text(value);
196
196
  return rb_enc_str_new(text, len, UTF8_ENCODING);
197
197
  }
198
198
  default:
@@ -339,7 +339,7 @@ VALUE Changeset_to_a(VALUE self) {
339
339
 
340
340
  // copied from: https://sqlite.org/sessionintro.html
341
341
  static int xConflict(void *pCtx, int eConflict, sqlite3_changeset_iter *pIter){
342
- int ret = (long)pCtx;
342
+ int ret = (int)pCtx;
343
343
  return ret;
344
344
  }
345
345