extralite-bundle 2.5 → 2.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CHANGELOG.md +34 -13
  4. data/Gemfile +4 -0
  5. data/Gemfile-bundle +1 -1
  6. data/LICENSE +1 -1
  7. data/README.md +1059 -247
  8. data/Rakefile +18 -0
  9. data/TODO.md +0 -7
  10. data/examples/kv_store.rb +49 -0
  11. data/examples/multi_fiber.rb +16 -0
  12. data/examples/on_progress.rb +9 -0
  13. data/examples/pubsub_store_polyphony.rb +194 -0
  14. data/examples/pubsub_store_threads.rb +204 -0
  15. data/ext/extralite/changeset.c +463 -0
  16. data/ext/extralite/common.c +177 -91
  17. data/ext/extralite/database.c +745 -276
  18. data/ext/extralite/extconf-bundle.rb +10 -4
  19. data/ext/extralite/extconf.rb +34 -34
  20. data/ext/extralite/extralite.h +104 -47
  21. data/ext/extralite/extralite_ext.c +6 -0
  22. data/ext/extralite/iterator.c +14 -86
  23. data/ext/extralite/query.c +171 -264
  24. data/extralite-bundle.gemspec +1 -1
  25. data/extralite.gemspec +1 -1
  26. data/gemspec.rb +10 -11
  27. data/lib/extralite/version.rb +1 -1
  28. data/lib/extralite.rb +69 -10
  29. data/lib/sequel/adapters/extralite.rb +1 -1
  30. data/test/helper.rb +9 -1
  31. data/test/perf_argv_transform.rb +74 -0
  32. data/test/perf_ary.rb +14 -12
  33. data/test/perf_hash.rb +17 -15
  34. data/test/perf_hash_prepared.rb +58 -0
  35. data/test/perf_hash_transform.rb +66 -0
  36. data/test/perf_polyphony.rb +74 -0
  37. data/test/test_changeset.rb +161 -0
  38. data/test/test_database.rb +720 -104
  39. data/test/test_extralite.rb +2 -2
  40. data/test/test_iterator.rb +28 -13
  41. data/test/test_query.rb +352 -110
  42. data/test/test_sequel.rb +4 -4
  43. metadata +24 -16
  44. data/Gemfile.lock +0 -37
  45. data/test/perf_prepared.rb +0 -64
data/Rakefile CHANGED
@@ -48,4 +48,22 @@ end
48
48
  task :build_bundled do
49
49
  puts 'Building extralite-bundle...'
50
50
  `gem build extralite-bundle.gemspec`
51
+ end
52
+
53
+ test_config = lambda do |t|
54
+ t.libs << "test"
55
+ t.libs << "lib"
56
+ t.test_files = FileList["test/**/test_*.rb"]
57
+ end
58
+
59
+ # Rake::TestTask.new(:test, &test_config)
60
+
61
+ begin
62
+ require "ruby_memcheck"
63
+
64
+ namespace :test do
65
+ RubyMemcheck::TestTask.new(:valgrind, &test_config)
66
+ end
67
+ rescue LoadError => e
68
+ warn("NOTE: ruby_memcheck is not available in this environment: #{e}")
51
69
  end
data/TODO.md CHANGED
@@ -1,10 +1,3 @@
1
- - Transactions and savepoints:
2
-
3
- - `DB#savepoint(name)` - creates a savepoint
4
- - `DB#release(name)` - releases a savepoint
5
- - `DB#rollback` - raises `Extralite::Rollback`, which is rescued by `DB#transaction`
6
- - `DB#rollback_to(name)` - rolls back to a savepoint
7
-
8
1
  - More database methods:
9
2
 
10
3
  - `Database#quote`
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/inline'
4
+
5
+ gemfile do
6
+ gem 'polyphony'
7
+ gem 'extralite', path: '.'
8
+ end
9
+
10
+ class KVStore
11
+ def initialize(db)
12
+ @db = db
13
+ setup_kv_tables
14
+ setup_queries
15
+ end
16
+
17
+ SETUP_SQL = <<~SQL
18
+ create table if not exists kv (key text primary key, value, expires float);
19
+ create index if not exists idx_kv_expires on kv (expires) where expires is not null;
20
+ SQL
21
+
22
+ def setup_kv_tables
23
+ @db.execute SETUP_SQL
24
+ end
25
+
26
+ def setup_queries
27
+ @q_get = @db.prepare_argv('select value from kv where key = ?')
28
+ @q_set = @db.prepare('insert into kv (key, value) values($1, $2) on conflict (key) do update set value = $2')
29
+ end
30
+
31
+ def get(key)
32
+ @q_get.bind(key).next
33
+ end
34
+
35
+ def set(key, value)
36
+ @q_set.bind(key, value).execute
37
+ end
38
+ end
39
+
40
+ db = Extralite::Database.new(':memory:')
41
+ kv = KVStore.new(db)
42
+
43
+ p get: kv.get('foo')
44
+ p set: kv.set('foo', 42)
45
+ p get: kv.get('foo')
46
+ p set: kv.set('foo', 43)
47
+ p get: kv.get('foo')
48
+
49
+ p db.query('select * from kv order by key')
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require './lib/extralite'
4
+ require 'fiber'
5
+
6
+ f1 = Fiber.new do |f2|
7
+ while true
8
+ STDOUT << '.'
9
+ f2 = f2.transfer
10
+ end
11
+ end
12
+
13
+ db = Extralite::Database.new(':memory:')
14
+ db.on_progress(1) { f1.transfer(Fiber.current) }
15
+
16
+ p db.query('select 1, 2, 3')
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require './lib/extralite'
4
+
5
+ db = Extralite::Database.new(':memory:')
6
+ count = 0
7
+ db.on_progress(10) { count += 1 }
8
+ 10000.times { db.query('select 1') }
9
+ p count: count
@@ -0,0 +1,194 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/inline'
4
+
5
+ gemfile do
6
+ gem 'polyphony'
7
+ gem 'extralite', path: '.'
8
+ gem 'benchmark-ips'
9
+ end
10
+
11
+ require 'tempfile'
12
+
13
+ class PubSub
14
+ def initialize(db)
15
+ @db = db
16
+ end
17
+
18
+ def attach
19
+ @db.execute('insert into subscribers (stamp) values (?)', Time.now.to_i)
20
+ @id = @db.last_insert_rowid
21
+ end
22
+
23
+ def detach
24
+ @db.execute('delete from subscribers where id = ?', @id)
25
+ end
26
+
27
+ def subscribe(*topics)
28
+ @db.transaction do
29
+ topics.each do |t|
30
+ @db.execute('insert into subscriber_topics (subscriber_id, topic) values (?, ?)', @id, t)
31
+ end
32
+ end
33
+ end
34
+
35
+ def unsubscribe(*topics)
36
+ @db.transaction do
37
+ topics.each do |t|
38
+ @db.execute('delete from subscriber_topics where subscriber_id = ? and topic = ?', @id, t)
39
+ end
40
+ end
41
+ end
42
+
43
+ def get_messages(&block)
44
+ # @db.execute('update subscribers set stamp = ? where id = ?', Time.now.to_i, @id)
45
+ @db.query_argv('delete from messages where subscriber_id = ? returning topic, message', @id, &block)
46
+ end
47
+
48
+ SCHEMA = <<~SQL
49
+ PRAGMA foreign_keys = ON;
50
+
51
+ create table if not exists subscribers (
52
+ id integer primary key,
53
+ stamp float
54
+ );
55
+ create index if not exists idx_subscribers_stamp on subscribers (stamp);
56
+
57
+ create table if not exists subscriber_topics (
58
+ subscriber_id integer references subscribers(id) on delete cascade,
59
+ topic text
60
+ );
61
+
62
+ create table if not exists messages(
63
+ subscriber_id integer,
64
+ topic text,
65
+ message text,
66
+ foreign key (subscriber_id, topic)
67
+ references subscriber_topics(subscriber_id, topic)
68
+ on delete cascade
69
+ );
70
+ create index if not exists idx_messages_subscriber_id_topic on messages (subscriber_id, topic);
71
+ SQL
72
+
73
+ def setup
74
+ @db.transaction do
75
+ @db.execute(SCHEMA)
76
+ end
77
+ end
78
+
79
+ def publish(topic, message)
80
+ @db.execute("
81
+ with subscribed as (
82
+ select subscriber_id
83
+ from subscriber_topics
84
+ where topic = $1
85
+ )
86
+ insert into messages
87
+ select subscriber_id, $1, $2
88
+ from subscribed
89
+ ", topic, message)
90
+ end
91
+
92
+ def prune_subscribers
93
+ @db.execute('delete from subscribers where stamp < ?', Time.now.to_i - 3600)
94
+ end
95
+ end
96
+
97
+ fn = Tempfile.new('pubsub_store').path
98
+ p fn: fn
99
+ db1 = Extralite::Database.new(fn)
100
+ db1.pragma(journal_mode: :wal, synchronous: 1)
101
+ db2 = Extralite::Database.new(fn)
102
+ db2.pragma(journal_mode: :wal, synchronous: 1)
103
+ db3 = Extralite::Database.new(fn)
104
+ db3.pragma(journal_mode: :wal, synchronous: 1)
105
+
106
+ db1.on_progress(1000) { |b| b ? sleep(0.0001) : snooze }
107
+ db2.on_progress(1000) { |b| b ? sleep(0.0001) : snooze }
108
+ db3.on_progress(1000) { |b| b ? sleep(0.0001) : snooze }
109
+
110
+ producer = PubSub.new(db1)
111
+ producer.setup
112
+
113
+ consumer1 = PubSub.new(db2)
114
+ consumer1.setup
115
+ consumer1.attach
116
+ consumer1.subscribe('foo', 'bar')
117
+
118
+ consumer2 = PubSub.new(db3)
119
+ consumer2.setup
120
+ consumer2.attach
121
+ consumer2.subscribe('foo', 'baz')
122
+
123
+
124
+ # producer.publish('foo', 'foo1')
125
+ # producer.publish('bar', 'bar1')
126
+ # producer.publish('baz', 'baz1')
127
+
128
+ # puts "- get messages"
129
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
130
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
131
+
132
+ # puts "- get messages again"
133
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
134
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
135
+
136
+ topics = %w{bar baz}
137
+
138
+ publish_count = 0
139
+ f1 = spin do
140
+ while true
141
+ message = "message#{rand(1000)}"
142
+ producer.publish(topics.sample, message)
143
+
144
+ publish_count += 1
145
+ snooze
146
+ end
147
+ end
148
+
149
+ receive_count = 0
150
+ f2 = spin do
151
+ while true
152
+ consumer1.get_messages { |t, m| receive_count += 1 }
153
+ sleep 0.1
154
+ end
155
+ end
156
+
157
+ f3 = spin do
158
+ while true
159
+ consumer2.get_messages { |t, m| receive_count += 1 }
160
+ sleep 0.1
161
+ end
162
+ end
163
+
164
+ db4 = Extralite::Database.new(fn)
165
+ db4.pragma(journal_mode: :wal, synchronous: 1)
166
+ db4.on_progress(1000) { |busy| busy ? sleep(0.05) : snooze }
167
+
168
+ last_t = Time.now
169
+ last_publish_count = 0
170
+ last_receive_count = 0
171
+ while true
172
+ sleep 1
173
+ now = Time.now
174
+ elapsed = now - last_t
175
+ d_publish = publish_count - last_publish_count
176
+ d_receive = receive_count - last_receive_count
177
+ pending = db4.query_single_argv('select count(*) from messages')
178
+ puts "#{Time.now} publish: #{d_publish/elapsed}/s receive: #{d_receive/elapsed}/s pending: #{pending}"
179
+ last_t = now
180
+ last_publish_count = publish_count
181
+ last_receive_count = receive_count
182
+ end
183
+
184
+ # bm = Benchmark.ips do |x|
185
+ # x.config(:time => 10, :warmup => 2)
186
+
187
+ # x.report("pubsub") do
188
+ # db.transaction { 10.times { |i| producer.publish('foo', "foo#{i}") } }
189
+ # consumer1.get_messages
190
+ # consumer2.get_messages
191
+ # end
192
+
193
+ # x.compare!
194
+ # end
@@ -0,0 +1,204 @@
1
+ # frozen_string_literal: true
2
+
3
+ require './lib/extralite'
4
+ require 'tempfile'
5
+
6
+ class PubSub
7
+ def initialize(db)
8
+ @db = db
9
+ end
10
+
11
+ def attach
12
+ @db.execute('insert into subscribers (stamp) values (?)', Time.now.to_i)
13
+ @id = @db.last_insert_rowid
14
+ end
15
+
16
+ def detach
17
+ @db.execute('delete from subscribers where id = ?', @id)
18
+ end
19
+
20
+ def subscribe(*topics)
21
+ @db.transaction do
22
+ topics.each do |t|
23
+ @db.execute('insert into subscriber_topics (subscriber_id, topic) values (?, ?)', @id, t)
24
+ end
25
+ end
26
+ end
27
+
28
+ def unsubscribe(*topics)
29
+ @db.transaction do
30
+ topics.each do |t|
31
+ @db.execute('delete from subscriber_topics where subscriber_id = ? and topic = ?', @id, t)
32
+ end
33
+ end
34
+ end
35
+
36
+ def get_messages(&block)
37
+ @db.transaction(:deferred) do
38
+ results = @db.query_ary('select topic, message from messages where subscriber_id = ?', @id)
39
+ return [] if results.empty?
40
+
41
+ @db.execute('delete from messages where subscriber_id = ?', @id)
42
+ results
43
+ end
44
+
45
+ # messages = @db.query_ary('delete from messages where subscriber_id = ? returning topic, message', @id)
46
+ # if block
47
+ # messages.each(&block)
48
+ # nil
49
+ # else
50
+ # messages
51
+ # end
52
+ rescue Extralite::BusyError
53
+ p busy: :get_message
54
+ block_given? ? nil : []
55
+ end
56
+
57
+ SCHEMA = <<~SQL
58
+ PRAGMA foreign_keys = ON;
59
+
60
+ create table if not exists subscribers (
61
+ id integer primary key,
62
+ stamp float
63
+ );
64
+ create index if not exists idx_subscribers_stamp on subscribers (stamp);
65
+
66
+ create table if not exists subscriber_topics (
67
+ subscriber_id integer references subscribers(id) on delete cascade,
68
+ topic text
69
+ );
70
+
71
+ create table if not exists messages(
72
+ subscriber_id integer,
73
+ topic text,
74
+ message text,
75
+ foreign key (subscriber_id, topic)
76
+ references subscriber_topics(subscriber_id, topic)
77
+ on delete cascade
78
+ );
79
+ create index if not exists idx_messages_subscriber_id_topic on messages (subscriber_id, topic);
80
+ SQL
81
+
82
+ def setup
83
+ @db.transaction do
84
+ @db.execute(SCHEMA)
85
+ end
86
+ end
87
+
88
+ def publish(topic, message)
89
+ # @db.transaction do
90
+ @db.execute("
91
+ with subscribed as (
92
+ select subscriber_id
93
+ from subscriber_topics
94
+ where topic = $1
95
+ )
96
+ insert into messages
97
+ select subscriber_id, $1, $2
98
+ from subscribed
99
+ ", topic, message)
100
+ # end
101
+ rescue Extralite::BusyError
102
+ p busy: :publish
103
+ retry
104
+ end
105
+
106
+ def prune_subscribers
107
+ @db.execute('delete from subscribers where stamp < ?', Time.now.to_i - 3600)
108
+ end
109
+ end
110
+
111
+ fn = Tempfile.new('pubsub_store').path
112
+ p fn
113
+ db1 = Extralite::Database.new(fn)
114
+ db1.busy_timeout = 0.1
115
+ db1.pragma(journal_mode: :wal, synchronous: 1)
116
+ db1.gvl_release_threshold = -1
117
+ db2 = Extralite::Database.new(fn)
118
+ db2.pragma(journal_mode: :wal, synchronous: 1)
119
+ db2.busy_timeout = 0.1
120
+ db2.gvl_release_threshold = -1
121
+ db3 = Extralite::Database.new(fn)
122
+ db3.pragma(journal_mode: :wal, synchronous: 1)
123
+ db3.busy_timeout = 0.1
124
+ db3.gvl_release_threshold = -1
125
+
126
+ # db1.on_progress(100) { Thread.pass }
127
+ # db2.on_progress(100) { Thread.pass }
128
+ # db3.on_progress(100) { Thread.pass }
129
+
130
+ producer = PubSub.new(db1)
131
+ producer.setup
132
+
133
+ consumer1 = PubSub.new(db2)
134
+ consumer1.setup
135
+ consumer1.attach
136
+ consumer1.subscribe('foo', 'bar')
137
+
138
+ consumer2 = PubSub.new(db3)
139
+ consumer2.setup
140
+ consumer2.attach
141
+ consumer2.subscribe('foo', 'baz')
142
+
143
+ # producer.publish('foo', 'foo1')
144
+ # producer.publish('bar', 'bar1')
145
+ # producer.publish('baz', 'baz1')
146
+
147
+ # puts "- get messages"
148
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
149
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
150
+
151
+ # puts "- get messages again"
152
+ # consumer1.get_messages { |t, m| p consumer: 1, topic: t, message: m }
153
+ # consumer2.get_messages { |t, m| p consumer: 2, topic: t, message: m }
154
+
155
+ topics = %w{bar baz}
156
+
157
+ publish_count = 0
158
+ t1 = Thread.new do
159
+ while true
160
+ message = "message#{rand(1000)}"
161
+ producer.publish(topics.sample, message)
162
+
163
+ publish_count += 1
164
+ Thread.pass
165
+ end
166
+ end
167
+
168
+ receive_count = 0
169
+
170
+ t2 = Thread.new do
171
+ while true
172
+ consumer1.get_messages { |t, m| receive_count += 1 }
173
+ sleep(0.1)
174
+ end
175
+ end
176
+
177
+ t3 = Thread.new do
178
+ while true
179
+ consumer2.get_messages { |t, m| receive_count += 1 }
180
+ sleep(0.1)
181
+ end
182
+ end
183
+
184
+ db4 = Extralite::Database.new(fn)
185
+ db4.pragma(journal_mode: :wal, synchronous: 1)
186
+ db4.gvl_release_threshold = -1
187
+ db4.busy_timeout = 3
188
+
189
+ last_t = Time.now
190
+ last_publish_count = 0
191
+ last_receive_count = 0
192
+ while true
193
+ sleep 1
194
+ now = Time.now
195
+ elapsed = now - last_t
196
+ d_publish = publish_count - last_publish_count
197
+ d_receive = receive_count - last_receive_count
198
+
199
+ count = db4.query_single_argv('select count(*) from messages')
200
+ puts "#{Time.now} publish: #{d_publish/elapsed}/s receive: #{d_receive/elapsed}/s pending: #{count}"
201
+ last_t = now
202
+ last_publish_count = publish_count
203
+ last_receive_count = receive_count
204
+ end