litestack 0.2.3 → 0.2.6

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,48 @@
1
+ <h5><a href="/?res=<%=@res%>">All Topics</a> > <%= @topic %></h5>
2
+ <% if @snapshot and not @snapshot.empty? and not @snapshot[0].nil?%>
3
+ <div><b>Snapshot</b> captured on <%= @snapshot[1] %></div>
4
+ <% @snapshot[0].each_pair do |k, v| %>
5
+ <table class ="table summary">
6
+ <tr>
7
+ <th colspan="2"><%=k.to_s.capitalize%></th>
8
+ </tr>
9
+ <% v.each do |ss| %>
10
+ <tr>
11
+ <td><%= ss[0].to_s.capitalize.gsub("_", " ") %></td>
12
+ <td><%= ss[1] %></td>
13
+ </tr>
14
+ <% end %>
15
+ </table>
16
+ <br/>
17
+ <% end %>
18
+ <% end %>
19
+ <div><b>Dynamic event data</b></div>
20
+ <div id="search"><form><input id="search-field" type="text" placeholder="Search events" onkeydown="search_kd(this)" onkeyup="search_ku(this)" value="<%=@search%>"/></form></div> <table class="table sortable">
21
+ <tr>
22
+ <th width="16%" class="<%='sorted' if @order == 'name'%>"><a href="<%=topic_sort_url('name')%>">Event</a> <%=dir('name')%></th>
23
+ <th width="8%" class="<%='sorted' if @order == 'rcount'%>"><a href="<%=topic_sort_url('rcount')%>">Event Count</a> <%=dir('rcount')%></th>
24
+ <th width="8%" class="<%='sorted' if @order == 'ravg'%>"><a href="<%=topic_sort_url('ravg')%>">Avg Value</a> <%=dir('ravg')%></th>
25
+ <th width="8%" class="<%='sorted' if @order == 'rtotal'%>"><a href="<%=topic_sort_url('rtotal')%>">Total Value</a> <%=dir('rtotal')%></th>
26
+ <th width="8%" class="<%='sorted' if @order == 'rmin'%>"><a href="<%=topic_sort_url('rmin')%>">Min Value</a> <%=dir('rmin')%></th>
27
+ <th width="8%" class="<%='sorted' if @order == 'rmax'%>"><a href="<%=topic_sort_url('rmax')%>">Max Value</a> <%=dir('rmax')%></th>
28
+ <th width="22%">Events over time</th>
29
+ <th width="22%">Average value over time</th>
30
+ </tr>
31
+ <% @events.each do |event|%>
32
+ <tr>
33
+ <td title="<%=event[0]%>"><div class="label"><a href="/topics/<%=encode(@topic)%>/events/<%=encode(event[0])%>?res=<%=@res%>"><%=event[0]%></a></div></td>
34
+ <td><%=event[2]%></td>
35
+ <td><%="%0.2f" % [event[3]] if event[3] %></td>
36
+ <td><%="%0.2f" % [event[4]] if event[4] %></td>
37
+ <td><%="%0.2f" % [event[5]] if event[5] %></td>
38
+ <td><%="%0.2f" % [event[6]] if event[6] %></td>
39
+ <td class="chart"><span class="inlinecolumn hidden" data-label="Count"><%=Oj.dump(event[7]) if event[7]%></span></td>
40
+ <td class="chart"><span class="inlinecolumn hidden" data-label="Avg Value"><%=Oj.dump(event[8]) if event[8]%></span></td>
41
+ </tr>
42
+ <% end %>
43
+ <% if @events.empty? %>
44
+ <tr>
45
+ <td class="empty" colspan="9">No data to display</td>
46
+ </tr>
47
+ <% end %>
48
+ </table>
@@ -15,7 +15,7 @@ class Litecable
15
15
 
16
16
  DEFAULT_OPTIONS = {
17
17
  config_path: "./litecable.yml",
18
- path: "./cable.db",
18
+ path: Litesupport.root.join("cable.sqlite3"),
19
19
  sync: 0,
20
20
  mmap_size: 16 * 1024 * 1024, # 16MB
21
21
  expire_after: 5, # remove messages older than 5 seconds
@@ -32,7 +32,7 @@ class Litecache
32
32
  # sleep_interval: 1 -> 1 second of sleep between cleanup runs
33
33
 
34
34
  DEFAULT_OPTIONS = {
35
- path: "./cache.db",
35
+ path: Litesupport.root.join("cache.sqlite3"),
36
36
  config_path: "./litecache.yml",
37
37
  sync: 0,
38
38
  expiry: 60 * 60 * 24 * 30, # one month
@@ -64,8 +64,8 @@ class Litecache
64
64
 
65
65
  def initialize(options = {})
66
66
  options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]
67
- init(options)
68
67
  @last_visited = {}
68
+ init(options)
69
69
  collect_metrics if @options[:metrics]
70
70
  end
71
71
 
@@ -76,7 +76,7 @@ class Litecache
76
76
  @conn.acquire do |cache|
77
77
  begin
78
78
  cache.stmts[:setter].execute!(key, value, expires_in)
79
- capture(:write, key)
79
+ capture(:set, key)
80
80
  rescue SQLite3::FullException
81
81
  cache.stmts[:extra_pruner].execute!(0.2)
82
82
  cache.execute("vacuum")
@@ -97,7 +97,7 @@ class Litecache
97
97
  cache.stmts[:inserter].execute!(key, value, expires_in)
98
98
  changes = cache.changes
99
99
  end
100
- capture(:write, key)
100
+ capture(:set, key)
101
101
  rescue SQLite3::FullException
102
102
  cache.stmts[:extra_pruner].execute!(0.2)
103
103
  cache.execute("vacuum")
@@ -113,10 +113,10 @@ class Litecache
113
113
  key = key.to_s
114
114
  if record = @conn.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
115
115
  @last_visited[key] = true
116
- capture(:hit, key)
116
+ capture(:get, key, 1)
117
117
  return record[1]
118
118
  end
119
- capture(:miss, key)
119
+ capture(:get, key, 0)
120
120
  nil
121
121
  end
122
122
 
@@ -160,9 +160,9 @@ class Litecache
160
160
  end
161
161
 
162
162
  # return the actual size of the cache file
163
- def size
164
- run_stmt(:sizer)[0][0]
165
- end
163
+ #def size
164
+ # run_stmt(:sizer)[0][0]
165
+ #end
166
166
 
167
167
  # delete all key, value pairs in the cache
168
168
  def clear
@@ -177,9 +177,23 @@ class Litecache
177
177
 
178
178
  # return the maximum size of the cache
179
179
  def max_size
180
- run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0]
180
+ run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0].to_f / (1024 * 1024)
181
181
  end
182
182
 
183
+ def snapshot
184
+ {
185
+ summary: {
186
+ path: path,
187
+ journal_mode: journal_mode,
188
+ synchronous: synchronous,
189
+ size: size,
190
+ max_size: max_size,
191
+ entries: count
192
+ }
193
+ }
194
+ end
195
+
196
+
183
197
  # low level access to SQLite transactions, use with caution
184
198
  def transaction(mode, acquire=true)
185
199
  return cache.transaction(mode){yield} unless acquire
@@ -239,6 +253,6 @@ class Litecache
239
253
  end
240
254
  sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
241
255
  conn
242
- end
256
+ end
243
257
 
244
258
  end
@@ -4,6 +4,9 @@ require_relative 'litesupport'
4
4
  # Litedb inherits from the SQLite3::Database class and adds a few initialization options
5
5
  class Litedb < ::SQLite3::Database
6
6
 
7
+ # add litemetric support
8
+ include Litemetric::Measurable
9
+
7
10
  # overrride the original initilaizer to allow for connection configuration
8
11
  def initialize(file, options = {}, zfs = nil )
9
12
  if block_given?
@@ -15,6 +18,13 @@ class Litedb < ::SQLite3::Database
15
18
  super(file, options, zfs)
16
19
  init unless options[:noinit] == true
17
20
  end
21
+ @running = true
22
+ @collecting_metrics = options[:metrics]
23
+ collect_metrics if @collecting_metrics
24
+ end
25
+
26
+ def collecting_metrics?
27
+ @collecting_metrics
18
28
  end
19
29
 
20
30
  # enforce immediate mode to avoid deadlocks for a small performance penalty
@@ -22,6 +32,61 @@ class Litedb < ::SQLite3::Database
22
32
  super(mode)
23
33
  end
24
34
 
35
+ # return the size of the database file
36
+ def size
37
+ execute("SELECT s.page_size * c.page_count FROM pragma_page_size() as s, pragma_page_count() as c")[0][0]
38
+ end
39
+
40
+ # collect snapshot information
41
+ def snapshot
42
+ {
43
+ summary: {
44
+ path: filename,
45
+ journal_mode: journal_mode,
46
+ synchronous: synchronous,
47
+ size: size
48
+ }
49
+ }
50
+ end
51
+
52
+ # override prepare to return Litedb::Statement (and pass the sql to it)
53
+ def prepare(sql)
54
+ stmt = Litedb::Statement.new(self, sql)
55
+ stmt.sql = sql.strip.upcase
56
+ return stmt unless block_given?
57
+ begin
58
+ yield stmt
59
+ ensure
60
+ stmt.close unless stmt.closed?
61
+ end
62
+ end
63
+
64
+ # override execute to capture metrics
65
+ def execute(sql, bind_vars = [], *args, &block)
66
+ if bind_vars.nil? || !args.empty?
67
+ if args.empty?
68
+ bind_vars = []
69
+ else
70
+ bind_vars = [bind_vars] + args
71
+ end
72
+ end
73
+
74
+ prepare(sql) do |stmt|
75
+ measure(stmt.stmt_type, stmt.sql) do
76
+ stmt.bind_params(bind_vars)
77
+ stmt = SQLite3::ResultSet.new self, stmt
78
+ end
79
+ if block_given?
80
+ stmt.each do |row|
81
+ yield row
82
+ end
83
+ else
84
+ stmt.to_a
85
+ end
86
+ end
87
+
88
+ end
89
+
25
90
  private
26
91
 
27
92
  # default connection configuration values
@@ -41,7 +106,56 @@ class Litedb < ::SQLite3::Database
41
106
  # increase the local connection cache to 2000 pages
42
107
  self.cache_size = 2000
43
108
  end
44
-
109
+
45
110
  end
46
111
 
112
+ # the Litedb::Statement also inherits from SQLite3::Statement
113
+ class Litedb::Statement < SQLite3::Statement
114
+
115
+ include Litemetric::Measurable
116
+
117
+ attr_accessor :sql
118
+
119
+ def initialize(db, sql)
120
+ super(db, sql)
121
+ collect_metrics if db.collecting_metrics?
122
+ end
47
123
 
124
+ def metrics_identifier
125
+ "Litedb" # overridden to match the parent class
126
+ end
127
+
128
+ # return the type of the statement
129
+ def stmt_type
130
+ @stmt_type ||= detect_stmt_type
131
+ end
132
+
133
+ def detect_stmt_type
134
+ if @sql.start_with?("SEL") || @sql.start_with?("WITH")
135
+ "Read"
136
+ elsif @sql.start_with?("CRE") || @sql.start_with?("ALT") || @sql.start_with?("DRO")
137
+ "Schema change"
138
+ elsif @sql.start_with?("PRA")
139
+ "Pragma"
140
+ else
141
+ "Write"
142
+ end
143
+ end
144
+
145
+ # overriding each to measure the query time (plus the processing time as well, sadly)
146
+ def each
147
+ measure(stmt_type, @sql) do
148
+ super
149
+ end
150
+ end
151
+
152
+ # overriding execute to measure the query time
153
+ def execute(*bind_vars)
154
+ res = nil
155
+ measure(stmt_type, @sql) do
156
+ res = super(*bind_vars)
157
+ end
158
+ res
159
+ end
160
+
161
+ end
@@ -74,11 +74,11 @@ module Litejob
74
74
  end
75
75
 
76
76
  def queue
77
- @@queue ||= "default"
77
+ @queue_name ||= "default"
78
78
  end
79
79
 
80
80
  def queue=(queue_name)
81
- @@queue = queue_name.to_s
81
+ @queue_name = queue_name.to_s
82
82
  end
83
83
 
84
84
  def options
@@ -19,7 +19,7 @@ class Litejobqueue < Litequeue
19
19
  # can be overriden by passing new options in a hash
20
20
  # to Litejobqueue.new, it will also be then passed to the underlying Litequeue object
21
21
  # config_path: "./litejob.yml" -> were to find the configuration file (if any)
22
- # path: "./queue.db"
22
+ # path: "./db/queue.db"
23
23
  # mmap_size: 128 * 1024 * 1024 -> 128MB to be held in memory
24
24
  # sync: 1 -> sync only when checkpointing
25
25
  # queues: [["default", 1, "spawn"]] -> an array of queues to process
@@ -32,7 +32,7 @@ class Litejobqueue < Litequeue
32
32
  # This can be particularly useful for long running, IO bound jobs. It is not recommended though for threaded environments, as it can result in creating many threads that may consudme a lot of memory.
33
33
  DEFAULT_OPTIONS = {
34
34
  config_path: "./litejob.yml",
35
- path: "./queue.db",
35
+ path: Litesupport.root.join("queue.sqlite3"),
36
36
  queues: [["default", 1]],
37
37
  workers: 5,
38
38
  retries: 5,
@@ -79,6 +79,11 @@ class Litejobqueue < Litequeue
79
79
  pgroups[q[1]] << [q[0], q[2] == "spawn"]
80
80
  end
81
81
  @queues = pgroups.keys.sort.reverse.collect{|p| [p, pgroups[p]]}
82
+ collect_metrics if @options[:metrics]
83
+ end
84
+
85
+ def metrics_identifier
86
+ "Litejob" # overrides default identifier
82
87
  end
83
88
 
84
89
  # push a job to the queue
@@ -164,12 +169,7 @@ class Litejobqueue < Litequeue
164
169
  def job_finished
165
170
  Litesupport.synchronize(@mutex){@jobs_in_flight -= 1}
166
171
  end
167
-
168
- # return a hash encapsulating the info about the current jobqueue
169
- def snapshot
170
- info
171
- end
172
-
172
+
173
173
  # optionally run a job in its own context
174
174
  def schedule(spawn = false, &block)
175
175
  if spawn
@@ -13,11 +13,12 @@ class Litemetric
13
13
 
14
14
  DEFAULT_OPTIONS = {
15
15
  config_path: "./litemetric.yml",
16
- path: "./metrics.db",
16
+ path: Litesupport.root.join("metrics.sqlite3"),
17
17
  sync: 1,
18
- mmap_size: 16 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
19
- flush_interval: 10, # flush data every 1 minute
20
- summarize_interval: 10 # summarize data every 1 minute
18
+ mmap_size: 128 * 1024 * 1024, # 16MB of memory to easily process 1 year worth of data
19
+ flush_interval: 10, # flush data every 10 seconds
20
+ summarize_interval: 30, # summarize data every 1/2 minute
21
+ snapshot_interval: 10*60 # snapshot every 10 minutes
21
22
  }
22
23
 
23
24
  RESOLUTIONS = {
@@ -27,8 +28,20 @@ class Litemetric
27
28
  week: 7*24*3600 # 1 week (lowest resolution)
28
29
  }
29
30
 
31
+ # :nodoc:
32
+ def self.options=(options)
33
+ # an ugly hack to pass options to a singleton
34
+ # need to rethink the whole singleton thing
35
+ @options = options
36
+ end
37
+
38
+ def self.options
39
+ @options
40
+ end
41
+
30
42
  # :nodoc:
31
43
  def initialize(options = {})
44
+ options = options.merge(Litemetric.options) if Litemetric.options
32
45
  init(options)
33
46
  end
34
47
 
@@ -41,36 +54,15 @@ class Litemetric
41
54
 
42
55
  ## event capturing
43
56
  ##################
44
-
57
+
58
+
45
59
  def capture(topic, event, key=event, value=nil)
46
- if key.is_a? Array
47
- key.each{|k| capture_single_key(topic, event, k, value)}
48
- else
49
- capture_single_key(topic, event, key, value)
50
- end
60
+ @collector.capture(topic, event, key, value)
51
61
  end
52
-
53
- def capture_single_key(topic, event, key=event, value=nil)
54
- @mutex.synchronize do
55
- time_slot = current_time_slot # should that be 5 minutes?
56
- topic_slot = @metrics[topic]
57
- if event_slot = topic_slot[event]
58
- if key_slot = event_slot[key]
59
- if key_slot[time_slot]
60
- key_slot[time_slot][:count] += 1
61
- key_slot[time_slot][:value] += value unless value.nil?
62
- else # new time slot
63
- key_slot[time_slot] = {count: 1, value: value}
64
- end
65
- else
66
- event_slot[key] = {time_slot => {count: 1, value: value}}
67
- end
68
- else # new event
69
- topic_slot[event] = {key => {time_slot => {count: 1, value: value}}}
70
- end
71
- end
62
+
63
+ def capture_snapshot(topic, state)
64
+ run_stmt(:capture_state, topic, Oj.dump(state))
72
65
  end
73
-
74
66
 
75
67
  ## event reporting
76
68
  ##################
@@ -78,17 +70,48 @@ class Litemetric
78
70
  def topics
79
71
  run_stmt(:list_topics).to_a
80
72
  end
81
-
82
- def event_names(resolution, topic)
83
- run_stmt(:list_event_names, resolution, topic).to_a
73
+
74
+ def topic_summaries(resolution, count, order, dir, search)
75
+ search = "%#{search}%" if search
76
+ if dir.downcase == "desc"
77
+ run_stmt(:topics_summaries, resolution, count, order, search).to_a
78
+ else
79
+ run_stmt(:topics_summaries_asc, resolution, count, order, search).to_a
80
+ end
84
81
  end
85
-
86
- def keys(resolution, topic, event_name)
87
- run_stmt(:list_event_keys, resolution, topic, event_name).to_a
82
+
83
+ def events_summaries(topic, resolution, order, dir, search, count)
84
+ search = "%#{search}%" if search
85
+ if dir.downcase == "desc"
86
+ run_stmt(:events_summaries, topic, resolution, order, search, count).to_a
87
+ else
88
+ run_stmt(:events_summaries_asc, topic, resolution, order, search, count).to_a
89
+ end
90
+ end
91
+
92
+ def keys_summaries(topic, event, resolution, order, dir, search, count)
93
+ search = "%#{search}%" if search
94
+ if dir.downcase == "desc"
95
+ run_stmt(:keys_summaries, topic, event, resolution, order, search, count).to_a
96
+ else
97
+ run_stmt(:keys_summaries_asc, topic, event, resolution, order, search, count).to_a
98
+ end
88
99
  end
89
100
 
90
- def event_data(resolution, topic, event_name, key)
91
- run_stmt(:list_events_by_key, resolution, topic, event_name, key).to_a
101
+ def topic_data_points(step, count, resolution, topic)
102
+ run_stmt(:topic_data_points, step, count, resolution, topic).to_a
103
+ end
104
+
105
+ def event_data_points(step, count, resolution, topic, event)
106
+ run_stmt(:event_data_points, step, count, resolution, topic, event).to_a
107
+ end
108
+
109
+ def key_data_points(step, count, resolution, topic, event, key)
110
+ run_stmt(:key_data_points, step, count, resolution, topic, event, key).to_a
111
+ end
112
+
113
+ def snapshot(topic)
114
+ run_stmt(:snapshot, topic)[0].to_a
92
115
  end
93
116
 
94
117
  ## summarize data
@@ -109,7 +132,7 @@ class Litemetric
109
132
  private
110
133
 
111
134
  def exit_callback
112
- puts "--- Litemetric detected an exit, flushing metrics"
135
+ STDERR.puts "--- Litemetric detected an exit, flushing metrics"
113
136
  @running = false
114
137
  flush
115
138
  end
@@ -118,9 +141,10 @@ class Litemetric
118
141
  super
119
142
  @metrics = {}
120
143
  @registered = {}
121
- @flusher = create_flusher
122
- @summarizer = create_summarizer
123
144
  @mutex = Litesupport::Mutex.new
145
+ @collector = Litemetric::Collector.new({dbpath: @options[:path]})
146
+ @summarizer = create_summarizer
147
+ @flusher = create_flusher
124
148
  end
125
149
 
126
150
  def current_time_slot
@@ -128,39 +152,24 @@ class Litemetric
128
152
  end
129
153
 
130
154
  def flush
131
- to_delete = []
132
- @conn.acquire do |conn|
133
- conn.transaction(:immediate) do
134
- @metrics.each_pair do |topic, event_hash|
135
- event_hash.each_pair do |event, key_hash|
136
- key_hash.each_pair do |key, time_hash|
137
- time_hash.each_pair do |time, data|
138
- conn.stmts[:capture_event].execute!(topic, event.to_s, key, time, data[:count], data[:value]) if data
139
- time_hash[time] = nil
140
- to_delete << [topic, event, key, time]
141
- end
142
- end
143
- end
144
- end
145
- end
146
- end
147
- to_delete.each do |r|
148
- @metrics[r[0]][r[1]][r[2]].delete(r[3])
149
- @metrics[r[0]][r[1]].delete(r[2]) if @metrics[r[0]][r[1]][r[2]].empty?
150
- @metrics[r[0]].delete(r[1]) if @metrics[r[0]][r[1]].empty?
151
- end
152
- end
155
+ @collector.flush
156
+ end
153
157
 
154
158
  def create_connection
155
159
  conn = super
156
- conn.wal_autocheckpoint = 10000
160
+ conn.wal_autocheckpoint = 10000 # checkpoint after 10000 pages are written
157
161
  sql = YAML.load_file("#{__dir__}/litemetric.sql.yml")
158
162
  version = conn.get_first_value("PRAGMA user_version")
159
163
  sql["schema"].each_pair do |v, obj|
160
164
  if v > version
161
- conn.transaction do
162
- obj.each{|k, s| conn.execute(s)}
163
- conn.user_version = v
165
+ begin
166
+ conn.transaction do
167
+ obj.each{|k, s| conn.execute(s)}
168
+ conn.user_version = v
169
+ end
170
+ rescue Exception => e
171
+ STDERR.puts e.message
172
+ raise e
164
173
  end
165
174
  end
166
175
  end
@@ -172,9 +181,7 @@ class Litemetric
172
181
  Litesupport.spawn do
173
182
  while @running do
174
183
  sleep @options[:flush_interval]
175
- @mutex.synchronize do
176
- flush
177
- end
184
+ flush
178
185
  end
179
186
  end
180
187
  end
@@ -199,6 +206,16 @@ class Litemetric
199
206
  def collect_metrics
200
207
  @litemetric = Litemetric.instance
201
208
  @litemetric.register(metrics_identifier)
209
+ @snapshotter = create_snapshotter
210
+ end
211
+
212
+ def create_snapshotter
213
+ Litesupport.spawn do
214
+ while @running do
215
+ sleep @litemetric.options[:snapshot_interval]
216
+ capture_snapshot
217
+ end
218
+ end
202
219
  end
203
220
 
204
221
  def metrics_identifier
@@ -220,9 +237,104 @@ class Litemetric
220
237
  res
221
238
  end
222
239
 
223
- def snapshot
224
- raise Litestack::NotImplementedError
240
+ def capture_snapshot
241
+ return unless @litemetric
242
+ state = snapshot if defined? snapshot
243
+ if state
244
+ @litemetric.capture_snapshot(metrics_identifier, state)
245
+ end
246
+ end
247
+
248
+ end
249
+ end
250
+
251
+ class Litemetric
252
+
253
+ class Collector
254
+
255
+ include Litesupport::Liteconnection
256
+
257
+ DEFAULT_OPTIONS = {
258
+ path: ":memory:",
259
+ sync: 1,
260
+ flush_interval: 3, # flush data every 1 minute
261
+ summarize_interval: 10, # summarize data every 1 minute
262
+ snapshot_interval: 1 # snapshot every 10 minutes
263
+ }
264
+
265
+ RESOLUTIONS = {
266
+ minute: 300, # 5 minutes (highest resolution)
267
+ hour: 3600, # 1 hour
268
+ day: 24*3600, # 1 day
269
+ week: 7*24*3600 # 1 week (lowest resolution)
270
+ }
271
+
272
+ def initialize(options = {})
273
+ init(options)
274
+ end
275
+
276
+ def capture(topic, event, key, value=nil)
277
+ if key.is_a? Array
278
+ key.each{|k| capture_single_key(topic, event, k, value)}
279
+ else
280
+ capture_single_key(topic, event, key, value)
281
+ end
282
+ end
283
+
284
+ def capture_single_key(topic, event, key, value)
285
+ run_stmt(:capture_event, topic.to_s, event.to_s, key.to_s, nil ,1, value)
286
+ end
287
+
288
+ def flush
289
+ t = Time.now
290
+ limit = 1000 # migrate 1000 records at a time
291
+ count = run_stmt(:event_count)[0][0]
292
+ while count > 0
293
+ @conn.acquire do |conn|
294
+ conn.transaction(:immediate) do
295
+ conn.stmts[:migrate_events].execute!(limit)
296
+ conn.stmts[:delete_migrated_events].execute!(limit)
297
+ count = conn.stmts[:event_count].execute![0][0]
298
+ end
299
+ end
300
+ sleep 0.005 #give other threads a chance to run
301
+ end
302
+ end
303
+
304
+ def create_connection
305
+ conn = super
306
+ conn.execute("ATTACH ? as m", @options[:dbpath].to_s)
307
+ conn.wal_autocheckpoint = 10000
308
+ sql = YAML.load_file("#{__dir__}/litemetric_collector.sql.yml")
309
+ version = conn.get_first_value("PRAGMA user_version")
310
+ sql["schema"].each_pair do |v, obj|
311
+ if v > version
312
+ conn.transaction do
313
+ obj.each do |k, s|
314
+ begin
315
+ conn.execute(s)
316
+ rescue Exception => e
317
+ STDERR.puts "Error parsing #{k}"
318
+ STDERR.puts s
319
+ raise e
320
+ end
321
+ end
322
+ conn.user_version = v
323
+ end
324
+ end
325
+ end
326
+ sql["stmts"].each do |k, v|
327
+ begin
328
+ conn.stmts[k.to_sym] = conn.prepare(v)
329
+ rescue Exception => e
330
+ STDERR.puts "Error parsing #{k}"
331
+ STDERR.puts v
332
+ raise e
333
+ end
334
+ end
335
+ conn
225
336
  end
226
337
 
227
338
  end
339
+
228
340
  end