litestack 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/Gemfile +2 -0
- data/README.md +5 -11
- data/assets/event_page.png +0 -0
- data/assets/index_page.png +0 -0
- data/assets/topic_page.png +0 -0
- data/bench/bench_jobs_rails.rb +1 -1
- data/bench/bench_jobs_raw.rb +1 -1
- data/bin/liteboard +81 -0
- data/lib/action_cable/subscription_adapter/litecable.rb +1 -11
- data/lib/generators/litestack/install/USAGE +11 -0
- data/lib/generators/litestack/install/install_generator.rb +35 -0
- data/lib/generators/litestack/install/templates/cable.yml +11 -0
- data/lib/generators/litestack/install/templates/database.yml +34 -0
- data/lib/litestack/liteboard/liteboard.rb +305 -0
- data/lib/litestack/liteboard/views/event.erb +32 -0
- data/lib/litestack/liteboard/views/index.erb +54 -0
- data/lib/litestack/liteboard/views/layout.erb +303 -0
- data/lib/litestack/liteboard/views/litecable.erb +118 -0
- data/lib/litestack/liteboard/views/litecache.erb +144 -0
- data/lib/litestack/liteboard/views/litedb.erb +168 -0
- data/lib/litestack/liteboard/views/litejob.erb +151 -0
- data/lib/litestack/liteboard/views/topic.erb +48 -0
- data/lib/litestack/litecable.rb +25 -35
- data/lib/litestack/litecable.sql.yml +1 -1
- data/lib/litestack/litecache.rb +31 -28
- data/lib/litestack/litedb.rb +124 -1
- data/lib/litestack/litejob.rb +2 -2
- data/lib/litestack/litejobqueue.rb +8 -8
- data/lib/litestack/litemetric.rb +177 -88
- data/lib/litestack/litemetric.sql.yml +312 -42
- data/lib/litestack/litemetric_collector.sql.yml +56 -0
- data/lib/litestack/litequeue.rb +28 -29
- data/lib/litestack/litequeue.sql.yml +11 -0
- data/lib/litestack/litesupport.rb +137 -57
- data/lib/litestack/railtie.rb +10 -0
- data/lib/litestack/version.rb +1 -1
- data/lib/litestack.rb +1 -0
- data/lib/sequel/adapters/litedb.rb +1 -1
- data/template.rb +7 -0
- metadata +81 -5
- data/lib/litestack/metrics_app.rb +0 -5
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
<div class = "row">
|
|
2
|
+
|
|
3
|
+
<div class = "col">
|
|
4
|
+
<div class="card">
|
|
5
|
+
<div class="card-header">
|
|
6
|
+
Size
|
|
7
|
+
</div>
|
|
8
|
+
<div class="card-body">
|
|
9
|
+
<h1><%=round(@size)%>MB</h1>
|
|
10
|
+
</div>
|
|
11
|
+
</div>
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
<div class = "col">
|
|
15
|
+
<div class="card">
|
|
16
|
+
<div class="card-header">
|
|
17
|
+
Tables
|
|
18
|
+
</div>
|
|
19
|
+
<div class="card-body">
|
|
20
|
+
<h1><%=@tables%></h1>
|
|
21
|
+
</div>
|
|
22
|
+
</div>
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
<div class = "col">
|
|
26
|
+
<div class="card">
|
|
27
|
+
<div class="card-header">
|
|
28
|
+
Indexes
|
|
29
|
+
</div>
|
|
30
|
+
<div class="card-body">
|
|
31
|
+
<h1><%=@indexes%></h1>
|
|
32
|
+
</div>
|
|
33
|
+
</div>
|
|
34
|
+
</div>
|
|
35
|
+
|
|
36
|
+
<div class = "col">
|
|
37
|
+
<div class="card">
|
|
38
|
+
<div class="card-header">
|
|
39
|
+
Read queries
|
|
40
|
+
</div>
|
|
41
|
+
<div class="card-body">
|
|
42
|
+
<h1><%=format(@reads)%></h1>
|
|
43
|
+
</div>
|
|
44
|
+
</div>
|
|
45
|
+
</div>
|
|
46
|
+
|
|
47
|
+
<div class = "col">
|
|
48
|
+
<div class="card">
|
|
49
|
+
<div class="card-header">
|
|
50
|
+
Write queries
|
|
51
|
+
</div>
|
|
52
|
+
<div class="card-body">
|
|
53
|
+
<h1><%=format(@writes)%></h1>
|
|
54
|
+
</div>
|
|
55
|
+
</div>
|
|
56
|
+
</div>
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
</div>
|
|
60
|
+
|
|
61
|
+
<div class = "row">
|
|
62
|
+
<br/>
|
|
63
|
+
</div>
|
|
64
|
+
|
|
65
|
+
<div class = "row">
|
|
66
|
+
|
|
67
|
+
<div class = "col">
|
|
68
|
+
<div class="card">
|
|
69
|
+
<div class="card-header">
|
|
70
|
+
Reads vs writes over time
|
|
71
|
+
</div>
|
|
72
|
+
<div class="card-body">
|
|
73
|
+
<span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads", "Writes"]] + @reads_vs_writes)%></span>
|
|
74
|
+
</div>
|
|
75
|
+
</div>
|
|
76
|
+
</div>
|
|
77
|
+
|
|
78
|
+
<div class = "col-6">
|
|
79
|
+
<div class="card">
|
|
80
|
+
<div class="card-header">
|
|
81
|
+
Reads vs writes
|
|
82
|
+
</div>
|
|
83
|
+
<div class="card-body">
|
|
84
|
+
<span class="hidden inlinepie"><%=[["name", "value"],["reads", @reads],["writes", @writes]]%></span>
|
|
85
|
+
</div>
|
|
86
|
+
</div>
|
|
87
|
+
</div>
|
|
88
|
+
|
|
89
|
+
</div>
|
|
90
|
+
|
|
91
|
+
<div class = "row">
|
|
92
|
+
<br/>
|
|
93
|
+
</div>
|
|
94
|
+
|
|
95
|
+
<div class = "row">
|
|
96
|
+
|
|
97
|
+
<div class = "col">
|
|
98
|
+
<div class="card">
|
|
99
|
+
<div class="card-header">
|
|
100
|
+
Query execution time over time
|
|
101
|
+
</div>
|
|
102
|
+
<div class="card-body">
|
|
103
|
+
<span class="hidden inlinecolumn"><%=Oj.dump([["Time", "Reads execution time", {'role' => 'tooltip'}, "Writes execution time", {'role' => 'tooltip'}]] + @reads_vs_writes_times)%></span>
|
|
104
|
+
</div>
|
|
105
|
+
</div>
|
|
106
|
+
</div>
|
|
107
|
+
|
|
108
|
+
<div class = "col-6">
|
|
109
|
+
<div class="card">
|
|
110
|
+
<div class="card-header">
|
|
111
|
+
Query time
|
|
112
|
+
</div>
|
|
113
|
+
<div class="card-body">
|
|
114
|
+
<span class="hidden inlinepie"><%=[["name", "value"],["total reads time", @read_times],["total writes time", @write_times]]%></span>
|
|
115
|
+
</div>
|
|
116
|
+
</div>
|
|
117
|
+
</div>
|
|
118
|
+
|
|
119
|
+
</div>
|
|
120
|
+
|
|
121
|
+
<div class = "row">
|
|
122
|
+
<br/>
|
|
123
|
+
</div>
|
|
124
|
+
|
|
125
|
+
<div class = "row">
|
|
126
|
+
|
|
127
|
+
<div class = "col-6">
|
|
128
|
+
<div class="card">
|
|
129
|
+
<div class="card-header">
|
|
130
|
+
Slowest queries
|
|
131
|
+
</div>
|
|
132
|
+
<div class="card-body">
|
|
133
|
+
<table class="table">
|
|
134
|
+
<%@slowest.each do |r| %>
|
|
135
|
+
<tr>
|
|
136
|
+
<td><%=r['key']%></td>
|
|
137
|
+
<td align="right"><h6><%=format(r['ravg'].truncate(4))%> sec</h6></td>
|
|
138
|
+
</tr>
|
|
139
|
+
<% end %>
|
|
140
|
+
</table>
|
|
141
|
+
</div>
|
|
142
|
+
</div>
|
|
143
|
+
</div>
|
|
144
|
+
|
|
145
|
+
<div class = "col-6">
|
|
146
|
+
<div class="card">
|
|
147
|
+
<div class="card-header">
|
|
148
|
+
Most expensive queries (execution time x frequency)
|
|
149
|
+
</div>
|
|
150
|
+
<div class="card-body">
|
|
151
|
+
<table class="table">
|
|
152
|
+
<%@popular.each do |r| %>
|
|
153
|
+
<tr>
|
|
154
|
+
<td><%=r['key']%></td>
|
|
155
|
+
<td align="right"><h6><%=format(r['rtotal'].truncate(4))%> sec</h6></td>
|
|
156
|
+
</tr>
|
|
157
|
+
<% end %>
|
|
158
|
+
</table>
|
|
159
|
+
</div>
|
|
160
|
+
</div>
|
|
161
|
+
</div>
|
|
162
|
+
|
|
163
|
+
</div>
|
|
164
|
+
|
|
165
|
+
<div class = "row">
|
|
166
|
+
<br/>
|
|
167
|
+
</div>
|
|
168
|
+
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
<div class = "row">
|
|
2
|
+
|
|
3
|
+
<div class = "col">
|
|
4
|
+
<div class="card">
|
|
5
|
+
<div class="card-header">
|
|
6
|
+
Size
|
|
7
|
+
</div>
|
|
8
|
+
<div class="card-body">
|
|
9
|
+
<h1><%=format(round(@size))%> MB</h1>
|
|
10
|
+
</div>
|
|
11
|
+
</div>
|
|
12
|
+
</div>
|
|
13
|
+
|
|
14
|
+
<div class = "col">
|
|
15
|
+
<div class="card">
|
|
16
|
+
<div class="card-header">
|
|
17
|
+
Active queues
|
|
18
|
+
</div>
|
|
19
|
+
<div class="card-body">
|
|
20
|
+
<h1><%=format(@queues.length)%></h1>
|
|
21
|
+
</div>
|
|
22
|
+
</div>
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
<div class = "col">
|
|
26
|
+
<div class="card">
|
|
27
|
+
<div class="card-header">
|
|
28
|
+
Outstanding jobs
|
|
29
|
+
</div>
|
|
30
|
+
<div class="card-body">
|
|
31
|
+
<h1><%=format(@jobs)%></h1>
|
|
32
|
+
</div>
|
|
33
|
+
</div>
|
|
34
|
+
</div>
|
|
35
|
+
|
|
36
|
+
</div>
|
|
37
|
+
|
|
38
|
+
<div class = "row">
|
|
39
|
+
<br/>
|
|
40
|
+
</div>
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
<div class="row">
|
|
44
|
+
|
|
45
|
+
<div class = "col">
|
|
46
|
+
<div class="card">
|
|
47
|
+
<div class="card-header">
|
|
48
|
+
Processed jobs
|
|
49
|
+
</div>
|
|
50
|
+
<div class="card-body">
|
|
51
|
+
<h1><%=format(@processed_count)%> jobs</h1>
|
|
52
|
+
</div>
|
|
53
|
+
</div>
|
|
54
|
+
</div>
|
|
55
|
+
|
|
56
|
+
<div class = "col">
|
|
57
|
+
<div class="card">
|
|
58
|
+
<div class="card-header">
|
|
59
|
+
Total processing time
|
|
60
|
+
</div>
|
|
61
|
+
<div class="card-body">
|
|
62
|
+
<h1><%=format(@processing_time)%> sec</h1>
|
|
63
|
+
</div>
|
|
64
|
+
</div>
|
|
65
|
+
</div>
|
|
66
|
+
|
|
67
|
+
<div class = "col">
|
|
68
|
+
<div class="card">
|
|
69
|
+
<div class="card-header">
|
|
70
|
+
Average processing time per job
|
|
71
|
+
</div>
|
|
72
|
+
<div class="card-body">
|
|
73
|
+
<h1><%=(@processing_time.to_f / @processed_count).truncate(4) rescue 0%> sec</h1>
|
|
74
|
+
</div>
|
|
75
|
+
</div>
|
|
76
|
+
</div>
|
|
77
|
+
|
|
78
|
+
</div>
|
|
79
|
+
|
|
80
|
+
<div class = "row">
|
|
81
|
+
<br/>
|
|
82
|
+
</div>
|
|
83
|
+
|
|
84
|
+
<div class="row">
|
|
85
|
+
|
|
86
|
+
<div class = "col">
|
|
87
|
+
<div class="card">
|
|
88
|
+
<div class="card-header">
|
|
89
|
+
Processed jobs by queue
|
|
90
|
+
</div>
|
|
91
|
+
<div class="card-body">
|
|
92
|
+
<span class="hidden inlinepie">
|
|
93
|
+
<%=[["name", "value"]] + @processed_count_by_queue.to_a%>
|
|
94
|
+
</span>
|
|
95
|
+
</div>
|
|
96
|
+
</div>
|
|
97
|
+
</div>
|
|
98
|
+
|
|
99
|
+
<div class = "col">
|
|
100
|
+
<div class="card">
|
|
101
|
+
<div class="card-header">
|
|
102
|
+
Job processing time by queue
|
|
103
|
+
</div>
|
|
104
|
+
<div class="card-body">
|
|
105
|
+
<span class="hidden inlinepie">
|
|
106
|
+
<%=[["name", "value"]] + @processing_time_by_queue.to_a%>
|
|
107
|
+
</span>
|
|
108
|
+
</div>
|
|
109
|
+
</div>
|
|
110
|
+
</div>
|
|
111
|
+
|
|
112
|
+
</div>
|
|
113
|
+
|
|
114
|
+
<div class = "row">
|
|
115
|
+
<br/>
|
|
116
|
+
</div>
|
|
117
|
+
|
|
118
|
+
<div class="row">
|
|
119
|
+
|
|
120
|
+
<div class = "col">
|
|
121
|
+
<div class="card">
|
|
122
|
+
<div class="card-header">
|
|
123
|
+
Processed jobs by queue over time
|
|
124
|
+
</div>
|
|
125
|
+
<div class="card-body">
|
|
126
|
+
<span class="hidden inlinestackedcolumn">
|
|
127
|
+
<%=@processed_count_over_time_by_queues.to_a%>
|
|
128
|
+
</span>
|
|
129
|
+
</div>
|
|
130
|
+
</div>
|
|
131
|
+
</div>
|
|
132
|
+
|
|
133
|
+
<div class = "col">
|
|
134
|
+
<div class="card">
|
|
135
|
+
<div class="card-header">
|
|
136
|
+
Job processing time by queue over time
|
|
137
|
+
</div>
|
|
138
|
+
<div class="card-body">
|
|
139
|
+
<span class="hidden inlinestackedcolumn">
|
|
140
|
+
<%=@processing_time_over_time_by_queues.to_a%>
|
|
141
|
+
</span>
|
|
142
|
+
</div>
|
|
143
|
+
</div>
|
|
144
|
+
</div>
|
|
145
|
+
|
|
146
|
+
</div>
|
|
147
|
+
|
|
148
|
+
<div class = "row">
|
|
149
|
+
<br/>
|
|
150
|
+
</div>
|
|
151
|
+
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
<h5><a href="/?res=<%=@res%>">All Topics</a> > <%= @topic %></h5>
|
|
2
|
+
<% if @snapshot and not @snapshot.empty? and not @snapshot[0].nil?%>
|
|
3
|
+
<div><b>Snapshot</b> captured on <%= @snapshot[1] %></div>
|
|
4
|
+
<% @snapshot[0].each_pair do |k, v| %>
|
|
5
|
+
<table class ="table summary">
|
|
6
|
+
<tr>
|
|
7
|
+
<th colspan="2"><%=k.to_s.capitalize%></th>
|
|
8
|
+
</tr>
|
|
9
|
+
<% v.each do |ss| %>
|
|
10
|
+
<tr>
|
|
11
|
+
<td><%= ss[0].to_s.capitalize.gsub("_", " ") %></td>
|
|
12
|
+
<td><%= ss[1] %></td>
|
|
13
|
+
</tr>
|
|
14
|
+
<% end %>
|
|
15
|
+
</table>
|
|
16
|
+
<br/>
|
|
17
|
+
<% end %>
|
|
18
|
+
<% end %>
|
|
19
|
+
<div><b>Dynamic event data</b></div>
|
|
20
|
+
<div id="search"><form><input id="search-field" type="text" placeholder="Search events" onkeydown="search_kd(this)" onkeyup="search_ku(this)" value="<%=@search%>"/></form></div> <table class="table sortable">
|
|
21
|
+
<tr>
|
|
22
|
+
<th width="16%" class="<%='sorted' if @order == 'name'%>"><a href="<%=topic_sort_url('name')%>">Event</a> <%=dir('name')%></th>
|
|
23
|
+
<th width="8%" class="<%='sorted' if @order == 'rcount'%>"><a href="<%=topic_sort_url('rcount')%>">Event Count</a> <%=dir('rcount')%></th>
|
|
24
|
+
<th width="8%" class="<%='sorted' if @order == 'ravg'%>"><a href="<%=topic_sort_url('ravg')%>">Avg Value</a> <%=dir('ravg')%></th>
|
|
25
|
+
<th width="8%" class="<%='sorted' if @order == 'rtotal'%>"><a href="<%=topic_sort_url('rtotal')%>">Total Value</a> <%=dir('rtotal')%></th>
|
|
26
|
+
<th width="8%" class="<%='sorted' if @order == 'rmin'%>"><a href="<%=topic_sort_url('rmin')%>">Min Value</a> <%=dir('rmin')%></th>
|
|
27
|
+
<th width="8%" class="<%='sorted' if @order == 'rmax'%>"><a href="<%=topic_sort_url('rmax')%>">Max Value</a> <%=dir('rmax')%></th>
|
|
28
|
+
<th width="22%">Events over time</th>
|
|
29
|
+
<th width="22%">Average value over time</th>
|
|
30
|
+
</tr>
|
|
31
|
+
<% @events.each do |event|%>
|
|
32
|
+
<tr>
|
|
33
|
+
<td title="<%=event[0]%>"><div class="label"><a href="/topics/<%=encode(@topic)%>/events/<%=encode(event[0])%>?res=<%=@res%>"><%=event[0]%></a></div></td>
|
|
34
|
+
<td><%=event[2]%></td>
|
|
35
|
+
<td><%="%0.2f" % [event[3]] if event[3] %></td>
|
|
36
|
+
<td><%="%0.2f" % [event[4]] if event[4] %></td>
|
|
37
|
+
<td><%="%0.2f" % [event[5]] if event[5] %></td>
|
|
38
|
+
<td><%="%0.2f" % [event[6]] if event[6] %></td>
|
|
39
|
+
<td class="chart"><span class="inlinecolumn hidden" data-label="Count"><%=Oj.dump(event[7]) if event[7]%></span></td>
|
|
40
|
+
<td class="chart"><span class="inlinecolumn hidden" data-label="Avg Value"><%=Oj.dump(event[8]) if event[8]%></span></td>
|
|
41
|
+
</tr>
|
|
42
|
+
<% end %>
|
|
43
|
+
<% if @events.empty? %>
|
|
44
|
+
<tr>
|
|
45
|
+
<td class="empty" colspan="9">No data to display</td>
|
|
46
|
+
</tr>
|
|
47
|
+
<% end %>
|
|
48
|
+
</table>
|
data/lib/litestack/litecable.rb
CHANGED
|
@@ -15,7 +15,7 @@ class Litecable
|
|
|
15
15
|
|
|
16
16
|
DEFAULT_OPTIONS = {
|
|
17
17
|
config_path: "./litecable.yml",
|
|
18
|
-
path: "
|
|
18
|
+
path: Litesupport.root.join("cable.sqlite3"),
|
|
19
19
|
sync: 0,
|
|
20
20
|
mmap_size: 16 * 1024 * 1024, # 16MB
|
|
21
21
|
expire_after: 5, # remove messages older than 5 seconds
|
|
@@ -24,52 +24,54 @@ class Litecable
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
def initialize(options = {})
|
|
27
|
-
@messages = []
|
|
27
|
+
@messages = Litesupport::Pool.new(1){[]}
|
|
28
28
|
init(options)
|
|
29
|
+
collect_metrics if @options[:metrics]
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
# broadcast a message to a specific channel
|
|
32
33
|
def broadcast(channel, payload=nil)
|
|
33
34
|
# group meesages and only do broadcast every 10 ms
|
|
34
|
-
#run_stmt(:publish, channel.to_s, Oj.dump(payload), @pid)
|
|
35
35
|
# but broadcast locally normally
|
|
36
|
-
@
|
|
36
|
+
@messages.acquire{|msgs| msgs << [channel.to_s, Oj.dump(payload)]}
|
|
37
|
+
capture(:broadcast, channel)
|
|
37
38
|
local_broadcast(channel, payload)
|
|
38
39
|
end
|
|
39
40
|
|
|
40
41
|
# subscribe to a channel, optionally providing a success callback proc
|
|
41
42
|
def subscribe(channel, subscriber, success_callback = nil)
|
|
42
|
-
@
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
@subscribers.acquire do |subs|
|
|
44
|
+
subs[channel] = {} unless subs[channel]
|
|
45
|
+
subs[channel][subscriber] = true
|
|
45
46
|
end
|
|
47
|
+
capture(:subscribe, channel)
|
|
46
48
|
end
|
|
47
49
|
|
|
48
50
|
# unsubscribe from a channel
|
|
49
51
|
def unsubscribe(channel, subscriber)
|
|
50
|
-
@
|
|
51
|
-
|
|
52
|
-
end
|
|
52
|
+
@subscribers.acquire{|subs| subs[channel].delete(subscriber) rescue nil }
|
|
53
|
+
capture(:unsubscribe, channel)
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
private
|
|
56
|
-
|
|
57
|
+
|
|
58
|
+
# broadcast the message to local subscribers
|
|
57
59
|
def local_broadcast(channel, payload=nil)
|
|
58
|
-
return unless @subscribers[channel]
|
|
59
60
|
subscribers = []
|
|
60
|
-
@
|
|
61
|
-
|
|
61
|
+
@subscribers.acquire do |subs|
|
|
62
|
+
return unless subs[channel]
|
|
63
|
+
subscribers = subs[channel].keys
|
|
62
64
|
end
|
|
63
65
|
subscribers.each do |subscriber|
|
|
64
66
|
subscriber.call(payload)
|
|
67
|
+
capture(:message, channel)
|
|
65
68
|
end
|
|
66
69
|
end
|
|
67
70
|
|
|
68
71
|
def setup
|
|
69
72
|
super # create connection
|
|
70
73
|
@pid = Process.pid
|
|
71
|
-
@subscribers = {}
|
|
72
|
-
@mutex = Litesupport::Mutex.new
|
|
74
|
+
@subscribers = Litesupport::Pool.new(1){{}}
|
|
73
75
|
@running = true
|
|
74
76
|
@listener = create_listener
|
|
75
77
|
@pruner = create_pruner
|
|
@@ -80,14 +82,14 @@ class Litecable
|
|
|
80
82
|
def create_broadcaster
|
|
81
83
|
Litesupport.spawn do
|
|
82
84
|
while @running do
|
|
83
|
-
@
|
|
84
|
-
if
|
|
85
|
+
@messages.acquire do |msgs|
|
|
86
|
+
if msgs.length > 0
|
|
85
87
|
run_sql("BEGIN IMMEDIATE")
|
|
86
|
-
while msg =
|
|
88
|
+
while msg = msgs.shift
|
|
87
89
|
run_stmt(:publish, msg[0], msg[1], @pid)
|
|
88
90
|
end
|
|
89
91
|
run_sql("END")
|
|
90
|
-
end
|
|
92
|
+
end
|
|
91
93
|
end
|
|
92
94
|
sleep 0.02
|
|
93
95
|
end
|
|
@@ -107,7 +109,6 @@ class Litecable
|
|
|
107
109
|
Litesupport.spawn do
|
|
108
110
|
while @running do
|
|
109
111
|
@last_fetched_id ||= (run_stmt(:last_id)[0][0] || 0)
|
|
110
|
-
@logger.info @last_fetched_id
|
|
111
112
|
run_stmt(:fetch, @last_fetched_id, @pid).to_a.each do |msg|
|
|
112
113
|
@logger.info "RECEIVED #{msg}"
|
|
113
114
|
@last_fetched_id = msg[0]
|
|
@@ -119,20 +120,9 @@ class Litecable
|
|
|
119
120
|
end
|
|
120
121
|
|
|
121
122
|
def create_connection
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
version = conn.get_first_value("PRAGMA user_version")
|
|
126
|
-
sql["schema"].each_pair do |v, obj|
|
|
127
|
-
if v > version
|
|
128
|
-
conn.transaction do
|
|
129
|
-
obj.each{|k, s| conn.execute(s)}
|
|
130
|
-
conn.user_version = v
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
end
|
|
134
|
-
sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
|
|
135
|
-
conn
|
|
123
|
+
super("#{__dir__}/litecable.sql.yml") do |conn|
|
|
124
|
+
conn.wal_autocheckpoint = 10000
|
|
125
|
+
end
|
|
136
126
|
end
|
|
137
127
|
|
|
138
128
|
end
|
|
@@ -17,7 +17,7 @@ stmts:
|
|
|
17
17
|
|
|
18
18
|
last_id: SELECT max(id) FROM messages
|
|
19
19
|
|
|
20
|
-
fetch: SELECT id, channel, value FROM messages WHERE id > $1 and pid != $2
|
|
20
|
+
fetch: SELECT id, channel, value, created_at FROM messages WHERE id > $1 and pid != $2
|
|
21
21
|
|
|
22
22
|
prune: DELETE FROM messages WHERE created_at < (unixepoch() - $1)
|
|
23
23
|
|
data/lib/litestack/litecache.rb
CHANGED
|
@@ -32,7 +32,7 @@ class Litecache
|
|
|
32
32
|
# sleep_interval: 1 -> 1 second of sleep between cleanup runs
|
|
33
33
|
|
|
34
34
|
DEFAULT_OPTIONS = {
|
|
35
|
-
path: "
|
|
35
|
+
path: Litesupport.root.join("cache.sqlite3"),
|
|
36
36
|
config_path: "./litecache.yml",
|
|
37
37
|
sync: 0,
|
|
38
38
|
expiry: 60 * 60 * 24 * 30, # one month
|
|
@@ -64,8 +64,8 @@ class Litecache
|
|
|
64
64
|
|
|
65
65
|
def initialize(options = {})
|
|
66
66
|
options[:size] = DEFAULT_OPTIONS[:min_size] if options[:size] && options[:size] < DEFAULT_OPTIONS[:min_size]
|
|
67
|
-
init(options)
|
|
68
67
|
@last_visited = {}
|
|
68
|
+
init(options)
|
|
69
69
|
collect_metrics if @options[:metrics]
|
|
70
70
|
end
|
|
71
71
|
|
|
@@ -76,7 +76,7 @@ class Litecache
|
|
|
76
76
|
@conn.acquire do |cache|
|
|
77
77
|
begin
|
|
78
78
|
cache.stmts[:setter].execute!(key, value, expires_in)
|
|
79
|
-
capture(:
|
|
79
|
+
capture(:set, key)
|
|
80
80
|
rescue SQLite3::FullException
|
|
81
81
|
cache.stmts[:extra_pruner].execute!(0.2)
|
|
82
82
|
cache.execute("vacuum")
|
|
@@ -97,7 +97,7 @@ class Litecache
|
|
|
97
97
|
cache.stmts[:inserter].execute!(key, value, expires_in)
|
|
98
98
|
changes = cache.changes
|
|
99
99
|
end
|
|
100
|
-
capture(:
|
|
100
|
+
capture(:set, key)
|
|
101
101
|
rescue SQLite3::FullException
|
|
102
102
|
cache.stmts[:extra_pruner].execute!(0.2)
|
|
103
103
|
cache.execute("vacuum")
|
|
@@ -113,10 +113,10 @@ class Litecache
|
|
|
113
113
|
key = key.to_s
|
|
114
114
|
if record = @conn.acquire{|cache| cache.stmts[:getter].execute!(key)[0] }
|
|
115
115
|
@last_visited[key] = true
|
|
116
|
-
capture(:
|
|
116
|
+
capture(:get, key, 1)
|
|
117
117
|
return record[1]
|
|
118
118
|
end
|
|
119
|
-
capture(:
|
|
119
|
+
capture(:get, key, 0)
|
|
120
120
|
nil
|
|
121
121
|
end
|
|
122
122
|
|
|
@@ -160,9 +160,9 @@ class Litecache
|
|
|
160
160
|
end
|
|
161
161
|
|
|
162
162
|
# return the actual size of the cache file
|
|
163
|
-
def size
|
|
164
|
-
|
|
165
|
-
end
|
|
163
|
+
#def size
|
|
164
|
+
# run_stmt(:sizer)[0][0]
|
|
165
|
+
#end
|
|
166
166
|
|
|
167
167
|
# delete all key, value pairs in the cache
|
|
168
168
|
def clear
|
|
@@ -177,8 +177,22 @@ class Litecache
|
|
|
177
177
|
|
|
178
178
|
# return the maximum size of the cache
|
|
179
179
|
def max_size
|
|
180
|
-
run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0]
|
|
180
|
+
run_sql("SELECT s.page_size * c.max_page_count FROM pragma_page_size() as s, pragma_max_page_count() as c")[0][0].to_f / (1024 * 1024)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def snapshot
|
|
184
|
+
{
|
|
185
|
+
summary: {
|
|
186
|
+
path: path,
|
|
187
|
+
journal_mode: journal_mode,
|
|
188
|
+
synchronous: synchronous,
|
|
189
|
+
size: size,
|
|
190
|
+
max_size: max_size,
|
|
191
|
+
entries: count
|
|
192
|
+
}
|
|
193
|
+
}
|
|
181
194
|
end
|
|
195
|
+
|
|
182
196
|
|
|
183
197
|
# low level access to SQLite transactions, use with caution
|
|
184
198
|
def transaction(mode, acquire=true)
|
|
@@ -222,23 +236,12 @@ class Litecache
|
|
|
222
236
|
end
|
|
223
237
|
|
|
224
238
|
def create_connection
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
sql["schema"].each_pair do |v, obj|
|
|
233
|
-
if v > version
|
|
234
|
-
conn.transaction do
|
|
235
|
-
obj.each{|k, s| conn.execute(s)}
|
|
236
|
-
conn.user_version = v
|
|
237
|
-
end
|
|
238
|
-
end
|
|
239
|
-
end
|
|
240
|
-
sql["stmts"].each { |k, v| conn.stmts[k.to_sym] = conn.prepare(v) }
|
|
241
|
-
conn
|
|
242
|
-
end
|
|
239
|
+
super("#{__dir__}/litecache.sql.yml") do |conn|
|
|
240
|
+
conn.cache_size = 2000
|
|
241
|
+
conn.journal_size_limit = [(@options[:size]/2).to_i, @options[:min_size]].min
|
|
242
|
+
conn.max_page_count = (@options[:size] / conn.page_size).to_i
|
|
243
|
+
conn.case_sensitive_like = true
|
|
244
|
+
end
|
|
245
|
+
end
|
|
243
246
|
|
|
244
247
|
end
|