scout_agent 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/AUTHORS +4 -0
- data/CHANGELOG +3 -0
- data/COPYING +340 -0
- data/INSTALL +17 -0
- data/LICENSE +6 -0
- data/README +3 -0
- data/Rakefile +123 -0
- data/TODO +3 -0
- data/bin/scout_agent +11 -0
- data/lib/scout_agent.rb +73 -0
- data/lib/scout_agent/agent.rb +42 -0
- data/lib/scout_agent/agent/communication_agent.rb +85 -0
- data/lib/scout_agent/agent/master_agent.rb +301 -0
- data/lib/scout_agent/api.rb +241 -0
- data/lib/scout_agent/assignment.rb +105 -0
- data/lib/scout_agent/assignment/configuration.rb +30 -0
- data/lib/scout_agent/assignment/identify.rb +110 -0
- data/lib/scout_agent/assignment/queue.rb +95 -0
- data/lib/scout_agent/assignment/reset.rb +91 -0
- data/lib/scout_agent/assignment/snapshot.rb +92 -0
- data/lib/scout_agent/assignment/start.rb +149 -0
- data/lib/scout_agent/assignment/status.rb +44 -0
- data/lib/scout_agent/assignment/stop.rb +60 -0
- data/lib/scout_agent/assignment/upload_log.rb +61 -0
- data/lib/scout_agent/core_extensions.rb +260 -0
- data/lib/scout_agent/database.rb +386 -0
- data/lib/scout_agent/database/mission_log.rb +282 -0
- data/lib/scout_agent/database/queue.rb +126 -0
- data/lib/scout_agent/database/snapshots.rb +187 -0
- data/lib/scout_agent/database/statuses.rb +65 -0
- data/lib/scout_agent/dispatcher.rb +157 -0
- data/lib/scout_agent/id_card.rb +143 -0
- data/lib/scout_agent/lifeline.rb +243 -0
- data/lib/scout_agent/mission.rb +212 -0
- data/lib/scout_agent/order.rb +58 -0
- data/lib/scout_agent/order/check_in_order.rb +32 -0
- data/lib/scout_agent/order/snapshot_order.rb +33 -0
- data/lib/scout_agent/plan.rb +306 -0
- data/lib/scout_agent/server.rb +123 -0
- data/lib/scout_agent/tracked.rb +59 -0
- data/lib/scout_agent/wire_tap.rb +513 -0
- data/setup.rb +1360 -0
- data/test/tc_core_extensions.rb +89 -0
- data/test/tc_id_card.rb +115 -0
- data/test/tc_plan.rb +285 -0
- data/test/test_helper.rb +22 -0
- data/test/ts_all.rb +7 -0
- metadata +171 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
module ScoutAgent
|
4
|
+
class Assignment
|
5
|
+
class Stop < Assignment
|
6
|
+
def execute
|
7
|
+
@agent = IDCard.new(:lifeline)
|
8
|
+
if @agent.pid_file.exist?
|
9
|
+
puts "Stopping #{ScoutAgent.proper_agent_name} (PID #{@agent.pid})..."
|
10
|
+
signal_and_wait("TERM")
|
11
|
+
if @agent.pid_file.exist?
|
12
|
+
puts "TERM signal was ignored, sending KILL..."
|
13
|
+
signal_and_wait("KILL")
|
14
|
+
if @agent.pid_file.exist?
|
15
|
+
abort_with_failed_to_stop
|
16
|
+
end
|
17
|
+
end
|
18
|
+
puts "Stopped."
|
19
|
+
else
|
20
|
+
abort_with_not_running_notice
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def signal_and_wait(signal_name, wait_count = 10, wait_delay = 0.5)
|
27
|
+
begin
|
28
|
+
@agent.signal(signal_name)
|
29
|
+
rescue Errno::EPERM # we don't have permission
|
30
|
+
abort_with_no_permission
|
31
|
+
end
|
32
|
+
wait_count.times do
|
33
|
+
sleep wait_delay
|
34
|
+
break unless @agent.pid_file.exist?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def abort_with_not_running_notice
|
39
|
+
puts "#{ScoutAgent.proper_agent_name} is not currently running."
|
40
|
+
end
|
41
|
+
|
42
|
+
def abort_with_no_permission
|
43
|
+
abort <<-END_PERMISSION.trim
|
44
|
+
Unable to signal the daemon. Please rerun this command with
|
45
|
+
super user privileges:
|
46
|
+
|
47
|
+
sudo #{$PROGRAM_NAME} stop
|
48
|
+
|
49
|
+
END_PERMISSION
|
50
|
+
end
|
51
|
+
|
52
|
+
def abort_with_failed_to_stop
|
53
|
+
abort <<-END_FAILED.trim
|
54
|
+
Unable to stop the daemon. You may need to use the PID files
|
55
|
+
in #{Plan.pid_dir} to clean up stay processes.
|
56
|
+
END_FAILED
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
# require standard libraries
|
4
|
+
require "tempfile"
|
5
|
+
|
6
|
+
module ScoutAgent
|
7
|
+
class Assignment
|
8
|
+
class UploadLog < Assignment
|
9
|
+
def execute
|
10
|
+
file_name = "#{ScoutAgent.agent_name}.log"
|
11
|
+
if date = Array(other_args).shift
|
12
|
+
file_name += ".#{date.delete('^0-9')}"
|
13
|
+
end
|
14
|
+
log_file = Plan.log_dir + file_name
|
15
|
+
|
16
|
+
unless log_file.exist?
|
17
|
+
abort_with_not_found(file_name)
|
18
|
+
end
|
19
|
+
|
20
|
+
puts "Preparing file for the server. This may take a moment..."
|
21
|
+
begin
|
22
|
+
upload_file = Tempfile.new("#{file_name}.gz")
|
23
|
+
gzipped = Zlib::GzipWriter.new(upload_file)
|
24
|
+
log_file.each_line do |line|
|
25
|
+
gzipped << line
|
26
|
+
end
|
27
|
+
gzipped.close
|
28
|
+
upload_file.open # reopen what Zlib closed
|
29
|
+
rescue Exception => error # Zlib or IOError
|
30
|
+
abort_with_preparation_error(error)
|
31
|
+
end
|
32
|
+
puts "Done."
|
33
|
+
|
34
|
+
|
35
|
+
puts "Sending file to the server. This may take a moment..."
|
36
|
+
server = Server.new
|
37
|
+
if server.post_log(upload_file)
|
38
|
+
puts "Log '#{file_name}' received. Thanks."
|
39
|
+
else
|
40
|
+
abort_with_server_error
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def abort_with_not_found(log_file_name)
|
47
|
+
abort "'#{log_file_name}' could not be found to upload."
|
48
|
+
end
|
49
|
+
|
50
|
+
def abort_with_preparation_error(error)
|
51
|
+
abort <<-END_PREPARATION_ERROR
|
52
|
+
Could not prepare file for upload: #{error.message} (#{error.class})
|
53
|
+
END_PREPARATION_ERROR
|
54
|
+
end
|
55
|
+
|
56
|
+
def abort_with_server_error
|
57
|
+
abort "A networking error prevented the file from being sent."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,260 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
module ScoutAgent
|
4
|
+
#
|
5
|
+
# This Module holds a handful of extensions to the Ruby core used by our
|
6
|
+
# agent. This is in no way meant to be an ActiveSupport or Facets level
|
7
|
+
# support library. Instead, this should house a few simple conveniences that
|
8
|
+
# are intristic to the agent's needs in some way.
|
9
|
+
#
|
10
|
+
# For example, the agent tries to load matching source files for Classes and
|
11
|
+
# Modules in multiple places. Given that, it makes sense for us to add helper
|
12
|
+
# methods for these case coversions.
|
13
|
+
#
|
14
|
+
module CoreExtensions
|
15
|
+
# Extensions for the Array class.
|
16
|
+
module Array
|
17
|
+
#
|
18
|
+
# This method converts an Array into a human readable word list String.
|
19
|
+
# These lists are of the form:
|
20
|
+
#
|
21
|
+
# one
|
22
|
+
# one and two
|
23
|
+
# one, two, and three
|
24
|
+
# etc.
|
25
|
+
#
|
26
|
+
# You can change the +conjuction+ used to join the last two elements.
|
27
|
+
#
|
28
|
+
def to_word_list(conjuction = "and")
|
29
|
+
case size
|
30
|
+
when 0 then ""
|
31
|
+
when 1 then self[0].to_s
|
32
|
+
when 2 then join(" #{conjuction} ")
|
33
|
+
else ( self[0..-3] +
|
34
|
+
[self[-2..-1].join(", #{conjuction} ")] ).join(", ")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Extensions for the Module class.
|
40
|
+
module Module
|
41
|
+
#
|
42
|
+
# Returns just the last element of a full Module or Class name as a
|
43
|
+
# String. For example, ScoutAgent::CoreExtensions::Module would be
|
44
|
+
# simplified to <tt>"Module"</tt>.
|
45
|
+
#
|
46
|
+
def short_name
|
47
|
+
name[/[^:]+\z/]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Globally available extensions.
|
52
|
+
module Object
|
53
|
+
#
|
54
|
+
# Tries to require a library with +name+. If that fails, attempts to load
|
55
|
+
# RubyGems and tries the require again. If that too fails, this process
|
56
|
+
# will exit with a message about the missing library.
|
57
|
+
#
|
58
|
+
def require_lib_or_gem(name)
|
59
|
+
require name
|
60
|
+
rescue LoadError # library not found
|
61
|
+
begin
|
62
|
+
require "rubygems"
|
63
|
+
require name
|
64
|
+
rescue LoadError # library not found
|
65
|
+
abort <<-END_LOAD_ERROR.trim
|
66
|
+
Unable to load the required '#{name}' library. Try:
|
67
|
+
|
68
|
+
sudo gem install #{name.tr('_', '-')}
|
69
|
+
|
70
|
+
END_LOAD_ERROR
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
#
|
75
|
+
# This is a helper for those libraries that aren't well enough behaved not
|
76
|
+
# to spit out warnings as they work. Warnings are disabled, your block is
|
77
|
+
# run, then warnings are restored to their previous level.
|
78
|
+
#
|
79
|
+
def no_warnings
|
80
|
+
old_verbose = $VERBOSE
|
81
|
+
$VERBOSE = nil
|
82
|
+
yield
|
83
|
+
ensure
|
84
|
+
$VERBOSE = old_verbose
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Sets up an at_exit() hander that will only run when this process exits.
|
89
|
+
# Child processes do not inherit +exit_code+.
|
90
|
+
#
|
91
|
+
def at_my_exit(&exit_code)
|
92
|
+
me = ::Process.pid
|
93
|
+
at_exit do
|
94
|
+
exit_code.call if me == ::Process.pid
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
#
|
99
|
+
# This method installs +handler+ for +TERM+ signals. If the agent is
|
100
|
+
# running as a daemon, the same code will be installed for +INT+ signals
|
101
|
+
# as well. Otherwise, +INT+ signals will be set to +IGNORE+.
|
102
|
+
#
|
103
|
+
def install_shutdown_handler(&handler)
|
104
|
+
shutdown_signals = %w[TERM]
|
105
|
+
if ScoutAgent::Plan.run_as_daemon?
|
106
|
+
shutdown_signals << "INT"
|
107
|
+
else
|
108
|
+
trap("INT", "IGNORE")
|
109
|
+
end
|
110
|
+
shutdown_signals.each do |signal|
|
111
|
+
trap(signal, &handler)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
# Module extensions for Process.
|
117
|
+
module ProcessModule
|
118
|
+
#
|
119
|
+
# A convenience method to ensure +child_pid+ is stopped. First a friendly
|
120
|
+
# +TERM+ signal is sent. Then, after +pause+ seconds, +KILL+ will be sent
|
121
|
+
# if the process has not yet exited. The exit code for the process is
|
122
|
+
# returned if it can be determined (+nil+ is returned otherwise).
|
123
|
+
#
|
124
|
+
def term_or_kill(child_pid, pause = 1)
|
125
|
+
%w[TERM KILL].each { |signal|
|
126
|
+
begin
|
127
|
+
::Process.kill(signal, child_pid) # attempt to stop process
|
128
|
+
rescue Errno::ECHILD, Errno::ESRCH # no such process
|
129
|
+
break # the process is stopped
|
130
|
+
end
|
131
|
+
if signal == "TERM"
|
132
|
+
# give them a chance to respond
|
133
|
+
begin
|
134
|
+
Timeout.timeout(pause) {
|
135
|
+
begin
|
136
|
+
return ::Process.wait2(child_pid).last # response to signal
|
137
|
+
rescue Errno::ECHILD # no such child
|
138
|
+
return nil # we have already caught the child
|
139
|
+
end
|
140
|
+
}
|
141
|
+
rescue Timeout::Error # the process didn't exit in time
|
142
|
+
# try again with KILL
|
143
|
+
end
|
144
|
+
end
|
145
|
+
}
|
146
|
+
begin
|
147
|
+
::Process.wait2(child_pid).last
|
148
|
+
rescue Errno::ECHILD # no such child
|
149
|
+
nil # we have already caught the child
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
# Extensions for the String class.
|
155
|
+
module String
|
156
|
+
########################
|
157
|
+
### Case Conversions ###
|
158
|
+
########################
|
159
|
+
|
160
|
+
#
|
161
|
+
# Converts a String like "class_name" into "ClassName". Symbols and
|
162
|
+
# whitespace are removed.
|
163
|
+
#
|
164
|
+
def CamelCase
|
165
|
+
tr("^A-Za-z0-9_", "_").
|
166
|
+
gsub(/(?:\A|_)([a-z])/) { $1.capitalize }.
|
167
|
+
delete("_")
|
168
|
+
end
|
169
|
+
alias_method :camel_case, :CamelCase
|
170
|
+
|
171
|
+
#
|
172
|
+
# Converts a String like "ClassName" into "class_name". All runs of
|
173
|
+
# symbols and whitespace are replaced with a single "_".
|
174
|
+
#
|
175
|
+
def snake_case
|
176
|
+
tr("^A-Za-z0-9_", "_").
|
177
|
+
gsub(/(\D)(\d)/, "\\1_\\2").
|
178
|
+
gsub(/(\d)(\D)/, "\\1_\\2").
|
179
|
+
gsub(/([a-z])([A-Z])/, "\\1_\\2").
|
180
|
+
gsub(/_{2,}/, "_").
|
181
|
+
downcase
|
182
|
+
end
|
183
|
+
|
184
|
+
######################
|
185
|
+
### String Cleanup ###
|
186
|
+
######################
|
187
|
+
|
188
|
+
#
|
189
|
+
# This helper will trim all lines in a String of leading whitespace. By
|
190
|
+
# default, it trims the amount of space present on the first line, but you
|
191
|
+
# can override this by setting +width+ manually.
|
192
|
+
#
|
193
|
+
# This method is handy for removing unwanted spacing in indented heredocs
|
194
|
+
# and thus helps keep the source clean:
|
195
|
+
#
|
196
|
+
# def whatever
|
197
|
+
# puts <<-END_MESSAGE.trim # clean up message
|
198
|
+
# <- removes this space
|
199
|
+
# <- and this
|
200
|
+
# END_MESSAGE
|
201
|
+
# end
|
202
|
+
#
|
203
|
+
def trim(width = self[/\A[ \t]*/].size)
|
204
|
+
gsub(/^[ \t]{#{width}}/, "")
|
205
|
+
end
|
206
|
+
|
207
|
+
#
|
208
|
+
# This helper swaps all trailing whitespace for two simple spaces. This
|
209
|
+
# is another method used to cleanup heredocs.
|
210
|
+
#
|
211
|
+
def to_question
|
212
|
+
rstrip + " "
|
213
|
+
end
|
214
|
+
|
215
|
+
#
|
216
|
+
# Wraps a String to fit within the specified +width+, if possible.
|
217
|
+
# Newlines are inserted where spaces were, as needed.
|
218
|
+
#
|
219
|
+
def word_wrap(width = 60)
|
220
|
+
strip.
|
221
|
+
gsub(/\s+/, " ").
|
222
|
+
gsub(/(.{1,#{width}}|\S{#{width + 1},})(?: +|$\n?)/, "\\1\n").
|
223
|
+
strip
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
# Class extensions for Time.
|
228
|
+
module TimeClass
|
229
|
+
#
|
230
|
+
# Converts a String (+str+) in the form <tt>"NNNN-NN-NN NN:NN"</tt> or
|
231
|
+
# <tt>"NNNN-NN-NN NN:NN:NN"</tt> into a proper Time object.
|
232
|
+
#
|
233
|
+
def from_db_s(str)
|
234
|
+
if str and str.to_s =~ /\A\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}(?::\d{2})?\z/
|
235
|
+
::Time.local(*str.to_s.scan(/\d+/))
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
# Extensions for the Time class.
|
241
|
+
module Time
|
242
|
+
#
|
243
|
+
# Returns a String version of this Time in the format expected by SQLite.
|
244
|
+
# If +trim_seconds+ is +true+, the sec() of this Time object will not be
|
245
|
+
# included in the String representation.
|
246
|
+
#
|
247
|
+
def to_db_s(trim_seconds = false)
|
248
|
+
strftime("%Y-%m-%d %H:%M#{':%S' unless trim_seconds}")
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# load all core extensions
|
255
|
+
ScoutAgent::CoreExtensions.constants.each do |name|
|
256
|
+
extension = ScoutAgent::CoreExtensions.const_get(name)
|
257
|
+
mixer = name.sub!(/([a-z])(?:Module|Class)\z/, "\\1") ? :extend : :include
|
258
|
+
core = Object.const_get(name)
|
259
|
+
core.send(mixer, extension)
|
260
|
+
end
|
@@ -0,0 +1,386 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
module ScoutAgent
|
4
|
+
#
|
5
|
+
# This wrapper over Amalgalite::Database adds database loading by name, schema
|
6
|
+
# migrations when they are loaded, installation of standardized busy handlers
|
7
|
+
# and taps for the agent as a connection is prepared, and helper methods for
|
8
|
+
# managing database locks and building schemas.
|
9
|
+
#
|
10
|
+
class Database
|
11
|
+
#
|
12
|
+
# Returns the path to a database file based on the configured data storage
|
13
|
+
# location and the passed +name+ of the database.
|
14
|
+
#
|
15
|
+
def self.path(name)
|
16
|
+
Plan.db_dir + "#{name}.sqlite"
|
17
|
+
end
|
18
|
+
|
19
|
+
#
|
20
|
+
# Loads a database by +name+, optionally preparing +log+ to receive any
|
21
|
+
# error messages.
|
22
|
+
#
|
23
|
+
# The process loads the proper code wrapper for the database by +name+,
|
24
|
+
# installs the handlers and taps used for all connections by the agent,
|
25
|
+
# migrates the schema up to the latest code, and returns the connection
|
26
|
+
# handle. If anything goes wrong in this process, +nil+ is returned
|
27
|
+
# instead.
|
28
|
+
#
|
29
|
+
def self.load(name, log = WireTap.new(nil))
|
30
|
+
require LIB_DIR + "database/#{name}"
|
31
|
+
|
32
|
+
begin
|
33
|
+
db = ScoutAgent::Database.const_get(name.to_s.CamelCase).new(log)
|
34
|
+
db.prepare_connection
|
35
|
+
db.migrate
|
36
|
+
rescue Amalgalite::SQLite3::Error => error # failed to migrate database
|
37
|
+
log.error("Database migration error: #{error.message}.")
|
38
|
+
return nil # cannot load database
|
39
|
+
end
|
40
|
+
|
41
|
+
db
|
42
|
+
end
|
43
|
+
|
44
|
+
#
|
45
|
+
# Builds a new database instance, optionally tied to a +log+.
|
46
|
+
#
|
47
|
+
# This is very low-level, bypassing the standard preparation and migration
|
48
|
+
# process. Thus databases should usually be created with load() instead.
|
49
|
+
#
|
50
|
+
def initialize(log = WireTap.new(nil))
|
51
|
+
@log = log
|
52
|
+
@sqlite = Amalgalite::Database.new(path)
|
53
|
+
@read_locked = false
|
54
|
+
@write_locked = false
|
55
|
+
end
|
56
|
+
|
57
|
+
# The log file (or bit buckect by default) this database reports to.
|
58
|
+
attr_reader :log
|
59
|
+
|
60
|
+
#
|
61
|
+
# This is a shortcut for the class method of the same name, passing a
|
62
|
+
# snake_case version of this Class name as the database name.
|
63
|
+
#
|
64
|
+
def path
|
65
|
+
self.class.path(self.class.short_name.snake_case)
|
66
|
+
end
|
67
|
+
|
68
|
+
#
|
69
|
+
# This method is invoked after a database is openned, but before a
|
70
|
+
# connection is used. This provides a chance to install handles, taps, and
|
71
|
+
# functions before SQL is executed through the connection.
|
72
|
+
#
|
73
|
+
def prepare_connection
|
74
|
+
#
|
75
|
+
# Wait up to 60 seconds for a database lock, attempting to grab it every
|
76
|
+
# 100 milliseconds.
|
77
|
+
#
|
78
|
+
@sqlite.busy_handler(Amalgalite::BusyTimeout.new(600, 100))
|
79
|
+
# Install a trace tap for SQL when debugging.
|
80
|
+
if log.debug?
|
81
|
+
@sqlite.trace_tap = Amalgalite::TraceTap.new(log, :debug)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Returns the current schema version number tracked by SQLite.
|
86
|
+
def schema_version
|
87
|
+
read_from_sqlite { |sqlite| sqlite.pragma(:schema_version).first.first }
|
88
|
+
end
|
89
|
+
|
90
|
+
#
|
91
|
+
# This method updates a database schema. It does that by grabbing a write
|
92
|
+
# lock and then pulling the current schema_version() and feeding that to
|
93
|
+
# update_schema() for SQL to upgrade the database with. SQL is passed
|
94
|
+
# through filters for type and trigger expansions, then batch processed by
|
95
|
+
# SQLite. This process will be repeated until update_schema() returns +nil+
|
96
|
+
# to indicate that the schema is now up-to-date.
|
97
|
+
#
|
98
|
+
# Some maintenance SQL is also inserted with the first migration to
|
99
|
+
# initialize database maintenance tracking.
|
100
|
+
#
|
101
|
+
def migrate
|
102
|
+
write_to_sqlite do |sqlite|
|
103
|
+
loop do
|
104
|
+
version = schema_version
|
105
|
+
sql = update_schema(version) or break
|
106
|
+
sql = "#{add_maintenance_sql(version)}#{sql}"
|
107
|
+
sql.gsub!(/^([ \t]*)(\w+)[ \t]+(\w+_TYPE)\b/) { # types
|
108
|
+
begin
|
109
|
+
send($3.downcase, $2).gsub(/^/, $1.to_s)
|
110
|
+
rescue NoMethodError # unsupported type
|
111
|
+
$&
|
112
|
+
end
|
113
|
+
}
|
114
|
+
sql.gsub!(/^[ \t]*(\w+_TRIGGER)[ \t]+(\S.*?)[ \t]*$/) { # triggers
|
115
|
+
begin
|
116
|
+
send($1.downcase, *$2.split)
|
117
|
+
rescue NoMethodError # unsupported trigger
|
118
|
+
$&
|
119
|
+
end
|
120
|
+
}
|
121
|
+
sqlite.execute_batch(sql)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
#
|
127
|
+
# Subclasses override this method to setup schema migrations as described in
|
128
|
+
# migrate().
|
129
|
+
#
|
130
|
+
def update_schema(version = schema_version)
|
131
|
+
nil
|
132
|
+
end
|
133
|
+
|
134
|
+
#
|
135
|
+
# This method should be called periodically to +VACUUM+ databases and
|
136
|
+
# reclaim the space they have consumed on the hard disk. Returns +true+ if
|
137
|
+
# maintenance was successful, +false+ if it wasn't needed, and +nil+ if
|
138
|
+
# errors prevented the process from completing. The next scheduled run for
|
139
|
+
# maintenance is stored in a table of the database added during the
|
140
|
+
# migration process.
|
141
|
+
#
|
142
|
+
# This method uses some external synchronization to ensure that only one
|
143
|
+
# process can be in here at once, thus processes may block for a time when
|
144
|
+
# they call it.
|
145
|
+
#
|
146
|
+
def maintain
|
147
|
+
#
|
148
|
+
# This cannot be in a transaction (+VACUUM+ fails). Given that, we use an
|
149
|
+
# external locking mechanism on the database code file to prevent doubled
|
150
|
+
# +VACUUM+ race conditions.
|
151
|
+
#
|
152
|
+
(LIB_DIR + "database/#{self.class.short_name.snake_case}.rb").open do |db|
|
153
|
+
db.flock(File::LOCK_EX)
|
154
|
+
begin
|
155
|
+
if @sqlite.first_value_from(
|
156
|
+
"SELECT ROWID FROM maintenance WHERE next_run_at <= ?",
|
157
|
+
Time.now.to_db_s
|
158
|
+
)
|
159
|
+
@sqlite.execute("VACUUM")
|
160
|
+
@sqlite.execute(<<-END_UPDATE_MAINTENANCE_TIME.trim)
|
161
|
+
INSERT OR REPLACE INTO
|
162
|
+
maintenance( ROWID, next_run_at )
|
163
|
+
VALUES( 1, datetime('now', 'localtime', '+1 day') );
|
164
|
+
END_UPDATE_MAINTENANCE_TIME
|
165
|
+
true # maintenance successful
|
166
|
+
else
|
167
|
+
false # maintenance not needed
|
168
|
+
end
|
169
|
+
ensure
|
170
|
+
db.flock(File::LOCK_UN)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
rescue Amalgalite::SQLite3::Error => error # failed to +VACUUM+ database
|
174
|
+
log.error("Database maintenance error: #{error.message}.")
|
175
|
+
nil # maintenance failed, we will try again later
|
176
|
+
end
|
177
|
+
|
178
|
+
# Returns +true+ if this connection is currently read locked.
|
179
|
+
def read_locked?
|
180
|
+
@read_locked
|
181
|
+
end
|
182
|
+
|
183
|
+
# Returns +true+ if this connection is currently write locked.
|
184
|
+
def write_locked?
|
185
|
+
@write_locked
|
186
|
+
end
|
187
|
+
|
188
|
+
# Returns +true+ if this connection is currently read or write locked.
|
189
|
+
def locked?
|
190
|
+
read_locked? or write_locked?
|
191
|
+
end
|
192
|
+
|
193
|
+
#
|
194
|
+
# This method is used to wrap some code in a read lock transaction. The
|
195
|
+
# block you pass is ensured to be run inside of a transaction. If one is
|
196
|
+
# already is active, the code will just be run normally. Otherwise, a new
|
197
|
+
# transaction is started and it will be completed after your block runs.
|
198
|
+
# This ensures that multiple calls to this method nest properly.
|
199
|
+
#
|
200
|
+
def read_from_sqlite
|
201
|
+
if locked?
|
202
|
+
yield @sqlite
|
203
|
+
else
|
204
|
+
begin
|
205
|
+
@sqlite.transaction("IMMEDIATE") {
|
206
|
+
@read_locked = true
|
207
|
+
yield @sqlite
|
208
|
+
}
|
209
|
+
ensure
|
210
|
+
@read_locked = false
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
#
|
216
|
+
# Works just like read_from_sqlite(), but with a write lock.
|
217
|
+
#
|
218
|
+
# This method will +raise+ a +RuntimeError+ if called inside a
|
219
|
+
# read_from_sqlite() block. It's too late to upgrade to a write lock at
|
220
|
+
# that point as a deadlock condition could be introduced. Such code must be
|
221
|
+
# reworked to aquire the write lock first.
|
222
|
+
#
|
223
|
+
def write_to_sqlite
|
224
|
+
if read_locked?
|
225
|
+
raise "Cannot upgrade a read lock to a write lock"
|
226
|
+
elsif write_locked?
|
227
|
+
yield @sqlite
|
228
|
+
else
|
229
|
+
begin
|
230
|
+
@sqlite.transaction("EXCLUSIVE") {
|
231
|
+
@write_locked = true
|
232
|
+
yield @sqlite
|
233
|
+
}
|
234
|
+
ensure
|
235
|
+
@write_locked = false
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
#
|
241
|
+
# A convenience for running +sql+, with any +params+, in a read lock and
|
242
|
+
# then applying +transform+ to each row before the result set is returned.
|
243
|
+
#
|
244
|
+
def query(sql, *params, &transform)
|
245
|
+
read_from_sqlite { |sqlite|
|
246
|
+
results = sqlite.execute(sql, *params)
|
247
|
+
results.each(&transform) unless transform.nil?
|
248
|
+
results
|
249
|
+
}
|
250
|
+
end
|
251
|
+
|
252
|
+
#######
|
253
|
+
private
|
254
|
+
#######
|
255
|
+
|
256
|
+
#
|
257
|
+
# Returns the SQL to create and initialize a maintenance table if +version+
|
258
|
+
# is <tt>0</tt>.
|
259
|
+
#
|
260
|
+
def add_maintenance_sql(version)
|
261
|
+
case version
|
262
|
+
when 0
|
263
|
+
<<-END_MAINTENANCE.trim + "\n"
|
264
|
+
CREATE TABLE maintenance (
|
265
|
+
next_run_at REQUIRED_DATETIME_TYPE
|
266
|
+
);
|
267
|
+
INSERT OR REPLACE INTO
|
268
|
+
maintenance( ROWID, next_run_at )
|
269
|
+
VALUES( 1, datetime('now', 'localtime', '+1 day') );
|
270
|
+
END_MAINTENANCE
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
# A text SQL type with content validation.
|
275
|
+
def required_text_type(field)
|
276
|
+
"#{field} TEXT NOT NULL CHECK(length(trim(#{field})) > 0)"
|
277
|
+
end
|
278
|
+
|
279
|
+
#
|
280
|
+
# An integer SQL type with a default (which should immediately follow
|
281
|
+
# the type definition).
|
282
|
+
#
|
283
|
+
def default_integer_type(field)
|
284
|
+
"#{field} INTEGER NOT NULL DEFAULT"
|
285
|
+
end
|
286
|
+
|
287
|
+
# An integer SQL type with validation of a positive value.
|
288
|
+
def positive_integer_type(field)
|
289
|
+
<<-END_POSITIVE_INTEGER.trim.rstrip
|
290
|
+
#{field} INTEGER NOT NULL
|
291
|
+
CHECK(typeof(#{field}) = 'integer' AND #{field} > 0)
|
292
|
+
END_POSITIVE_INTEGER
|
293
|
+
end
|
294
|
+
|
295
|
+
# A real SQL type with validation of a zero or positive value.
|
296
|
+
def zero_or_positive_real_type(field)
|
297
|
+
<<-END_POSITIVE_REAL.trim.rstrip
|
298
|
+
#{field} REAL NOT NULL
|
299
|
+
CHECK(typeof(#{field}) = 'real' AND #{field} >= 0.0)
|
300
|
+
END_POSITIVE_REAL
|
301
|
+
end
|
302
|
+
|
303
|
+
# A date and time SQL type with format validation.
|
304
|
+
def required_datetime_type(field)
|
305
|
+
<<-END_DATETIME.trim.rstrip
|
306
|
+
#{field} TEXT NOT NULL COLLATE NOCASE
|
307
|
+
CHECK(datetime(#{field}) IS NOT NULL)
|
308
|
+
END_DATETIME
|
309
|
+
end
|
310
|
+
|
311
|
+
#
|
312
|
+
# As required_datetime_type(), but allowing +NULL+ which a trigger can
|
313
|
+
# replace.
|
314
|
+
#
|
315
|
+
def datetime_type(field)
|
316
|
+
<<-END_DEFAULT_DATETIME.trim.rstrip
|
317
|
+
#{field} TEXT COLLATE NOCASE
|
318
|
+
CHECK(#{field} IS NULL OR datetime(#{field}) IS NOT NULL)
|
319
|
+
END_DEFAULT_DATETIME
|
320
|
+
end
|
321
|
+
|
322
|
+
# A trigger for defaulting a date and time field to a local time.
|
323
|
+
def default_localtime_trigger(table, field, second_mode = "include_seconds")
|
324
|
+
now = if second_mode == "trim_seconds"
|
325
|
+
"strftime('%Y-%m-%d %H:%M', 'now', 'localtime')"
|
326
|
+
else
|
327
|
+
"datetime('now', 'localtime')"
|
328
|
+
end
|
329
|
+
<<-END_DEFAULT_LOCALTIME_TRIGGER.trim
|
330
|
+
CREATE TRIGGER default_#{field}_on_insert
|
331
|
+
AFTER INSERT ON #{table}
|
332
|
+
BEGIN
|
333
|
+
UPDATE #{table}
|
334
|
+
SET #{field} = #{now}
|
335
|
+
WHERE ROWID = NEW.ROWID AND #{field} IS NULL;
|
336
|
+
END;
|
337
|
+
END_DEFAULT_LOCALTIME_TRIGGER
|
338
|
+
end
|
339
|
+
|
340
|
+
#
|
341
|
+
# A trigger for ensuring that a foreign key is valid during +INSERT+ and
|
342
|
+
# +UPDATE+.
|
343
|
+
#
|
344
|
+
def foreign_key_check_trigger(table, field, foreign_table, foreign_field)
|
345
|
+
<<-END_FOREIGN_KEY_CHECK_TRIGGER.trim
|
346
|
+
CREATE TRIGGER check_#{field}_on_insert
|
347
|
+
BEFORE INSERT ON #{table}
|
348
|
+
BEGIN
|
349
|
+
SELECT CASE
|
350
|
+
WHEN ( SELECT #{foreign_table}.#{foreign_field}
|
351
|
+
FROM #{foreign_table}
|
352
|
+
WHERE #{foreign_table}.#{foreign_field} = NEW.#{field}
|
353
|
+
LIMIT 1 ) IS NULL
|
354
|
+
THEN RAISE(ABORT, '#{table}.#{field} not in database.')
|
355
|
+
END;
|
356
|
+
END;
|
357
|
+
CREATE TRIGGER check_#{field}_on_update
|
358
|
+
BEFORE UPDATE OF #{field} ON #{table}
|
359
|
+
BEGIN
|
360
|
+
SELECT CASE
|
361
|
+
WHEN ( SELECT #{foreign_table}.#{foreign_field}
|
362
|
+
FROM #{foreign_table}
|
363
|
+
WHERE #{foreign_table}.#{foreign_field} = NEW.#{field}
|
364
|
+
LIMIT 1 ) IS NULL
|
365
|
+
THEN RAISE(ABORT, '#{table}.#{field} not in database.')
|
366
|
+
END;
|
367
|
+
END;
|
368
|
+
END_FOREIGN_KEY_CHECK_TRIGGER
|
369
|
+
end
|
370
|
+
|
371
|
+
#
|
372
|
+
# A trigger that eliminates old records in +table+ once +limit+ records have
|
373
|
+
# been inserted.
|
374
|
+
#
|
375
|
+
def limit_table_size_trigger(table, limit)
|
376
|
+
<<-END_LIMIT_TABLE_SIZE_TRIGGER.trim
|
377
|
+
CREATE TRIGGER limit_#{table}_size
|
378
|
+
AFTER INSERT ON #{table}
|
379
|
+
BEGIN
|
380
|
+
DELETE FROM #{table}
|
381
|
+
WHERE ROWID <= (SELECT MAX(ROWID) FROM #{table}) - #{limit};
|
382
|
+
END;
|
383
|
+
END_LIMIT_TABLE_SIZE_TRIGGER
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|