scout_agent 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/AUTHORS +4 -0
- data/CHANGELOG +3 -0
- data/COPYING +340 -0
- data/INSTALL +17 -0
- data/LICENSE +6 -0
- data/README +3 -0
- data/Rakefile +123 -0
- data/TODO +3 -0
- data/bin/scout_agent +11 -0
- data/lib/scout_agent.rb +73 -0
- data/lib/scout_agent/agent.rb +42 -0
- data/lib/scout_agent/agent/communication_agent.rb +85 -0
- data/lib/scout_agent/agent/master_agent.rb +301 -0
- data/lib/scout_agent/api.rb +241 -0
- data/lib/scout_agent/assignment.rb +105 -0
- data/lib/scout_agent/assignment/configuration.rb +30 -0
- data/lib/scout_agent/assignment/identify.rb +110 -0
- data/lib/scout_agent/assignment/queue.rb +95 -0
- data/lib/scout_agent/assignment/reset.rb +91 -0
- data/lib/scout_agent/assignment/snapshot.rb +92 -0
- data/lib/scout_agent/assignment/start.rb +149 -0
- data/lib/scout_agent/assignment/status.rb +44 -0
- data/lib/scout_agent/assignment/stop.rb +60 -0
- data/lib/scout_agent/assignment/upload_log.rb +61 -0
- data/lib/scout_agent/core_extensions.rb +260 -0
- data/lib/scout_agent/database.rb +386 -0
- data/lib/scout_agent/database/mission_log.rb +282 -0
- data/lib/scout_agent/database/queue.rb +126 -0
- data/lib/scout_agent/database/snapshots.rb +187 -0
- data/lib/scout_agent/database/statuses.rb +65 -0
- data/lib/scout_agent/dispatcher.rb +157 -0
- data/lib/scout_agent/id_card.rb +143 -0
- data/lib/scout_agent/lifeline.rb +243 -0
- data/lib/scout_agent/mission.rb +212 -0
- data/lib/scout_agent/order.rb +58 -0
- data/lib/scout_agent/order/check_in_order.rb +32 -0
- data/lib/scout_agent/order/snapshot_order.rb +33 -0
- data/lib/scout_agent/plan.rb +306 -0
- data/lib/scout_agent/server.rb +123 -0
- data/lib/scout_agent/tracked.rb +59 -0
- data/lib/scout_agent/wire_tap.rb +513 -0
- data/setup.rb +1360 -0
- data/test/tc_core_extensions.rb +89 -0
- data/test/tc_id_card.rb +115 -0
- data/test/tc_plan.rb +285 -0
- data/test/test_helper.rb +22 -0
- data/test/ts_all.rb +7 -0
- metadata +171 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
module ScoutAgent
|
4
|
+
class Assignment
|
5
|
+
class Stop < Assignment
|
6
|
+
def execute
|
7
|
+
@agent = IDCard.new(:lifeline)
|
8
|
+
if @agent.pid_file.exist?
|
9
|
+
puts "Stopping #{ScoutAgent.proper_agent_name} (PID #{@agent.pid})..."
|
10
|
+
signal_and_wait("TERM")
|
11
|
+
if @agent.pid_file.exist?
|
12
|
+
puts "TERM signal was ignored, sending KILL..."
|
13
|
+
signal_and_wait("KILL")
|
14
|
+
if @agent.pid_file.exist?
|
15
|
+
abort_with_failed_to_stop
|
16
|
+
end
|
17
|
+
end
|
18
|
+
puts "Stopped."
|
19
|
+
else
|
20
|
+
abort_with_not_running_notice
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def signal_and_wait(signal_name, wait_count = 10, wait_delay = 0.5)
|
27
|
+
begin
|
28
|
+
@agent.signal(signal_name)
|
29
|
+
rescue Errno::EPERM # we don't have permission
|
30
|
+
abort_with_no_permission
|
31
|
+
end
|
32
|
+
wait_count.times do
|
33
|
+
sleep wait_delay
|
34
|
+
break unless @agent.pid_file.exist?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def abort_with_not_running_notice
|
39
|
+
puts "#{ScoutAgent.proper_agent_name} is not currently running."
|
40
|
+
end
|
41
|
+
|
42
|
+
def abort_with_no_permission
|
43
|
+
abort <<-END_PERMISSION.trim
|
44
|
+
Unable to signal the daemon. Please rerun this command with
|
45
|
+
super user privileges:
|
46
|
+
|
47
|
+
sudo #{$PROGRAM_NAME} stop
|
48
|
+
|
49
|
+
END_PERMISSION
|
50
|
+
end
|
51
|
+
|
52
|
+
def abort_with_failed_to_stop
|
53
|
+
abort <<-END_FAILED.trim
|
54
|
+
Unable to stop the daemon. You may need to use the PID files
|
55
|
+
in #{Plan.pid_dir} to clean up stay processes.
|
56
|
+
END_FAILED
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
# require standard libraries
|
4
|
+
require "tempfile"
|
5
|
+
|
6
|
+
module ScoutAgent
|
7
|
+
class Assignment
|
8
|
+
class UploadLog < Assignment
|
9
|
+
def execute
|
10
|
+
file_name = "#{ScoutAgent.agent_name}.log"
|
11
|
+
if date = Array(other_args).shift
|
12
|
+
file_name += ".#{date.delete('^0-9')}"
|
13
|
+
end
|
14
|
+
log_file = Plan.log_dir + file_name
|
15
|
+
|
16
|
+
unless log_file.exist?
|
17
|
+
abort_with_not_found(file_name)
|
18
|
+
end
|
19
|
+
|
20
|
+
puts "Preparing file for the server. This may take a moment..."
|
21
|
+
begin
|
22
|
+
upload_file = Tempfile.new("#{file_name}.gz")
|
23
|
+
gzipped = Zlib::GzipWriter.new(upload_file)
|
24
|
+
log_file.each_line do |line|
|
25
|
+
gzipped << line
|
26
|
+
end
|
27
|
+
gzipped.close
|
28
|
+
upload_file.open # reopen what Zlib closed
|
29
|
+
rescue Exception => error # Zlib or IOError
|
30
|
+
abort_with_preparation_error(error)
|
31
|
+
end
|
32
|
+
puts "Done."
|
33
|
+
|
34
|
+
|
35
|
+
puts "Sending file to the server. This may take a moment..."
|
36
|
+
server = Server.new
|
37
|
+
if server.post_log(upload_file)
|
38
|
+
puts "Log '#{file_name}' received. Thanks."
|
39
|
+
else
|
40
|
+
abort_with_server_error
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def abort_with_not_found(log_file_name)
|
47
|
+
abort "'#{log_file_name}' could not be found to upload."
|
48
|
+
end
|
49
|
+
|
50
|
+
def abort_with_preparation_error(error)
|
51
|
+
abort <<-END_PREPARATION_ERROR
|
52
|
+
Could not prepare file for upload: #{error.message} (#{error.class})
|
53
|
+
END_PREPARATION_ERROR
|
54
|
+
end
|
55
|
+
|
56
|
+
def abort_with_server_error
|
57
|
+
abort "A networking error prevented the file from being sent."
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,260 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
module ScoutAgent
|
4
|
+
#
|
5
|
+
# This Module holds a handful of extensions to the Ruby core used by our
|
6
|
+
# agent. This is in no way meant to be an ActiveSupport or Facets level
|
7
|
+
# support library. Instead, this should house a few simple conveniences that
|
8
|
+
# are intristic to the agent's needs in some way.
|
9
|
+
#
|
10
|
+
# For example, the agent tries to load matching source files for Classes and
|
11
|
+
# Modules in multiple places. Given that, it makes sense for us to add helper
|
12
|
+
# methods for these case coversions.
|
13
|
+
#
|
14
|
+
module CoreExtensions
|
15
|
+
# Extensions for the Array class.
|
16
|
+
module Array
|
17
|
+
#
|
18
|
+
# This method converts an Array into a human readable word list String.
|
19
|
+
# These lists are of the form:
|
20
|
+
#
|
21
|
+
# one
|
22
|
+
# one and two
|
23
|
+
# one, two, and three
|
24
|
+
# etc.
|
25
|
+
#
|
26
|
+
# You can change the +conjuction+ used to join the last two elements.
|
27
|
+
#
|
28
|
+
def to_word_list(conjuction = "and")
|
29
|
+
case size
|
30
|
+
when 0 then ""
|
31
|
+
when 1 then self[0].to_s
|
32
|
+
when 2 then join(" #{conjuction} ")
|
33
|
+
else ( self[0..-3] +
|
34
|
+
[self[-2..-1].join(", #{conjuction} ")] ).join(", ")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Extensions for the Module class.
|
40
|
+
module Module
|
41
|
+
#
|
42
|
+
# Returns just the last element of a full Module or Class name as a
|
43
|
+
# String. For example, ScoutAgent::CoreExtensions::Module would be
|
44
|
+
# simplified to <tt>"Module"</tt>.
|
45
|
+
#
|
46
|
+
def short_name
|
47
|
+
name[/[^:]+\z/]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Globally available extensions.
|
52
|
+
module Object
|
53
|
+
#
|
54
|
+
# Tries to require a library with +name+. If that fails, attempts to load
|
55
|
+
# RubyGems and tries the require again. If that too fails, this process
|
56
|
+
# will exit with a message about the missing library.
|
57
|
+
#
|
58
|
+
def require_lib_or_gem(name)
|
59
|
+
require name
|
60
|
+
rescue LoadError # library not found
|
61
|
+
begin
|
62
|
+
require "rubygems"
|
63
|
+
require name
|
64
|
+
rescue LoadError # library not found
|
65
|
+
abort <<-END_LOAD_ERROR.trim
|
66
|
+
Unable to load the required '#{name}' library. Try:
|
67
|
+
|
68
|
+
sudo gem install #{name.tr('_', '-')}
|
69
|
+
|
70
|
+
END_LOAD_ERROR
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
#
|
75
|
+
# This is a helper for those libraries that aren't well enough behaved not
|
76
|
+
# to spit out warnings as they work. Warnings are disabled, your block is
|
77
|
+
# run, then warnings are restored to their previous level.
|
78
|
+
#
|
79
|
+
def no_warnings
|
80
|
+
old_verbose = $VERBOSE
|
81
|
+
$VERBOSE = nil
|
82
|
+
yield
|
83
|
+
ensure
|
84
|
+
$VERBOSE = old_verbose
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Sets up an at_exit() hander that will only run when this process exits.
|
89
|
+
# Child processes do not inherit +exit_code+.
|
90
|
+
#
|
91
|
+
def at_my_exit(&exit_code)
|
92
|
+
me = ::Process.pid
|
93
|
+
at_exit do
|
94
|
+
exit_code.call if me == ::Process.pid
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
#
|
99
|
+
# This method installs +handler+ for +TERM+ signals. If the agent is
|
100
|
+
# running as a daemon, the same code will be installed for +INT+ signals
|
101
|
+
# as well. Otherwise, +INT+ signals will be set to +IGNORE+.
|
102
|
+
#
|
103
|
+
def install_shutdown_handler(&handler)
|
104
|
+
shutdown_signals = %w[TERM]
|
105
|
+
if ScoutAgent::Plan.run_as_daemon?
|
106
|
+
shutdown_signals << "INT"
|
107
|
+
else
|
108
|
+
trap("INT", "IGNORE")
|
109
|
+
end
|
110
|
+
shutdown_signals.each do |signal|
|
111
|
+
trap(signal, &handler)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
# Module extensions for Process.
|
117
|
+
module ProcessModule
|
118
|
+
#
|
119
|
+
# A convenience method to ensure +child_pid+ is stopped. First a friendly
|
120
|
+
# +TERM+ signal is sent. Then, after +pause+ seconds, +KILL+ will be sent
|
121
|
+
# if the process has not yet exited. The exit code for the process is
|
122
|
+
# returned if it can be determined (+nil+ is returned otherwise).
|
123
|
+
#
|
124
|
+
def term_or_kill(child_pid, pause = 1)
|
125
|
+
%w[TERM KILL].each { |signal|
|
126
|
+
begin
|
127
|
+
::Process.kill(signal, child_pid) # attempt to stop process
|
128
|
+
rescue Errno::ECHILD, Errno::ESRCH # no such process
|
129
|
+
break # the process is stopped
|
130
|
+
end
|
131
|
+
if signal == "TERM"
|
132
|
+
# give them a chance to respond
|
133
|
+
begin
|
134
|
+
Timeout.timeout(pause) {
|
135
|
+
begin
|
136
|
+
return ::Process.wait2(child_pid).last # response to signal
|
137
|
+
rescue Errno::ECHILD # no such child
|
138
|
+
return nil # we have already caught the child
|
139
|
+
end
|
140
|
+
}
|
141
|
+
rescue Timeout::Error # the process didn't exit in time
|
142
|
+
# try again with KILL
|
143
|
+
end
|
144
|
+
end
|
145
|
+
}
|
146
|
+
begin
|
147
|
+
::Process.wait2(child_pid).last
|
148
|
+
rescue Errno::ECHILD # no such child
|
149
|
+
nil # we have already caught the child
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
# Extensions for the String class.
|
155
|
+
module String
|
156
|
+
########################
|
157
|
+
### Case Conversions ###
|
158
|
+
########################
|
159
|
+
|
160
|
+
#
|
161
|
+
# Converts a String like "class_name" into "ClassName". Symbols and
|
162
|
+
# whitespace are removed.
|
163
|
+
#
|
164
|
+
def CamelCase
|
165
|
+
tr("^A-Za-z0-9_", "_").
|
166
|
+
gsub(/(?:\A|_)([a-z])/) { $1.capitalize }.
|
167
|
+
delete("_")
|
168
|
+
end
|
169
|
+
alias_method :camel_case, :CamelCase
|
170
|
+
|
171
|
+
#
|
172
|
+
# Converts a String like "ClassName" into "class_name". All runs of
|
173
|
+
# symbols and whitespace are replaced with a single "_".
|
174
|
+
#
|
175
|
+
def snake_case
|
176
|
+
tr("^A-Za-z0-9_", "_").
|
177
|
+
gsub(/(\D)(\d)/, "\\1_\\2").
|
178
|
+
gsub(/(\d)(\D)/, "\\1_\\2").
|
179
|
+
gsub(/([a-z])([A-Z])/, "\\1_\\2").
|
180
|
+
gsub(/_{2,}/, "_").
|
181
|
+
downcase
|
182
|
+
end
|
183
|
+
|
184
|
+
######################
|
185
|
+
### String Cleanup ###
|
186
|
+
######################
|
187
|
+
|
188
|
+
#
|
189
|
+
# This helper will trim all lines in a String of leading whitespace. By
|
190
|
+
# default, it trims the amount of space present on the first line, but you
|
191
|
+
# can override this by setting +width+ manually.
|
192
|
+
#
|
193
|
+
# This method is handy for removing unwanted spacing in indented heredocs
|
194
|
+
# and thus helps keep the source clean:
|
195
|
+
#
|
196
|
+
# def whatever
|
197
|
+
# puts <<-END_MESSAGE.trim # clean up message
|
198
|
+
# <- removes this space
|
199
|
+
# <- and this
|
200
|
+
# END_MESSAGE
|
201
|
+
# end
|
202
|
+
#
|
203
|
+
def trim(width = self[/\A[ \t]*/].size)
|
204
|
+
gsub(/^[ \t]{#{width}}/, "")
|
205
|
+
end
|
206
|
+
|
207
|
+
#
|
208
|
+
# This helper swaps all trailing whitespace for two simple spaces. This
|
209
|
+
# is another method used to cleanup heredocs.
|
210
|
+
#
|
211
|
+
def to_question
|
212
|
+
rstrip + " "
|
213
|
+
end
|
214
|
+
|
215
|
+
#
|
216
|
+
# Wraps a String to fit within the specified +width+, if possible.
|
217
|
+
# Newlines are inserted where spaces were, as needed.
|
218
|
+
#
|
219
|
+
def word_wrap(width = 60)
|
220
|
+
strip.
|
221
|
+
gsub(/\s+/, " ").
|
222
|
+
gsub(/(.{1,#{width}}|\S{#{width + 1},})(?: +|$\n?)/, "\\1\n").
|
223
|
+
strip
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
# Class extensions for Time.
|
228
|
+
module TimeClass
|
229
|
+
#
|
230
|
+
# Converts a String (+str+) in the form <tt>"NNNN-NN-NN NN:NN"</tt> or
|
231
|
+
# <tt>"NNNN-NN-NN NN:NN:NN"</tt> into a proper Time object.
|
232
|
+
#
|
233
|
+
def from_db_s(str)
|
234
|
+
if str and str.to_s =~ /\A\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}(?::\d{2})?\z/
|
235
|
+
::Time.local(*str.to_s.scan(/\d+/))
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
# Extensions for the Time class.
|
241
|
+
module Time
|
242
|
+
#
|
243
|
+
# Returns a String version of this Time in the format expected by SQLite.
|
244
|
+
# If +trim_seconds+ is +true+, the sec() of this Time object will not be
|
245
|
+
# included in the String representation.
|
246
|
+
#
|
247
|
+
def to_db_s(trim_seconds = false)
|
248
|
+
strftime("%Y-%m-%d %H:%M#{':%S' unless trim_seconds}")
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# load all core extensions
|
255
|
+
ScoutAgent::CoreExtensions.constants.each do |name|
|
256
|
+
extension = ScoutAgent::CoreExtensions.const_get(name)
|
257
|
+
mixer = name.sub!(/([a-z])(?:Module|Class)\z/, "\\1") ? :extend : :include
|
258
|
+
core = Object.const_get(name)
|
259
|
+
core.send(mixer, extension)
|
260
|
+
end
|
@@ -0,0 +1,386 @@
|
|
1
|
+
#!/usr/bin/env ruby -wKU
|
2
|
+
|
3
|
+
module ScoutAgent
|
4
|
+
#
|
5
|
+
# This wrapper over Amalgalite::Database adds database loading by name, schema
|
6
|
+
# migrations when they are loaded, installation of standardized busy handlers
|
7
|
+
# and taps for the agent as a connection is prepared, and helper methods for
|
8
|
+
# managing database locks and building schemas.
|
9
|
+
#
|
10
|
+
class Database
|
11
|
+
#
|
12
|
+
# Returns the path to a database file based on the configured data storage
|
13
|
+
# location and the passed +name+ of the database.
|
14
|
+
#
|
15
|
+
def self.path(name)
|
16
|
+
Plan.db_dir + "#{name}.sqlite"
|
17
|
+
end
|
18
|
+
|
19
|
+
#
|
20
|
+
# Loads a database by +name+, optionally preparing +log+ to receive any
|
21
|
+
# error messages.
|
22
|
+
#
|
23
|
+
# The process loads the proper code wrapper for the database by +name+,
|
24
|
+
# installs the handlers and taps used for all connections by the agent,
|
25
|
+
# migrates the schema up to the latest code, and returns the connection
|
26
|
+
# handle. If anything goes wrong in this process, +nil+ is returned
|
27
|
+
# instead.
|
28
|
+
#
|
29
|
+
def self.load(name, log = WireTap.new(nil))
|
30
|
+
require LIB_DIR + "database/#{name}"
|
31
|
+
|
32
|
+
begin
|
33
|
+
db = ScoutAgent::Database.const_get(name.to_s.CamelCase).new(log)
|
34
|
+
db.prepare_connection
|
35
|
+
db.migrate
|
36
|
+
rescue Amalgalite::SQLite3::Error => error # failed to migrate database
|
37
|
+
log.error("Database migration error: #{error.message}.")
|
38
|
+
return nil # cannot load database
|
39
|
+
end
|
40
|
+
|
41
|
+
db
|
42
|
+
end
|
43
|
+
|
44
|
+
#
|
45
|
+
# Builds a new database instance, optionally tied to a +log+.
|
46
|
+
#
|
47
|
+
# This is very low-level, bypassing the standard preparation and migration
|
48
|
+
# process. Thus databases should usually be created with load() instead.
|
49
|
+
#
|
50
|
+
def initialize(log = WireTap.new(nil))
|
51
|
+
@log = log
|
52
|
+
@sqlite = Amalgalite::Database.new(path)
|
53
|
+
@read_locked = false
|
54
|
+
@write_locked = false
|
55
|
+
end
|
56
|
+
|
57
|
+
# The log file (or bit buckect by default) this database reports to.
|
58
|
+
attr_reader :log
|
59
|
+
|
60
|
+
#
|
61
|
+
# This is a shortcut for the class method of the same name, passing a
|
62
|
+
# snake_case version of this Class name as the database name.
|
63
|
+
#
|
64
|
+
def path
|
65
|
+
self.class.path(self.class.short_name.snake_case)
|
66
|
+
end
|
67
|
+
|
68
|
+
#
|
69
|
+
# This method is invoked after a database is openned, but before a
|
70
|
+
# connection is used. This provides a chance to install handles, taps, and
|
71
|
+
# functions before SQL is executed through the connection.
|
72
|
+
#
|
73
|
+
def prepare_connection
|
74
|
+
#
|
75
|
+
# Wait up to 60 seconds for a database lock, attempting to grab it every
|
76
|
+
# 100 milliseconds.
|
77
|
+
#
|
78
|
+
@sqlite.busy_handler(Amalgalite::BusyTimeout.new(600, 100))
|
79
|
+
# Install a trace tap for SQL when debugging.
|
80
|
+
if log.debug?
|
81
|
+
@sqlite.trace_tap = Amalgalite::TraceTap.new(log, :debug)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Returns the current schema version number tracked by SQLite.
|
86
|
+
def schema_version
|
87
|
+
read_from_sqlite { |sqlite| sqlite.pragma(:schema_version).first.first }
|
88
|
+
end
|
89
|
+
|
90
|
+
#
|
91
|
+
# This method updates a database schema. It does that by grabbing a write
|
92
|
+
# lock and then pulling the current schema_version() and feeding that to
|
93
|
+
# update_schema() for SQL to upgrade the database with. SQL is passed
|
94
|
+
# through filters for type and trigger expansions, then batch processed by
|
95
|
+
# SQLite. This process will be repeated until update_schema() returns +nil+
|
96
|
+
# to indicate that the schema is now up-to-date.
|
97
|
+
#
|
98
|
+
# Some maintenance SQL is also inserted with the first migration to
|
99
|
+
# initialize database maintenance tracking.
|
100
|
+
#
|
101
|
+
def migrate
|
102
|
+
write_to_sqlite do |sqlite|
|
103
|
+
loop do
|
104
|
+
version = schema_version
|
105
|
+
sql = update_schema(version) or break
|
106
|
+
sql = "#{add_maintenance_sql(version)}#{sql}"
|
107
|
+
sql.gsub!(/^([ \t]*)(\w+)[ \t]+(\w+_TYPE)\b/) { # types
|
108
|
+
begin
|
109
|
+
send($3.downcase, $2).gsub(/^/, $1.to_s)
|
110
|
+
rescue NoMethodError # unsupported type
|
111
|
+
$&
|
112
|
+
end
|
113
|
+
}
|
114
|
+
sql.gsub!(/^[ \t]*(\w+_TRIGGER)[ \t]+(\S.*?)[ \t]*$/) { # triggers
|
115
|
+
begin
|
116
|
+
send($1.downcase, *$2.split)
|
117
|
+
rescue NoMethodError # unsupported trigger
|
118
|
+
$&
|
119
|
+
end
|
120
|
+
}
|
121
|
+
sqlite.execute_batch(sql)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
#
|
127
|
+
# Subclasses override this method to setup schema migrations as described in
|
128
|
+
# migrate().
|
129
|
+
#
|
130
|
+
def update_schema(version = schema_version)
|
131
|
+
nil
|
132
|
+
end
|
133
|
+
|
134
|
+
#
|
135
|
+
# This method should be called periodically to +VACUUM+ databases and
|
136
|
+
# reclaim the space they have consumed on the hard disk. Returns +true+ if
|
137
|
+
# maintenance was successful, +false+ if it wasn't needed, and +nil+ if
|
138
|
+
# errors prevented the process from completing. The next scheduled run for
|
139
|
+
# maintenance is stored in a table of the database added during the
|
140
|
+
# migration process.
|
141
|
+
#
|
142
|
+
# This method uses some external synchronization to ensure that only one
|
143
|
+
# process can be in here at once, thus processes may block for a time when
|
144
|
+
# they call it.
|
145
|
+
#
|
146
|
+
def maintain
|
147
|
+
#
|
148
|
+
# This cannot be in a transaction (+VACUUM+ fails). Given that, we use an
|
149
|
+
# external locking mechanism on the database code file to prevent doubled
|
150
|
+
# +VACUUM+ race conditions.
|
151
|
+
#
|
152
|
+
(LIB_DIR + "database/#{self.class.short_name.snake_case}.rb").open do |db|
|
153
|
+
db.flock(File::LOCK_EX)
|
154
|
+
begin
|
155
|
+
if @sqlite.first_value_from(
|
156
|
+
"SELECT ROWID FROM maintenance WHERE next_run_at <= ?",
|
157
|
+
Time.now.to_db_s
|
158
|
+
)
|
159
|
+
@sqlite.execute("VACUUM")
|
160
|
+
@sqlite.execute(<<-END_UPDATE_MAINTENANCE_TIME.trim)
|
161
|
+
INSERT OR REPLACE INTO
|
162
|
+
maintenance( ROWID, next_run_at )
|
163
|
+
VALUES( 1, datetime('now', 'localtime', '+1 day') );
|
164
|
+
END_UPDATE_MAINTENANCE_TIME
|
165
|
+
true # maintenance successful
|
166
|
+
else
|
167
|
+
false # maintenance not needed
|
168
|
+
end
|
169
|
+
ensure
|
170
|
+
db.flock(File::LOCK_UN)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
rescue Amalgalite::SQLite3::Error => error # failed to +VACUUM+ database
|
174
|
+
log.error("Database maintenance error: #{error.message}.")
|
175
|
+
nil # maintenance failed, we will try again later
|
176
|
+
end
|
177
|
+
|
178
|
+
# Returns +true+ if this connection is currently read locked.
|
179
|
+
def read_locked?
|
180
|
+
@read_locked
|
181
|
+
end
|
182
|
+
|
183
|
+
# Returns +true+ if this connection is currently write locked.
|
184
|
+
def write_locked?
|
185
|
+
@write_locked
|
186
|
+
end
|
187
|
+
|
188
|
+
# Returns +true+ if this connection is currently read or write locked.
|
189
|
+
def locked?
|
190
|
+
read_locked? or write_locked?
|
191
|
+
end
|
192
|
+
|
193
|
+
#
|
194
|
+
# This method is used to wrap some code in a read lock transaction. The
|
195
|
+
# block you pass is ensured to be run inside of a transaction. If one is
|
196
|
+
# already is active, the code will just be run normally. Otherwise, a new
|
197
|
+
# transaction is started and it will be completed after your block runs.
|
198
|
+
# This ensures that multiple calls to this method nest properly.
|
199
|
+
#
|
200
|
+
def read_from_sqlite
|
201
|
+
if locked?
|
202
|
+
yield @sqlite
|
203
|
+
else
|
204
|
+
begin
|
205
|
+
@sqlite.transaction("IMMEDIATE") {
|
206
|
+
@read_locked = true
|
207
|
+
yield @sqlite
|
208
|
+
}
|
209
|
+
ensure
|
210
|
+
@read_locked = false
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
#
|
216
|
+
# Works just like read_from_sqlite(), but with a write lock.
|
217
|
+
#
|
218
|
+
# This method will +raise+ a +RuntimeError+ if called inside a
|
219
|
+
# read_from_sqlite() block. It's too late to upgrade to a write lock at
|
220
|
+
# that point as a deadlock condition could be introduced. Such code must be
|
221
|
+
# reworked to aquire the write lock first.
|
222
|
+
#
|
223
|
+
def write_to_sqlite
|
224
|
+
if read_locked?
|
225
|
+
raise "Cannot upgrade a read lock to a write lock"
|
226
|
+
elsif write_locked?
|
227
|
+
yield @sqlite
|
228
|
+
else
|
229
|
+
begin
|
230
|
+
@sqlite.transaction("EXCLUSIVE") {
|
231
|
+
@write_locked = true
|
232
|
+
yield @sqlite
|
233
|
+
}
|
234
|
+
ensure
|
235
|
+
@write_locked = false
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
#
|
241
|
+
# A convenience for running +sql+, with any +params+, in a read lock and
|
242
|
+
# then applying +transform+ to each row before the result set is returned.
|
243
|
+
#
|
244
|
+
def query(sql, *params, &transform)
|
245
|
+
read_from_sqlite { |sqlite|
|
246
|
+
results = sqlite.execute(sql, *params)
|
247
|
+
results.each(&transform) unless transform.nil?
|
248
|
+
results
|
249
|
+
}
|
250
|
+
end
|
251
|
+
|
252
|
+
#######
|
253
|
+
private
|
254
|
+
#######
|
255
|
+
|
256
|
+
#
|
257
|
+
# Returns the SQL to create and initialize a maintenance table if +version+
|
258
|
+
# is <tt>0</tt>.
|
259
|
+
#
|
260
|
+
def add_maintenance_sql(version)
|
261
|
+
case version
|
262
|
+
when 0
|
263
|
+
<<-END_MAINTENANCE.trim + "\n"
|
264
|
+
CREATE TABLE maintenance (
|
265
|
+
next_run_at REQUIRED_DATETIME_TYPE
|
266
|
+
);
|
267
|
+
INSERT OR REPLACE INTO
|
268
|
+
maintenance( ROWID, next_run_at )
|
269
|
+
VALUES( 1, datetime('now', 'localtime', '+1 day') );
|
270
|
+
END_MAINTENANCE
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
# A text SQL type with content validation.
|
275
|
+
def required_text_type(field)
|
276
|
+
"#{field} TEXT NOT NULL CHECK(length(trim(#{field})) > 0)"
|
277
|
+
end
|
278
|
+
|
279
|
+
#
|
280
|
+
# An integer SQL type with a default (which should immediately follow
|
281
|
+
# the type definition).
|
282
|
+
#
|
283
|
+
def default_integer_type(field)
|
284
|
+
"#{field} INTEGER NOT NULL DEFAULT"
|
285
|
+
end
|
286
|
+
|
287
|
+
# An integer SQL type with validation of a positive value.
|
288
|
+
def positive_integer_type(field)
|
289
|
+
<<-END_POSITIVE_INTEGER.trim.rstrip
|
290
|
+
#{field} INTEGER NOT NULL
|
291
|
+
CHECK(typeof(#{field}) = 'integer' AND #{field} > 0)
|
292
|
+
END_POSITIVE_INTEGER
|
293
|
+
end
|
294
|
+
|
295
|
+
# A real SQL type with validation of a zero or positive value.
|
296
|
+
def zero_or_positive_real_type(field)
|
297
|
+
<<-END_POSITIVE_REAL.trim.rstrip
|
298
|
+
#{field} REAL NOT NULL
|
299
|
+
CHECK(typeof(#{field}) = 'real' AND #{field} >= 0.0)
|
300
|
+
END_POSITIVE_REAL
|
301
|
+
end
|
302
|
+
|
303
|
+
# A date and time SQL type with format validation.
|
304
|
+
def required_datetime_type(field)
|
305
|
+
<<-END_DATETIME.trim.rstrip
|
306
|
+
#{field} TEXT NOT NULL COLLATE NOCASE
|
307
|
+
CHECK(datetime(#{field}) IS NOT NULL)
|
308
|
+
END_DATETIME
|
309
|
+
end
|
310
|
+
|
311
|
+
#
|
312
|
+
# As required_datetime_type(), but allowing +NULL+ which a trigger can
|
313
|
+
# replace.
|
314
|
+
#
|
315
|
+
def datetime_type(field)
|
316
|
+
<<-END_DEFAULT_DATETIME.trim.rstrip
|
317
|
+
#{field} TEXT COLLATE NOCASE
|
318
|
+
CHECK(#{field} IS NULL OR datetime(#{field}) IS NOT NULL)
|
319
|
+
END_DEFAULT_DATETIME
|
320
|
+
end
|
321
|
+
|
322
|
+
# A trigger for defaulting a date and time field to a local time.
|
323
|
+
def default_localtime_trigger(table, field, second_mode = "include_seconds")
|
324
|
+
now = if second_mode == "trim_seconds"
|
325
|
+
"strftime('%Y-%m-%d %H:%M', 'now', 'localtime')"
|
326
|
+
else
|
327
|
+
"datetime('now', 'localtime')"
|
328
|
+
end
|
329
|
+
<<-END_DEFAULT_LOCALTIME_TRIGGER.trim
|
330
|
+
CREATE TRIGGER default_#{field}_on_insert
|
331
|
+
AFTER INSERT ON #{table}
|
332
|
+
BEGIN
|
333
|
+
UPDATE #{table}
|
334
|
+
SET #{field} = #{now}
|
335
|
+
WHERE ROWID = NEW.ROWID AND #{field} IS NULL;
|
336
|
+
END;
|
337
|
+
END_DEFAULT_LOCALTIME_TRIGGER
|
338
|
+
end
|
339
|
+
|
340
|
+
#
|
341
|
+
# A trigger for ensuring that a foreign key is valid during +INSERT+ and
|
342
|
+
# +UPDATE+.
|
343
|
+
#
|
344
|
+
def foreign_key_check_trigger(table, field, foreign_table, foreign_field)
|
345
|
+
<<-END_FOREIGN_KEY_CHECK_TRIGGER.trim
|
346
|
+
CREATE TRIGGER check_#{field}_on_insert
|
347
|
+
BEFORE INSERT ON #{table}
|
348
|
+
BEGIN
|
349
|
+
SELECT CASE
|
350
|
+
WHEN ( SELECT #{foreign_table}.#{foreign_field}
|
351
|
+
FROM #{foreign_table}
|
352
|
+
WHERE #{foreign_table}.#{foreign_field} = NEW.#{field}
|
353
|
+
LIMIT 1 ) IS NULL
|
354
|
+
THEN RAISE(ABORT, '#{table}.#{field} not in database.')
|
355
|
+
END;
|
356
|
+
END;
|
357
|
+
CREATE TRIGGER check_#{field}_on_update
|
358
|
+
BEFORE UPDATE OF #{field} ON #{table}
|
359
|
+
BEGIN
|
360
|
+
SELECT CASE
|
361
|
+
WHEN ( SELECT #{foreign_table}.#{foreign_field}
|
362
|
+
FROM #{foreign_table}
|
363
|
+
WHERE #{foreign_table}.#{foreign_field} = NEW.#{field}
|
364
|
+
LIMIT 1 ) IS NULL
|
365
|
+
THEN RAISE(ABORT, '#{table}.#{field} not in database.')
|
366
|
+
END;
|
367
|
+
END;
|
368
|
+
END_FOREIGN_KEY_CHECK_TRIGGER
|
369
|
+
end
|
370
|
+
|
371
|
+
#
|
372
|
+
# A trigger that eliminates old records in +table+ once +limit+ records have
|
373
|
+
# been inserted.
|
374
|
+
#
|
375
|
+
def limit_table_size_trigger(table, limit)
|
376
|
+
<<-END_LIMIT_TABLE_SIZE_TRIGGER.trim
|
377
|
+
CREATE TRIGGER limit_#{table}_size
|
378
|
+
AFTER INSERT ON #{table}
|
379
|
+
BEGIN
|
380
|
+
DELETE FROM #{table}
|
381
|
+
WHERE ROWID <= (SELECT MAX(ROWID) FROM #{table}) - #{limit};
|
382
|
+
END;
|
383
|
+
END_LIMIT_TABLE_SIZE_TRIGGER
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|