thoughtafter-lockfile 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/lockfile.rb ADDED
@@ -0,0 +1,536 @@
1
+ unless(defined?($__lockfile__) or defined?(Lockfile))
2
+
3
+ require 'socket'
4
+ require 'timeout'
5
+ require 'fileutils'
6
+
7
+ class Lockfile
8
+
9
+ VERSION = '2.0.0'
10
+ def Lockfile.version() Lockfile::VERSION end
11
+ def version() Lockfile::VERSION end
12
+
13
+ class LockError < StandardError; end
14
+ class StolenLockError < LockError; end
15
+ class StackingLockError < LockError; end
16
+ class StatLockError < LockError; end
17
+ class MaxTriesLockError < LockError; end
18
+ class TimeoutLockError < LockError; end
19
+ class NFSLockError < LockError; end
20
+ class UnLockError < LockError; end
21
+
22
+ class SleepCycle < Array
23
+ attr :min
24
+ attr :max
25
+ attr :range
26
+ attr :inc
27
+
28
+ def initialize(min, max, inc)
29
+ @min, @max, @inc = Float(min), Float(max), Float(inc)
30
+ @range = @max - @min
31
+ raise RangeError, "max(#{ @max }) <= min(#{ @min })" if @max <= @min
32
+ raise RangeError, "inc(#{ @inc }) > range(#{ @range })" if @inc > @range
33
+ raise RangeError, "inc(#{ @inc }) <= 0" if @inc <= 0
34
+ raise RangeError, "range(#{ @range }) <= 0" if @range <= 0
35
+ s = @min
36
+ push(s) and s += @inc while(s <= @max)
37
+ self[-1] = @max if self[-1] < @max
38
+ reset
39
+ end
40
+
41
+ def next
42
+ ret = self[@idx]
43
+ @idx = ((@idx + 1) % self.size)
44
+ ret
45
+ end
46
+
47
+ def reset
48
+ @idx = 0
49
+ end
50
+ end
51
+
52
+ HOSTNAME = Socket.gethostname
53
+
54
+ DEFAULT_RETRIES = nil # maximum number of attempts
55
+ DEFAULT_TIMEOUT = nil # the longest we will try
56
+ DEFAULT_MAX_AGE = 3600 # lockfiles older than this are stale
57
+ DEFAULT_SLEEP_INC = 2 # sleep cycle is this much longer each time
58
+ DEFAULT_MIN_SLEEP = 2 # shortest sleep time
59
+ DEFAULT_MAX_SLEEP = 32 # longest sleep time
60
+ DEFAULT_SUSPEND = 1800 # iff we steal a lock wait this long before we go on
61
+ DEFAULT_REFRESH = 8 # how often we touch/validate the lock
62
+ DEFAULT_DONT_CLEAN = false # iff we leave lock files lying around
63
+ DEFAULT_POLL_RETRIES = 16 # this many polls makes one 'try'
64
+ DEFAULT_POLL_MAX_SLEEP = 0.08 # the longest we'll sleep between polls
65
+ DEFAULT_DONT_SWEEP = false # if we cleanup after other process on our host
66
+ DEFAULT_DONT_USE_LOCK_ID = false # if we dump lock info into lockfile
67
+
68
+ DEFAULT_DEBUG = ENV['LOCKFILE_DEBUG'] || false
69
+
70
+ class << Lockfile
71
+ attr :retries, true
72
+ attr :max_age, true
73
+ attr :sleep_inc, true
74
+ attr :min_sleep, true
75
+ attr :max_sleep, true
76
+ attr :suspend, true
77
+ attr :timeout, true
78
+ attr :refresh, true
79
+ attr :debug, true
80
+ attr :dont_clean, true
81
+ attr :poll_retries, true
82
+ attr :poll_max_sleep, true
83
+ attr :dont_sweep, true
84
+ attr :dont_use_lock_id, true
85
+
86
+ def init
87
+ @retries = DEFAULT_RETRIES
88
+ @max_age = DEFAULT_MAX_AGE
89
+ @sleep_inc = DEFAULT_SLEEP_INC
90
+ @min_sleep = DEFAULT_MIN_SLEEP
91
+ @max_sleep = DEFAULT_MAX_SLEEP
92
+ @suspend = DEFAULT_SUSPEND
93
+ @timeout = DEFAULT_TIMEOUT
94
+ @refresh = DEFAULT_REFRESH
95
+ @dont_clean = DEFAULT_DONT_CLEAN
96
+ @poll_retries = DEFAULT_POLL_RETRIES
97
+ @poll_max_sleep = DEFAULT_POLL_MAX_SLEEP
98
+ @dont_sweep = DEFAULT_DONT_SWEEP
99
+ @dont_use_lock_id = DEFAULT_DONT_USE_LOCK_ID
100
+
101
+ @debug = DEFAULT_DEBUG
102
+
103
+ STDOUT.sync = true if @debug
104
+ STDERR.sync = true if @debug
105
+ end
106
+ end
107
+
108
+ Lockfile.init
109
+
110
+ attr :klass
111
+ attr :path
112
+ attr :opts
113
+ attr :locked
114
+ attr :thief
115
+ attr :refresher
116
+ attr :dirname
117
+ attr :basename
118
+ attr :clean
119
+ attr :retries
120
+ attr :max_age
121
+ attr :sleep_inc
122
+ attr :min_sleep
123
+ attr :max_sleep
124
+ attr :suspend
125
+ attr :refresh
126
+ attr :timeout
127
+ attr :dont_clean
128
+ attr :poll_retries
129
+ attr :poll_max_sleep
130
+ attr :dont_sweep
131
+ attr :dont_use_lock_id
132
+
133
+ attr_accessor :debug
134
+
135
+ alias thief? thief
136
+ alias locked? locked
137
+ alias debug? debug
138
+
139
+ def Lockfile.create(path, *a, &b)
140
+ opts = {
141
+ 'retries' => 0,
142
+ 'min_sleep' => 0,
143
+ 'max_sleep' => 1,
144
+ 'sleep_inc' => 1,
145
+ 'max_age' => nil,
146
+ 'suspend' => 0,
147
+ 'refresh' => nil,
148
+ 'timeout' => nil,
149
+ 'poll_retries' => 0,
150
+ 'dont_clean' => true,
151
+ 'dont_sweep' => false,
152
+ 'dont_use_lock_id' => true,
153
+ }
154
+ begin
155
+ new(path, opts).lock
156
+ rescue LockError
157
+ raise Errno::EEXIST, path
158
+ end
159
+ open(path, *a, &b)
160
+ end
161
+
162
+ def initialize(path, opts = {}, &block)
163
+ @klass = self.class
164
+ @path = path
165
+ @opts = opts
166
+
167
+ @retries = getopt 'retries' , @klass.retries
168
+ @max_age = getopt 'max_age' , @klass.max_age
169
+ @sleep_inc = getopt 'sleep_inc' , @klass.sleep_inc
170
+ @min_sleep = getopt 'min_sleep' , @klass.min_sleep
171
+ @max_sleep = getopt 'max_sleep' , @klass.max_sleep
172
+ @suspend = getopt 'suspend' , @klass.suspend
173
+ @timeout = getopt 'timeout' , @klass.timeout
174
+ @refresh = getopt 'refresh' , @klass.refresh
175
+ @dont_clean = getopt 'dont_clean' , @klass.dont_clean
176
+ @poll_retries = getopt 'poll_retries' , @klass.poll_retries
177
+ @poll_max_sleep = getopt 'poll_max_sleep' , @klass.poll_max_sleep
178
+ @dont_sweep = getopt 'dont_sweep' , @klass.dont_sweep
179
+ @dont_use_lock_id = getopt 'dont_use_lock_id' , @klass.dont_use_lock_id
180
+ @debug = getopt 'debug' , @klass.debug
181
+
182
+ @sleep_cycle = SleepCycle.new @min_sleep, @max_sleep, @sleep_inc
183
+
184
+ @clean = @dont_clean ? nil : lambda{ File.unlink @path rescue nil }
185
+ @dirname = File.dirname @path
186
+ @basename = File.basename @path
187
+ @thief = false
188
+ @locked = false
189
+ @refrsher = nil
190
+
191
+ lock(&block) if block
192
+ end
193
+
194
+ def lock
195
+ raise StackingLockError, "<#{ @path }> is locked!" if @locked
196
+
197
+ sweep unless @dont_sweep
198
+
199
+ ret = nil
200
+
201
+ attempt do
202
+ begin
203
+ @sleep_cycle.reset
204
+ create_tmplock do |f|
205
+ begin
206
+ Timeout.timeout(@timeout) do
207
+ tmp_path = f.path
208
+ tmp_stat = f.lstat
209
+ n_retries = 0
210
+ trace{ "attempting to lock <#{ @path }>..." }
211
+ begin
212
+ i = 0
213
+ begin
214
+ trace{ "polling attempt <#{ i }>..." }
215
+ begin
216
+ File.link tmp_path, @path
217
+ rescue Errno::ENOENT
218
+ try_again!
219
+ end
220
+ lock_stat = File.lstat @path
221
+ raise StatLockError, "stat's do not agree" unless
222
+ tmp_stat.rdev == lock_stat.rdev and tmp_stat.ino == lock_stat.ino
223
+ trace{ "aquired lock <#{ @path }>" }
224
+ @locked = true
225
+ rescue => e
226
+ i += 1
227
+ unless i >= @poll_retries
228
+ t = [rand(@poll_max_sleep), @poll_max_sleep].min
229
+ trace{ "poll sleep <#{ t }>..." }
230
+ sleep t
231
+ retry
232
+ end
233
+ raise
234
+ end
235
+
236
+ rescue => e
237
+ n_retries += 1
238
+ trace{ "n_retries <#{ n_retries }>" }
239
+ case validlock?
240
+ when true
241
+ raise MaxTriesLockError, "surpased retries <#{ @retries }>" if
242
+ @retries and n_retries >= @retries
243
+ trace{ "found valid lock" }
244
+ sleeptime = @sleep_cycle.next
245
+ trace{ "sleep <#{ sleeptime }>..." }
246
+ sleep sleeptime
247
+ when false
248
+ trace{ "found invalid lock and removing" }
249
+ begin
250
+ File.unlink @path
251
+ @thief = true
252
+ warn "<#{ @path }> stolen by <#{ Process.pid }> at <#{ timestamp }>"
253
+ trace{ "i am a thief!" }
254
+ rescue Errno::ENOENT
255
+ end
256
+ trace{ "suspending <#{ @suspend }>" }
257
+ sleep @suspend
258
+ when nil
259
+ raise MaxTriesLockError, "surpased retries <#{ @retries }>" if
260
+ @retries and n_retries >= @retries
261
+ end
262
+ retry
263
+ end # begin
264
+ end # timeout
265
+ rescue Timeout::Error
266
+ raise TimeoutLockError, "surpassed timeout <#{ @timeout }>"
267
+ end # begin
268
+ end # create_tmplock
269
+
270
+ if block_given?
271
+ stolen = false
272
+ @refresher = (@refresh ? new_refresher : nil)
273
+ begin
274
+ begin
275
+ ret = yield @path
276
+ rescue StolenLockError
277
+ stolen = true
278
+ raise
279
+ end
280
+ ensure
281
+ begin
282
+ @refresher.kill if @refresher and @refresher.status
283
+ @refresher = nil
284
+ ensure
285
+ unlock unless stolen
286
+ end
287
+ end
288
+ else
289
+ @refresher = (@refresh ? new_refresher : nil)
290
+ ObjectSpace.define_finalizer self, @clean if @clean
291
+ ret = self
292
+ end
293
+ rescue Errno::ESTALE, Errno::EIO => e
294
+ raise(NFSLockError, errmsg(e))
295
+ end
296
+ end
297
+
298
+ return ret
299
+ end
300
+
301
+ def sweep
302
+ begin
303
+ glob = File.join(@dirname, ".*lck")
304
+ paths = Dir[glob]
305
+ paths.each do |path|
306
+ begin
307
+ basename = File.basename path
308
+ pat = %r/^\s*\.([^_]+)_([^_]+)/o
309
+ if pat.match(basename)
310
+ host, pid = $1, $2
311
+ else
312
+ next
313
+ end
314
+ host.gsub!(%r/^\.+|\.+$/,'')
315
+ quad = host.split %r/\./
316
+ host = quad.first
317
+ pat = %r/^\s*#{ host }/i
318
+ if pat.match(HOSTNAME) and %r/^\s*\d+\s*$/.match(pid)
319
+ unless alive?(pid)
320
+ trace{ "process <#{ pid }> on <#{ host }> is no longer alive" }
321
+ trace{ "sweeping <#{ path }>" }
322
+ FileUtils.rm_f path
323
+ else
324
+ trace{ "process <#{ pid }> on <#{ host }> is still alive" }
325
+ trace{ "ignoring <#{ path }>" }
326
+ end
327
+ else
328
+ trace{ "ignoring <#{ path }> generated by <#{ host }>" }
329
+ end
330
+ rescue
331
+ next
332
+ end
333
+ end
334
+ rescue => e
335
+ warn(errmsg(e))
336
+ end
337
+ end
338
+
339
+ def alive? pid
340
+ pid = Integer("#{ pid }")
341
+ begin
342
+ Process.kill 0, pid
343
+ true
344
+ rescue Errno::ESRCH
345
+ false
346
+ end
347
+ end
348
+
349
+ def unlock
350
+ raise UnLockError, "<#{ @path }> is not locked!" unless @locked
351
+
352
+ @refresher.kill if @refresher and @refresher.status
353
+ @refresher = nil
354
+
355
+ begin
356
+ File.unlink @path
357
+ rescue Errno::ENOENT
358
+ raise StolenLockError, @path
359
+ ensure
360
+ @thief = false
361
+ @locked = false
362
+ ObjectSpace.undefine_finalizer self if @clean
363
+ end
364
+ end
365
+
366
+ def new_refresher
367
+ Thread.new(Thread.current, @path, @refresh, @dont_use_lock_id) do |thread, path, refresh, dont_use_lock_id|
368
+ loop do
369
+ begin
370
+ touch path
371
+ trace{"touched <#{ path }> @ <#{ Time.now.to_f }>"}
372
+ unless dont_use_lock_id
373
+ loaded = load_lock_id(IO.read(path))
374
+ trace{"loaded <\n#{ loaded.inspect }\n>"}
375
+ raise unless loaded == @lock_id
376
+ end
377
+ sleep refresh
378
+ rescue Exception => e
379
+ trace{errmsg e}
380
+ thread.raise StolenLockError
381
+ Thread.exit
382
+ end
383
+ end
384
+ end
385
+ end
386
+
387
+ def validlock?
388
+ if @max_age
389
+ uncache @path rescue nil
390
+ begin
391
+ return((Time.now - File.stat(@path).mtime) < @max_age)
392
+ rescue Errno::ENOENT
393
+ return nil
394
+ end
395
+ else
396
+ exist = File.exist?(@path)
397
+ return(exist ? true : nil)
398
+ end
399
+ end
400
+
401
+ def uncache file
402
+ refresh = nil
403
+ begin
404
+ is_a_file = File === file
405
+ path = (is_a_file ? file.path : file.to_s)
406
+ stat = (is_a_file ? file.stat : File.stat(file.to_s))
407
+ refresh = tmpnam(File.dirname(path))
408
+ File.link path, refresh
409
+ File.chmod stat.mode, path
410
+ File.utime stat.atime, stat.mtime, path
411
+ ensure
412
+ begin
413
+ File.unlink refresh if refresh
414
+ rescue Errno::ENOENT
415
+ end
416
+ end
417
+ end
418
+
419
+ def create_tmplock
420
+ tmplock = tmpnam @dirname
421
+ begin
422
+ create(tmplock) do |f|
423
+ unless dont_use_lock_id
424
+ @lock_id = gen_lock_id
425
+ dumped = dump_lock_id
426
+ trace{"lock_id <\n#{ @lock_id.inspect }\n>"}
427
+ f.write dumped
428
+ f.flush
429
+ end
430
+ yield f
431
+ end
432
+ ensure
433
+ begin; File.unlink tmplock; rescue Errno::ENOENT; end if tmplock
434
+ end
435
+ end
436
+
437
+ def gen_lock_id
438
+ Hash[
439
+ 'host' => "#{ HOSTNAME }",
440
+ 'pid' => "#{ Process.pid }",
441
+ 'ppid' => "#{ Process.ppid }",
442
+ 'time' => timestamp,
443
+ ]
444
+ end
445
+
446
+ def timestamp
447
+ time = Time.now
448
+ usec = time.usec.to_s
449
+ usec << '0' while usec.size < 6
450
+ "#{ time.strftime('%Y-%m-%d %H:%M:%S') }.#{ usec }"
451
+ end
452
+
453
+ def dump_lock_id(lock_id = @lock_id)
454
+ "host: %s\npid: %s\nppid: %s\ntime: %s\n" %
455
+ lock_id.values_at('host','pid','ppid','time')
456
+ end
457
+
458
+ def load_lock_id(buf)
459
+ lock_id = {}
460
+ kv = %r/([^:]+):(.*)/o
461
+ buf.each_line do |line|
462
+ m = kv.match line
463
+ k, v = m[1], m[2]
464
+ next unless m and k and v
465
+ lock_id[k.strip] = v.strip
466
+ end
467
+ lock_id
468
+ end
469
+
470
+ def tmpnam(dir, seed = File.basename($0))
471
+ pid = Process.pid
472
+ time = Time.now
473
+ sec = time.to_i
474
+ usec = time.usec
475
+ "%s%s.%s_%d_%s_%d_%d_%d.lck" %
476
+ [dir, File::SEPARATOR, HOSTNAME, pid, seed, sec, usec, rand(sec)]
477
+ end
478
+
479
+ def create(path)
480
+ umask = nil
481
+ f = nil
482
+ begin
483
+ umask = File.umask 022
484
+ f = open path, File::WRONLY|File::CREAT|File::EXCL, 0644
485
+ ensure
486
+ File.umask umask if umask
487
+ end
488
+ return(block_given? ? begin; yield f; ensure; f.close; end : f)
489
+ end
490
+
491
+ def touch(path)
492
+ FileUtils.touch path
493
+ end
494
+
495
+ def getopt(key, default = nil)
496
+ [ key, key.to_s, key.to_s.intern ].each do |k|
497
+ return @opts[k] if @opts.has_key?(k)
498
+ end
499
+ return default
500
+ end
501
+
502
+ def to_str
503
+ @path
504
+ end
505
+ alias to_s to_str
506
+
507
+ def trace(s = nil)
508
+ STDERR.puts((s ? s : yield)) if @debug
509
+ end
510
+
511
+ def errmsg(e)
512
+ "%s (%s)\n%s\n" % [e.class, e.message, e.backtrace.join("\n")]
513
+ end
514
+
515
+ def attempt
516
+ ret = nil
517
+ loop{ break unless catch('attempt'){ ret = yield } == 'try_again' }
518
+ ret
519
+ end
520
+
521
+ def try_again!
522
+ throw 'attempt', 'try_again'
523
+ end
524
+ alias again! try_again!
525
+
526
+ def give_up!
527
+ throw 'attempt', 'give_up'
528
+ end
529
+ end
530
+
531
+ def Lockfile(path, *a, &b)
532
+ Lockfile.new(path, *a, &b)
533
+ end
534
+
535
+ $__lockfile__ = __FILE__
536
+ end