scout-gear 7.2.0 → 8.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +51 -6
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +33 -29
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/exceptions.rb +1 -0
  10. data/lib/scout/log/color.rb +4 -2
  11. data/lib/scout/log/progress/report.rb +1 -1
  12. data/lib/scout/log/progress/util.rb +71 -2
  13. data/lib/scout/log/progress.rb +1 -1
  14. data/lib/scout/log/trap.rb +107 -0
  15. data/lib/scout/log.rb +56 -21
  16. data/lib/scout/meta_extension.rb +13 -6
  17. data/lib/scout/misc/digest.rb +1 -1
  18. data/lib/scout/misc/format.rb +12 -0
  19. data/lib/scout/misc/helper.rb +31 -0
  20. data/lib/scout/misc/insist.rb +1 -1
  21. data/lib/scout/misc/monitor.rb +12 -1
  22. data/lib/scout/misc/system.rb +10 -0
  23. data/lib/scout/misc.rb +1 -0
  24. data/lib/scout/named_array.rb +65 -3
  25. data/lib/scout/open/lock/lockfile.rb +587 -0
  26. data/lib/scout/open/lock.rb +28 -2
  27. data/lib/scout/open/remote.rb +4 -0
  28. data/lib/scout/open/stream.rb +111 -42
  29. data/lib/scout/open/util.rb +13 -3
  30. data/lib/scout/path/find.rb +9 -1
  31. data/lib/scout/path/util.rb +35 -0
  32. data/lib/scout/persist/serialize.rb +18 -5
  33. data/lib/scout/persist.rb +60 -30
  34. data/lib/scout/resource/path.rb +53 -0
  35. data/lib/scout/resource/produce.rb +0 -8
  36. data/lib/scout/resource/util.rb +2 -1
  37. data/lib/scout/semaphore.rb +8 -1
  38. data/lib/scout/tmpfile.rb +7 -8
  39. data/lib/scout/tsv/attach.rb +177 -0
  40. data/lib/scout/tsv/change_id.rb +40 -0
  41. data/lib/scout/tsv/dumper.rb +85 -54
  42. data/lib/scout/tsv/index.rb +188 -20
  43. data/lib/scout/tsv/open.rb +182 -0
  44. data/lib/scout/tsv/parser.rb +200 -118
  45. data/lib/scout/tsv/path.rb +5 -6
  46. data/lib/scout/tsv/persist/adapter.rb +26 -37
  47. data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
  48. data/lib/scout/tsv/persist/serialize.rb +117 -0
  49. data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
  50. data/lib/scout/tsv/persist.rb +4 -2
  51. data/lib/scout/tsv/transformer.rb +141 -0
  52. data/lib/scout/tsv/traverse.rb +136 -37
  53. data/lib/scout/tsv/util/filter.rb +312 -0
  54. data/lib/scout/tsv/util/process.rb +73 -0
  55. data/lib/scout/tsv/util/reorder.rb +81 -0
  56. data/lib/scout/tsv/util/select.rb +265 -0
  57. data/lib/scout/tsv/util/unzip.rb +86 -0
  58. data/lib/scout/tsv/util.rb +126 -19
  59. data/lib/scout/tsv.rb +28 -5
  60. data/lib/scout/work_queue/socket.rb +6 -1
  61. data/lib/scout/work_queue/worker.rb +5 -2
  62. data/lib/scout/work_queue.rb +15 -8
  63. data/lib/scout/workflow/definition.rb +29 -2
  64. data/lib/scout/workflow/step/dependencies.rb +24 -4
  65. data/lib/scout/workflow/step/info.rb +40 -5
  66. data/lib/scout/workflow/step/progress.rb +14 -0
  67. data/lib/scout/workflow/step/provenance.rb +8 -7
  68. data/lib/scout/workflow/step/status.rb +45 -0
  69. data/lib/scout/workflow/step.rb +104 -33
  70. data/lib/scout/workflow/task/inputs.rb +14 -20
  71. data/lib/scout/workflow/task.rb +86 -47
  72. data/lib/scout/workflow/usage.rb +10 -6
  73. data/scout-gear.gemspec +30 -3
  74. data/scout_commands/workflow/task +37 -9
  75. data/scout_commands/workflow/task_old +2 -2
  76. data/test/scout/open/test_stream.rb +61 -59
  77. data/test/scout/path/test_find.rb +10 -1
  78. data/test/scout/resource/test_produce.rb +15 -0
  79. data/test/scout/test_meta_extension.rb +25 -0
  80. data/test/scout/test_named_array.rb +18 -0
  81. data/test/scout/test_persist.rb +67 -0
  82. data/test/scout/test_tmpfile.rb +1 -1
  83. data/test/scout/test_tsv.rb +222 -3
  84. data/test/scout/test_work_queue.rb +21 -18
  85. data/test/scout/tsv/persist/test_adapter.rb +11 -1
  86. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  87. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  88. data/test/scout/tsv/test_attach.rb +227 -0
  89. data/test/scout/tsv/test_change_id.rb +98 -0
  90. data/test/scout/tsv/test_dumper.rb +1 -1
  91. data/test/scout/tsv/test_index.rb +127 -3
  92. data/test/scout/tsv/test_open.rb +167 -0
  93. data/test/scout/tsv/test_parser.rb +45 -3
  94. data/test/scout/tsv/test_persist.rb +9 -0
  95. data/test/scout/tsv/test_transformer.rb +108 -0
  96. data/test/scout/tsv/test_traverse.rb +195 -3
  97. data/test/scout/tsv/test_util.rb +24 -0
  98. data/test/scout/tsv/util/test_filter.rb +188 -0
  99. data/test/scout/tsv/util/test_process.rb +47 -0
  100. data/test/scout/tsv/util/test_reorder.rb +94 -0
  101. data/test/scout/tsv/util/test_select.rb +58 -0
  102. data/test/scout/tsv/util/test_unzip.rb +112 -0
  103. data/test/scout/work_queue/test_socket.rb +0 -1
  104. data/test/scout/work_queue/test_worker.rb +63 -6
  105. data/test/scout/workflow/step/test_load.rb +3 -3
  106. data/test/scout/workflow/step/test_status.rb +31 -0
  107. data/test/scout/workflow/task/test_inputs.rb +14 -14
  108. data/test/scout/workflow/test_step.rb +13 -13
  109. data/test/scout/workflow/test_task.rb +168 -32
  110. data/test/scout/workflow/test_usage.rb +33 -6
  111. data/test/test_helper.rb +3 -1
  112. metadata +29 -2
@@ -0,0 +1,587 @@
1
+ # Originaly from here https://github.com/ahoward/lockfile
2
+ # Author: Ara T. Howard (Ara T. Howard)
3
+ # Licence: Ruby (as listed [here](https://github.com/ahoward/lockfile/blob/20ab06f29bda69b9773d799510d00585b6a27e3b/lockfile.gemspec#L10))
4
+ # with modifications for scout-gear and rbbt by Miguel Vazquez
5
+ unless(defined?($__lockfile__) or defined?(Lockfile))
6
+
7
+ require 'socket'
8
+ require 'timeout'
9
+ require 'fileutils'
10
+
11
+ class Lockfile
12
+
13
+ VERSION = '2.1.8'
14
+ def Lockfile.version() Lockfile::VERSION end
15
+ def version() Lockfile::VERSION end
16
+
17
+ def Lockfile.description
18
+ 'a ruby library for creating perfect and NFS safe lockfiles'
19
+ end
20
+
21
+ class LockError < StandardError; end
22
+ class StolenLockError < LockError; end
23
+ class StackingLockError < LockError; end
24
+ class StatLockError < LockError; end
25
+ class MaxTriesLockError < LockError; end
26
+ class TimeoutLockError < LockError; end
27
+ class NFSLockError < LockError; end
28
+ class UnLockError < LockError; end
29
+
30
+ class SleepCycle < Array
31
+ attr_reader :min
32
+ attr_reader :max
33
+ attr_reader :range
34
+ attr_reader :inc
35
+
36
+ def initialize(min, max, inc)
37
+ @min, @max, @inc = Float(min), Float(max), Float(inc)
38
+ @range = @max - @min
39
+ raise RangeError, "max(#{ @max }) <= min(#{ @min })" if @max <= @min
40
+ raise RangeError, "inc(#{ @inc }) > range(#{ @range })" if @inc > @range
41
+ raise RangeError, "inc(#{ @inc }) <= 0" if @inc <= 0
42
+ raise RangeError, "range(#{ @range }) <= 0" if @range <= 0
43
+ s = @min
44
+ push(s) and s += @inc while(s <= @max)
45
+ self[-1] = @max if self[-1] < @max
46
+ reset
47
+ end
48
+
49
+ def next
50
+ ret = self[@idx]
51
+ @idx = ((@idx + 1) % self.size)
52
+ ret
53
+ end
54
+
55
+ def reset
56
+ @idx = 0
57
+ end
58
+ end
59
+
60
+ HOSTNAME = Socket.gethostname
61
+
62
+ DEFAULT_RETRIES = nil # maximum number of attempts
63
+ DEFAULT_TIMEOUT = nil # the longest we will try
64
+ DEFAULT_MAX_AGE = 3600 # lockfiles older than this are stale
65
+ DEFAULT_SLEEP_INC = 2 # sleep cycle is this much longer each time
66
+ DEFAULT_MIN_SLEEP = 2 # shortest sleep time
67
+ DEFAULT_MAX_SLEEP = 32 # longest sleep time
68
+ DEFAULT_SUSPEND = 1800 # iff we steal a lock wait this long before we go on
69
+ DEFAULT_REFRESH = 8 # how often we touch/validate the lock
70
+ DEFAULT_DONT_CLEAN = false # iff we leave lock files lying around
71
+ DEFAULT_POLL_RETRIES = 16 # this many polls makes one 'try'
72
+ DEFAULT_POLL_MAX_SLEEP = 0.08 # the longest we'll sleep between polls
73
+ DEFAULT_DONT_SWEEP = false # if we cleanup after other process on our host
74
+ DEFAULT_DONT_USE_LOCK_ID = false # if we dump lock info into lockfile
75
+
76
+ DEFAULT_DEBUG = ENV['LOCKFILE_DEBUG'] || false
77
+
78
+ class << Lockfile
79
+ attr_accessor :retries
80
+ attr_accessor :max_age
81
+ attr_accessor :sleep_inc
82
+ attr_accessor :min_sleep
83
+ attr_accessor :max_sleep
84
+ attr_accessor :suspend
85
+ attr_accessor :timeout
86
+ attr_accessor :refresh
87
+ attr_accessor :debug
88
+ attr_accessor :dont_clean
89
+ attr_accessor :poll_retries
90
+ attr_accessor :poll_max_sleep
91
+ attr_accessor :dont_sweep
92
+ attr_accessor :dont_use_lock_id
93
+
94
+ def init
95
+ @retries = DEFAULT_RETRIES
96
+ @max_age = DEFAULT_MAX_AGE
97
+ @sleep_inc = DEFAULT_SLEEP_INC
98
+ @min_sleep = DEFAULT_MIN_SLEEP
99
+ @max_sleep = DEFAULT_MAX_SLEEP
100
+ @suspend = DEFAULT_SUSPEND
101
+ @timeout = DEFAULT_TIMEOUT
102
+ @refresh = DEFAULT_REFRESH
103
+ @dont_clean = DEFAULT_DONT_CLEAN
104
+ @poll_retries = DEFAULT_POLL_RETRIES
105
+ @poll_max_sleep = DEFAULT_POLL_MAX_SLEEP
106
+ @dont_sweep = DEFAULT_DONT_SWEEP
107
+ @dont_use_lock_id = DEFAULT_DONT_USE_LOCK_ID
108
+
109
+ @debug = DEFAULT_DEBUG
110
+
111
+ STDOUT.sync = true if @debug
112
+ STDERR.sync = true if @debug
113
+ end
114
+ end
115
+
116
+ Lockfile.init
117
+
118
+ attr_reader :klass
119
+ attr_reader :path
120
+ attr_reader :opts
121
+ attr_reader :locked
122
+ attr_reader :thief
123
+ attr_reader :refresher
124
+ attr_reader :dirname
125
+ attr_reader :basename
126
+ attr_reader :clean
127
+ attr_reader :retries
128
+ attr_reader :max_age
129
+ attr_reader :sleep_inc
130
+ attr_reader :min_sleep
131
+ attr_reader :max_sleep
132
+ attr_reader :suspend
133
+ attr_reader :refresh
134
+ attr_reader :timeout
135
+ attr_reader :dont_clean
136
+ attr_reader :poll_retries
137
+ attr_reader :poll_max_sleep
138
+ attr_reader :dont_sweep
139
+ attr_reader :dont_use_lock_id
140
+
141
+ attr_accessor :debug
142
+
143
+ alias thief? thief
144
+ alias locked? locked
145
+ alias debug? debug
146
+
147
+ def Lockfile.create(path, *a, &b)
148
+ opts = {
149
+ 'retries' => 0,
150
+ 'min_sleep' => 0,
151
+ 'max_sleep' => 1,
152
+ 'sleep_inc' => 1,
153
+ 'max_age' => nil,
154
+ 'suspend' => 0,
155
+ 'refresh' => nil,
156
+ 'timeout' => nil,
157
+ 'poll_retries' => 0,
158
+ 'dont_clean' => true,
159
+ 'dont_sweep' => false,
160
+ 'dont_use_lock_id' => true,
161
+ }
162
+ begin
163
+ new(path, opts).lock
164
+ rescue LockError
165
+ raise Errno::EEXIST, path
166
+ end
167
+ open(path, *a, &b)
168
+ end
169
+
170
+ def self.finalizer_proc(file)
171
+ pid = Process.pid
172
+ lambda do |id|
173
+ File.unlink file if Process.pid == pid
174
+ rescue
175
+ nil
176
+ end
177
+ end
178
+
179
+ def initialize(path, opts = {}, &block)
180
+ @klass = self.class
181
+ @path = path
182
+ @opts = opts
183
+
184
+ @retries = getopt 'retries' , @klass.retries
185
+ @max_age = getopt 'max_age' , @klass.max_age
186
+ @sleep_inc = getopt 'sleep_inc' , @klass.sleep_inc
187
+ @min_sleep = getopt 'min_sleep' , @klass.min_sleep
188
+ @max_sleep = getopt 'max_sleep' , @klass.max_sleep
189
+ @suspend = getopt 'suspend' , @klass.suspend
190
+ @timeout = getopt 'timeout' , @klass.timeout
191
+ @refresh = getopt 'refresh' , @klass.refresh
192
+ @dont_clean = getopt 'dont_clean' , @klass.dont_clean
193
+ @poll_retries = getopt 'poll_retries' , @klass.poll_retries
194
+ @poll_max_sleep = getopt 'poll_max_sleep' , @klass.poll_max_sleep
195
+ @dont_sweep = getopt 'dont_sweep' , @klass.dont_sweep
196
+ @dont_use_lock_id = getopt 'dont_use_lock_id' , @klass.dont_use_lock_id
197
+ @debug = getopt 'debug' , @klass.debug
198
+
199
+ @semaphore = Mutex.new
200
+
201
+ @sleep_cycle = SleepCycle.new @min_sleep, @max_sleep, @sleep_inc
202
+
203
+ @clean = @dont_clean ? nil : Lockfile.finalizer_proc(@path)
204
+
205
+ @dirname = File.dirname @path
206
+ @basename = File.basename @path
207
+ @thief = false
208
+ @locked = false
209
+ @refrsher = nil
210
+
211
+ lock(&block) if block
212
+ end
213
+
214
+ ##
215
+ # Executes the given block after acquiring the lock and
216
+ # ensures that the lock is relinquished afterwards.
217
+ #
218
+ def synchronize
219
+ raise ArgumentError, 'block must be given' unless block_given?
220
+ begin
221
+ lock
222
+ yield
223
+ ensure
224
+ unlock
225
+ end
226
+ end
227
+
228
+ def lock
229
+ raise StackingLockError, "<#{ @path }> is locked!" if @locked
230
+
231
+ sweep unless @dont_sweep
232
+
233
+ ret = nil
234
+
235
+ attempt do
236
+ begin
237
+ @sleep_cycle.reset
238
+ create_tmplock do |f|
239
+ begin
240
+ Timeout.timeout(@timeout) do
241
+ tmp_path = f.path
242
+ tmp_stat = f.lstat
243
+ n_retries = 0
244
+ trace{ "attempting to lock <#{ @path }>..." }
245
+ begin
246
+ i = 0
247
+ begin
248
+ trace{ "polling attempt <#{ i }>..." }
249
+ begin
250
+ File.link tmp_path, @path
251
+ rescue Errno::ENOENT
252
+ try_again!
253
+ end
254
+ lock_stat = File.lstat @path
255
+ raise StatLockError, "stat's do not agree" unless
256
+ tmp_stat.rdev == lock_stat.rdev and tmp_stat.ino == lock_stat.ino
257
+ trace{ "aquired lock <#{ @path }>" }
258
+ @locked = true
259
+ rescue => e
260
+ i += 1
261
+ unless i >= @poll_retries
262
+ t = [rand(@poll_max_sleep), @poll_max_sleep].min
263
+ trace{ "poll sleep <#{ t }>..." }
264
+ sleep t
265
+ retry
266
+ end
267
+ raise
268
+ end
269
+
270
+ rescue => e
271
+ n_retries += 1
272
+ trace{ "n_retries <#{ n_retries }>" }
273
+ case validlock?
274
+ when true
275
+ raise MaxTriesLockError, "surpased retries <#{ @retries }>" if
276
+ @retries and n_retries >= @retries
277
+ trace{ "found valid lock" }
278
+ sleeptime = @sleep_cycle.next
279
+ trace{ "sleep <#{ sleeptime }>..." }
280
+ sleep sleeptime
281
+ when false
282
+ trace{ "found invalid lock and removing" }
283
+ begin
284
+ File.unlink @path
285
+ @thief = true
286
+ warn "<#{ @path }> stolen by <#{ Process.pid }> at <#{ timestamp }>"
287
+ trace{ "i am a thief!" }
288
+ rescue Errno::ENOENT
289
+ end
290
+ trace{ "suspending <#{ @suspend }>" }
291
+ sleep @suspend
292
+ when nil
293
+ raise MaxTriesLockError, "surpased retries <#{ @retries }>" if
294
+ @retries and n_retries >= @retries
295
+ end
296
+ retry
297
+ end # begin
298
+ end # timeout
299
+ rescue Timeout::Error
300
+ raise TimeoutLockError, "surpassed timeout <#{ @timeout }>"
301
+ end # begin
302
+ end # create_tmplock
303
+
304
+ if block_given?
305
+ stolen = false
306
+ @refresher = (@refresh ? new_refresher : nil)
307
+ begin
308
+ begin
309
+ ret = yield @path
310
+ rescue StolenLockError
311
+ stolen = true
312
+ raise
313
+ end
314
+ ensure
315
+ begin
316
+ begin
317
+ @semaphore.synchronize do
318
+ @refresher.kill
319
+ end
320
+ rescue
321
+ @refresher.kill
322
+ end if @refresher and @refresher.status
323
+ @refresher = nil
324
+ ensure
325
+ unlock unless stolen
326
+ end
327
+ end
328
+ else
329
+ @refresher = (@refresh ? new_refresher : nil)
330
+ ObjectSpace.define_finalizer self, @clean if @clean
331
+ ret = self
332
+ end
333
+ rescue Errno::ESTALE, Errno::EIO => e
334
+ raise(NFSLockError, errmsg(e))
335
+ end
336
+ end
337
+
338
+ return ret
339
+ end
340
+
341
+ def sweep
342
+ begin
343
+ glob = File.join(@dirname, ".*lck")
344
+ paths = Dir[glob]
345
+ paths.each do |path|
346
+ begin
347
+ basename = File.basename path
348
+ pat = %r/^\s*\.([^_]+)_([^_]+)/o
349
+ if pat.match(basename)
350
+ host, pid = $1, $2
351
+ else
352
+ next
353
+ end
354
+ host.gsub!(%r/^\.+|\.+$/,'')
355
+ quad = host.split %r/\./
356
+ host = quad.first
357
+ pat = %r/^\s*#{ host }/i
358
+ if pat.match(HOSTNAME) and %r/^\s*\d+\s*$/.match(pid)
359
+ unless alive?(pid)
360
+ trace{ "process <#{ pid }> on <#{ host }> is no longer alive" }
361
+ trace{ "sweeping <#{ path }>" }
362
+ FileUtils.rm_f path
363
+ else
364
+ trace{ "process <#{ pid }> on <#{ host }> is still alive" }
365
+ trace{ "ignoring <#{ path }>" }
366
+ end
367
+ else
368
+ trace{ "ignoring <#{ path }> generated by <#{ host }>" }
369
+ end
370
+ rescue
371
+ next
372
+ end
373
+ end
374
+ rescue => e
375
+ warn(errmsg(e))
376
+ end
377
+ end
378
+
379
+ def alive? pid
380
+ pid = Integer("#{ pid }")
381
+ begin
382
+ Process.kill 0, pid
383
+ true
384
+ rescue Errno::ESRCH
385
+ false
386
+ end
387
+ end
388
+
389
+ def unlock
390
+ raise UnLockError, "<#{ @path }> is not locked!" unless @locked
391
+
392
+ begin
393
+ @semaphore.synchronize do
394
+ @refresher.kill
395
+ end
396
+ rescue
397
+ @refresher.kill
398
+ end if @refresher and @refresher.status
399
+
400
+ @refresher = nil
401
+
402
+ begin
403
+ File.unlink @path
404
+ rescue Errno::ENOENT
405
+ raise StolenLockError, @path
406
+ ensure
407
+ @thief = false
408
+ @locked = false
409
+ ObjectSpace.undefine_finalizer self if @clean
410
+ end
411
+ end
412
+
413
+ def new_refresher
414
+ Thread.new(Thread.current, @path, @refresh, @dont_use_lock_id) do |thread, path, refresh, dont_use_lock_id|
415
+ loop do
416
+ begin
417
+ touch path
418
+ trace{"touched <#{ path }> @ <#{ Time.now.to_f }>"}
419
+ unless dont_use_lock_id
420
+ txt = nil
421
+ @semaphore.synchronize do
422
+ txt = IO.read(path)
423
+ end
424
+ loaded = load_lock_id(txt)
425
+ trace{"loaded <\n#{ loaded.inspect }\n>"}
426
+ raise unless loaded == @lock_id
427
+ end
428
+ sleep refresh
429
+ rescue Exception => e
430
+ trace{errmsg e}
431
+ thread.raise StolenLockError
432
+ Thread.exit
433
+ end
434
+ end
435
+ end
436
+ end
437
+
438
+ def validlock?
439
+ if @max_age
440
+ uncache @path rescue nil
441
+ begin
442
+ return((Time.now - File.stat(@path).mtime) < @max_age)
443
+ rescue Errno::ENOENT
444
+ return nil
445
+ end
446
+ else
447
+ exist = File.exist?(@path)
448
+ return(exist ? true : nil)
449
+ end
450
+ end
451
+
452
+ def uncache file
453
+ refresh = nil
454
+ begin
455
+ is_a_file = File === file
456
+ path = (is_a_file ? file.path : file.to_s)
457
+ stat = (is_a_file ? file.stat : File.stat(file.to_s))
458
+ refresh = tmpnam(File.dirname(path))
459
+ File.link path, refresh
460
+ File.chmod stat.mode, path
461
+ File.utime stat.atime, stat.mtime, path
462
+ ensure
463
+ begin
464
+ File.unlink refresh if refresh
465
+ rescue Errno::ENOENT
466
+ end
467
+ end
468
+ end
469
+
470
+ def create_tmplock
471
+ tmplock = tmpnam @dirname
472
+ begin
473
+ create(tmplock) do |f|
474
+ unless dont_use_lock_id
475
+ @lock_id = gen_lock_id
476
+ dumped = dump_lock_id
477
+ trace{"lock_id <\n#{ @lock_id.inspect }\n>"}
478
+ f.write dumped
479
+ f.flush
480
+ end
481
+ yield f
482
+ end
483
+ ensure
484
+ begin; File.unlink tmplock; rescue Errno::ENOENT; end if tmplock
485
+ end
486
+ end
487
+
488
+ def gen_lock_id
489
+ Hash[
490
+ 'host' => "#{ HOSTNAME }",
491
+ 'pid' => "#{ Process.pid }",
492
+ 'ppid' => "#{ Process.ppid }",
493
+ 'time' => timestamp,
494
+ ]
495
+ end
496
+
497
+ def timestamp
498
+ time = Time.now
499
+ usec = time.usec.to_s
500
+ usec << '0' while usec.size < 6
501
+ "#{ time.strftime('%Y-%m-%d %H:%M:%S') }.#{ usec }"
502
+ end
503
+
504
+ def dump_lock_id(lock_id = @lock_id)
505
+ "host: %s\npid: %s\nppid: %s\ntime: %s\n" %
506
+ lock_id.values_at('host','pid','ppid','time')
507
+ end
508
+
509
+ def load_lock_id(buf)
510
+ lock_id = {}
511
+ kv = %r/([^:]+):(.*)/o
512
+ buf.each_line do |line|
513
+ m = kv.match line
514
+ k, v = m[1], m[2]
515
+ next unless m and k and v
516
+ lock_id[k.strip] = v.strip
517
+ end
518
+ lock_id
519
+ end
520
+
521
+ def tmpnam(dir, seed = File.basename($0))
522
+ pid = Process.pid
523
+ time = Time.now
524
+ sec = time.to_i
525
+ usec = time.usec
526
+ "%s%s.%s_%d_%s_%d_%d_%d.lck" %
527
+ [dir, File::SEPARATOR, HOSTNAME, pid, seed, sec, usec, rand(sec)]
528
+ end
529
+
530
+ def create(path)
531
+ umask = nil
532
+ f = nil
533
+ begin
534
+ umask = File.umask 022
535
+ f = open path, File::WRONLY|File::CREAT|File::EXCL, 0644
536
+ ensure
537
+ File.umask umask if umask
538
+ end
539
+ return(block_given? ? begin; yield f; ensure; f.close; end : f)
540
+ end
541
+
542
+ def touch(path)
543
+ FileUtils.touch path
544
+ end
545
+
546
+ def getopt(key, default = nil)
547
+ [ key, key.to_s, key.to_s.intern ].each do |k|
548
+ return @opts[k] if @opts.has_key?(k)
549
+ end
550
+ return default
551
+ end
552
+
553
+ def to_str
554
+ @path
555
+ end
556
+ alias to_s to_str
557
+
558
+ def trace(s = nil)
559
+ STDERR.puts((s ? s : yield)) if @debug
560
+ end
561
+
562
+ def errmsg(e)
563
+ "%s (%s)\n%s\n" % [e.class, e.message, e.backtrace.join("\n")]
564
+ end
565
+
566
+ def attempt
567
+ ret = nil
568
+ loop{ break unless catch('attempt'){ ret = yield } == 'try_again' }
569
+ ret
570
+ end
571
+
572
+ def try_again!
573
+ throw 'attempt', 'try_again'
574
+ end
575
+ alias again! try_again!
576
+
577
+ def give_up!
578
+ throw 'attempt', 'give_up'
579
+ end
580
+ end
581
+
582
+ def Lockfile(path, *a, &b)
583
+ Lockfile.new(path, *a, &b)
584
+ end
585
+
586
+ $__lockfile__ = __FILE__
587
+ end
@@ -1,9 +1,31 @@
1
1
  require_relative '../path'
2
2
  require_relative '../log'
3
3
  require_relative '../exceptions'
4
- require 'lockfile'
4
+ require_relative 'lock/lockfile'
5
5
 
6
6
  module Open
7
+ def self.thread_in_lock(t)
8
+ t.backtrace.select{|l| l.include?("lockfile") }.any?
9
+ end
10
+
11
+ def self.unlock_thread(t, exception = nil)
12
+ while t.alive? && t.backtrace.select{|l| l.include?("lockfile") }.any?
13
+ iii :UNLOCK
14
+ t.raise(exception)
15
+ Log.stack t.backtrace
16
+ locks = t["locks"]
17
+ return if locks.nil? || locks.empty?
18
+ locks.each do |lock|
19
+ lock.unlock if lock.locked?
20
+ Open.rm(lock.path)
21
+ end
22
+ iii Thread.current
23
+ Thread.list.each{|t| iii [t, t["locks"]]; Log.stack t.backtrace }
24
+ Log.stack t.backtrace
25
+ end
26
+ end
27
+
28
+
7
29
  def self.lock(file, unlock = true, options = {})
8
30
  unlock, options = true, unlock if Hash === unlock
9
31
  return yield if file.nil? and not Lockfile === options[:lock]
@@ -35,6 +57,9 @@ module Open
35
57
  raise LockInterrupted
36
58
  end
37
59
 
60
+ iii Thread.current["lock_exception"] if Thread.current["lock_exception"]
61
+ raise Thread.current["lock_exception"] if Thread.current["lock_exception"]
62
+
38
63
  res = nil
39
64
 
40
65
  begin
@@ -47,8 +72,9 @@ module Open
47
72
  begin
48
73
  if lockfile.locked?
49
74
  lockfile.unlock
50
- else
51
75
  end
76
+ iii Thread.current["lock_exception"] if Thread.current["lock_exception"]
77
+ raise Thread.current["lock_exception"] if Thread.current["lock_exception"]
52
78
  rescue Exception
53
79
  Log.warn "Exception unlocking: #{lockfile.path}"
54
80
  Log.exception $!
@@ -81,6 +81,10 @@ module Open
81
81
  end
82
82
  end
83
83
 
84
+ def self.download(url, file)
85
+ CMD.cmd_log(:wget, "'#{url}' -O '#{file}'")
86
+ end
87
+
84
88
  def self.digest_url(url, options = {})
85
89
  params = [url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]).split("\n").sort * "\n" : "")]
86
90
  Misc.digest([url, params])