higgs 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/higgs/storage.rb CHANGED
@@ -1,13 +1,14 @@
1
1
  # = transactional storage core
2
2
  #
3
3
  # Author:: $Author: toki $
4
- # Date:: $Date: 2007-10-10 00:49:33 +0900 (Wed, 10 Oct 2007) $
5
- # Revision:: $Revision: 626 $
4
+ # Date:: $Date: 2007-11-11 11:07:25 +0900 (Sun, 11 Nov 2007) $
5
+ # Revision:: $Revision: 681 $
6
6
  #
7
7
  # == license
8
8
  # :include:LICENSE
9
9
  #
10
10
 
11
+ require 'digest'
11
12
  require 'forwardable'
12
13
  require 'higgs/block'
13
14
  require 'higgs/cache'
@@ -23,7 +24,7 @@ module Higgs
23
24
  # = transactional storage core
24
25
  class Storage
25
26
  # for ident(1)
26
- CVS_ID = '$Id: storage.rb 626 2007-10-09 15:49:33Z toki $'
27
+ CVS_ID = '$Id: storage.rb 681 2007-11-11 02:07:25Z toki $'
27
28
 
28
29
  extend Forwardable
29
30
  include Exceptions
@@ -44,26 +45,17 @@ module Higgs
44
45
  PROPERTIES_CKSUM_BITS = 16
45
46
 
46
47
  DATA_HASH = {}
47
- [ [ :SUM16, proc{|s| s.sum(16).to_s }, nil ],
48
- [ :MD5, proc{|s| Digest::MD5.hexdigest(s) }, 'digest/md5' ],
49
- [ :RMD160, proc{|s| Digest::RMD160.hexdigest(s) }, 'digest/rmd160' ],
50
- [ :SHA1, proc{|s| Digest::SHA1.hexdigest(s) }, 'digest/sha1' ],
51
- [ :SHA256, proc{|s| Digest::SHA256.hexdigest(s) }, 'digest/sha2' ],
52
- [ :SHA384, proc{|s| Digest::SHA384.hexdigest(s) }, 'digest/sha2' ],
53
- [ :SHA512, proc{|s| Digest::SHA512.hexdigest(s) }, 'digest/sha2' ]
54
- ].each do |hash_symbol, hash_proc, hash_lib|
55
- if (hash_lib) then
56
- begin
57
- require(hash_lib)
58
- rescue LoadError
59
- next
60
- end
61
- end
62
- DATA_HASH[hash_symbol] = hash_proc
63
- end
64
-
65
48
  DATA_HASH_BIN = {}
66
- DATA_HASH.each do |hash_symbol, hash_proc|
49
+
50
+ [ [ :SUM16, proc{|s| s.sum(16).to_s } ],
51
+ [ :MD5, proc{|s| Digest::MD5.hexdigest(s) } ],
52
+ [ :RMD160, proc{|s| Digest::RMD160.hexdigest(s) } ],
53
+ [ :SHA1, proc{|s| Digest::SHA1.hexdigest(s) } ],
54
+ [ :SHA256, proc{|s| Digest::SHA256.hexdigest(s) } ],
55
+ [ :SHA384, proc{|s| Digest::SHA384.hexdigest(s) } ],
56
+ [ :SHA512, proc{|s| Digest::SHA512.hexdigest(s) } ]
57
+ ].each do |hash_symbol, hash_proc|
58
+ DATA_HASH[hash_symbol] = hash_proc
67
59
  DATA_HASH_BIN[hash_symbol.to_s] = hash_proc
68
60
  end
69
61
 
@@ -72,6 +64,10 @@ module Higgs
72
64
  # these options are defined.
73
65
  # [<tt>:number_of_read_io</tt>] number of read I/O handle of pool. default is <tt>2</tt>.
74
66
  # [<tt>:read_only</tt>] if <tt>true</tt> then storage is read-only. default is <tt>false</tt>.
67
+ # <tt>:standby</tt> is standby mode. in standby mode, storage is read-only
68
+ # and Higgs::Storage#apply_journal_log is callable.
69
+ # if Higgs::Storage#switch_to_write is called in standby mode then
70
+ # state of storage changes from standby mode to read-write mode.
75
71
  # [<tt>:properties_cache</tt>] read-cache for properties. default is a new instance of Higgs::LRUCache.
76
72
  # [<tt>:data_hash_type</tt>] hash type (<tt>:SUM16</tt>, <tt>:MD5</tt>, <tt>:RMD160</tt>,
77
73
  # <tt>:SHA1</tt>, <tt>:SHA256</tt>, <tt>:SHA384</tt> or <tt>:SHA512</tt>)
@@ -84,9 +80,6 @@ module Higgs
84
80
  # if <tt>:jlog_rotate_max</tt> is <tt>0</tt>, old journal log is
85
81
  # not deleted. if online-backup is used, <tt>:jlog_rotate_max</tt>
86
82
  # should be <tt>0</tt>. default is <tt>1</tt>.
87
- # [<tt>:jlog_rotate_service_uri</tt>] URI for DRb remote call to switch journal log to a new file.
88
- # if online-backup is used, <tt>:jlog_rotate_service_uri</tt>
89
- # should be defined. default is not defined.
90
83
  # [<tt>:logger</tt>] procedure to create a logger. default is a procedure to create a new
91
84
  # instance of Logger with logging level <tt>Logger::WARN</tt>.
92
85
  def init_options(options)
@@ -122,7 +115,6 @@ module Higgs
122
115
 
123
116
  @jlog_rotate_size = options[:jlog_rotate_size] || 1024 * 256
124
117
  @jlog_rotate_max = options[:jlog_rotate_max] || 1
125
- @jlog_rotate_service_uri = options[:jlog_rotate_service_uri]
126
118
 
127
119
  if (options.key? :logger) then
128
120
  @Logger = options[:logger]
@@ -144,43 +136,9 @@ module Higgs
144
136
  attr_reader :jlog_hash_type
145
137
  attr_reader :jlog_rotate_size
146
138
  attr_reader :jlog_rotate_max
147
- attr_reader :jlog_rotate_service_uri
148
139
  end
149
140
  include InitOptions
150
141
 
151
- # export Higgs::Storage methods from <tt>@storage</tt> instance variable.
152
- #
153
- # these methods are delegated.
154
- # * Higgs::Storage#name
155
- # * Higgs::Storage#read_only
156
- # * Higgs::Storage#number_of_read_io
157
- # * Higgs::Storage#data_hash_type
158
- # * Higgs::Storage#jlog_sync
159
- # * Higgs::Storage#jlog_hash_type
160
- # * Higgs::Storage#jlog_rotate_size
161
- # * Higgs::Storage#jlog_rotate_max
162
- # * Higgs::Storage#jlog_rotate_service_uri
163
- # * Higgs::Storage#shutdown
164
- # * Higgs::Storage#shutdown?
165
- # * Higgs::Storage#rotate_journal_log
166
- #
167
- module Export
168
- extend Forwardable
169
-
170
- def_delegator :@storage, :name
171
- def_delegator :@storage, :read_only
172
- def_delegator :@storage, :number_of_read_io
173
- def_delegator :@storage, :data_hash_type
174
- def_delegator :@storage, :jlog_sync
175
- def_delegator :@storage, :jlog_hash_type
176
- def_delegator :@storage, :jlog_rotate_size
177
- def_delegator :@storage, :jlog_rotate_max
178
- def_delegator :@storage, :jlog_rotate_service_uri
179
- def_delegator :@storage, :shutdown
180
- def_delegator :@storage, :shutdown?
181
- def_delegator :@storage, :rotate_journal_log
182
- end
183
-
184
142
  def self.load_conf(path)
185
143
  conf = YAML.load(IO.read(path))
186
144
  options = {}
@@ -217,7 +175,7 @@ module Higgs
217
175
  # storage is composed of the following 5 files.
218
176
  # <tt>name.log</tt>:: event log. default logging level is <tt>WARN</tt>.
219
177
  # <tt>name.tar</tt>:: data file compatible with unix TAR format.
220
- # <tt>name.idx</tt>:: index snapshot. genuine index is Hash in the memory.
178
+ # <tt>name.idx</tt>:: index snapshot. genuine index is Hash in memory.
221
179
  # see Higgs::Index for detail.
222
180
  # <tt>name.jlog</tt>:: transaction journal log. see Higgs::JournalLogger for detail.
223
181
  # <tt>name.lock</tt>:: lock file for File#flock. see Higgs::FileLock for detail.
@@ -238,9 +196,10 @@ module Higgs
238
196
  init_options(options)
239
197
 
240
198
  init_completed = false
199
+ read_only = @read_only && @read_only != :standby
241
200
  begin
242
- @flock = FileLock.new(@lock_name, @read_only)
243
- if (@read_only) then
201
+ @flock = FileLock.new(@lock_name, read_only)
202
+ if (read_only) then
244
203
  @flock.read_lock
245
204
  else
246
205
  @flock.write_lock
@@ -248,7 +207,7 @@ module Higgs
248
207
 
249
208
  @logger = @Logger.call(@log_name)
250
209
  @logger.info("storage open start...")
251
- if (@read_only) then
210
+ if (read_only) then
252
211
  @logger.info("get file lock for read")
253
212
  else
254
213
  @logger.info("get file lock for write")
@@ -265,7 +224,7 @@ module Higgs
265
224
  value = read_record_body(key, :p) and decode_properties(key, value)
266
225
  }
267
226
 
268
- unless (@read_only) then
227
+ unless (read_only) then
269
228
  begin
270
229
  w_io = File.open(@tar_name, File::WRONLY | File::CREAT | File::EXCL, 0660)
271
230
  @logger.info("create and open I/O handle for write: #{@tar_name}")
@@ -289,25 +248,25 @@ module Higgs
289
248
  if (File.exist? @idx_name) then
290
249
  @logger.info("load index: #{@idx_name}")
291
250
  @index.load(@idx_name)
251
+ unless (@index.storage_id) then # for migration to new index format of version 0.2 from old version
252
+ @index.storage_id = create_storage_id
253
+ @logger.info("save storage id: #{@index.storage_id}")
254
+ @index.save(@idx_name)
255
+ end
256
+ else
257
+ @index.storage_id = create_storage_id
258
+ @logger.info("save storage id: #{@index.storage_id}")
259
+ @index.save(@idx_name)
292
260
  end
293
261
  if (JournalLogger.need_for_recovery? @jlog_name) then
294
262
  recover
295
263
  end
296
- unless (@read_only) then
264
+ unless (read_only) then
297
265
  @logger.info("journal log sync mode: #{@jlog_sync}")
298
266
  @logger.info("open journal log for write: #{@jlog_name}")
299
267
  @jlog = JournalLogger.open(@jlog_name, @jlog_sync, @jlog_hash_type)
300
268
  end
301
269
 
302
- if (@jlog_rotate_service_uri) then
303
- @logger.info("start journal log rotation service: #{@jlog_rotate_service_uri}")
304
- require 'drb'
305
- @jlog_rotate_service = DRb::DRbServer.new(@jlog_rotate_service_uri,
306
- method(:rotate_journal_log))
307
- else
308
- @jlog_rotate_service = nil
309
- end
310
-
311
270
  init_completed = true
312
271
  ensure
313
272
  if (init_completed) then
@@ -331,7 +290,7 @@ module Higgs
331
290
  end
332
291
  end
333
292
 
334
- unless (@read_only) then
293
+ unless (read_only) then
335
294
  if (@jlog) then
336
295
  begin
337
296
  @jlog.close(false)
@@ -351,7 +310,7 @@ module Higgs
351
310
  }
352
311
  end
353
312
 
354
- unless (@read_only) then
313
+ unless (read_only) then
355
314
  if (@w_tar) then
356
315
  begin
357
316
  @w_tar.close(false)
@@ -381,27 +340,61 @@ module Higgs
381
340
  end
382
341
  end
383
342
 
343
+ def create_storage_id
344
+ hash = Digest::MD5.new
345
+ now = Time.now
346
+ hash.update(now.to_s)
347
+ hash.update(String(now.usec))
348
+ hash.update(String(rand(0)))
349
+ hash.update(String($$))
350
+ hash.update(CVS_ID)
351
+ hash.update(@name)
352
+ hash.update('toki')
353
+ hash.hexdigest
354
+ end
355
+ private :create_storage_id
356
+
384
357
  def check_panic
358
+ if (@shutdown) then
359
+ raise ShutdownException, 'storage shutdown'
360
+ end
361
+ if (@panic) then
362
+ raise PanicError, 'broken storage'
363
+ end
364
+ end
365
+ private :check_panic
366
+
367
+ def check_read
385
368
  @state_lock.synchronize{
386
- if (@shutdown) then
387
- raise ShutdownException, 'storage shutdown'
369
+ check_panic
370
+ }
371
+ end
372
+ private :check_read
373
+
374
+ def check_standby
375
+ @state_lock.synchronize{
376
+ check_panic
377
+ if (@read_only && @read_only != :standby) then
378
+ raise NotWritableError, 'failed to write to read only storage'
388
379
  end
389
- if (@panic) then
390
- raise PanicError, 'broken storage'
380
+ }
381
+ end
382
+ private :check_standby
383
+
384
+ def check_read_write
385
+ @state_lock.synchronize{
386
+ check_panic
387
+ if (@read_only) then
388
+ raise NotWritableError, 'failed to write to read only storage'
391
389
  end
392
390
  }
393
391
  end
394
- private :check_panic
392
+ private :check_read_write
395
393
 
396
394
  def recover
397
395
  @logger.warn('incompleted storage and recover from journal log...')
398
396
 
399
- check_panic
400
- if (@read_only) then
401
- @logger.warn('read only storage is not recoverable.')
402
- raise NotWritableError, 'need for recovery'
403
- end
404
-
397
+ check_standby
405
398
  recover_completed = false
406
399
  begin
407
400
  safe_pos = 0
@@ -421,6 +414,9 @@ module Higgs
421
414
  }
422
415
  @logger.info("last safe point of journal log: #{safe_pos}")
423
416
 
417
+ @logger.info("flush storage.")
418
+ @w_tar.flush
419
+
424
420
  File.open(@jlog_name, File::WRONLY, 0660) {|f|
425
421
  f.binmode
426
422
  @logger.info("shrink journal log to erase last broken segment.")
@@ -430,10 +426,6 @@ module Higgs
430
426
  JournalLogger.eof_mark(f)
431
427
  }
432
428
 
433
- @logger.info("write EOA to storage: #{@index.eoa}")
434
- @w_tar.seek(@index.eoa)
435
- @w_tar.write_EOA
436
-
437
429
  recover_completed = true
438
430
  ensure
439
431
  unless (recover_completed) then
@@ -452,6 +444,8 @@ module Higgs
452
444
  def shutdown
453
445
  @commit_lock.synchronize{
454
446
  @state_lock.synchronize{
447
+ read_only = @read_only && @read_only != :standby
448
+
455
449
  if (@shutdown) then
456
450
  raise ShutdownException, 'storage shutdown'
457
451
  end
@@ -463,7 +457,7 @@ module Higgs
463
457
  @jlog_rotate_service.stop_service
464
458
  end
465
459
 
466
- unless (@read_only) then
460
+ unless (read_only) then
467
461
  if (@panic) then
468
462
  @logger.warn("abort journal log: #{@jlog_name}")
469
463
  @jlog.close(false)
@@ -473,7 +467,7 @@ module Higgs
473
467
  end
474
468
  end
475
469
 
476
- if (! @panic && ! @read_only) then
470
+ if (! @panic && ! read_only) then
477
471
  @logger.info("save index: #{@idx_name}")
478
472
  @index.save(@idx_name)
479
473
  end
@@ -482,7 +476,7 @@ module Higgs
482
476
  @logger.info("close I/O handle for read: #{@tar_name}")
483
477
  r_tar.close
484
478
  }
485
- unless (@read_only) then
479
+ unless (read_only) then
486
480
  @logger.info("sync write data: #{@tar_name}")
487
481
  @w_tar.fsync
488
482
  @logger.info("close I/O handle for write: #{@tar_name}")
@@ -503,7 +497,13 @@ module Higgs
503
497
  @state_lock.synchronize{ @shutdown }
504
498
  end
505
499
 
506
- def self.rotate_entries(name)
500
+ def alive?
501
+ @state_lock.synchronize{
502
+ ! @shutdown && ! @panic
503
+ }
504
+ end
505
+
506
+ def self.rotated_entries(name)
507
507
  rotate_list = Dir["#{name}.*"].map{|nm|
508
508
  n = Integer(nm[(name.length + 1)..-1])
509
509
  [ nm, n ]
@@ -516,7 +516,7 @@ module Higgs
516
516
  end
517
517
 
518
518
  def internal_rotate_journal_log(save_index)
519
- @logger.info("start journal log rotation...")
519
+ @logger.info("start journal log rotation.")
520
520
 
521
521
  commit_log = []
522
522
  while (File.exist? "#{@jlog_name}.#{@index.change_number}")
@@ -526,7 +526,7 @@ module Higgs
526
526
  end
527
527
  unless (commit_log.empty?) then
528
528
  @logger.debug("write journal log: #{@index.change_number}") if @logger.debug?
529
- @jlog.write([ @index.change_number, commit_log ])
529
+ @jlog.write([ @index.change_number, commit_log, @index.storage_id ])
530
530
  end
531
531
  rot_jlog_name = "#{@jlog_name}.#{@index.change_number}"
532
532
 
@@ -548,7 +548,7 @@ module Higgs
548
548
  @logger.info("rename journal log: #{@jlog_name} -> #{rot_jlog_name}")
549
549
  File.rename(@jlog_name, rot_jlog_name)
550
550
  if (@jlog_rotate_max > 0) then
551
- rotate_list = Storage.rotate_entries(@jlog_name)
551
+ rotate_list = Storage.rotated_entries(@jlog_name)
552
552
  while (rotate_list.length > @jlog_rotate_max)
553
553
  unlink_jlog_name = rotate_list.shift
554
554
  @logger.info("unlink old journal log: #{unlink_jlog_name}")
@@ -564,11 +564,7 @@ module Higgs
564
564
 
565
565
  def rotate_journal_log(save_index=true)
566
566
  @commit_lock.synchronize{
567
- check_panic
568
- if (@read_only) then
569
- raise NotWritableError, 'failed to write to read only storage'
570
- end
571
-
567
+ check_read_write
572
568
  rotate_completed = false
573
569
  begin
574
570
  internal_rotate_journal_log(save_index)
@@ -589,16 +585,16 @@ module Higgs
589
585
  @commit_lock.synchronize{
590
586
  @logger.debug("start raw_write_and_commit.") if @logger.debug?
591
587
 
592
- check_panic
593
- if (@read_only) then
594
- raise NotWritableError, 'failed to write to read only storage'
595
- end
596
-
588
+ check_read_write
597
589
  commit_log = []
598
590
  commit_completed = false
599
591
  eoa = @index.eoa
600
592
 
601
593
  begin
594
+ @index.succ!
595
+ @logger.debug("index succ: #{@index.change_number}") if @logger.debug?
596
+ commit_log << { :ope => :succ, :cnum => @index.change_number }
597
+
602
598
  for ope, key, type, name, value in write_list
603
599
  case (ope)
604
600
  when :write
@@ -636,11 +632,12 @@ module Higgs
636
632
  @index.free_store(j[:pos], j[:siz])
637
633
  j[:pos] = pos
638
634
  j[:siz] = blocked_size
635
+ j[:cnum] = @index.change_number
639
636
  else
640
- i[type] = { :pos => pos, :siz => blocked_size }
637
+ i[type] = { :pos => pos, :siz => blocked_size, :cnum => @index.change_number }
641
638
  end
642
639
  else
643
- @index[key] = { type => { :pos => pos, :siz => blocked_size } }
640
+ @index[key] = { type => { :pos => pos, :siz => blocked_size, :cnum => @index.change_number } }
644
641
  end
645
642
  next
646
643
  end
@@ -659,6 +656,7 @@ module Higgs
659
656
  :nam => name,
660
657
  :val => value
661
658
  }
659
+ j[:cnum] = @index.change_number
662
660
  if (j[:siz] > blocked_size) then
663
661
  commit_log << {
664
662
  :ope => :free_store,
@@ -697,11 +695,12 @@ module Higgs
697
695
  @index.free_store(j[:pos], j[:siz])
698
696
  j[:pos] = eoa
699
697
  j[:siz] = blocked_size
698
+ j[:cnum] = @index.change_number
700
699
  else
701
- i[type] = { :pos => eoa, :siz => blocked_size }
700
+ i[type] = { :pos => eoa, :siz => blocked_size, :cnum => @index.change_number }
702
701
  end
703
702
  else
704
- @index[key] = { type => { :pos => eoa, :siz => blocked_size } }
703
+ @index[key] = { type => { :pos => eoa, :siz => blocked_size, :cnum => @index.change_number } }
705
704
  end
706
705
  eoa += blocked_size
707
706
  commit_log << {
@@ -730,12 +729,8 @@ module Higgs
730
729
  end
731
730
  end
732
731
 
733
- @index.succ!
734
- @logger.debug("index succ: #{@index.change_number}") if @logger.debug?
735
- commit_log << { :ope => :succ, :cnum => @index.change_number }
736
-
737
732
  @logger.debug("write journal log: #{@index.change_number}") if @logger.debug?
738
- @jlog.write([ @index.change_number, commit_log ])
733
+ @jlog.write([ @index.change_number, commit_log, @index.storage_id ])
739
734
 
740
735
  for cmd in commit_log
741
736
  case (cmd[:ope])
@@ -784,7 +779,14 @@ module Higgs
784
779
  end
785
780
 
786
781
  def self.apply_journal(w_tar, index, log)
787
- change_number, commit_log = log
782
+ change_number, commit_log, storage_id = log
783
+
784
+ if (storage_id) then # check for backward compatibility
785
+ if (storage_id != index.storage_id) then
786
+ raise PanicError, "unexpected storage id: expected <#{index.storage_id}> but was <#{storage_id}>"
787
+ end
788
+ end
789
+
788
790
  if (change_number - 1 < index.change_number) then
789
791
  # skip old jounal log
790
792
  elsif (change_number - 1 > index.change_number) then
@@ -793,6 +795,7 @@ module Higgs
793
795
  for cmd in commit_log
794
796
  case (cmd[:ope])
795
797
  when :write
798
+ yield(cmd[:key]) if block_given?
796
799
  name = "#{cmd[:key]}.#{cmd[:typ]}"[0, Tar::Block::MAX_LEN]
797
800
  w_tar.seek(cmd[:pos])
798
801
  w_tar.add(cmd[:nam], cmd[:val], :mtime => cmd[:mod])
@@ -801,13 +804,15 @@ module Higgs
801
804
  if (j = i[cmd[:typ]]) then
802
805
  j[:pos] = cmd[:pos]
803
806
  j[:siz] = blocked_size
807
+ j[:cnum] = index.change_number
804
808
  else
805
- i[cmd[:typ]] = { :pos => cmd[:pos], :siz => blocked_size }
809
+ i[cmd[:typ]] = { :pos => cmd[:pos], :siz => blocked_size, :cnum => index.change_number }
806
810
  end
807
811
  else
808
- index[cmd[:key]] = { cmd[:typ] => { :pos => cmd[:pos], :siz => blocked_size } }
812
+ index[cmd[:key]] = { cmd[:typ] => { :pos => cmd[:pos], :siz => blocked_size, :cnum => index.change_number } }
809
813
  end
810
814
  when :delete
815
+ yield(cmd[:key]) if block_given?
811
816
  index.delete(cmd[:key])
812
817
  when :free_fetch
813
818
  index.free_fetch_at(cmd[:pos], cmd[:siz])
@@ -818,6 +823,8 @@ module Higgs
818
823
  w_tar.write_header(:name => name, :size => cmd[:siz] - Tar::Block::BLKSIZ, :mtime => cmd[:mod])
819
824
  when :eoa
820
825
  index.eoa = cmd[:pos]
826
+ w_tar.seek(cmd[:pos])
827
+ w_tar.write_EOA
821
828
  when :succ
822
829
  index.succ!
823
830
  if (index.change_number != cmd[:cnum]) then
@@ -828,6 +835,7 @@ module Higgs
828
835
  end
829
836
  end
830
837
  end
838
+
831
839
  nil
832
840
  end
833
841
 
@@ -851,7 +859,7 @@ module Higgs
851
859
  index.load(idx_name) if (File.exist? idx_name)
852
860
 
853
861
  out << "recovery target: #{name}\n" if (out && verbose_level >= 1)
854
- jlog_list = rotate_entries(jlog_name)
862
+ jlog_list = rotated_entries(jlog_name)
855
863
  jlog_list << jlog_name if (File.exist? jlog_name)
856
864
  for curr_name in jlog_list
857
865
  begin
@@ -864,8 +872,6 @@ module Higgs
864
872
  out << "warning: incompleted journal log and stopped at #{curr_name}\n" if out
865
873
  end
866
874
  end
867
- w_tar.seek(index.eoa)
868
- w_tar.write_EOA
869
875
 
870
876
  index.save(idx_name)
871
877
  w_tar.fsync
@@ -876,11 +882,72 @@ module Higgs
876
882
  nil
877
883
  end
878
884
 
885
+ def apply_journal_log(path)
886
+ @commit_lock.synchronize{
887
+ @logger.info("start to apply journal log.")
888
+
889
+ check_standby
890
+ apply_completed = false
891
+ begin
892
+ JournalLogger.each_log(path) do |log|
893
+ change_number, commit_log, storage_id = log
894
+
895
+ if (storage_id) then # check for backward compatibility
896
+ if (storage_id != @index.storage_id) then
897
+ raise PanicError, "unexpected storage id: expected <#{@index.storage_id}> but was <#{storage_id}>"
898
+ end
899
+ end
900
+
901
+ if (change_number - 1 < @index.change_number) then
902
+ @logger.debug("skip journal log: #{change_number}") if @logger.debug?
903
+ elsif (change_number - 1 > @index.change_number) then
904
+ raise PanicError, "lost journal log (cnum: #{@index.change_number + 1})"
905
+ else # if (change_number - 1 == @index.change_number) then
906
+ @logger.debug("write journal log: #{change_number}") if @logger.debug?
907
+ @jlog.write(log)
908
+
909
+ @logger.debug("apply journal log: #{change_number}") if @logger.debug?
910
+ Storage.apply_journal(@w_tar, @index, log) {|key|
911
+ @properties_cache.delete(key)
912
+ yield(key) if block_given?
913
+ }
914
+
915
+ if (@jlog_rotate_size > 0 && @jlog.size >= @jlog_rotate_size) then
916
+ internal_rotate_journal_log(true)
917
+ end
918
+ end
919
+ end
920
+
921
+ @logger.debug("flush storage.")
922
+ @w_tar.flush
923
+
924
+ apply_completed = true
925
+ ensure
926
+ unless (apply_completed) then
927
+ @state_lock.synchronize{ @panic = true }
928
+ @logger.error("panic: failed to apply journal log.")
929
+ @logger.error($!) if $!
930
+ end
931
+ end
932
+
933
+ @logger.info("completed to apply journal log.")
934
+ }
935
+
936
+ nil
937
+ end
938
+
939
+ def switch_to_write
940
+ @state_lock.synchronize{
941
+ if (@read_only != :standby) then
942
+ raise "not standby mode: #{@read_only}"
943
+ end
944
+ @read_only = false
945
+ }
946
+ nil
947
+ end
948
+
879
949
  def write_and_commit(write_list, commit_time=Time.now)
880
- check_panic
881
- if (@read_only) then
882
- raise NotWritableError, 'failed to write to read only storage'
883
- end
950
+ check_read_write
884
951
 
885
952
  raw_write_list = []
886
953
  deleted_entries = {}
@@ -1013,12 +1080,12 @@ module Higgs
1013
1080
  private :internal_fetch_properties
1014
1081
 
1015
1082
  def fetch_properties(key)
1016
- check_panic
1083
+ check_read
1017
1084
  internal_fetch_properties(key)
1018
1085
  end
1019
1086
 
1020
1087
  def fetch(key)
1021
- check_panic
1088
+ check_read
1022
1089
  value = read_record_body(key, :d) or return
1023
1090
  unless (properties = internal_fetch_properties(key)) then
1024
1091
  @state_lock.synchronize{ @panic = true }
@@ -1047,13 +1114,27 @@ module Higgs
1047
1114
 
1048
1115
  def_delegator :@index, :identity
1049
1116
 
1117
+ def data_change_number(key)
1118
+ i = @index[key] and i[:d][:cnum] || -1
1119
+ end
1120
+
1121
+ def properties_change_number(key)
1122
+ i = @index[key] and i[:p][:cnum] || -1
1123
+ end
1124
+
1125
+ def unique_data_id(key)
1126
+ id = identity(key) or return
1127
+ cnum = data_change_number(key) or return
1128
+ "#{id}\t#{cnum}"
1129
+ end
1130
+
1050
1131
  def key?(key)
1051
- check_panic
1132
+ check_read
1052
1133
  @index.key? key
1053
1134
  end
1054
1135
 
1055
1136
  def each_key
1056
- check_panic
1137
+ check_read
1057
1138
  @index.each_key do |key|
1058
1139
  yield(key)
1059
1140
  end
@@ -1070,7 +1151,7 @@ module Higgs
1070
1151
  ]
1071
1152
 
1072
1153
  def verify(out=nil, verbose_level=1)
1073
- check_panic
1154
+ check_read
1074
1155
 
1075
1156
  keys = @index.keys
1076
1157
  keys.sort!{|a, b|
@@ -1096,6 +1177,55 @@ module Higgs
1096
1177
 
1097
1178
  nil
1098
1179
  end
1180
+
1181
+ class ClientSideLocalhostCheckHandler
1182
+ def initialize(path, messg)
1183
+ @path = File.expand_path(path)
1184
+ @messg = messg
1185
+ end
1186
+
1187
+ def call
1188
+ unless (File.exist? @path) then
1189
+ raise 'client should exist in localhost.'
1190
+ end
1191
+
1192
+ File.open(@path, 'r') {|f|
1193
+ f.binmode
1194
+ if (f.read != @messg) then
1195
+ raise 'client should exist in localhost.'
1196
+ end
1197
+ }
1198
+ nil
1199
+ end
1200
+ end
1201
+
1202
+ # check that client exists in localhost.
1203
+ def localhost_check
1204
+ base_dir = File.dirname(@name)
1205
+ tmp_fname = File.join(base_dir, ".localhost_check.#{$$}")
1206
+ messg = (0...8).map{ rand(64) }.pack("C*").tr("\x00-\x3f", "A-Za-z0-9./")
1207
+
1208
+ begin
1209
+ f = File.open(tmp_fname, File::WRONLY | File::CREAT | File::EXCL, 0644)
1210
+ rescue Errno::EEXIST
1211
+ tmp_fname.succ!
1212
+ retry
1213
+ end
1214
+
1215
+ begin
1216
+ begin
1217
+ f.binmode
1218
+ f.write(messg)
1219
+ ensure
1220
+ f.close
1221
+ end
1222
+ yield(ClientSideLocalhostCheckHandler.new(tmp_fname, messg))
1223
+ ensure
1224
+ File.unlink(tmp_fname) if (File.exist? tmp_fname)
1225
+ end
1226
+
1227
+ nil
1228
+ end
1099
1229
  end
1100
1230
  end
1101
1231