sgfa 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,556 @@
1
+ #
2
+ # Simple Group of Filing Applications
3
+ # Jacket
4
+ #
5
+ # Copyright (C) 2015 by Graham A. Field.
6
+ #
7
+ # See LICENSE.txt for licensing information.
8
+ #
9
+ # This program is distributed WITHOUT ANY WARRANTY; without even the
10
+ # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
11
+
12
+ require 'digest/sha2'
13
+ require 'logger'
14
+
15
+ require_relative 'error'
16
+ require_relative 'history'
17
+ require_relative 'entry'
18
+
19
+ module Sgfa
20
+
21
+
22
+ #####################################################################
23
+ # A basic filing container which holds {Entry} items with attachments
24
+ # and maintains a record of changes made in the form of a linked chain
25
+ # of {History} items.
26
+ #
27
+ # This class provides the shared services in common to all
28
+ # implementations, which are provided by child classes.
29
+ #
30
+ # To be functional, a child class must provide:
31
+ # * @lock - a lock to protect access to the Jacket
32
+ # * @state - keep track of current history and entry revisions and
33
+ # track all tags active in the Jacket.
34
+ # * @store - actually keeps all the items in the Jacket.
35
+ # * implementations for open, close, create, etc.
36
+ #
37
+ class Jacket
38
+
39
+ # Max length of text ID
40
+ TextIdMax = 128
41
+
42
+ # Invalid characters in text ID
43
+ TextIdChars = /[[:cntrl:]]/
44
+
45
+ private_constant :TextIdMax, :TextIdChars
46
+
47
+
48
+ #####################################
49
+ # Limits on Text ID
50
+ def self.limits_id(txt)
51
+ Error.limits(txt, 1, TextIdMax, TextIdChars, 'Jacket text ID')
52
+ end # def self.limits_id()
53
+
54
+
55
+ ##########################################
56
+ # Initialize new jacket
57
+ def initialize
58
+ @id_hash = nil
59
+ @item_hash = Digest::SHA256.new
60
+ end # def initialize
61
+
62
+
63
+ ##########################################
64
+ # Get Hash ID
65
+ #
66
+ # @return [String] Hash ID of the jacket
67
+ # @raise [Error::Sanity] if jacket not open
68
+ def id_hash
69
+ raise Error::Sanity, 'Jacket not open' if !@id_hash
70
+ return @id_hash.dup
71
+ end # def id_hash
72
+
73
+
74
+ ##########################################
75
+ # Get Text ID
76
+ #
77
+ # @return [String] Text ID of the jacket
78
+ # @raise [Error::Sanity] if jacket not open
79
+ def id_text
80
+ raise Error::Sanity, 'Jacket not open' if !@id_hash
81
+ return @id_text.dup
82
+ end # def id_text
83
+
84
+
85
+
86
+ ##########################################
87
+ # History item
88
+ #
89
+ # @param [Fixnum] hnum History number
90
+ # @return [Array] [:history, item]
91
+ #
92
+ # @raise [Error::Sanity] if jacket not open
93
+ def item_history(hnum)
94
+ raise Error::Sanity, 'Jacket not open' if !@id_hash
95
+ txt = "%s history %d\n" % [@id_hash, hnum]
96
+ [:history, @item_hash.reset.update(txt).hexdigest]
97
+ end # def item_history()
98
+
99
+
100
+ ##########################################
101
+ # Entry item
102
+ #
103
+ # @param [Fixnum] enum Entry number
104
+ # @param [Fixnum] rnum Revision number
105
+ # @return [Array] [:entry, item]
106
+ #
107
+ # @raise [Error::Sanity] if jacket not open
108
+ def item_entry(enum, rnum)
109
+ raise Error::Sanity, 'Jacket not open' if !@id_hash
110
+ txt = "%s entry %d %d\n" % [@id_hash, enum, rnum]
111
+ [:entry, @item_hash.reset.update(txt).hexdigest]
112
+ end
113
+
114
+
115
+ ##########################################
116
+ # Attach item
117
+ #
118
+ # @param [Fixnum] enum Entry number
119
+ # @param [Fixnum] anum Attach number
120
+ # @param [Fixnum] hnum History number
121
+ # @return [Array] [:file, item]
122
+ #
123
+ # @raise [Error::Sanity] if jacket not open
124
+ def item_attach(enum, anum, hnum)
125
+ raise Error::Sanity, 'Jacket not open' if !@id_hash
126
+ txt = "%s attach %d %d %d\n" % [@id_hash, enum, anum, hnum]
127
+ [:file, @item_hash.reset.update(txt).hexdigest]
128
+ end # def item_attach()
129
+
130
+
131
+ #####################################
132
+ # Read an entry
133
+ #
134
+ # @param enum [Integer] The entry number to read
135
+ # @param rnum [Integer] The revision number, defaults to current
136
+ # @raise [Error::Sanity] if Jacket is not open
137
+ # @raise [Error::Corrupt] if the current Entry is missing
138
+ # @raise [Error::NonExistent] if the Entry is missing
139
+ # @raise (see Entry#canonical=)
140
+ # @return [Entry] the {Entry} item
141
+ def read_entry(enum, rnum=0)
142
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
143
+
144
+ # current entry
145
+ if rnum == 0
146
+ rnum = @lock.do_sh{ @state.get(enum) }
147
+ current = true
148
+ else
149
+ current = false
150
+ end
151
+
152
+ # read, process, and close
153
+ ent = _read_entry(enum, rnum)
154
+ if !ent
155
+ if current
156
+ raise Error::Corrupt, 'Jacket current entry not present'
157
+ else
158
+ raise Error::NonExistent, 'Jacket entry does not exist'
159
+ end
160
+ end
161
+
162
+ return ent
163
+ end # def read_entry()
164
+
165
+
166
+ #####################################
167
+ # Read an array of current entries
168
+ #
169
+ # @param enums [Array] Entry number list
170
+ # @return [Array] of {Entry} items
171
+ # @raise [Error::Corrupt] if current entries not present
172
+ # @raise [Error::NonExistent] if entry does not exist
173
+ def read_array(enums)
174
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
175
+
176
+ # get current entries
177
+ rnums = @lock.do_sh{ enums.map{|enum| @state.get(enum) } }
178
+
179
+ # get entries
180
+ ents = []
181
+ enums.each_index do |idx|
182
+ ent = _read_entry(enums[idx], rnums[idx])
183
+ raise Error::Corrupt, 'Jacket current entry not present' if !ent
184
+ ents.push ent
185
+ end
186
+
187
+ return ents
188
+ end # def read_array()
189
+
190
+
191
+ #####################################
192
+ # Read individual entry
193
+ def _read_entry(enum, rnum)
194
+ ent = Entry.new
195
+ type, item = item_entry(enum, rnum)
196
+ fi = @store.read(type, item)
197
+ return nil if !fi
198
+ begin
199
+ ent.canonical = fi.read
200
+ ensure
201
+ fi.close
202
+ end
203
+ return ent
204
+ end # def _read_entry()
205
+ private :_read_entry
206
+
207
+
208
+ #####################################
209
+ # Read history item
210
+ #
211
+ # @param hnum [Integer] the history number to read, defaults to most recent
212
+ # @raise [Error::Sanity] if Jacket is not open
213
+ # @raise [Error::NonExistent] if the history does not exist
214
+ # @raise (see History#canonical=)
215
+ # @return [History] the {History} item
216
+ def read_history(hnum=0)
217
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
218
+
219
+ hst = History.new
220
+ hnum = @lock.do_sh{ @state.get(0) } if hnum == 0
221
+ return nil if hnum == 0
222
+ type, item = item_history(hnum)
223
+ fi = @store.read(type, item)
224
+ raise Error::NonExistent, 'Jacket history does not exist' if !fi
225
+ begin
226
+ hst.canonical = fi.read
227
+ ensure
228
+ fi.close
229
+ end
230
+
231
+ return hst
232
+ end # def read_history()
233
+
234
+
235
+ #####################################
236
+ # Read attachment
237
+ #
238
+ # @note Remember to close the returned file
239
+ #
240
+ # @param enum [Integer] the entry number
241
+ # @param anum [Integer] the attachemnt number
242
+ # @param hnum [Integer] the history number
243
+ # @return [File] the attachment opened read only
244
+ # @raise [Error::Sanity] if Jacket is not open
245
+ # @raise [Error::NonExistent] if attachment does not exist
246
+ def read_attach(enum, anum, hnum)
247
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
248
+
249
+ type, item = item_attach(enum, anum, hnum)
250
+ fi = @store.read(type, item)
251
+ raise Error::NonExistent, 'Jacket attachment does not exist' if !fi
252
+ return fi
253
+ end # def read_attach()
254
+
255
+
256
+ #####################################
257
+ # Read a tag, just getting the list of entry numbers
258
+ #
259
+ # @note You probably want to use {#read_tag} instead of this method.
260
+ #
261
+ # @param tag [String] Tag name
262
+ # @param offs [Integer] Offset to begin reading
263
+ # @param max [Integer] Maximum entries to return
264
+ # @return [Array] Total number of entries in the tag, and possibly empty
265
+ # array of entry numbers
266
+ # @raise [Error::Sanity] if Jacket is not open
267
+ # @raise [Error::NonExistent] if tag does not exist
268
+ # @raise [Error::Corrupt] if file format for tag is bad #
269
+ def read_tag_raw(tag, offs, max)
270
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
271
+ return @lock.do_sh{ @state.tag(tag, offs, max) }
272
+ end # def read_tag_raw()
273
+
274
+
275
+ #####################################
276
+ # Read a tag
277
+ #
278
+ # @param (see #read_tag_raw)
279
+ # @return [Array] Total number of entries in the tag, possibly empty
280
+ # array of {Entry} items
281
+ # @raise (see #read_tag_raw)
282
+ # @raise [Error::Corrupt] if current entry is not available
283
+ def read_tag(tag, offs, max)
284
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
285
+ ents = []
286
+ size = nil
287
+ @lock.do_sh do
288
+ size, elst = @state.tag(tag, offs, max)
289
+
290
+ elst.each do |enum|
291
+ rnum = @state.get(enum)
292
+ type, item = item_entry(enum, rnum)
293
+ fi = @store.read(type, item)
294
+ raise Error::Corrupt, 'Jacket current entry not present' if !fi
295
+ ent = Entry.new
296
+ begin
297
+ ent.canonical = fi.read
298
+ ensure
299
+ fi.close
300
+ end
301
+ ents.push ent
302
+ end
303
+ end
304
+ return size, ents
305
+ end # def read_tag()
306
+
307
+
308
+ #####################################
309
+ # Read list of all tags
310
+ #
311
+ # @return [Array] list of tag names
312
+ # @raise [Error::Sanity] if Jacket is not open
313
+ # @raise [Error::Corrupt] if tag list is missing
314
+ def read_list
315
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
316
+ return @lock.do_sh{ @state.list }
317
+ end # def read_list
318
+
319
+
320
+ #####################################
321
+ # Write entires to a Jacket
322
+ #
323
+ # @param user [String] User
324
+ # @param ents [Array] {Entry}s to write
325
+ # @param tme [Time] Time of the write
326
+ # @return [History] The history item just created
327
+ # @raise [Error::Sanity] if Jacket is not open
328
+ # @raise [Error::Conflict] if entry revision is not one up from current
329
+ # @raise (see History#process)
330
+ def write(user, ents, tme=nil)
331
+ raise Error::Sanity, 'Jacket is not open' if !@id_hash
332
+
333
+ hst = nil
334
+
335
+ @lock.do_ex do
336
+ # Check entries to ensure they don't conflict
337
+ ents.each do |ent|
338
+ next if !ent.entry
339
+ if ent.revision != @state.get(ent.entry) + 1
340
+ raise Error::Conflict, 'Entry revision conflict'
341
+ end
342
+ end
343
+
344
+ # Update history
345
+ hnum = @state.get(0)
346
+ if hnum == 0
347
+ hst = History.new(@id_hash)
348
+ cng = hst.process(1, '00000000'*8, 0, user, ents, tme)
349
+ else
350
+ prv = History.new
351
+ type, item = item_history(hnum)
352
+ fi = @store.read(type, item)
353
+ raise Error::Corrupt, 'Missing history' if !fi
354
+ begin
355
+ prv.canonical = fi.read
356
+ ensure
357
+ fi.close
358
+ end
359
+ hst, cng = prv.next(user, ents, tme)
360
+ end
361
+ hnum = hst.history
362
+
363
+ # store entries
364
+ hle = []
365
+ cng[:entry].each do |ent|
366
+ type, item = item_entry(ent.entry, ent.revision)
367
+ hle.push [item, ent]
368
+ fi = @store.temp
369
+ begin
370
+ fi.write ent.canonical
371
+ rescue
372
+ fi.close!
373
+ raise
374
+ end
375
+ @store.write(type, item, fi)
376
+ @state.set(ent.entry, ent.revision)
377
+ end
378
+
379
+ # store attachments
380
+ hla = []
381
+ cng[:attach].each do |enum, anum, file|
382
+ type, item = item_attach(enum, anum, hnum)
383
+ hla.push [item, enum, anum, hnum]
384
+ @store.write(type, item, file)
385
+ end
386
+
387
+ # tags
388
+ @state.update(cng[:tag])
389
+
390
+ # store history and set state
391
+ type, item = item_history(hnum)
392
+ fi = @store.temp
393
+ fi.write hst.canonical
394
+ @store.write(type, item, fi)
395
+ @state.set(0, hnum)
396
+
397
+ end
398
+
399
+ return hst
400
+ end # def write()
401
+
402
+
403
+ #####################################
404
+ # Validate history chain
405
+ #
406
+ # @param opts [Hash] Option hash
407
+ # @option opts [Boolean] :hash_entry Validate entries by checking their
408
+ # hash
409
+ # @option opts [Boolean] :hash_attach Validate attachments by checking
410
+ # their hash
411
+ # @option opts [Fixnum] :max_history History number to stop checking.
412
+ # Defaults to not stopping until missing history items stop the check.
413
+ # @option opts [Fixnum] :min_history History number to start checking.
414
+ # Defaults to 1.
415
+ # @option opts [Fixnum] :miss_history Number of allowable missing history
416
+ # items before checking stops. Defaults to zero.
417
+ # @option opts [String] :max_hash Known good hash for :max_history item
418
+ # @option opts [Logger] :log The logger to use. Defaults to STDERR log
419
+ # @return [Boolean] true if valid history chain
420
+ def check(opts={})
421
+ max = opts[:max_history] || 1000000000
422
+ min = opts[:min_history] || 1
423
+ stop = opts[:miss_history] || 0
424
+ if opts[:log]
425
+ log = opts[:log]
426
+ else
427
+ log = Logger.new(STDERR)
428
+ log.level = Logger::WARN
429
+ end
430
+
431
+ log.info('Begin validate jacket %s at %d' % [@id_hash, min])
432
+
433
+ miss = 0
434
+ hnum = min-1
435
+ prev = nil
436
+ good = true
437
+ while (hnum += 1) <= max
438
+
439
+ # get history item
440
+ begin
441
+ hst = read_history(hnum)
442
+ rescue Error::NonExistent
443
+ miss += 1
444
+ if miss <= stop
445
+ next
446
+ else
447
+ hnum = hnum-miss+1
448
+ break
449
+ end
450
+ rescue Error::Corrupt => exp
451
+ log.error('History item corrupt %d' % hnum)
452
+ miss += 1
453
+ if miss <= stop
454
+ next
455
+ else
456
+ num = hnum-miss+1
457
+ break
458
+ end
459
+ end
460
+
461
+ # missing history items
462
+ if miss != 0
463
+ good = false
464
+ if miss == 1
465
+ log.error('History item missing %d' % (hnum-1))
466
+ else
467
+ log.error('HIstory items missing %d-%d' % [hnum-miss, hnum-1])
468
+ end
469
+ miss = 0
470
+ prev = nil
471
+ end
472
+
473
+ # check previous
474
+ if prev
475
+ if prev != hst.previous
476
+ good = false
477
+ log.error('History chain broken %d' % hnum)
478
+ else
479
+ log.debug('History chain matches %d' % hnum)
480
+ end
481
+ elsif hnum != min
482
+ log.warn('History chain not checked %d' % hnum)
483
+ end
484
+ prev = hst.hash
485
+
486
+ # entries
487
+ if opts[:hash_entry]
488
+ hst.entries.each do |enum, rnum, hash|
489
+ begin
490
+ ent = read_entry(enum, rnum)
491
+ rescue Error::NonExistent
492
+ log.info('Entry missing %d-%d' % [enum, rnum])
493
+ next
494
+ rescue Error::Corrupt
495
+ log.error('Entry corrupt %d-%d' % [enum, rnum])
496
+ good = false
497
+ next
498
+ end
499
+ if ent.hash != hash
500
+ log.error('Entry invalid %d-%d' % [enum, rnum])
501
+ good = false
502
+ else
503
+ log.debug('Entry is valid %d-%d' % [enum, rnum])
504
+ end
505
+ end
506
+ end
507
+
508
+ # attachments
509
+ if opts[:hash_attach]
510
+ hst.attachments.each do |enum, anum, hash|
511
+ begin
512
+ fil = read_attach(enum, anum, hnum)
513
+ rescue Error::NonExistent
514
+ log.info('Attachment missing %d-%d-%d' % [enum, anum, hnum])
515
+ next
516
+ end
517
+ begin
518
+ calc = Digest::SHA256.file(fil.path).hexdigest
519
+ ensure
520
+ fil.close
521
+ end
522
+ if calc != hash
523
+ log.error('Attachment invalid %d-%d-%d' % [enum, anum, hnum])
524
+ good = false
525
+ else
526
+ log.debug('Attachment is valid %d-%d-%d' % [enum, anum, hnum])
527
+ end
528
+ end
529
+ end
530
+
531
+ # last history check known good hash
532
+ if hnum == max && opts[:max_hash]
533
+ if hst.hash != opts[:max_hash]
534
+ log.error('Max history does not match known hash')
535
+ good = false
536
+ else
537
+ log.debug('Max history matches known hash')
538
+ end
539
+ end
540
+
541
+ end
542
+
543
+ log.info('History chain validation max %d' % (hnum-1))
544
+
545
+ if opts[:max_history] && hnum != opts[:max_history]
546
+ return false
547
+ else
548
+ return good
549
+ end
550
+
551
+ end # def check()
552
+
553
+
554
+ end # class Jacket
555
+
556
+ end # module Sgfa
@@ -0,0 +1,136 @@
1
+ #
2
+ # Simple Group of Filing Applications
3
+ # Jacket implemented using filesystem storage and locking
4
+ #
5
+ # Copyright (C) 2015 by Graham A. Field.
6
+ #
7
+ # See LICENSE.txt for licensing information.
8
+ #
9
+ # This program is distributed WITHOUT ANY WARRANTY; without even the
10
+ # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
11
+
12
+ require 'json'
13
+
14
+ require_relative 'error'
15
+ require_relative 'jacket'
16
+ require_relative 'lock_fs'
17
+ require_relative 'state_fs'
18
+ require_relative 'store_fs'
19
+
20
+ module Sgfa
21
+
22
+
23
+ #####################################################################
24
+ # An implementation of {Jacket} using file system storage and locking
25
+ # provided by {LockFs}, {StoreFs}, and {StateFs}
26
+ class JacketFs < Jacket
27
+
28
+
29
+ #####################################
30
+ # Create a new jacket
31
+ #
32
+ # @param path [String] Path to create the jacket
33
+ # @param id_text [String] Text ID of the jacket
34
+ # @return [String] Hash ID of the jacket
35
+ # @raise [Error::Limits] if id_text exceeds allowed limits
36
+ # @raise [Error::Conflict] if path already exists
37
+ def self.create(path, id_text)
38
+ Jacket.limits_id(id_text)
39
+
40
+ # info
41
+ id_hash = Digest::SHA256.new.update(id_text).hexdigest
42
+ info = {
43
+ 'sgfa_jacket_ver' => 1,
44
+ 'id_hash' => id_hash,
45
+ 'id_text' => id_text,
46
+ }
47
+ json = JSON.pretty_generate(info) + "\n"
48
+
49
+ # create
50
+ begin
51
+ Dir.mkdir(path)
52
+ rescue Errno::EEXIST
53
+ raise Error::Conflict, 'Jacket path already exists'
54
+ end
55
+ fn_info = File.join(path, 'sgfa_jacket.json')
56
+ File.open(fn_info, 'w', :encoding => 'utf-8'){|fi| fi.write json }
57
+
58
+ # create state and store
59
+ StateFs.create(File.join(path, 'state'))
60
+ StoreFs.create(File.join(path, 'store'))
61
+
62
+ return id_hash
63
+ end # def self.create()
64
+
65
+
66
+ #####################################
67
+ # Initialize new Jacket, optionally opening
68
+ #
69
+ # @param path [String] path Path to the jacket
70
+ # @raise (see #open)
71
+ def initialize(path=nil)
72
+ super()
73
+ @path = nil
74
+ @state = nil
75
+ @store = nil
76
+ @lock = nil
77
+ open(path) if path
78
+ end # def initialize
79
+
80
+
81
+ #####################################
82
+ # Open a jacket
83
+ #
84
+ # @param path [String] Path to the jacket
85
+ # @return [JacketFs] self
86
+ # @raise [Error::Sanity] if jacket already open
87
+ # @raise [Error::Corrupt] if Jacket info is corrupt
88
+ # @raise [Error::NonExistent] if path does not exist and contain a valid
89
+ # jacket info file
90
+ def open(path)
91
+ raise Error::Sanity, 'Jacket already open' if @path
92
+
93
+ @lock = LockFs.new
94
+ begin
95
+ json = @lock.open(File.join(path, 'sgfa_jacket.json'))
96
+ info = JSON.parse(json, :symbolize_names => true)
97
+ rescue Errno::ENOENT
98
+ raise Error::NonExistent, 'Jacket does not exist'
99
+ rescue JSON::NestingError, JSON::ParserError
100
+ @lock.close
101
+ @lock = nil
102
+ raise Error::Corrupt, 'Jacket info corrupt'
103
+ end
104
+
105
+ @store = StoreFs.new(File.join(path, 'store'))
106
+ @state = StateFs.new(File.join(path, 'state'))
107
+
108
+ @id_text = info[:id_text]
109
+ @id_hash = info[:id_hash]
110
+ @path = path.dup
111
+
112
+ return self
113
+ end # def open()
114
+
115
+
116
+ #####################################
117
+ # Close a jacket
118
+ #
119
+ # @raise [Error::Sanity] if jacket not open
120
+ def close
121
+ raise Error::Sanity, 'Jacket not open' if !@path
122
+ @lock.close
123
+ @lock = nil
124
+ @state.close
125
+ @state = nil
126
+ @store.close
127
+ @store = nil
128
+ @id_hash = nil
129
+ @id_text = nil
130
+ @path = nil
131
+ end # def close
132
+
133
+
134
+ end # class JacketFs
135
+
136
+ end # module Sgfa