icfs 0.1.0 → 0.1.1

data/lib/icfs/store_fs.rb

@@ -57,6 +57,16 @@ class StoreFs < Store
   end
 
 
+  ###############################################
+  # (see Store#file_size)
+  #
+  def file_size(cid, enum, lnum, fnum)
+    File.size(_file(cid, enum, lnum, fnum))
+  rescue Errno::ENOENT
+    return nil
+  end
+
+
   ###############################################
   # (see Store#tempfile)
   #

data/lib/icfs/store_s3.rb

@@ -64,6 +64,17 @@ class StoreS3 < Store
   end
 
 
+  ###############################################
+  # (see Store#file_size)
+  def file_size(cid, enum, lnum, fnum)
+    key = _file(cid, enum, lnum, fnum)
+    resp = @s3.head_object( bucket: @bck, key: key )
+    return resp.content_length
+  rescue Aws::S3::Errors::NotFound
+    return nil
+  end
+
+
   ###############################################
   # (see Store#tempfile)
   #

data/lib/icfs/users.rb

@@ -19,6 +19,8 @@ module ICFS
 #
 # @abstract
 #
+# @todo Add cache flush method
+#
 class Users
 
   ###############################################

data/lib/icfs/users_elastic.rb

@@ -17,6 +17,8 @@ module ICFS
 ##########################################################################
 # Implements {ICFS::Users Users} using Elasticsearch to cache
 # details from another {ICFS::Users Users} instance.
+# 
+# @todo Add logging
 #
 class UsersElastic < Users
 
@@ -120,12 +122,12 @@ class UsersElastic < Users
       obj['active'] = true
 
       # store in cache
-      json = Validate.generate(obj, 'User/Role/Group'.freeze, ValUserCache)
+      json = Items.generate(obj, 'User/Role/Group'.freeze, ValUserCache)
       _write(:users, urg, json)
 
     # use cached version
     else
-      obj = Validate.parse(json, 'User/Role/Group'.freeze, ValUserCache)
+      obj = Items.parse(json, 'User/Role/Group'.freeze, ValUserCache)
     end
 
     # expired
@@ -146,7 +148,7 @@ class UsersElastic < Users
       obj['last'] = now
 
       # and store in cache
-      json = Validate.generate(obj, 'User/Role/Group'.freeze, ValUserCache)
+      json = Items.generate(obj, 'User/Role/Group'.freeze, ValUserCache)
       _write(:users, urg, json)
     end
 

data/lib/icfs/users_fs.rb

@@ -26,7 +26,7 @@ class UsersFs < Users
   # read a raw file
   def _read(fn)
     json = File.read(File.join(@path, fn + '.json'.freeze))
-    obj = Validate.parse(json, 'User/Role/Group'.freeze, Users::ValUser)
+    obj = Items.parse(json, 'User/Role/Group'.freeze, Users::ValUser)
     if obj['name'] != fn
       raise(Error::Values, 'UsersFs user %s name mismatch'.freeze % fn)
     end
@@ -51,7 +51,7 @@ class UsersFs < Users
   # (see Users#read)
   #
   def read(urg)
-    Validate.validate(urg, 'User/Role/Group'.freeze, Items::FieldUsergrp)
+    Items.validate(urg, 'User/Role/Group'.freeze, Items::FieldUsergrp)
 
     # get the base user
     usr = _read(urg)
@@ -113,7 +113,7 @@ class UsersFs < Users
   # (see Users#write)
   #
   def write(obj)
-    Validate.validate(obj, 'User/Role/Group'.freeze, Users::ValUser)
+    Items.validate(obj, 'User/Role/Group'.freeze, Users::ValUser)
     json = JSON.pretty_generate(obj)
 
     # write to temp file

data/lib/icfs/utils/backup.rb

@@ -0,0 +1,177 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require_relative '../store_fs'
+#
+module ICFS
+
+module Utils
+
+##########################################################################
+# Backup and restore utilities
+#
+class Backup
+
+  ###############################################
+  # Instance
+  #
+  # @param cache [Cache] The live cache
+  # @param store [Store] The live store
+  # @param log [Logger] Where to log
+  #
+  def initialize(cache, store, log)
+    @cache = cache
+    @store = store
+    @log = log
+  end
+
+
+  ###############################################
+  # Copy an item
+  #
+  def _item(dest, title, read, write, args, val=nil)
+    @log.debug('ICFS copy: %s'.freeze % title)
+
+    # read the item
+    json = @store.send(read, *args)
+    if !json
+      @log.warn('ICFS copy: %s is missing'.freeze % title)
+      return nil
+    end
+
+    # parse the item if requested
+    if val
+      obj = JSON.parse(json)
+      err = Validate.check(obj, val)
+      if err
+        @log.error('ICFS copy: %s bad format'.freeze % title)
+        return nil
+      end
+    end
+
+    # write the item
+    dest.send(write, *args, json)
+    return obj
+
+  rescue JSON::ParserError
+    @log.error('ICFS copy: %s bad JSON'.freeze % title)
+    return nil
+  end # def _item
+  private :_item
+
+
+  ###############################################
+  # Transfer a case to another store
+  #
+  # @param cid [String] The case ID
+  # @param dest [Store] The destination store
+  # @param lnum_max [Integer] The highest log
+  # @param lnum_min [Integer] The lowest log
+  #
+  def copy(cid, dest, lnum_max, lnum_min=1)
+    @log.info('ICFS copy: %s %d-%d'.freeze % [cid, lnum_min, lnum_max])
+    if lnum_min > lnum_max
+      raise ArgumentError, 'ICFS copy, log num min is larger than max'.freeze
+    end
+
+    # each log
+    lnum = lnum_min
+    while lnum <= lnum_max
+
+      # copy the log
+      log = _copy_item(dest,
+        'log %d'.freeze % lnum,
+        :log_read,
+        :log_write,
+        [cid, lnum],
+        Items::ItemLog
+      )
+      if !log
+        lnum += 1
+        next
+      end
+
+      # entry
+      enum = log['entry']['num']
+      _copy_item(dest,
+        'entry %d-%d'.freeze % [enum, lnum],
+        :entry_read,
+        :entry_write,
+        [cid, lnum]
+      )
+
+      # index
+      if log['index']
+        xnum = log['index']['num']
+        _copy_item(dest,
+          'index %d-%d'.freeze % [xnum, lnum],
+          :index_read,
+          :index_write,
+          [cid, xnum, lnum]
+        )
+      end
+
+      # action
+      if log['action']
+        anum = log['action']['lnum']
+        _copy_item(dest,
+          'action %d-%d'.freeze % [anum, lnum],
+          :action_read,
+          :action_write,
+          [cid, anum, lnum]
+        )
+      end
+
+      # case
+      if log['case_hash']
+        _copy_item(dest,
+          'case %d'.freeze, % lnum,
+          :case_read,
+          :case_write,
+          [cid, lnum]
+        )
+      end
+
+      # files
+      if log['files_hash']
+        log['files_hash'].each_index do |fnum|
+
+          @log.debug('ICFS copy: file %d-%d-%d'.freeze % [enum, lnum, fnum])
+
+          # read
+          fi = @store.file_read(cid, enum, lnum, fnum)
+          if !fi
+            @log.warn('ICFS copy: file %d-%d-%d missing'.freeze %
+                [enum, lnum, fnum])
+            next
+          end
+
+          # copy
+          tmp = dest.tempfile
+          IO.copy_stream(fi, tmp)
+          @store.close(fi)
+
+          # write
+          dest.file_write(cid, enum, lnum, fnum, tmp)
+        end
+      end
+
+      lnum += 1
+    end
+
+  end # def copy()
+
+
+end # class ICFS::Utils::Backup
+
+end # module ICFS::Utils
+
+end # module ICFS

data/lib/icfs/utils/check.rb

@@ -0,0 +1,290 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+#
+module ICFS
+
+
+##########################################################################
+# Low level utilities for working directly with ICFS systems.
+#
+# @todo Archive/move case tool written
+#
+module Utils
+
+
+##########################################################################
+# Check a case for errors
+#
+# @todo Make a function to auto find the last log
+#
+class Check
+
+
+  ###############################################
+  # Instance
+  #
+  # @param store [Store] The store to check
+  # @param log [Logger] Where to log
+  #
+  def initialize(store, log)
+    @store = store
+    @log = log
+  end
+
+
+  ###############################################
+  # check an object
+  #
+  def _item(title, read, read_args, hist, hash, val, con)
+
+    @log.debug('ICFS check: %s'.freeze % title)
+
+    # read
+    json = @store.send(read, *read_args)
+    if !json
+      if hist
+        @log.warn('ICFS check: %s is missing and historical'.freeze % title)
+      else
+        @log.error('ICFS check: %s is missing and current'.freeze % title)
+      end
+      return nil
+    end
+
+    # hash
+    if hash
+      if hash != ICFS.hash(json)
+        @log.error('ICFS check: %s hash bad'.freeze % title)
+      end
+    else
+      @log.warn('ICFS check: %s hash unverified'.freeze % title)
+    end
+
+    # parse
+    obj = JSON.parse(json)
+    err = Validate.check(obj, val)
+    if err
+      @log.error('ICFS check: %s bad format'.freeze % title)
+      return nil
+    end
+
+    # inconsistent
+    con.each do |name, num|
+      if obj[name] != read_args[num]
+        @log.error('ICFS check: %s inconsistent'.freeze % title)
+        return nil
+      end
+    end
+
+    return obj
+
+  rescue JSON::ParserError
+    @log.error('ICFS check: %s bad JSON'.freeze % title)
+    return nil
+  end # def _item()
+  private :_item
+
+
+  ###############################################
+  # Check a case
+  #
+  # @param cur_log [Integer] The last log
+  # @param cur_hash [String] The hash of the last log
+  #
+  def check(cid, cur_log, cur_hash, opts={})
+    @log.info('ICFS check: case %s'.freeze % cid)
+
+    ent_cur = Set.new
+    cse_cur = false
+    idx_cur = Set.new
+    act_cur = Set.new
+    file_cur = Set.new
+
+
+    # go thru the logs from most current
+    lnum = cur_log
+    hash_log = cur_hash
+    time_log = Time.now.to_i
+    while( lnum > 0 )
+
+      # log
+      log = _item(
+        'log %d'.freeze % lnum,
+        :log_read,
+        [cid, lnum],
+        false,
+        hash_log,
+        Items::ItemLog,
+        [
+          ['caseid'.freeze, 0].freeze,
+          ['log'.freeze, 1].freeze
+        ].freeze,
+      )
+      if !log
+        hash_log = nil
+        lnum = lnum - 1
+        next
+      end
+
+      # check that time decreases
+      if log['time'] > time_log
+        @log.warn('ICFS check: log %d time inconsistent'.freeze % lnum)
+      end
+
+      # entry
+      enum = log['entry']['num']
+      ent = _item(
+        'entry %d-%d' % [enum, lnum],
+        :entry_read,
+        [cid, enum, lnum],
+        ent_cur.include?(enum),
+        log['entry']['hash'],
+        Items::ItemEntry,
+        [
+          ['caseid'.freeze, 0].freeze,
+          ['entry'.freeze, 1].freeze,
+          ['log'.freeze, 2].freeze
+        ].freeze,
+      )
+
+      # current entry
+      unless ent_cur.include?(enum)
+        ent_cur.add(enum)
+        if ent['files']
+          ent['files'].each do |fd|
+            file_cur.add( '%d-%d-%d'.freeze % [enum, fd['num'], fd['log']] )
+          end
+        end
+      end
+
+      # index
+      if log['index']
+        xnum = log['index']['num']
+        idx = _item(
+          'index %d-%d'. freeze % [xnum, lnum],
+          :index_read,
+          [cid, xnum, lnum],
+          idx_cur.include?(xnum),
+          log['index']['hash'],
+          Items::ItemIndex,
+          [
+            ['caseid'.freeze, 0].freeze,
+            ['index'.freeze, 1].freeze,
+            ['log'.freeze, 2].freeze
+          ]
+        )  
+        idx_cur.add(xnum)
+      end
+
+      # action
+      if log['action']
+        anum = log['action']['num']
+        act = _item(
+          'action %d-%d'.freeze % [anum, lnum],
+          :action_read,
+          [cid, anum, lnum],
+          act_cur.include?(anum),
+          log['action']['hash'],
+          Items::ItemAction,
+          [
+            ['caseid'.freeze, 0].freeze,
+            ['action'.freeze, 1].freeze,
+            ['log'.freeze, 2].freeze
+          ]
+        )
+        act_cur.add(anum)
+      end
+
+      # case
+      if log['case_hash']
+        cse = _item(
+          'case %d'.freeze % lnum,
+          :case_read,
+          [cid, lnum],
+          cse_cur,
+          log['case_hash'],
+          Items::ItemCase,
+          [
+            ['caseid'.freeze, 0].freeze,
+            ['log'.freeze, 1].freeze
+          ]
+        )
+        cse_cur = true
+      end
+
+      # files
+      if log['files_hash']
+        fnum = 0
+        log['files_hash'].each do |hash|
+          fnum = fnum + 1
+          fn = '%d-%d-%d'.freeze % [enum, fnum, lnum]
+          cur = file_cur.include?(fn)
+          file_cur.delete(fn) if cur
+
+          @log.debug('ICFS check: file %s'.freeze % fn)
+
+          # read/size
+          if opts[:hash_all] || (cur && opts[:hash_current])
+            fi = @store.file_read(cid, enum, lnum, fnum)
+          elsif opts[:stat_all] || (cur && opts[:stat_current])
+            fi = @store.file_size(cur, enum, lnum, fnum)
+          else
+            fi = true
+          end
+
+          # missing
+          if !fi
+            if cur
+              @log.error('ICFS check: file %s missing and current'.freeze %
+                fn)
+            else
+              @log.warn('ICFS check: file %s missing and historical'.freeze %
+                fn)
+            end
+          end
+
+          # hash
+          if fi.is_a?(File)
+            # check
+            if hash != ICFS.hash_temp(fi)
+              @log.error('ICFS check: file %s hash bad'.freeze % fn)
+            end
+
+            # close
+            if fi.respond_to?(:close!)
+              fi.close!
+            else
+              fi.close
+            end
+          end
+        end
+      end
+
+      # previous log
+      lnum = lnum - 1
+      hash_log = log['prev']
+    end
+
+    # check for any non-existant current files
+    unless file_cur.empty?
+      file_cur.each do |fn|
+        @log.error('ICFS check: file %s current but not logged'.freeze % fn)
+      end
+    end
+
+    @log.debug('ICFS check: case %s complete'.freeze % cid)
+  end # def check()
+
+
+end # class ICFS::Utils::Check
+
+end # module ICFS::Utils
+end # module ICFS