icfs 0.1.1 → 0.1.2

data/lib/icfs/store.rb

@@ -220,8 +220,7 @@ class Store
   # Path for case
   #
   def _case(cid, lnum)
-    [
-      @base,
+    @base + [
       cid,
       'c'.freeze,
       '%d.json'.freeze % lnum
@@ -233,8 +232,7 @@ class Store
   # Path for log
   #
   def _log(cid, lnum)
-    [
-      @base,
+    @base + [
       cid,
       'l'.freeze,
       '%d.json'.freeze % lnum
@@ -246,8 +244,7 @@ class Store
   # Path for entry
   #
   def _entry(cid, enum, lnum)
-    [
-      @base,
+    @base + [
       cid,
       'e'.freeze,
       enum.to_s,
@@ -260,8 +257,7 @@ class Store
   # Path for file
   #
   def _file(cid, enum, lnum, fnum)
-    [
-      @base,
+    @base + [
       cid,
       'e'.freeze,
       enum.to_s,
@@ -274,8 +270,7 @@ class Store
   # Filename for action
   #
   def _action(cid, anum, lnum)
-    [
-      @base,
+    @base + [
       cid,
       'a'.freeze,
       anum.to_s,
@@ -288,8 +283,7 @@ class Store
   # Filename for index
   #
   def _index(cid, xnum, lnum)
-    [
-      @base,
+    @base + [
       cid,
       'i'.freeze,
       xnum.to_s,

data/lib/icfs/store_fs.rb

@@ -29,7 +29,11 @@ class StoreFs < Store
   # @param base [String] the base directory
   #
   def initialize(base)
-    @base = base.freeze
+    if base[-1] == '/'.freeze
+      @base = base.freeze
+    else
+      @base = (base + '/').freeze
+    end
   end
 
 

data/lib/icfs/users.rb

@@ -16,11 +16,8 @@ module ICFS
 ##########################################################################
 # User, Role, Group, and Global Perms
 #
-#
 # @abstract
 #
-# @todo Add cache flush method
-#
 class Users
 
   ###############################################

data/lib/icfs/users_fs.rb

@@ -11,7 +11,9 @@
 
 require 'tempfile'
 require 'fileutils'
-require 'set'
+require 'json'
+
+require_relative 'users'
 
 module ICFS
 
@@ -20,22 +22,6 @@ module ICFS
 #
 class UsersFs < Users
 
-  private
-
-  ###############################################
-  # read a raw file
-  def _read(fn)
-    json = File.read(File.join(@path, fn + '.json'.freeze))
-    obj = Items.parse(json, 'User/Role/Group'.freeze, Users::ValUser)
-    if obj['name'] != fn
-      raise(Error::Values, 'UsersFs user %s name mismatch'.freeze % fn)
-    end
-    return obj
-  end # _read
-
-
-  public
-
 
   ###############################################
   # New instance
@@ -47,63 +33,26 @@ class UsersFs < Users
   end
 
 
+  ###############################################
+  # Path to store the file
+  #
+  def _path(urg)
+    File.join(@path, urg + '.json'.freeze)
+  end
+  private :_path
+
+
   ###############################################
   # (see Users#read)
   #
   def read(urg)
-    Items.validate(urg, 'User/Role/Group'.freeze, Items::FieldUsergrp)
-
-    # get the base user
-    usr = _read(urg)
-    return usr if usr['type'] != 'user'.freeze
-
-    # assemble
-    type = usr['type']
-    done_s = Set.new.add(urg)
-    ary = []
-    roles_s = Set.new
-    if usr['roles']
-      ary.concat usr['roles']
-      roles_s.merge usr['roles']
-    end
-    grps_s = Set.new
-    if usr['groups']
-      ary.concat usr['groups']
-      grps_s.merge usr['groups']
-    end
-    perms_s = Set.new
-    if usr['perms']
-      perms_s.merge usr['perms']
-    end
-
-    # roles & groups
-    while itm = ary.shift
-      next if done_s.include?(itm)
-      done_s.add(itm)
-
-      usr = _read(itm)
-      if usr['roles']
-        ary.concat usr['roles']
-        roles_s.merge usr['roles']
-      end
-      if usr['groups']
-        ary.concat usr['groups']
-        grps_s.merge usr['groups']
-      end
-      if usr['perms']
-        perms_s.merge usr['perms']
-      end
+    Items.validate(urg, 'User/Role/Group name'.freeze, Items::FieldUsergrp)
+    json = File.read(_path(urg))
+    obj = Items.parse(json, 'User/Role/Group'.freeze, Users::ValUser)
+    if obj['name'] != urg
+      raise(Error::Values, 'UsersFs user %s name mismatch'.freeze % fn)
     end
-
-    # assemble final
-    return {
-      'name' => urg.dup,
-      'type' => type,
-      'roles' => roles_s.to_a,
-      'groups' => grps_s.to_a,
-      'perms' => perms_s.to_a,
-    }
-
+    return obj
   rescue Errno::ENOENT
     return nil
   end # def read()
@@ -122,7 +71,7 @@ class UsersFs < Users
     tmp.close
 
     # move
-    FileUtils.mv(tmp.path, File.join(@path, obj['name'] + '.json'.freeze))
+    FileUtils.mv(tmp.path, _path(obj['name']))
     tmp.unlink
   end # def write()
 

data/lib/icfs/users_redis.rb

@@ -0,0 +1,127 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'redis'
+
+module ICFS
+
+##########################################################################
+# Implement Users with a Redis cache
+#
+class UsersRedis < Users
+
+  ###############################################
+  # New instance
+  #
+  # @param redis [Redis] The redis client
+  # @param base [Users] The base Users store
+  # @param opts [Hash] Options
+  # @option opts [String] :prefix Prefix for Redis key
+  # @option opts [Integer] :expires Expiration time in seconds
+  #
+  def initialize(redis, base, opts={})
+    @redis = redis
+    @base = base
+    @pre = opts[:prefix] || ''.freeze
+    @exp = opts[:expires] || 1*60*60 # 1 hour default
+  end
+
+
+  ###############################################
+  # Where to store in Redis
+  #
+  def _key(urg)
+    @pre + urg
+  end
+  private :_key
+
+
+  ###############################################
+  # (see Users#read)
+  #
+  def read(urg)
+    Validate.check(urg, Items::FieldUsergrp) # FIXME
+    key = _key(urg)
+
+    # try cache
+    json = @redis.get(key)
+    return JSON.parse(json) if json
+
+    # get base object from base store
+    bse = @base.read(urg)
+    return nil if !bse
+
+    # assemble
+    seen = Set.new.add(urg)
+    ary = []
+    roles = Set.new
+    grps = Set.new
+    perms = Set.new
+    if bse['roles']
+      ary.concat bse['roles']
+      roles.merge bse['roles']
+    end
+    if bse['groups']
+      ary.concat bse['groups']
+      grps.merge bse['groups']
+    end
+    if bse['perms']
+      perms.merge bse['perms']
+    end
+
+    # call ourself recursively for any un-expanded memberships
+    while itm = ary.shift
+      next if seen.include?(itm)
+      seen.add(itm)
+      ikey = _key(itm)
+
+      # all included u/r/g have been seen & expanded
+      mem = self.read(itm)
+      next if !mem
+      if mem['roles']
+        roles.merge mem['roles']
+        seen.merge mem['roles']
+      end
+      if mem['groups']
+        grps.merge mem['groups']
+        seen.merge mem['groups']
+      end
+      if mem['perms']
+        perms.merge mem['perms']
+      end
+    end
+
+    # final result
+    bse['roles'] = roles.to_a unless roles.empty?
+    bse['groups'] = grps.to_a unless grps.empty?
+    bse['perms'] = perms.to_a unless perms.empty?
+    json = JSON.pretty_generate(bse)
+
+    # save to cache
+    @redis.set(key, json)
+    @redis.expire(key, @exp)
+    return bse
+  end # def read()
+
+
+  ###############################################
+  # (see Users#write)
+  #
+  def write(obj)
+    json = Items.generate(obj, 'User/Role/Group'.freeze, Users::ValUser)
+    key = _key(obj['name'])
+    @redis.del(key)
+    @base.write(obj)
+  end # def write()
+
+end # class ICFS::UsersRedis
+
+end # module ICFS

data/lib/icfs/users_s3.rb

@@ -0,0 +1,68 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'aws-sdk-s3'
+
+module ICFS
+
+##########################################################################
+# Implements {ICFS::Users Users} from AWS S3
+#
+class UsersS3 < Users
+
+  ###############################################
+  # New instance
+  #
+  # @param s3 [Aws::S3::Client] the configured S3 client
+  # @param bucket [String] The bucket name
+  # @param prefix [String] Prefix to use for object keys
+  #
+  def initialize(s3, bucket, prefix=nil)
+    @s3 = s3
+    @bck = bucket
+    @pre = prefix || ''.freeze
+  end
+
+
+  ###############################################
+  # Where to store onfig
+  #
+  def _path(user)
+    @pre + user
+  end # def _path()
+  private :_path
+
+
+  ###############################################
+  # (see Users#read)
+  #
+  def read(urg)
+    Items.validate(urg, 'User/Role/Group name'.freeze, Items::FieldUsergrp)
+    json = @s3.get_object( bucket: @bck, key: _path(urg) ).body.read
+    return JSON.parse(json)
+  rescue
+    return nil
+  end # def read()
+
+
+  ###############################################
+  # (see Users#write)
+  #
+  def write(obj)
+    Items.validate(obj, 'User/Role/Group'.freeze, Users::ValUser)
+    json = JSON.pretty_generate(obj)
+    @s3.put_object( bucket: @bck, key: _path(obj['name']), body: json )
+  end # def write()
+
+
+end # class ICFS::UsersS3
+
+end # module ICFS

data/lib/icfs/utils/backup.rb

@@ -10,6 +10,7 @@
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
 require_relative '../store_fs'
+
 #
 module ICFS
 
@@ -20,6 +21,7 @@ module Utils
 #
 class Backup
 
+
   ###############################################
   # Instance
   #
@@ -37,7 +39,7 @@ class Backup
   ###############################################
   # Copy an item
   #
-  def _item(dest, title, read, write, args, val=nil)
+  def _copy_item(dest, title, read, write, args, val=nil)
     @log.debug('ICFS copy: %s'.freeze % title)
 
     # read the item
@@ -65,7 +67,7 @@ class Backup
     @log.error('ICFS copy: %s bad JSON'.freeze % title)
     return nil
   end # def _item
-  private :_item
+  private :_copy_item
 
 
   ###############################################
@@ -76,8 +78,16 @@ class Backup
   # @param lnum_max [Integer] The highest log
   # @param lnum_min [Integer] The lowest log
   #
-  def copy(cid, dest, lnum_max, lnum_min=1)
+  def copy(cid, dest, lnum_min=1, lnum_max=0)
     @log.info('ICFS copy: %s %d-%d'.freeze % [cid, lnum_min, lnum_max])
+
+    # if no max specified, pull from current
+    if lnum_max == 0
+      json = @cache.current_read(cid)
+      cur = Items.parse(json, 'current'.freeze, Items::ItemCurrent)
+      lnum_max = cur['log']
+    end
+
     if lnum_min > lnum_max
       raise ArgumentError, 'ICFS copy, log num min is larger than max'.freeze
     end
@@ -105,7 +115,7 @@ class Backup
         'entry %d-%d'.freeze % [enum, lnum],
         :entry_read,
         :entry_write,
-        [cid, lnum]
+        [cid, enum, lnum]
       )
 
       # index
@@ -121,7 +131,7 @@ class Backup
 
       # action
       if log['action']
-        anum = log['action']['lnum']
+        anum = log['action']['num']
         _copy_item(dest,
           'action %d-%d'.freeze % [anum, lnum],
           :action_read,
@@ -133,7 +143,7 @@ class Backup
       # case
       if log['case_hash']
         _copy_item(dest,
-          'case %d'.freeze, % lnum,
+          'case %d'.freeze % lnum,
           :case_read,
           :case_write,
           [cid, lnum]
@@ -142,7 +152,8 @@ class Backup
 
       # files
       if log['files_hash']
-        log['files_hash'].each_index do |fnum|
+        log['files_hash'].each_index do |fraw|
+          fnum = fraw + 1
 
           @log.debug('ICFS copy: file %d-%d-%d'.freeze % [enum, lnum, fnum])
 
@@ -170,6 +181,200 @@ class Backup
   end # def copy()
 
 
+  ###############################################
+  # Restore an item
+  #
+  def _restore_item(src, title, read, write, args_st, args_ca, val=nil)
+    @log.debug('ICFS restore: %s'.freeze % title)
+
+    # read the item
+    json = src.send(read, *args_st)
+    if !json
+      @log.warn('ICFS restore: %s is missing'.freeze % title)
+      return nil
+    end
+
+    # parse item if requested
+    if val
+      obj = JSON.parse(json)
+      err = Validate.check(obj, val)
+      if err
+        @log.error('ICFS restore: %s bad format'.freeze % title)
+        return nil
+      end
+    end
+
+    # write the item to store & cache
+    @store.send(write, *args_st, json)
+    @cache.send(write, *args_ca, json)
+    return [obj, json]
+
+  end # def _restore_item()
+  private :_restore_item
+
+
+  ###############################################
+  # Restore a backup into a case
+  #
+  # @param cid [String] The case ID
+  # @param src [Store] Source store
+  # @param lnum_max [Integer] The highest log
+  # @param lnum_min [Integer] The lowest log
+  #
+  def restore(cid, src, lnum_min=0, lnum_max=0)
+    @log.info('ICFS restore: %s %d-%d'.freeze % [cid, lnum_min, lnum_max])
+
+    # take lock
+    @cache.lock_take(cid)
+    begin
+
+      # read current
+      json = @cache.current_read(cid)
+      if json
+        cur = Items.parse(json, 'current'.freeze, Items::ItemCurrent)
+      else
+        cur = {
+          'icfs' => 1,
+          'caseid' => cid,
+          'log' => 1,
+          'entry' => 1,
+          'action' => 0,
+          'index' => 0
+        }
+      end
+
+      # if no min specified, pull from current or default to 1
+      lnum_min = cur['log'] if lnum_min == 0
+
+      # sanity check min & max
+      if (lnum_min > lnum_max) && (lnum_max != 0)
+        raise ArgumentError: 'ICFS restore, log min is larger than max'.freeze
+      end
+
+      # max entry, action, index
+      emax = cur['entry']
+      amax = cur['action']
+      imax = cur['index']
+
+      # each log
+      lnum = lnum_min
+      llast = nil
+      while lnum != lnum_max
+
+        # copy the log
+        log, litem = _restore_item(src,
+          'log %d'.freeze % lnum,
+          :log_read,
+          :log_write,
+          [cid, lnum],
+          [cid, lnum],
+          Items::ItemLog
+        )
+
+        # no log - all done
+        if !log
+          break
+        else
+          llast = litem
+        end
+
+        # entry
+        enum = log['entry']['num']
+        _restore_item(src,
+          'entry %d-%d'.freeze % [enum, lnum],
+          :entry_read,
+          :entry_write,
+          [cid, enum, lnum],
+          [cid, enum]
+        )
+        emax = enum if enum > emax
+
+        # index
+        if log['index']
+          xnum = log['index']['num']
+          _restore_item(src,
+            'index %d-%d'.freeze % [xnum, lnum],
+            :index_read,
+            :index_write,
+            [cid, xnum, lnum],
+            [cid, xnum]
+          )
+          imax = xnum if xnum > imax
+        end
+
+        # action
+        if log['action']
+          anum = log['action']['num']
+          _restore_item(src,
+            'action %d-%d'.freeze % [anum, lnum],
+            :action_read,
+            :action_write,
+            [cid, anum, lnum],
+            [cid, anum]
+          )
+          amax = anum if anum > amax
+        end
+
+        # case
+        if log['case_hash']
+          _restore_item(src,
+            'case %d'.freeze % lnum,
+            :case_read,
+            :case_write,
+            [cid, lnum],
+            [cid]
+          )
+        end
+
+        # files
+        if log['files_hash']
+          log['files_hash'].each_index do |fraw|
+            fnum = fraw + 1
+
+            @log.debug('ICFS restore: file %d-%d-%d'.freeze % [enum, lnum, fnum])
+
+            # read
+            fi = src.file_read(cid, enum, lnum, fnum)
+            if !fi
+              @log.warn('ICFS restore: file %d-%d-%d missing'.freeze %
+                [enum, lnum, fnum])
+              next
+            end
+
+            # copy
+            tmp = @store.tempfile
+            IO.copy_stream(fi, tmp)
+            src.close(fi)
+
+            # write
+            @store.file_write(cid, enum, lnum, fnum, tmp)
+          end
+        end
+
+        lnum += 1
+      end
+
+      # write current
+      cur = {
+        'icfs' => 1,
+        'caseid' => cid,
+        'log' => lnum-1,
+        'entry' => emax,
+        'action' => amax,
+        'index' => imax,
+        'hash' => ICFS.hash(llast)
+      }
+      nitem = Items.generate(cur, 'current'.freeze, Items::ItemCurrent)
+      @cache.current_write(cid, nitem)
+
+    ensure
+      # release lock
+      @cache.lock_release(cid)
+    end
+
+  end # def restore()
+
+
 end # class ICFS::Utils::Backup
 
 end # module ICFS::Utils

data/lib/icfs/utils/check.rb

@@ -16,16 +16,12 @@ module ICFS
 ##########################################################################
 # Low level utilities for working directly with ICFS systems.
 #
-# @todo Archive/move case tool written
-#
 module Utils
 
 
 ##########################################################################
 # Check a case for errors
 #
-# @todo Make a function to auto find the last log
-#
 class Check
 
 
@@ -180,7 +176,7 @@ class Check
             ['index'.freeze, 1].freeze,
             ['log'.freeze, 2].freeze
           ]
-        )  
+        )
         idx_cur.add(xnum)
       end