icfs 0.1.2 → 0.1.3

data/devel/devel-webrick.yml

@@ -0,0 +1,49 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+version: '3'
+
+services:
+
+  # Elasticsearch instance
+  elastic:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.7.2
+    environment:
+      discovery-type: single-node
+    volume:
+      - elastic:/usr/share/elasticsearch/data
+    ports:
+      - "127.0.0.1:9200:9200"
+
+  # Minio S3-compatible object store
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ACCESS_KEY: minio_key
+      MINIO_SECRET_KEY: minio_secret
+    volume:
+      - minio:/data
+    command: ["server", "/data"]
+
+  # Redis cache
+  redis:
+    image: redis:alpine
+
+  # the app
+  app:
+    image: icfs-wrk
+    ports:
+      - "127.0.0.1:80:8080"
+
+
+volumes:
+  - elastic
+  - minio

data/devel/docker-compose.yml

@@ -0,0 +1,63 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+version: '3'
+
+services:
+
+  # Elasticsearch
+  elastic:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.7.2
+    environment:
+      discovery.type: single-node
+    volumes:
+      - elastic:/usr/share/elasticsearch/data
+    ports:
+      - "127.0.0.1:9200:9200"
+      - "127.0.0.1:9300:9300"
+      
+
+  # Minio - S3 compatible object store
+  minio:
+    image: minio/minio
+    volumes:
+      - objects:/data
+    environment:
+      MINIO_ACCESS_KEY: minio_key
+      MINIO_SECRET_KEY: minio_secret
+    ports:
+      - "127.0.0.1:9000:9000"
+    command: ["server", "/data"]
+
+
+  # Redis - cache
+  # to run the CLI: docker exec -it icfs-redis redis-cli
+  redis:
+    image: redis:alpine
+    ports:
+      - "127.0.0.1:6379:6379"
+
+
+  # App development
+  # just run a sleep for a week, then connect interactively using:
+  #   docker-compose exec icfs /bin/sh
+  icfs:
+    image: icfs-wrk
+    build: ./icfs-wrk
+    volumes:
+      - ./icfs:/icfs
+    ports:
+      - "127.0.0.1:8080:8080"
+    command: ["/bin/sleep", "7d"]
+
+volumes:
+  elastic:
+  objects:

data/devel/icfs

@@ -0,0 +1 @@
+devel/../

data/devel/run/copy-s3.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'aws-sdk-s3'
+require 'find'
+
+require_relative '../../lib/icfs'
+
+# Minio config
+Aws.config.update(
+  endpoint: 'http://minio:9000',
+  access_key_id: 'minio_key',
+  secret_access_key: 'minio_secret',
+  force_path_style: true,
+  region: 'us-east-1'
+)
+
+s3 = Aws::S3::Client.new
+
+dir = ARGV[0]
+prefix = ARGV[1]
+size = (dir[-1] == '/') ? dir.size : dir.size + 1
+
+# copy in all the files in the dir
+Find.find(dir) do |fn|
+  next unless File.file?(fn)
+
+  rel = fn[size..-1]
+  key = prefix + rel
+
+  cont = File.binread(fn)
+  s3.put_object( bucket: 'icfs', key: key, body: cont )
+  puts key
+end

data/devel/{test/init.rb → run/init-icfs.rb}

@@ -10,30 +10,22 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
-# Elasticsearch cache
-# S3 item store
-# S3 & Redis Users
-
 require 'json'
 require 'faraday'
-require 'fileutils'
 require 'aws-sdk-s3'
-require 'redis'
 
 require_relative '../../lib/icfs'
 require_relative '../../lib/icfs/cache_elastic'
 require_relative '../../lib/icfs/store_s3'
-require_relative '../../lib/icfs/users_fs'
-require_relative '../../lib/icfs/users_redis'
 
 
 # Minio config
 Aws.config.update(
-  endpoint: 'http://icfs-minio:9000',
+  endpoint: 'http://minio:9000',
   access_key_id: 'minio_key',
   secret_access_key: 'minio_secret',
   force_path_style: true,
-  region: 'use-east-1'
+  region: 'us-east-1'
 )
 
 # default mapping
@@ -49,15 +41,9 @@ map = {
 
 # base items
 s3 = Aws::S3::Client.new
-redis = Redis.new(host: 'icfs-redis')
-es = Faraday.new('http://icfs-elastic:9200')
+es = Faraday.new('http://elastic:9200')
 cache = ICFS::CacheElastic.new(map, es)
 store = ICFS::StoreS3.new(s3, 'icfs', 'case/')
-users_base = ICFS::UsersFs.new(ARGV[0])
-users = ICFS::UsersRedis.new(redis, users_base, {
-    prefix: 'users/'.freeze,
-    expires: 60, # one minute cache for testing
-  })
 
 # create the indexes
 cache.create(ICFS::CacheElastic::Maps)
@@ -67,8 +53,7 @@ puts "Indexes created"
 s3.create_bucket(bucket: 'icfs')
 puts "S3 bucket created"
 
-api = ICFS::Api.new([], users, cache, store)
-api.user = 'user1'
+api = ICFS::Api.new([], nil, cache, store)
 
 # add a template
 tmpl = {
@@ -89,5 +74,5 @@ ent = {
   'title' => 'Create template',
   'content' => 'To test'
 }
-api.case_create(ent, tmpl)
+api.case_create(ent, tmpl, nil, 'user1')
 puts "Create template"

data/devel/{test/run.rb → run/webrick.rb}

@@ -10,16 +10,14 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
-require 'json'
 require 'faraday'
-require 'fileutils'
 require 'aws-sdk-s3'
 require 'redis'
 
 require_relative '../../lib/icfs'
 require_relative '../../lib/icfs/cache_elastic'
 require_relative '../../lib/icfs/store_s3'
-require_relative '../../lib/icfs/users_fs'
+require_relative '../../lib/icfs/users_s3'
 require_relative '../../lib/icfs/users_redis'
 require_relative '../../lib/icfs/web/client'
 require_relative '../../lib/icfs/web/config_s3'
@@ -29,11 +27,11 @@ require_relative '../../lib/icfs/demo/static'
 
 # Minio config
 Aws.config.update(
-  endpoint: 'http://icfs-minio:9000',
+  endpoint: 'http://minio:9000',
   access_key_id: 'minio_key',
   secret_access_key: 'minio_secret',
   force_path_style: true,
-  region: 'use-east-1'
+  region: 'us-east-1'
 )
 
 # default mapping
@@ -52,16 +50,21 @@ defaults = {
   'tz' => '-04:00'
 }
 
+# the log
+log = Logger.new(STDERR)
+log.level = Logger::INFO
+
 # base items
 s3 = Aws::S3::Client.new
-redis = Redis.new(host: 'icfs-redis')
-es = Faraday.new('http://icfs-elastic:9200')
+redis = Redis.new(host: 'redis')
+es = Faraday.new('http://elastic:9200')
 cache = ICFS::CacheElastic.new(map, es)
-store = ICFS::StoreS3.new(s3, 'icfs', 'case/')
-users_base = ICFS::UsersFs.new(ARGV[0])
+store = ICFS::StoreS3.new(s3, 'icfs'.freeze, 'case/'.freeze)
+users_base = ICFS::UsersS3.new(s3, 'icfs'.freeze, 'users/'.freeze)
 users = ICFS::UsersRedis.new(redis, users_base, {
     prefix: 'users/'.freeze,
     expires: 60, # one minute cache for testing
+    log: log,
   })
 api = ICFS::Api.new([], users, cache, store)
 config_base = ICFS::Web::ConfigS3.new(defaults, s3, 'icfs', 'config/')
@@ -74,11 +77,11 @@ web = ICFS::Web::Client.new('/static/icfs.css', '/static/icfs.js')
 # static files
 static = {
   '/static/icfs.css' => {
-    'path' => 'data/icfs.css',
+    'path' => '/icfs/data/icfs.css',
     'mime' => 'text/css; charset=utf-8'
   },
   '/static/icfs.js' => {
-    'path' => 'data/icfs.js',
+    'path' => '/icfs/data/icfs.js',
     'mime' => 'application/javascript; charset=utf-8'
   }
 }

data/lib/icfs/api.rb

@@ -100,6 +100,14 @@ class Api
   end # def user=()
 
 
+  ###############################################
+  # Flush any user caching
+  def user_flush()
+    @users.flush(@user)
+    self.user = @user
+  end # def user_flush
+
+
   ###############################################
   # User
   #
@@ -878,8 +886,9 @@ class Api
   # @param ent [Hash] the first entry
   # @param cse [Hash] the case
   # @param tid [String] the template name
+  # @param unam [String] the user name if not using the Users API
   #
-  def case_create(ent, cse, tid=nil)
+  def case_create(ent, cse, tid=nil, unam=nil)
 
     ####################
     # Sanity checks
@@ -888,12 +897,14 @@ class Api
     Items.validate(ent, 'entry'.freeze, Items::ItemEntryNew)
     Items.validate(cse, 'case'.freeze, Items::ItemCaseEdit)
 
-    # access users/roles/groups are valid
-    cse["access"].each do |acc|
-      acc["grant"].each do |gnt|
-        urg = @users.read(gnt)
-        if !urg
-          raise(Error::NotFound, 'User/role/group %s not found'.freeze % urg)
+    # access users/roles/groups are valid, unless manually specifying user
+    unless unam
+      cse["access"].each do |acc|
+        acc["grant"].each do |gnt|
+          urg = @users.read(gnt)
+          if !urg
+            raise(Error::NotFound, 'User/role/group %s not found'.freeze % urg)
+          end
         end
       end
     end
@@ -923,6 +934,9 @@ class Api
       raise(Error::Value, 'No Index for a new case entry'.freeze)
     end
 
+    # Allow case creation without a Users system in place
+    user = @user ? @user : unam
+    raise(ArgumentError, 'No user specified'.freeze) if user.nil?
 
     ####################
     # Prep
@@ -941,7 +955,7 @@ class Api
     ent['log'] = 1
     ent['tags'] ||= [ ]
     ent['tags'] << ICFS::TagCase
-    ent['user'] = @user
+    ent['user'] = user
     files, fhash = _pre_files(ent)
 
     # log
@@ -950,7 +964,7 @@ class Api
       'caseid' => cid,
       'log' => 1,
       'prev' => '0'*64,
-      'user' => @user,
+      'user' => user,
       'entry' => {
         'num' => 1,
        },

data/lib/icfs/users.rb

@@ -56,6 +56,15 @@ class Users
   }.freeze
 
 
+  ###############################################
+  # Flush a user/role/group from a cache, if any
+  #
+  # @param urg [String] User/Role/Group name
+  # @return [Boolean] if cached
+  #
+  def flush(urg); raise NotImplementedError; end
+
+
   ###############################################
   # Read a user/role/group
   #

data/lib/icfs/users_fs.rb

@@ -42,6 +42,12 @@ class UsersFs < Users
   private :_path
 
 
+  ###############################################
+  # (see Users#flush)
+  #
+  def flush(urg); false; end
+
+
   ###############################################
   # (see Users#read)
   #

data/lib/icfs/users_redis.rb

@@ -26,12 +26,14 @@ class UsersRedis < Users
   # @param opts [Hash] Options
   # @option opts [String] :prefix Prefix for Redis key
   # @option opts [Integer] :expires Expiration time in seconds
+  # @option opts [Logger] :log Logger which records info about user
   #
   def initialize(redis, base, opts={})
     @redis = redis
     @base = base
     @pre = opts[:prefix] || ''.freeze
     @exp = opts[:expires] || 1*60*60 # 1 hour default
+    @log = opts[:log]
   end
 
 
@@ -44,20 +46,38 @@ class UsersRedis < Users
   private :_key
 
 
+  ###############################################
+  # (see Users#flush)
+  #
+  def flush(urg)
+    Items.validate(urg, 'User/role/group name'.freeze, Items::FieldUsergrp)
+    @log.info('User/role/group cache flush: %s'.freeze % urg) if @log
+    @redis.del(_key(urg))
+    return true
+  end # def flush()
+
+
   ###############################################
   # (see Users#read)
   #
   def read(urg)
-    Validate.check(urg, Items::FieldUsergrp) # FIXME
+    Items.validate(urg, 'User/role/group name'.freeze, Items::FieldUsergrp)
     key = _key(urg)
+    @log.debug('User/role/group read: %s'.freeze % urg) if @log
 
     # try cache
     json = @redis.get(key)
-    return JSON.parse(json) if json
+    if json
+      @log.debug('User/role/group cache hit: %s'.freeze % urg) if @log
+      return JSON.parse(json)
+    end
 
     # get base object from base store
     bse = @base.read(urg)
-    return nil if !bse
+    if !bse
+      @log.warn('User/role/group not found: %s'.freeze % urg) if @log
+      return nil
+    end
 
     # assemble
     seen = Set.new.add(urg)
@@ -103,11 +123,12 @@ class UsersRedis < Users
     bse['roles'] = roles.to_a unless roles.empty?
     bse['groups'] = grps.to_a unless grps.empty?
     bse['perms'] = perms.to_a unless perms.empty?
-    json = JSON.pretty_generate(bse)
+    json = JSON.generate(bse)
 
     # save to cache
     @redis.set(key, json)
     @redis.expire(key, @exp)
+    @log.info('User/role/group cached: %s %s'.freeze % [urg, json]) if @log
     return bse
   end # def read()
 
@@ -118,6 +139,7 @@ class UsersRedis < Users
   def write(obj)
     json = Items.generate(obj, 'User/Role/Group'.freeze, Users::ValUser)
     key = _key(obj['name'])
+    @log.info('User/role/group write: %s'.freeze % urg) if @log
     @redis.del(key)
     @base.write(obj)
   end # def write()

data/lib/icfs/users_s3.rb

@@ -33,14 +33,20 @@ class UsersS3 < Users
 
 
   ###############################################
-  # Where to store onfig
+  # Where to store user
   #
   def _path(user)
-    @pre + user
+    @pre + user + '.json'.freeze
   end # def _path()
   private :_path
 
 
+  ###############################################
+  # (see Users#flush)
+  #
+  def flush(urg); false; end
+
+
   ###############################################
   # (see Users#read)
   #

data/lib/icfs/web/auth_ssl.rb

@@ -24,11 +24,13 @@ class AuthSsl
   # @param app [Object] The rack app
   # @param map [Object] Maps DN to user name
   # @param api [ICFS::Api] the Api
+  # @param cfg [ICFS::Web::Config] the config settings
   #
-  def initialize(app, map, api)
+  def initialize(app, map, api, cfg)
     @app = app
     @map = map
     @api = api
+    @cfg = cfg
   end
 
 
@@ -70,6 +72,8 @@ class AuthSsl
       ]
     end
     env['icfs'] = @api
+    @cfg.load(user)
+    env['icfs.config'] = @cfg
     return @app.call(env)
   end # def call()
 

data/lib/icfs/web/client.rb

@@ -780,6 +780,7 @@ class Client
       para = _util_post(env)
       _post_config(env, para).each{|key, val| cfg.set(key,val) }
       cfg.save
+      api.user_flush()
 
       # display the index
       body = [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: icfs
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Graham A. Field
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-07 00:00:00.000000000 Z
+date: 2019-06-11 00:00:00.000000000 Z
 dependencies: []
 description: |2-
 
@@ -24,41 +24,23 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE.txt
-- bin/icfs_demo_check.rb
-- bin/icfs_demo_create.rb
 - bin/icfs_demo_fcgi.rb
 - bin/icfs_demo_ssl_gen.rb
-- bin/icfs_demo_web.rb
-- data/demo_config.yml
-- data/docker/build-web.sh
-- data/docker/compose-demo.yml
-- data/docker/compose-init.yml
-- data/docker/icfs-app.rb
-- data/docker/icfs-cfg.yml
-- data/docker/icfs-init.rb
-- data/docker/icfs-ruby/Dockerfile
-- data/docker/icfs-ruby/build.sh
-- data/docker/nginx.conf
 - data/icfs.css
 - data/icfs.js
-- devel/create.sh
-- devel/elastic.sh
+- devel/devel-webrick.yml
+- devel/docker-compose.yml
+- devel/icfs
 - devel/icfs-wrk/Dockerfile
-- devel/minio.sh
-- devel/redis.sh
-- devel/server.sh
-- devel/test/es-s3-fs-init.rb
-- devel/test/es-s3-fs-webrick.rb
-- devel/test/es-s3-restore.rb
-- devel/test/init.rb
-- devel/test/run.rb
+- devel/run/copy-s3.rb
+- devel/run/init-icfs.rb
+- devel/run/webrick.rb
 - lib/icfs.rb
 - lib/icfs/api.rb
 - lib/icfs/cache.rb
 - lib/icfs/cache_elastic.rb
 - lib/icfs/demo/auth.rb
 - lib/icfs/demo/static.rb
-- lib/icfs/demo/timezone.rb
 - lib/icfs/elastic.rb
 - lib/icfs/items.rb
 - lib/icfs/store.rb

data/bin/icfs_demo_check.rb

@@ -1,29 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Investigative Case File System
-#
-# Copyright 2019 by Graham A. Field
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3.
-#
-# This program is distributed WITHOUT ANY WARRANTY; without even the
-# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-
-require 'yaml'
-require 'logger'
-
-require_relative '../lib/icfs'
-require_relative '../lib/icfs/utils/check'
-require_relative '../lib/icfs/store_fs'
-
-# load the config file
-cfg = YAML.load_file(ARGV[0])
-
-# objects
-store = ICFS::StoreFs.new(cfg['store']['dir'])
-log = Logger.new(STDOUT, level: Logger::INFO)
-check = ICFS::Utils::Check.new(store, log)
-
-# check
-check.check(ARGV[1], ARGV[2].to_i, nil, {hash_all: true})