icfs 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecd555ee72ea33e646323a37019652a7fb111c4aec34146c92825fbe30fd7ded
-  data.tar.gz: 2c53ec9caf008aa4a6a1f187a7b2d2d4a631d19081ed83c497717f4386facbca
+  metadata.gz: b0469a0b363219f00dc2955e61bc1dcd6a89743f5cceb53f82a340956f2eac5d
+  data.tar.gz: f2091feb2f38b17ea38d870dd35d287fe6a2a8deada2e51b4e14e58ce45e4c3e
 SHA512:
-  metadata.gz: 5ae5fc9857d14845419257717150815b3deb7ae55e17d72e0990928916c0561dbe6c56e99bdaa4a74c3d8d92b99ead2b3f3bfcb9b34190de0dd1ac12c3d56e32
-  data.tar.gz: 4d98034fe142019aaed7cd7208fa6d8841942361755e58eb49fa04dbfd6b678aeea44299800ed8536088b7e5addcaa4ccc68adb94c39e63f60ded95d7bccf263
+  metadata.gz: 867f463393cac2ce8c22ef835ce86ab36ce6fc96167427c118c332d0dc465a5ec09effd43bff7011c6ae6388c458b34f7e2e82953c6757e4fb60d80939f67a21
+  data.tar.gz: 46f9e799ebbb39eb0d2c87affd84efa2fc1205e88a9c777f9d5c5dc67ca4bd8a3ab9712ccee45fca26fe1fc1ff18b10dd5cc36ffb31118043fde3d917f6204c7

data/data/icfs.css

@@ -79,6 +79,7 @@ div.nav-tab {
   border: 1px;
   width: 7em;
   border-style: solid solid none solid;
+  border-radius: 5px 5px 0 0;
 }
 
 

data/{bin/icfs_dev_todo.rb → devel/create.sh}

@@ -1,4 +1,4 @@
-#!/usr/bin/env ruby
+#!/bin/bash
 #
 # Investigative Case File System
 #
@@ -10,11 +10,6 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
-
-require 'yard'
-
-YARD::Registry.load!.all.each do |o|
-  todo = o.tags(:todo)
-  next if todo.empty?
-  todo.each{|tg| puts "%s\n  %s\n\n" % [o.path, tg.text] }
-end
+docker network create icfs
+docker volume create icfs-es
+docker volume create icfs-obj

data/devel/elastic.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# run elasticsearch
+docker run -d --rm \
+  -e "discovery.type=single-node" \
+  -v icfs-es:/usr/share/elasticsearch/data \
+  -p "127.0.0.1:9200:9200" \
+  --network icfs \
+  --name icfs-elastic \
+  docker.elastic.co/elasticsearch/elasticsearch:6.7.2

data/devel/icfs-wrk/Dockerfile

@@ -0,0 +1,27 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+FROM alpine:latest
+
+# install software
+RUN apk update && \
+    apk upgrade && \
+    apk --update add ruby fcgi ruby-json tzdata vim curl git bash && \
+    apk --update add --virtual build-deps ruby-dev build-base fcgi-dev && \
+    gem install -N rack webrick etc faraday yard aws-sdk-s3 redis && \
+    apk del build-deps && \
+    rm -rf /var/cache/apk/*
+
+# add user
+RUN addgroup -S icfsgrp && \
+    adduser -S icfsusr -G icfsgrp
+
+USER icfsusr

data/devel/minio.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# run minio
+# to debug add -e "MINIO_HTTP_TRACE=/dev/stdout" \
+docker run -d --rm \
+  -v icfs-obj:/data \
+  -p "127.0.0.1:9000:9000" \
+  -e "MINIO_ACCESS_KEY=minio_key" \
+  -e "MINIO_SECRET_KEY=minio_secret" \
+  --network icfs \
+  --name icfs-minio \
+  minio/minio server /data

data/devel/redis.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# run redis
+docker run -d --rm \
+  --network icfs \
+  --name icfs-redis \
+  redis:alpine
+# to run the CLI: docker exec -it icfs-redis redis-cli

data/devel/server.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# run the server
+docker run \
+  -v $1:/icfs \
+  -p "127.0.0.1:80:8080" \
+  --network icfs \
+  -it --rm icfs-wrk

data/devel/test/es-s3-fs-init.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'json'
+require 'faraday'
+require 'fileutils'
+require 'aws-sdk-s3'
+
+require_relative '../../lib/icfs'
+require_relative '../../lib/icfs/cache_elastic'
+require_relative '../../lib/icfs/store_s3'
+require_relative '../../lib/icfs/users_fs'
+
+# Minio config
+Aws.config.update(
+  endpoint: 'http://icfs-minio:9000',
+  access_key_id: 'minio_key',
+  secret_access_key: 'minio_secret',
+  force_path_style: true,
+  region: 'use-east-1'
+)
+
+# default mapping
+map = {
+  entry: 'entry',
+  case: 'case',
+  action: 'action',
+  index: 'index',
+  log: 'log',
+  lock: 'lock',
+  current: 'current',
+}.freeze
+
+# base items
+s3 = Aws::S3::Client.new
+es = Faraday.new('http://icfs-elastic:9200')
+cache = ICFS::CacheElastic.new(map, es)
+store = ICFS::StoreS3.new(s3, 'icfs-cache')
+users = ICFS::UsersFs.new('/var/lib/icfs/users')
+
+# create users dir
+FileUtils.mkdir_p('/var/lib/icfs/users')
+puts "Created users directory"
+
+# put a testuser
+tusr = {
+  'name' => 'testuser',
+  'type' => 'user',
+}
+users.write(tusr)
+puts "Added testuser"
+
+# create the indexes
+cache.create(ICFS::CacheElastic::Maps)
+puts "Indexes created"
+
+# create a bucket
+s3.create_bucket(bucket: 'icfs-cache')
+puts "Bucket created"
+
+api = ICFS::Api.new([], users, cache, store)
+api.user = 'testuser'
+
+# add a template
+tmpl = {
+  'template' => true,
+  'status' => true,
+  'title' => 'Test template',
+  'access' => [
+    {
+      'perm' => '[manage]',
+      'grant' => [
+        'testuser'
+      ]
+    }
+  ]
+}
+ent = {
+  'caseid' => 'test_tmpl',
+  'title' => 'Create template',
+  'content' => 'To test'
+}
+api.case_create(ent, tmpl)
+puts "Create template"

data/devel/test/es-s3-fs-webrick.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'json'
+require 'faraday'
+require 'fileutils'
+require 'aws-sdk-s3'
+
+require_relative '../../lib/icfs'
+require_relative '../../lib/icfs/cache_elastic'
+require_relative '../../lib/icfs/store_s3'
+require_relative '../../lib/icfs/users_fs'
+require_relative '../../lib/icfs/web/client'
+require_relative '../../lib/icfs/demo/auth'
+require_relative '../../lib/icfs/demo/static'
+require_relative '../../lib/icfs/demo/timezone'
+
+# Minio config
+Aws.config.update(
+  endpoint: 'http://icfs-minio:9000',
+  access_key_id: 'minio_key',
+  secret_access_key: 'minio_secret',
+  force_path_style: true,
+  region: 'use-east-1'
+)
+
+# default mapping
+map = {
+  entry: 'entry',
+  case: 'case',
+  action: 'action',
+  index: 'index',
+  log: 'log',
+  lock: 'lock',
+  current: 'current',
+}.freeze
+
+# base items
+s3 = Aws::S3::Client.new
+es = Faraday.new('http://icfs-elastic:9200')
+cache = ICFS::CacheElastic.new(map, es)
+store = ICFS::StoreS3.new(s3, 'icfs-cache')
+users = ICFS::UsersFs.new('/var/lib/icfs/users')
+api = ICFS::Api.new([], users, cache, store)
+web = ICFS::Web::Client.new('/static/icfs.css', '/static/icfs.js')
+
+# static files
+static = {
+  '/static/icfs.css' => {
+    'path' => 'data/icfs.css',
+    'mime' => 'text/css; charset=utf-8'
+  },
+  '/static/icfs.js' => {
+    'path' => 'data/icfs.js',
+    'mime' => 'application/javascript; charset=utf-8'
+  }
+}
+
+app = Rack::Builder.new do
+  use(ICFS::Demo::Auth, api)
+  use(ICFS::Demo::Static, static)
+  use(ICFS::Demo::Timezone, '-04:00')
+  run web
+end
+
+opts = {}
+opts[:Host] = "0.0.0.0"
+opts[:Port] = 8080
+
+Rack::Handler::WEBrick.run(app, opts)

data/devel/test/es-s3-restore.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+
+# usage <caseid> <store_fs>
+
+require 'json'
+require 'faraday'
+require 'aws-sdk-s3'
+
+require_relative '../../lib/icfs'
+require_relative '../../lib/icfs/cache_elastic'
+require_relative '../../lib/icfs/store_s3'
+require_relative '../../lib/icfs/store_fs'
+require_relative '../../lib/icfs/utils/backup'
+
+# Minio config
+Aws.config.update(
+  endpoint: 'http://icfs-minio:9000',
+  access_key_id: 'minio_key',
+  secret_access_key: 'minio_secret',
+  force_path_style: true,
+  region: 'use-east-1'
+)
+
+# default mapping
+map = {
+  entry: 'entry',
+  case: 'case',
+  action: 'action',
+  index: 'index',
+  log: 'log',
+  lock: 'lock',
+  current: 'current',
+}.freeze
+
+# base items
+s3 = Aws::S3::Client.new
+es = Faraday.new('http://icfs-elastic:9200')
+cache = ICFS::CacheElastic.new(map, es)
+store = ICFS::StoreS3.new(s3, 'icfs-cache')
+log = Logger.new(STDOUT, level: Logger::DEBUG)
+
+src = ICFS::StoreFs.new(ARGV[1])
+backup = ICFS::Utils::Backup.new(cache, store, log)
+backup.restore(ARGV[0], src)
+

data/devel/test/init.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# Elasticsearch cache
+# S3 item store
+# S3 & Redis Users
+
+require 'json'
+require 'faraday'
+require 'fileutils'
+require 'aws-sdk-s3'
+require 'redis'
+
+require_relative '../../lib/icfs'
+require_relative '../../lib/icfs/cache_elastic'
+require_relative '../../lib/icfs/store_s3'
+require_relative '../../lib/icfs/users_fs'
+require_relative '../../lib/icfs/users_redis'
+
+
+# Minio config
+Aws.config.update(
+  endpoint: 'http://icfs-minio:9000',
+  access_key_id: 'minio_key',
+  secret_access_key: 'minio_secret',
+  force_path_style: true,
+  region: 'use-east-1'
+)
+
+# default mapping
+map = {
+  entry: 'entry',
+  case: 'case',
+  action: 'action',
+  index: 'index',
+  log: 'log',
+  lock: 'lock',
+  current: 'current',
+}.freeze
+
+# base items
+s3 = Aws::S3::Client.new
+redis = Redis.new(host: 'icfs-redis')
+es = Faraday.new('http://icfs-elastic:9200')
+cache = ICFS::CacheElastic.new(map, es)
+store = ICFS::StoreS3.new(s3, 'icfs', 'case/')
+users_base = ICFS::UsersFs.new(ARGV[0])
+users = ICFS::UsersRedis.new(redis, users_base, {
+    prefix: 'users/'.freeze,
+    expires: 60, # one minute cache for testing
+  })
+
+# create the indexes
+cache.create(ICFS::CacheElastic::Maps)
+puts "Indexes created"
+
+# create a bucket
+s3.create_bucket(bucket: 'icfs')
+puts "S3 bucket created"
+
+api = ICFS::Api.new([], users, cache, store)
+api.user = 'user1'
+
+# add a template
+tmpl = {
+  'template' => true,
+  'status' => true,
+  'title' => 'Test template',
+  'access' => [
+    {
+      'perm' => '[manage]',
+      'grant' => [
+        'role2'
+      ]
+    }
+  ]
+}
+ent = {
+  'caseid' => 'test_tmpl',
+  'title' => 'Create template',
+  'content' => 'To test'
+}
+api.case_create(ent, tmpl)
+puts "Create template"

data/devel/test/run.rb

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'json'
+require 'faraday'
+require 'fileutils'
+require 'aws-sdk-s3'
+require 'redis'
+
+require_relative '../../lib/icfs'
+require_relative '../../lib/icfs/cache_elastic'
+require_relative '../../lib/icfs/store_s3'
+require_relative '../../lib/icfs/users_fs'
+require_relative '../../lib/icfs/users_redis'
+require_relative '../../lib/icfs/web/client'
+require_relative '../../lib/icfs/web/config_s3'
+require_relative '../../lib/icfs/web/config_redis'
+require_relative '../../lib/icfs/demo/auth'
+require_relative '../../lib/icfs/demo/static'
+
+# Minio config
+Aws.config.update(
+  endpoint: 'http://icfs-minio:9000',
+  access_key_id: 'minio_key',
+  secret_access_key: 'minio_secret',
+  force_path_style: true,
+  region: 'use-east-1'
+)
+
+# default mapping
+map = {
+  entry: 'entry',
+  case: 'case',
+  action: 'action',
+  index: 'index',
+  log: 'log',
+  lock: 'lock',
+  current: 'current',
+}.freeze
+
+# default config
+defaults = {
+  'tz' => '-04:00'
+}
+
+# base items
+s3 = Aws::S3::Client.new
+redis = Redis.new(host: 'icfs-redis')
+es = Faraday.new('http://icfs-elastic:9200')
+cache = ICFS::CacheElastic.new(map, es)
+store = ICFS::StoreS3.new(s3, 'icfs', 'case/')
+users_base = ICFS::UsersFs.new(ARGV[0])
+users = ICFS::UsersRedis.new(redis, users_base, {
+    prefix: 'users/'.freeze,
+    expires: 60, # one minute cache for testing
+  })
+api = ICFS::Api.new([], users, cache, store)
+config_base = ICFS::Web::ConfigS3.new(defaults, s3, 'icfs', 'config/')
+config = ICFS::Web::ConfigRedis.new(redis, config_base, {
+    prefix: 'config/',
+    expires: 60,  # debug, only cache for one minute
+  })
+web = ICFS::Web::Client.new('/static/icfs.css', '/static/icfs.js')
+
+# static files
+static = {
+  '/static/icfs.css' => {
+    'path' => 'data/icfs.css',
+    'mime' => 'text/css; charset=utf-8'
+  },
+  '/static/icfs.js' => {
+    'path' => 'data/icfs.js',
+    'mime' => 'application/javascript; charset=utf-8'
+  }
+}
+
+app = Rack::Builder.new do
+  use(ICFS::Demo::Auth, api, config)
+  use(ICFS::Demo::Static, static)
+  run web
+end
+
+opts = {}
+opts[:Host] = "0.0.0.0"
+opts[:Port] = 8080
+
+Rack::Handler::WEBrick.run(app, opts)

data/lib/icfs/api.rb

@@ -15,8 +15,6 @@ module ICFS
 ##########################################################################
 # Api
 #
-# @todo Add event logging
-#
 class Api
 
   # Validate a size
@@ -71,19 +69,31 @@ class Api
     urgp = @users.read(uname)
     raise(Error::NotFound, 'User name not found'.freeze) if !urgp
     raise(Error::Value, 'Not a user'.freeze) if urgp['type'] != 'user'
-    @roles = urgp['roles'].each{|rn| rn.freeze }
-    @groups = urgp['groups'].each{ |gn| gn.freeze }
-    @perms = urgp['perms'].each{|pn| pn.freeze }
+    if urgp['roles']
+      @roles = urgp['roles'].map{|rn| rn.freeze }.freeze
+    else
+      @roles = [].freeze
+    end
+    if urgp['groups']
+      @groups = urgp['groups'].map{ |gn| gn.freeze }.freeze
+    else
+      @groups = [].freeze
+    end
+    if urgp['perms']
+      @perms = urgp['perms'].map{|pn| pn.freeze }.freeze
+    else
+      @perms = [].freeze
+    end
 
     @urg = Set.new
-    @urg.add user
-    @urg.merge roles
-    @urg.merge groups
+    @urg.add @user
+    @urg.merge @roles
+    @urg.merge @groups
     @urg.freeze
 
     @ur = Set.new
-    @ur.add user
-    @ur.merge roles
+    @ur.add @user
+    @ur.merge @roles
     @ur.freeze
 
     reset
@@ -444,7 +454,7 @@ class Api
     end
 
     json = @cache.current_read(cid)
-    return Items.parse(json, 'current'.current, Items::ItemCurrent)
+    return Items.parse(json, 'current'.freeze, Items::ItemCurrent)
   end # end def current_read()
 
 
@@ -1424,6 +1434,7 @@ class Api
           fhash << ICFS.hash_temp(fi)
         end
       end
+      fhash = nil if fhash.empty?
     end
 
     return [files, fhash]

data/lib/icfs/cache_elastic.rb

@@ -685,11 +685,19 @@ class CacheElastic < Cache
         { 'title.raw' => 'desc' },
         { '_id' => 'desc' },
       ]
-    when 'title_asc', nil
+    when 'title_asc'
       req['sort'] = [
         { 'title.raw' => 'asc' },
         { '_id' => 'desc' },
       ]
+    else
+      # default if not a title/content query
+      if must.empty?
+        req['sort'] = [
+          { 'title.raw' => 'asc' },
+          { '_id' => 'desc' },
+        ]
+      end
     end
 
     # paging

data/lib/icfs/demo/auth.rb

@@ -27,10 +27,12 @@ class Auth
   #
   # @param app [Object] The rack app
   # @param api [Object] the ICFS API
+  # @param cfg [Object] the ICFS Web Config object
   #
-  def initialize(app, api)
+  def initialize(app, api, cfg)
     @app = app
     @api = api
+    @cfg = cfg
   end
 
 
@@ -59,8 +61,12 @@ class Auth
     if !user
       return [400, {'Content-Type' => 'text/plain'}, ['Login first'.freeze]]
     end
+
+    # set up for the call
     @api.user = user
     env['icfs'] = @api
+    @cfg.load(user)
+    env['icfs.config'] = @cfg
     return @app.call(env)
 
   rescue ICFS::Error::NotFound, ICFS::Error::Value => err