icfs 0.1.3 → 0.2.0

data/lib/icfs/{web/config_s3.rb → config_s3.rb}

@@ -9,10 +9,11 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
+# frozen_string_literal: true
+
 require_relative 'config'
 
 module ICFS
-module Web
 
 ##########################################################################
 # Configuration storage implemented in S3
@@ -32,7 +33,7 @@ class ConfigS3 < Config
     super(defaults)
     @s3 = s3
     @bck = bucket
-    @pre = prefix || ''.freeze
+    @pre = prefix || ''
   end
 
 
@@ -40,10 +41,10 @@ class ConfigS3 < Config
   # (see Config#load)
   #
   def load(unam)
-    Items.validate(unam, 'User/Role/Group name'.freeze, Items::FieldUsergrp)
+    Items.validate(unam, 'User/Role/Group name', Items::FieldUsergrp)
     @unam = unam.dup
     json = @s3.get_object( bucket: @bck, key: _key(unam) ).body.read
-    @data = Items.parse(json, 'Config values'.freeze, Config::ValConfig)
+    @data = Items.parse(json, 'Config values', Config::ValConfig)
     return true
   rescue
     @data = {}
@@ -55,12 +56,11 @@ class ConfigS3 < Config
   # (see Config#save)
   #
   def save()
-    raise(RuntimeError, 'Save requires a user name'.freeze) if !@unam
-    json = Items.generate(@data, 'Config values'.freeze, Config::ValConfig)
+    raise(RuntimeError, 'Save requires a user name') if !@unam
+    json = Items.generate(@data, 'Config values', Config::ValConfig)
     @s3.put_object( bucket: @bck, key: _key(@unam), body: json )
   end # def save()
 
-end # class ICFS::Web::ConfigS3
+end # class ICFS::ConfigS3
 
-end # module ICFS::Web
 end # module ICFS

data/lib/icfs/demo/auth.rb

@@ -8,6 +8,8 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
+# frozen_string_literal: true
+
 require 'rack'
 
 module ICFS
@@ -27,12 +29,10 @@ class Auth
   #
   # @param app [Object] The rack app
   # @param api [Object] the ICFS API
-  # @param cfg [Object] the ICFS Web Config object
   #
-  def initialize(app, api, cfg)
+  def initialize(app, api)
     @app = app
     @api = api
-    @cfg = cfg
   end
 
 
@@ -42,7 +42,7 @@ class Auth
   def call(env)
 
     # login
-    if env['PATH_INFO'] == '/login'.freeze
+    if env['PATH_INFO'] == '/login'
       user = env['QUERY_STRING']
       body = 'User set'
 
@@ -59,14 +59,12 @@ class Auth
     cookies = Rack::Request.new(env).cookies
     user = cookies['icfs-user']
     if !user
-      return [400, {'Content-Type' => 'text/plain'}, ['Login first'.freeze]]
+      return [400, {'Content-Type' => 'text/plain'}, ['Login first']]
     end
 
     # set up for the call
     @api.user = user
     env['icfs'] = @api
-    @cfg.load(user)
-    env['icfs.config'] = @cfg
     return @app.call(env)
 
   rescue ICFS::Error::NotFound, ICFS::Error::Value => err

data/lib/icfs/demo/static.rb

@@ -8,6 +8,8 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
+# frozen_string_literal: true
+
 #
 module ICFS
 

data/lib/icfs/elastic.rb

@@ -9,6 +9,8 @@
 # This program is distributed WITHOUT ANY WARRANTY; without even the
 # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 
+# frozen_string_literal: true
+
 #
 module ICFS
 
@@ -28,12 +30,12 @@ module Elastic
   # @return [String] JSON encoded object
   #
   def _read(ix, id)
-    url = '%s/_doc/%s/_source'.freeze % [ @map[ix], CGI.escape(id)]
-    resp = @es.run_request(:get, url, ''.freeze, {})
+    url = '%s/_doc/%s/_source' % [ @map[ix], CGI.escape(id)]
+    resp = @es.run_request(:get, url, '', {})
     if resp.status == 404
       return nil
     elsif !resp.success?
-      raise('Elasticsearch read failed'.freeze)
+      raise('Elasticsearch read failed')
     end
     return resp.body
   end # def _read()
@@ -47,11 +49,11 @@ module Elastic
   # @param item [String] JSON encoded object to write
   #
   def _write(ix, id, item)
-    url = '%s/_doc/%s'.freeze % [ @map[ix], CGI.escape(id)]
-    head = {'Content-Type'.freeze => 'application/json'.freeze}.freeze
+    url = '%s/_doc/%s' % [ @map[ix], CGI.escape(id)]
+    head = {'Content-Type' => 'application/json'}.freeze
     resp = @es.run_request(:put, url, item, head)
     if !resp.success?
-      raise('Elasticsearch index failed'.freeze)
+      raise('Elasticsearch index failed')
     end
   end # def _write()
 
@@ -64,7 +66,7 @@ module Elastic
   # @param maps [Hash] symbol to Elasticsearch mapping
   #
   def create(maps)
-    head = {'Content-Type'.freeze => 'application/json'.freeze}.freeze
+    head = {'Content-Type' => 'application/json'}.freeze
     maps.each do |ix, map|
       url = @map[ix]
       resp = @es.run_request(:put, url, map, head)
@@ -72,7 +74,7 @@ module Elastic
         puts 'URL: %s' % url
         puts map
         puts resp.body
-        raise('Elasticsearch index create failed: %s'.freeze % ix.to_s)
+        raise('Elasticsearch index create failed: %s' % ix.to_s)
       end
     end
   end # def create()

data/lib/icfs/email/basic.rb

@@ -0,0 +1,242 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# frozen_string_literal: true
+
+require_relative 'core'
+
+module ICFS
+module Email
+
+##########################################################################
+# Basic email processing
+#
+# This looks for ICFS email gateway instructions, and processes
+# attachments.
+#
+class Basic
+
+  ###############################################
+  # Strip regex
+  StripRx = /^[^[:graph:]]*([[:graph:]].*[[:graph:]])[^[:graph:]]*$/.freeze
+
+
+  ###############################################
+  # Strip spaces from collected lines
+  #
+  def _strip(collect)
+    collect.map{ |lr|
+      ma = StripRx.match(lr) # include wierd UNICODE spaces
+      ma ? ma[1] : nil
+    }.compact
+  end # def _strip()
+
+
+  ###############################################
+  # Check for a boolean
+  #
+  def _boolean(str)
+    case str.downcase
+    when 'true', 'yes'
+      return true
+    when 'false', 'no'
+      return false
+    else
+      return nil
+    end
+  end # def _boolean()
+
+
+  ###############################################
+  # Fields regex
+  FieldRx = /^ICFS ([^:[:blank:]]*)[[:blank:]]*:[[:blank:]]*(.*)[[:blank:]]*$/.freeze
+
+
+  ###############################################
+  # Regex for stat
+  StatRx = /^([+\-]?\d+(\.\d*)?)[^[:graph:]]+([[:graph:]].*[[:graph:]])$/.freeze
+
+
+  ###############################################
+  # Look for instructions in the email and process them
+  #
+  def receive(env)
+
+    # we only work on text/plain version of the email
+    if env[:msg].multipart?
+      txt = env[:msg].text_part
+    elsif env[:msg].mime_type == 'text/plain'
+      txt = env[:msg]
+    end
+    return [:continue, nil] if !txt
+    lines = txt.decoded.lines
+
+    # User specified values
+    collect = nil
+    term = nil
+    state = nil
+    stat_name = nil
+    stat_value = nil
+    lines.each do |ln|
+      # collecting lines
+      if collect
+        if ln.start_with?(term)
+          case state
+
+          when :tags
+            tags = _strip(collect)
+            unless tags.empty?
+              env[:tags] ||= []
+              env[:tags] = env[:tags] + tags
+            end
+            collect = nil
+
+          when :perms
+            perms = _strip(collect)
+            env[:perms] = perms unless perms.empty?
+            collect = nil
+
+          when :stat
+            credit = _strip(collect)
+            env[:stats] ||= []
+            env[:stats] << {
+              'name' => stat_name,
+              'value' => stat_value,
+              'credit' => credit
+            }
+            collect = nil
+
+          when :content
+            cont = collect.map{|lr| lr.delete("\r")}.join('')
+            env[:content] = cont unless cont.empty?
+            collect = nil
+
+          else
+            raise ScriptError
+          end
+        else
+          collect << ln
+          next
+        end
+      end
+
+      next unless ma = FieldRx.match(ln)
+      fn = ma[1].downcase
+
+      case fn
+      when 'case'
+        env[:caseid] = ma[2].strip
+
+      when 'entry'
+        enum = ma[2].strip.to_i
+        env[:entry] = enum if enum != 0
+
+      when 'title'
+        env[:title] = ma[2].strip
+
+      when 'time'
+        if env[:user]
+          env[:api].user = env[:user]
+          tm = ICFS.time_parse(ma[2].strip, env[:api].config)
+          env[:time] = tm if tm
+        end
+
+      when 'tag'
+        env[:tags] ||= []
+        env[:tags] << ma[2].strip
+
+      when 'tags'
+        collect = []
+        state = :tags
+        term = 'ICFS'
+
+      when 'perms'
+        collect = []
+        state = :perms
+        term = 'ICFS'
+
+      when 'stat'
+        next unless pm = StatRx.match(ma[2].strip)
+        stat_name = pm[3]
+        stat_value = pm[1].to_f
+        collect = []
+        state = :stat
+        term = 'ICFS'
+
+      when 'content'
+        collect = []
+        state = :content
+        term = ma[2].strip
+        term = 'ICFS' if term.empty?
+
+      when 'save_files'
+        val = _boolean(ma[2].strip)
+        env[:save_files] = val unless val.nil?
+
+      when 'save_original'
+        val = _boolean(ma[2].strip)
+        env[:save_original] = val unless val.nil?
+
+      when 'save_email'
+        val = _boolean(ma[2].strip)
+        env[:save_email] = val unless val.nil?
+      end
+
+    end
+
+    # time defaults to message date
+    unless env[:time]
+      env[:time] = env[:msg].date.to_time.to_i
+    end
+
+    # title defaults to subject if okay
+    unless env[:title]
+      # check the subject time
+      title = env[:msg].subject
+      err = Validate.check(title, Items::FieldTitle)
+      env[:title] = title unless err
+    end
+
+    # save the edited email defaults to yes
+    unless env.key?(:save_email)
+      env[:save_email] = true
+    end
+
+    # save the raw email defaults to no
+    unless env.key?(:save_original)
+      env[:save_original] = false
+    end
+
+    # save attachments as files
+    unless env.key?(:save_files) && !env[:save_files]
+      cnt = 0
+      env[:msg].attachments.each do |att|
+        type = att.header[:content_disposition].disposition_type
+        next if type == 'inline'
+        cnt += 1
+        name = att.filename
+        if !name
+          ext = MIME::Types[att.content_type].first.extensions.first
+          name = 'unnamed_%d.%s' % [cnt, ext]
+        end
+        env[:files] << { name: name, content: att.decoded }
+      end
+      env[:msg] = ::Mail.new(env[:msg].without_attachments!.encoded)
+    end
+
+    return :continue
+  end # def receive()
+
+
+end # class ICFS::Email::RxCore
+
+end # module ICFS::Email
+end # module ICFS

data/lib/icfs/email/core.rb

@@ -0,0 +1,293 @@
+#
+# Investigative Case File System
+#
+# Copyright 2019 by Graham A. Field
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+# frozen_string_literal: true
+
+require 'set'
+
+module ICFS
+
+
+##########################################################################
+# Email integration with ICFS
+#
+module Email
+
+
+##########################################################################
+# Core email processing engine.
+#
+class Core
+
+
+  ###############################################
+  # A file to attach
+  ValFile = {
+    method: :hash,
+    required: {
+      content: Validate::IsString,     # content of the file
+      name: Items::FieldFilename,      # name of the file
+    }.freeze
+  }.freeze
+
+
+  ###############################################
+  # Results of processing a valid message
+  ValReceive = {
+    method: :hash,
+    required: {
+      orig: Validate::IsString,        # the raw message
+      caseid: Items::FieldCaseid,      # case to write
+      user: Items::FieldUsergrp,       # user as author
+      files: {                         # files to attach
+        method: :array,
+        check: ValFile
+      }.freeze,
+    }.freeze,
+    optional: {
+      entry: Validate::IsIntPos,       # the entry number
+      time: Validate::IsIntPos,        # the time of the entry
+      title: Items::FieldTitle,        # title of the entry
+      content: Items::FieldContent,    # content of the entry
+      tags: {                          # tags to apply
+        method: :array,
+        check: Items::FieldTag
+      }.freeze,
+      perms: {                         # perms to apply
+        method: :array,
+        check: Items::FieldPermAny,
+      }.freeze,
+      stats: {                         # stats to apply
+        method: :array,
+        min: 1,
+        check: {
+          method: :hash,
+          required: {
+            "name" => Items::FieldStat,
+            "value" => Validate::IsFloat,
+            "credit" => {
+              method: :array,
+              min: 1,
+              max: 32,
+              check: Items::FieldUsergrp
+            }.freeze
+          }.freeze
+        }.freeze
+      }.freeze,
+      save_raw: Validate::IsBoolean,   # save the raw email as a File
+      save_msg: Validate::IsBoolean,   # save the processed message w/o attach
+    }.freeze,
+    others: true,
+  }.freeze
+
+
+  ###############################################
+  # Default title
+  DefaultTitle = 'Email gateway default title'
+
+
+  ###############################################
+  # Default content
+  DefaultContent = 'Entry generated via email gateway with no content.'
+
+
+  ###############################################
+  # Filename for original content
+  DefaultOrig = 'email_received.eml'
+
+
+  ###############################################
+  # Filename for processed content without attachments
+  DefaultEmail = 'email.eml'
+
+
+  ###############################################
+  # New instance
+  #
+  # @param api [ICFS::Api] the ICFS API
+  # @param log [Logger] The log
+  # @param st [Array] the middleware
+  #
+  def initialize(api, log, st=nil)
+    @api = api
+    @log = log
+    self.stack_set(st) if st
+  end # def initialize()
+
+
+  ###############################################
+  # Set the middleware stack
+  #
+  # Each middleware object must respond to #receive and return one of:
+  #  * :continue - process more middleware
+  #  * :success - stops further middleare, and records the entry
+  #  * :failure - stops further middlware and does not record
+  #
+  def stack_set(st)
+    @stack = st
+  end # def stack_set()
+
+
+  ###############################################
+  # Process a received email using the middleware stack
+  #
+  # @param msg [::Mail::Message] the email message
+  # @return [Array] results, first field is a Symbol, second field is error or
+  #   the recorded message
+  #
+  def receive(msg)
+    @log.debug('Email: Processing %s' % msg.message_id)
+
+    # setup the environment
+    env = {
+      orig: msg.raw_source.dup, # the original text email
+      msg: msg,                 # the email message being worked on
+      files: [],                # files to attach to the entry
+      api: @api,                # the ICFS API
+    }
+
+    # process all middleware
+    @stack.each do |mid|
+      resp, err = mid.receive(env)
+      case resp
+      when :continue
+        next
+      when :stop
+        break
+      when :failure
+        return [:failure, err]
+      else
+        raise ScriptError
+      end
+    end
+
+    # check that all required fields were completed
+    err = Validate.check(env, ValReceive)
+    if err
+      @log.info('Email: Invalid: %s' % err.inspect)
+      return [:invalid, err]
+    end
+
+    # API set to active user
+    @api.user = env[:user]
+
+    # if an entry was specified
+    if env[:entry] && env[:entry] != 0
+      ent = @api.entry_read(env[:caseid], env[:entry])
+      ent.delete('icfs')
+      ent.delete('log')
+      ent.delete('user')
+      ent.delete('tags') if ent['tags'][0] == ICFS::TagNone
+    else
+      ent = {}
+      ent['caseid'] = env[:caseid]
+    end
+
+    # build entry
+    ent['time'] = env[:time] if env[:time]
+    ent['title'] = env[:title] if env[:title]
+    ent['title'] ||= DefaultTitle
+    ent['content'] = env[:content] if env[:content]
+    ent['content'] ||= DefaultContent
+    if env[:tags]
+      ent['tags'] ||= []
+      ent['tags'] = (ent['tags'] + env[:tags]).uniq
+    end
+    ent['perms'] = env[:perms].uniq if env[:perms]
+    ent['stats'] = env[:stats] if env[:stats]
+
+    # files
+    files = env[:files].map do |fd|
+      tmp = @api.tempfile
+      tmp.write(fd[:content])
+      { 'name' => fd[:name], 'temp' => tmp }
+    end
+    if env[:save_original]
+      tmp = @api.tempfile
+      tmp.write(env[:orig])
+      files << { 'name' => DefaultOrig, 'temp' => tmp }
+    end
+    if env[:save_email]
+      tmp = @api.tempfile
+      env[:msg].header.fields.delete_if do |fi|
+        !FieldsSet.include?(fi.name.downcase)
+      end
+      tmp.write(env[:msg].encoded)
+      files << { 'name' => DefaultEmail, 'temp' => tmp }
+    end
+    unless files.empty?
+      ent['files'] ||= []
+      ent['files'] = ent['files'] + files
+    end
+
+    # try to record it
+    @api.record(ent, nil, nil, nil)
+
+    @log.info('Email: Success: %s %d-%d' %
+        [ent['caseid'], ent['entry'], ent['log']])
+    return [:success, ent]
+
+  rescue ICFS::Error::Conflict => ex
+    @log.warn('Email: Conflict: %s' % ex.message)
+    return [:conflict, ex.message]
+  rescue ICFS::Error::NotFound => ex
+    @log.warn('Email: Not Found: %s' % ex.message)
+    return [:notfound, ex.message]
+  rescue ICFS::Error::Perms => ex
+    @log.warn('Email: Permissions: %s' % ex.message)
+    return [:perms, ex.message]
+  rescue ICFS::Error::Value => ex
+    @log.warn('Email: Value: %s' % ex.message)
+    return [:value, ex.message]
+  end # def receive()
+
+
+  ###############################################
+  # Basic header fields to copy
+  #
+  CopyFields = [
+    "to",
+    "cc",
+    "message-id",
+    "in-reply-to",
+    "references",
+    "subject",
+    "comments",
+    "keywords",
+    "date",
+    "from",
+    "sender",
+    "reply-to",
+  ].freeze
+
+
+  ###############################################
+  # Content related fields
+  #
+  ContentFields = [
+    "content-transfer-encoding",
+    "content-description",
+    "content-disposition",
+    "content-type",
+    "content-id",
+    "content-location",
+  ].freeze
+
+
+  ###############################################
+  # Set of header fields to copy set
+  FieldsSet = Set.new(CopyFields).merge(ContentFields).freeze
+
+end # class ICFS::Email::Core
+
+
+end # module ICFS::Email
+end # module ICFS