log_analysis 0.1.0 → 0.1.1

data/lib/log_analysis.rb

@@ -2,28 +2,35 @@ require 'log_analysis/version'
 require 'log_analysis/preprocess'
 require 'log_analysis/user_identification'
 require 'log_analysis/session_identification'
+require 'log_analysis/transformation'
+require 'log_analysis/rule_generation'
 
 class LogAnalysis
   class Error < StandardError; end
   # Your code goes here...
 
-  attr_reader :path, :origin_logs
+  attr_reader :path, :type, :cleaned_data
 
   def initialize(path, type = nil)
-    @path        = path
-    @type        = type
-    @origin_logs = PreProcess.input(path, type)
-  end
-
-  def cleaned_data
-    PreProcess.data_cleaning(@origin_logs)
+    @path         = path
+    @type         = type
+    @cleaned_data = PreProcess.input(path, type)
+    system('mkdir', '-p', LogAnalysis::DATA_PATH)
   end
 
   def identified_user
-    UserIdentification.execute(cleaned_data)
+    UserIdentification.execute(@cleaned_data)
   end
 
   def identified_session
-    SessionIdentification.execute(cleaned_data)
+    SessionIdentification.execute(@cleaned_data)
+  end
+
+  def transformation
+    Transformation.execute(identified_session)
+  end
+
+  def rule_generation
+    RuleGeneration.execute(transformation)
   end
 end

data/lib/log_analysis/model/record.rb

@@ -25,7 +25,7 @@ class Record
 
   attr_reader   :params
 
-  DATA_TYPE = %w[.txt .json .js .css .jpg .jpeg .gif .woff2 .ico .png .bmp .mp3 .wav .avi .mpeg .vmw .mpg .map .pdf .doc .svg].freeze
+  DATA_TYPE = %w[.txt .json .js .css .jpg .jpeg .gif .woff2 .ico .png .bmp .mp3 .wav .avi .mpeg .vmw .mpg .map .pdf .doc .svg .otf].freeze
   REGEX_BOT = /facebookexternalhit|Mediapartners-Google|AWS|-|Crawler|spider|Detection/.freeze
 
   def initialize(params)
@@ -70,7 +70,7 @@ class Record
       p['apptime']        = @params['apptime'].to_f
       p['cache']          = validate_string(@params['cache'])
       p['vhost']          = validate_string(@params['vhost'])
-      p['user']           = UserIdentity.new(host: IPAddr.new(@params['host']), user_agent: UserAgent.parse(@params['ua']))
+      p['user']           = @params['user'] || nil
       p['forwardedfor']   = validate_string(@params['forwardedfor'])
       p['forwardedproto'] = validate_string(@params['forwardedproto'])
     end

data/lib/log_analysis/preprocess.rb

@@ -1,6 +1,6 @@
 require 'log_analysis/model/record'
+require 'log_analysis/model/user_identity'
 require 'json'
-require 'pry'
 
 module PreProcess
   class Error < StandardError; end
@@ -13,63 +13,54 @@ module PreProcess
   CONVERT_RECORD = { 'nginx' => 'convert_nginx_logs', 'apache' => 'convert_apache_logs', 'default' => 'to_records' }.freeze
 
   def self.input(file_path, type)
-    arr_logs = File.readlines(file_path).each_with_object([]) do |line, arr|
-      preprocess_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
-      arr.push(preprocess_log)
-    end
-
-    send(CONVERT_RECORD[type.nil? ? 'nginx' : type], arr_logs)
-  end
-
-  def self.data_cleaning(logs)
-    logs.select { |record| record.status_200? && record.method_get? && record.uri_without_data && !record.robot? }
-  end
-
-  def self.to_records(logs)
-    logs.each_with_object([]).with_index do |(log, arrays), i|
-      next if log.nil?
-
-      o = log.split(REGEX_KEYS)
-      o = o.map(&:strip)
-      o.delete('')
+    @users = []
 
-      log = o.each_slice(2).to_a.each_with_object({}) do |pair, log_obj|
-        log_obj.merge!(to_json(pair))
-      end
+    File.readlines(file_path).each_with_object([]).with_index do |(line, arr), i|
+      preprocessed_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
+      record           = Record.new(send(CONVERT_RECORD[type.nil? ? 'nginx' : type], preprocessed_log)) unless preprocessed_log.nil?
 
-      arrays << Record.new(log)
+      arr.push(record) if record.status_200? && record.method_get? && record.uri_without_data && !record.robot?
 
-      puts "#{i}/#{logs.size}"
+      puts arr.size
     end
   end
 
-  def self.convert_nginx_logs(logs)
-    logs.each_with_object([]).with_index do |(log, arrays), i|
-      next if log.nil?
-
-      o = log.split(REGEX_NGINX)
-      o.delete('')
-
-      obj = {}.tap do |p|
-        p['host']      = o[0]
-        p['user']      = o[2]
-        p['time']      = o[3]
-        p['method']    = o[4]
-        p['uri']       = o[5]
-        p['status']    = o[6]
-        p['size']      = o[7]
-        p['referer']   = o[8]
-        p['ua']        = o[9]
-        p['forwarded'] = o[10]
-      end
-
-      arrays << Record.new(obj)
+  def self.to_record(log)
+    o = log.gsub!('\t', ' ')
+    o = log.split(REGEX_KEYS)
+    o = o.map(&:strip)
+    o.delete('')
+    o.each_slice(2).to_a.each_with_object({}) { |pair, log_obj| log_obj.merge!(to_json(pair)) }
+  end
 
-      puts "#{i}/#{logs.size}"
+  def self.convert_nginx_logs(log)
+    o = log.split(REGEX_NGINX)
+    o.delete('')
+
+    {}.tap do |p|
+      p['host']      = o[0]
+      p['user']      = o[2]
+      p['time']      = o[3]
+      p['method']    = o[4]
+      p['uri']       = o[5]
+      p['status']    = o[6]
+      p['size']      = o[7]
+      p['referer']   = o[8]
+      p['ua']        = o[9]
+      p['forwarded'] = o[10]
+      p['user']      = save_user(o)
     end
   end
 
   def self.to_json(pair)
     { pair.first.delete(':') => pair.last }
   end
+
+  def self.save_user(log)
+    user = @users.find { |i| i.host == log[0] && i.user_agent.to_s == log[9] }
+    return user unless user.nil?
+
+    @users.push(UserIdentity.new(host: IPAddr.new(log[0]), user_agent: UserAgent.parse(log[9])))
+    @users.last
+  end
 end

data/lib/log_analysis/rule_generation.rb

@@ -0,0 +1,59 @@
+require 'time'
+require 'log_analysis/version'
+
+module RuleGeneration
+  JAR_FILE_PATH       = File.expand_path('spmf.jar')
+  TRANSFORM_DATA_PATH = File.expand_path("#{LogAnalysis::DATA_PATH}transform_data_#{Time.now.strftime('%Y%m%d')}.txt")
+  RULE_FILE_PATH      = File.expand_path("#{LogAnalysis::DATA_PATH}output_#{Time.now.strftime('%Y%m%d')}.txt")
+  MAP_URI_FILE_PATH   = File.expand_path("#{LogAnalysis::DATA_PATH}map_uri_#{Time.now.strftime('%Y%m%d')}.txt")
+
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(transform_data)
+    File.open(TRANSFORM_DATA_PATH, 'w+') { |f| transform_data.keys.each { |e| f.puts(transform_data[e].map { |i| i.is_a?(Array) ? i.join(' ') : i }.join(' -1 ').concat(' -1 -2')) } }
+    system("java -jar #{JAR_FILE_PATH} run SPADE #{TRANSFORM_DATA_PATH} #{RULE_FILE_PATH} 65%")
+    result = rule_gen(get_seq(File.read(RULE_FILE_PATH)), 0.5)
+    map_uri = File.read(MAP_URI_FILE_PATH).split(' ')
+
+    result.map do |rule|
+      seq, sub, rea = rule
+      [seq.map { |i| map_uri[i.to_i] }, sub.map { |i| map_uri[i.to_i] }, rea]
+    end
+  end
+
+  def self.rule_gen(seqs, min_conf)
+    seqs.each_with_object([]) { |seq, arr| seqs.each { |sub| arr.push([seq[0], sub[0], seq[1] / sub[1]]) if sub[0] != seq[0] && sub_seq?(sub[0], seq[0]) && seq[1] / sub[1] >= min_conf } }
+  end
+
+  def self.sub_seq?(first, second)
+    ptr = 0
+    first.each do |sub|
+      return false if ptr >= second.size
+
+      (ptr..second.size - 1).each do |n|
+        if sub?(second[n], sub)
+          ptr = n + 1
+          break
+        end
+        return false if ptr == second.size - 1
+      end
+    end
+    true
+  end
+
+  def self.sub?(str, sub)
+    mark_sub = 0
+    sub.split(' ').each { |char| mark_sub += 1 if str.include?(char) }
+
+    mark_sub == sub.split(' ').size
+  end
+
+  def self.get_seq(seq_str)
+    seq = seq_str.split("\n")
+    seq.each_with_object([]) do |s, arr|
+      split_seq = s.split('-1')
+      arr.push([split_seq[0..-2], split_seq[-1][-1].to_f])
+    end
+  end
+end

data/lib/log_analysis/session_identification.rb

@@ -8,18 +8,10 @@ module SessionIdentification
   # Your code goes here...
 
   def self.execute(cleaned_data)
-    session_identity = []
-    cleaned_data.each do |record|
-      isession = session_identity.rindex { |s| s.user.host == record.user.host && s.user.user_agent == record.user.user_agent }
-
-      if isession.present? && validate_time_session(session_identity[isession].records.last.time, record.time)
-        session_identity[isession].records << record
-      else
-        session_identity << SessionIdentity.new(session_identity_params(record))
-      end
+    cleaned_data.each_with_object([]) do |record, arr|
+      isession = arr.rindex { |s| s.user == record.user }
+      isession.present? && validate_time_session(arr[isession].records.last.time, record.time) ? arr[isession].records << record : arr << SessionIdentity.new(session_identity_params(record))
     end
-
-    session_identity.map { |i| i.records.map(&:uri) }
   end
 
   private

data/lib/log_analysis/transformation.rb

@@ -0,0 +1,26 @@
+require 'log_analysis/model/session_identity'
+require 'log_analysis/model/user_identity'
+require 'log_analysis/version'
+
+module Transformation
+  MAP_URI_FILE_PATH = File.expand_path("#{LogAnalysis::DATA_PATH}map_uri_#{Time.now.strftime('%Y%m%d')}.txt")
+
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(identified_session)
+    map_uri = []
+    transform = identified_session.each_with_object({}) do |v, hash|
+      uries = v.records.map(&:uri)
+      uries.each { |i| map_uri.push(i) unless map_uri.include?(i) }
+      if hash.key?(v.user.host.to_s)
+        uries.size == 1 ? hash[v.user.host.to_s] += v.records.map { |i| map_uri.index(i.uri) } : hash[v.user.host.to_s].push(v.records.map { |i| map_uri.index(i.uri) })
+      else
+        hash.merge!(v.user.host.to_s => v.records.map { |i| map_uri.index(i.uri) })
+      end
+    end
+
+    File.open(MAP_URI_FILE_PATH, 'w+') { |f| f.write(map_uri.join(' ')) }
+    transform
+  end
+end

data/lib/log_analysis/user_identification.rb

@@ -6,10 +6,6 @@ module UserIdentification
   # Your code goes here...
 
   def self.execute(cleaned_data)
-    data = cleaned_data.map { |record| [record.host, record.ua.to_s] }.uniq
-    data.each_with_object([]) do |record, arrs|
-      o = UserIdentity.new(host: record.first, user_agent: record.last)
-      arrs << o
-    end
+    cleaned_data.map(&:user).uniq
   end
 end

data/lib/log_analysis/version.rb

@@ -1,3 +1,4 @@
 class LogAnalysis
-  VERSION = '0.1.0'.freeze
+  VERSION   = '0.1.1'.freeze
+  DATA_PATH = '~/data/waazabag/'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_analysis
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Michael Tran
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-27 00:00:00.000000000 Z
+date: 2020-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: useragent
@@ -53,6 +53,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- access.log
 - bin/console
 - bin/setup
 - lib/log_analysis.rb
@@ -60,10 +61,13 @@ files:
 - lib/log_analysis/model/session_identity.rb
 - lib/log_analysis/model/user_identity.rb
 - lib/log_analysis/preprocess.rb
+- lib/log_analysis/rule_generation.rb
 - lib/log_analysis/session_identification.rb
+- lib/log_analysis/transformation.rb
 - lib/log_analysis/user_identification.rb
 - lib/log_analysis/version.rb
 - log_analysis.gemspec
+- spmf.jar
 homepage: https://github.com/michaelt0520/log_analysis_thesis
 licenses:
 - MIT