log_analysis 0.1.0 → 0.1.5

data/lib/log_analysis.rb

@@ -1,29 +1,48 @@
 require 'log_analysis/version'
-require 'log_analysis/preprocess'
+require 'log_analysis/loading_data'
 require 'log_analysis/user_identification'
 require 'log_analysis/session_identification'
+require 'log_analysis/transformation'
+require 'log_analysis/data_mining'
+require 'log_analysis/intepretation'
+require 'time'
 
 class LogAnalysis
   class Error < StandardError; end
   # Your code goes here...
 
-  attr_reader :path, :origin_logs
+  attr_accessor :path, :type, :match_uri, :conf, :sup, :origin_data
 
   def initialize(path, type = nil)
-    @path        = path
-    @type        = type
-    @origin_logs = PreProcess.input(path, type)
+    @path         = path
+    @type         = type
+    @origin_data  = LoadingData.input(path, type)
   end
 
-  def cleaned_data
-    PreProcess.data_cleaning(@origin_logs)
+  def selecting_data
+    return @origin_data if @match_uri.nil?
+
+    @origin_data.select { |record| record.uri.match?(@match_uri) }
+  end
+
+  def preprocessing_data
+    filter  = selecting_data.select { |record| record.status_200? && record.method_get? && record.uri_without_data && !record.robot? }
+    user    = UserIdentification.execute(filter)
+    session = SessionIdentification.execute(user)
+    session
+  end
+
+  def transformation
+    Transformation.execute(preprocessing_data)
   end
 
-  def identified_user
-    UserIdentification.execute(cleaned_data)
+  def data_mining
+    @conf ||= 0.5
+    @sup  ||= 60
+    DataMining.execute(transformation, @conf, @sup)
   end
 
-  def identified_session
-    SessionIdentification.execute(cleaned_data)
+  def intepretation
+    Intepretation.execute(data_mining)
   end
 end

data/lib/log_analysis/data_mining.rb

@@ -0,0 +1,48 @@
+require 'time'
+require 'log_analysis/version'
+
+module DataMining
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(transform_data, min_conf, min_sup)
+    File.open(LogAnalysis::TRANSFORM_DATA_PATH, 'w+') { |f| transform_data.keys.each { |e| f.puts(transform_data[e].map { |i| i.is_a?(Array) ? i.join(' ') : i }.join(' -1 ').concat(' -1 -2')) } }
+    system("java -jar #{LogAnalysis::JAR_FILE_PATH} run SPADE #{LogAnalysis::TRANSFORM_DATA_PATH} #{LogAnalysis::RULE_FILE_PATH} #{min_sup}%")
+    rule_gen(get_seq(File.read(LogAnalysis::RULE_FILE_PATH)), min_conf)
+  end
+
+  def self.rule_gen(seqs, min_conf)
+    seqs.each_with_object([]) { |seq, arr| seqs.each { |sub| arr.push([seq[0], sub[0], seq[1] / sub[1]]) if sub[0] != seq[0] && sub_seq?(sub[0], seq[0]) && seq[1] / sub[1] >= min_conf } }
+  end
+
+  def self.sub_seq?(first, second)
+    ptr = 0
+    first.each do |sub|
+      return false if ptr >= second.size
+
+      (ptr..second.size - 1).each do |n|
+        if sub?(second[n], sub)
+          ptr = n + 1
+          break
+        end
+        return false if ptr == second.size - 1
+      end
+    end
+    true
+  end
+
+  def self.sub?(str, sub)
+    mark_sub = 0
+    sub.split(' ').each { |char| mark_sub += 1 if str.include?(char) }
+
+    mark_sub == sub.split(' ').size
+  end
+
+  def self.get_seq(seq_str)
+    seq = seq_str.split("\n")
+    seq.each_with_object([]) do |s, arr|
+      split_seq = s.split('-1')
+      arr.push([split_seq[0..-2], split_seq[-1][-1].to_f])
+    end
+  end
+end

data/lib/log_analysis/intepretation.rb

@@ -0,0 +1,22 @@
+require 'log_analysis/version'
+
+module Intepretation
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(data_mining)
+    map_uri = File.read(LogAnalysis::MAP_URI_FILE_PATH).split(' ')
+    move_data
+
+    data_mining.map do |data|
+      seq, sub, rea = data
+      [seq.map { |i| map_uri[i.to_i] }, sub.map { |i| map_uri[i.to_i] }, rea]
+    end
+  end
+
+  def self.move_data
+    return unless File.directory?(LogAnalysis::DATA_PATH)
+
+    system('mv', "*_#{Time.now.strftime('%Y%m%d')}.txt", LogAnalysis::DATA_PATH)
+  end
+end

data/lib/log_analysis/loading_data.rb

@@ -0,0 +1,70 @@
+require 'log_analysis/model/record'
+require 'log_analysis/model/user_identity'
+require 'json'
+
+module LoadingData
+  class Error < StandardError; end
+  # Your code goes here...
+
+  REGEX_KEYS   = /(time:| host:| status:| size:| request_length:| req:| method:| uri:| referer:| ua:| reqtime:| runtime:| apptime:| cache:| vhost:| server:| user:| forwardedfor:| forwardedproto:)/.freeze
+  REGEX_NGINX  = /\A^(?<host>\S*) (?<identity>\S*) (?<user>\S*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?:\s+\S*)?)?" (?<code>\S*) (?<size>\S*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>\S+))?)?$/.freeze
+  REGEX_APACHE = %r{(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - (.{0})- \[([^\]]+?)\] "(GET|POST|PUT|DELETE) ([^\s]+?) (HTTP\/1\.1)" (\d+) (\d+) "-" "(.*)"}.freeze
+
+  CONVERT_RECORD = { 'nginx' => 'convert_nginx_logs', 'apache' => 'convert_apache_logs', 'default' => 'to_records' }.freeze
+
+  def self.input(file_path, type)
+    @users    = []
+    text_file = File.readlines(file_path)
+
+    text_file.each_with_object([]).with_index do |(line, arr), i|
+      preprocessed_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
+      record_params    = send(CONVERT_RECORD[type.nil? ? 'nginx' : type], preprocessed_log)
+      record           = Record.new(record_params) if record_params && preprocessed_log
+
+      system('clear')
+      puts "#{((i.to_f / text_file.size) * 100).round}/100"
+      arr.push(record) if record
+    end
+  end
+
+  def self.to_record(log)
+    o = log.gsub!('\t', ' ')
+    o = log.split(REGEX_KEYS)
+    o = o.map(&:strip)
+    o.delete('')
+    o.each_slice(2).to_a.each_with_object({}) { |pair, log_obj| log_obj.merge!(to_json(pair)) }
+  end
+
+  def self.convert_nginx_logs(log)
+    o = log.split(REGEX_NGINX)
+
+    return false if o.size <= 1
+    o.delete('')
+
+    {}.tap do |p|
+      p['host']      = o[0]
+      p['user']      = o[2]
+      p['time']      = o[3]
+      p['method']    = o[4]
+      p['uri']       = o[5]
+      p['status']    = o[6]
+      p['size']      = o[7]
+      p['referer']   = o[8]
+      p['ua']        = o[9]
+      p['forwarded'] = o[10]
+      p['user']      = save_user(o)
+    end
+  end
+
+  def self.to_json(pair)
+    { pair.first.delete(':') => pair.last }
+  end
+
+  def self.save_user(log)
+    user = @users.find { |i| i.host == log[0] && i.user_agent.to_s == log[9] }
+    return user unless user.nil?
+
+    @users.push(UserIdentity.new(host: IPAddr.new(log[0]), user_agent: UserAgent.parse(log[9])))
+    @users.last
+  end
+end

data/lib/log_analysis/model/record.rb

@@ -25,7 +25,7 @@ class Record
 
   attr_reader   :params
 
-  DATA_TYPE = %w[.txt .json .js .css .jpg .jpeg .gif .woff2 .ico .png .bmp .mp3 .wav .avi .mpeg .vmw .mpg .map .pdf .doc .svg].freeze
+  DATA_TYPE = %w[.txt .json .js .css .jpg .jpeg .gif .woff2 .ico .png .bmp .mp3 .wav .avi .mpeg .vmw .mpg .map .pdf .doc .svg .otf].freeze
   REGEX_BOT = /facebookexternalhit|Mediapartners-Google|AWS|-|Crawler|spider|Detection/.freeze
 
   def initialize(params)
@@ -70,7 +70,7 @@ class Record
       p['apptime']        = @params['apptime'].to_f
       p['cache']          = validate_string(@params['cache'])
       p['vhost']          = validate_string(@params['vhost'])
-      p['user']           = UserIdentity.new(host: IPAddr.new(@params['host']), user_agent: UserAgent.parse(@params['ua']))
+      p['user']           = @params['user'] || nil
       p['forwardedfor']   = validate_string(@params['forwardedfor'])
       p['forwardedproto'] = validate_string(@params['forwardedproto'])
     end

data/lib/log_analysis/model/user_identity.rb

@@ -1,13 +1,15 @@
 require 'active_support/core_ext/module/delegation'
+require 'log_analysis/model/record'
 require 'useragent'
 
 class UserIdentity
-  attr_accessor :host, :user_agent
+  attr_accessor :host, :user_agent, :records
 
   delegate :browser, :version, :os, :platform, :mobile?, :application, :localization, to: :user_agent
 
   def initialize(params)
     @host       = params[:host]
     @user_agent = params[:user_agent]
+    @records    = params[:records]
   end
 end

data/lib/log_analysis/session_identification.rb

@@ -7,19 +7,13 @@ module SessionIdentification
   class Error < StandardError; end
   # Your code goes here...
 
-  def self.execute(cleaned_data)
-    session_identity = []
-    cleaned_data.each do |record|
-      isession = session_identity.rindex { |s| s.user.host == record.user.host && s.user.user_agent == record.user.user_agent }
-
-      if isession.present? && validate_time_session(session_identity[isession].records.last.time, record.time)
-        session_identity[isession].records << record
-      else
-        session_identity << SessionIdentity.new(session_identity_params(record))
+  def self.execute(identified_user)
+    identified_user.each_with_object([]) do |user, arr|
+      user.records.each do |record|
+        isession = arr.rindex { |s| s.user == user }
+        isession.present? && validate_time_session(arr[isession].records.last.time, record.time) ? arr[isession].records << record : arr << SessionIdentity.new(session_identity_params(record))
       end
     end
-
-    session_identity.map { |i| i.records.map(&:uri) }
   end
 
   private

data/lib/log_analysis/transformation.rb

@@ -0,0 +1,24 @@
+require 'log_analysis/model/session_identity'
+require 'log_analysis/model/user_identity'
+require 'log_analysis/version'
+
+module Transformation
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(identified_session)
+    map_uri = []
+    transform = identified_session.each_with_object({}) do |v, hash|
+      uries = v.records.map(&:uri)
+      uries.each { |i| map_uri.push(i) unless map_uri.include?(i) }
+      if hash.key?(v.user.host.to_s)
+        uries.size == 1 ? hash[v.user.host.to_s] += v.records.map { |i| map_uri.index(i.uri) } : hash[v.user.host.to_s].push(v.records.map { |i| map_uri.index(i.uri) })
+      else
+        hash.merge!(v.user.host.to_s => v.records.map { |i| map_uri.index(i.uri) })
+      end
+    end
+
+    File.open(LogAnalysis::MAP_URI_FILE_PATH, 'w+') { |f| f.write(map_uri.join(' ')) }
+    transform
+  end
+end

data/lib/log_analysis/user_identification.rb

@@ -6,10 +6,14 @@ module UserIdentification
   # Your code goes here...
 
   def self.execute(cleaned_data)
-    data = cleaned_data.map { |record| [record.host, record.ua.to_s] }.uniq
-    data.each_with_object([]) do |record, arrs|
-      o = UserIdentity.new(host: record.first, user_agent: record.last)
-      arrs << o
+    cleaned_data.each_with_object([]) do |record, arr|
+      user = arr.detect { |i| i == record.user }
+      if user
+        user.records.push(record)
+      else
+        record.user.records = [record]
+        arr << record.user
+      end
     end
   end
 end

data/lib/log_analysis/version.rb

@@ -1,3 +1,8 @@
 class LogAnalysis
-  VERSION = '0.1.0'.freeze
+  VERSION             = '0.1.5'.freeze
+  TRANSFORM_DATA_PATH = "transform_data_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  RULE_FILE_PATH      = "output_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  MAP_URI_FILE_PATH   = "map_uri_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  JAR_FILE_PATH       = File.join(File.dirname(__FILE__), './files/spmf.jar')
+  DATA_PATH           = File.expand_path('data/log_analysis', '~')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_analysis
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Michael Tran
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-27 00:00:00.000000000 Z
+date: 2020-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: useragent
@@ -53,14 +53,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- access.log
 - bin/console
 - bin/setup
 - lib/log_analysis.rb
+- lib/log_analysis/data_mining.rb
+- lib/log_analysis/files/spmf.jar
+- lib/log_analysis/intepretation.rb
+- lib/log_analysis/loading_data.rb
 - lib/log_analysis/model/record.rb
 - lib/log_analysis/model/session_identity.rb
 - lib/log_analysis/model/user_identity.rb
-- lib/log_analysis/preprocess.rb
 - lib/log_analysis/session_identification.rb
+- lib/log_analysis/transformation.rb
 - lib/log_analysis/user_identification.rb
 - lib/log_analysis/version.rb
 - log_analysis.gemspec
@@ -70,7 +75,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/michaelt0520/log_analysis_thesis
   source_code_uri: https://github.com/michaelt0520/log_analysis_thesis
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -85,8 +90,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Log Analysis for thesis Huflit
 test_files: []

data/lib/log_analysis/preprocess.rb

@@ -1,75 +0,0 @@
-require 'log_analysis/model/record'
-require 'json'
-require 'pry'
-
-module PreProcess
-  class Error < StandardError; end
-  # Your code goes here...
-
-  REGEX_KEYS   = /(time:| host:| status:| size:| request_length:| req:| method:| uri:| referer:| ua:| reqtime:| runtime:| apptime:| cache:| vhost:| server:| user:| forwardedfor:| forwardedproto:)/.freeze
-  REGEX_NGINX  = /\A^(?<host>\S*) (?<identity>\S*) (?<user>\S*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?:\s+\S*)?)?" (?<code>\S*) (?<size>\S*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>\S+))?)?$/.freeze
-  REGEX_APACHE = %r{(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - (.{0})- \[([^\]]+?)\] "(GET|POST|PUT|DELETE) ([^\s]+?) (HTTP\/1\.1)" (\d+) (\d+) "-" "(.*)"}.freeze
-
-  CONVERT_RECORD = { 'nginx' => 'convert_nginx_logs', 'apache' => 'convert_apache_logs', 'default' => 'to_records' }.freeze
-
-  def self.input(file_path, type)
-    arr_logs = File.readlines(file_path).each_with_object([]) do |line, arr|
-      preprocess_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
-      arr.push(preprocess_log)
-    end
-
-    send(CONVERT_RECORD[type.nil? ? 'nginx' : type], arr_logs)
-  end
-
-  def self.data_cleaning(logs)
-    logs.select { |record| record.status_200? && record.method_get? && record.uri_without_data && !record.robot? }
-  end
-
-  def self.to_records(logs)
-    logs.each_with_object([]).with_index do |(log, arrays), i|
-      next if log.nil?
-
-      o = log.split(REGEX_KEYS)
-      o = o.map(&:strip)
-      o.delete('')
-
-      log = o.each_slice(2).to_a.each_with_object({}) do |pair, log_obj|
-        log_obj.merge!(to_json(pair))
-      end
-
-      arrays << Record.new(log)
-
-      puts "#{i}/#{logs.size}"
-    end
-  end
-
-  def self.convert_nginx_logs(logs)
-    logs.each_with_object([]).with_index do |(log, arrays), i|
-      next if log.nil?
-
-      o = log.split(REGEX_NGINX)
-      o.delete('')
-
-      obj = {}.tap do |p|
-        p['host']      = o[0]
-        p['user']      = o[2]
-        p['time']      = o[3]
-        p['method']    = o[4]
-        p['uri']       = o[5]
-        p['status']    = o[6]
-        p['size']      = o[7]
-        p['referer']   = o[8]
-        p['ua']        = o[9]
-        p['forwarded'] = o[10]
-      end
-
-      arrays << Record.new(obj)
-
-      puts "#{i}/#{logs.size}"
-    end
-  end
-
-  def self.to_json(pair)
-    { pair.first.delete(':') => pair.last }
-  end
-end