log_analysis 0.1.1 → 0.1.6

data/lib/log_analysis.rb

@@ -1,36 +1,54 @@
 require 'log_analysis/version'
-require 'log_analysis/preprocess'
+require 'log_analysis/loading_data'
 require 'log_analysis/user_identification'
 require 'log_analysis/session_identification'
 require 'log_analysis/transformation'
-require 'log_analysis/rule_generation'
+require 'log_analysis/data_mining'
+require 'log_analysis/intepretation'
+require 'time'
 
 class LogAnalysis
   class Error < StandardError; end
   # Your code goes here...
 
-  attr_reader :path, :type, :cleaned_data
+  attr_accessor :path, :type, :match_uri, :conf, :sup, :origin_data
 
   def initialize(path, type = nil)
     @path         = path
     @type         = type
-    @cleaned_data = PreProcess.input(path, type)
-    system('mkdir', '-p', LogAnalysis::DATA_PATH)
+    @origin_data  = LoadingData.input(path, type)
   end
 
-  def identified_user
-    UserIdentification.execute(@cleaned_data)
+  def selecting_data
+    return @origin_data if @match_uri.nil?
+
+    @origin_data.select { |record| record.uri.match?(@match_uri) }
   end
 
-  def identified_session
-    SessionIdentification.execute(@cleaned_data)
+  def preprocessing_data
+    data = selecting_data
+    return if data.nil? || data.empty?
+
+    filter  = data.select { |record| record.status_200? && record.method_get? && record.uri_without_data && !record.robot? }
+    user    = UserIdentification.execute(filter)
+    session = SessionIdentification.execute(user)
+    session
   end
 
   def transformation
-    Transformation.execute(identified_session)
+    data = preprocessing_data
+    Transformation.execute(data) unless data.nil? || data.empty?
+  end
+
+  def data_mining
+    data    = transformation
+    @conf ||= 0.5
+    @sup  ||= 60
+    DataMining.execute(data, @conf, @sup) unless data.nil? || data.empty?
   end
 
-  def rule_generation
-    RuleGeneration.execute(transformation)
+  def intepretation
+    data = data_mining
+    Intepretation.execute(data) unless data.nil? || data.empty?
   end
 end

data/lib/log_analysis/{rule_generation.rb → data_mining.rb}

@@ -1,25 +1,14 @@
 require 'time'
 require 'log_analysis/version'
 
-module RuleGeneration
-  JAR_FILE_PATH       = File.expand_path('spmf.jar')
-  TRANSFORM_DATA_PATH = File.expand_path("#{LogAnalysis::DATA_PATH}transform_data_#{Time.now.strftime('%Y%m%d')}.txt")
-  RULE_FILE_PATH      = File.expand_path("#{LogAnalysis::DATA_PATH}output_#{Time.now.strftime('%Y%m%d')}.txt")
-  MAP_URI_FILE_PATH   = File.expand_path("#{LogAnalysis::DATA_PATH}map_uri_#{Time.now.strftime('%Y%m%d')}.txt")
-
+module DataMining
   class Error < StandardError; end
   # Your code goes here...
 
-  def self.execute(transform_data)
-    File.open(TRANSFORM_DATA_PATH, 'w+') { |f| transform_data.keys.each { |e| f.puts(transform_data[e].map { |i| i.is_a?(Array) ? i.join(' ') : i }.join(' -1 ').concat(' -1 -2')) } }
-    system("java -jar #{JAR_FILE_PATH} run SPADE #{TRANSFORM_DATA_PATH} #{RULE_FILE_PATH} 65%")
-    result = rule_gen(get_seq(File.read(RULE_FILE_PATH)), 0.5)
-    map_uri = File.read(MAP_URI_FILE_PATH).split(' ')
-
-    result.map do |rule|
-      seq, sub, rea = rule
-      [seq.map { |i| map_uri[i.to_i] }, sub.map { |i| map_uri[i.to_i] }, rea]
-    end
+  def self.execute(transform_data, min_conf, min_sup)
+    File.open(LogAnalysis::TRANSFORM_DATA_PATH, 'w+') { |f| transform_data.keys.each { |e| f.puts(transform_data[e].map { |i| i.is_a?(Array) ? i.join(' ') : i }.join(' -1 ').concat(' -1 -2')) } }
+    system("java -jar #{LogAnalysis::JAR_FILE_PATH} run SPADE #{LogAnalysis::TRANSFORM_DATA_PATH} #{LogAnalysis::RULE_FILE_PATH} #{min_sup}%")
+    rule_gen(get_seq(File.read(LogAnalysis::RULE_FILE_PATH)), min_conf)
   end
 
   def self.rule_gen(seqs, min_conf)

data/lib/log_analysis/intepretation.rb

@@ -0,0 +1,22 @@
+require 'log_analysis/version'
+
+module Intepretation
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(data_mining)
+    map_uri = File.read(LogAnalysis::MAP_URI_FILE_PATH).split(' ')
+    move_data
+
+    data_mining.map do |data|
+      seq, sub, rea = data
+      [seq.map { |i| map_uri[i.to_i] }, sub.map { |i| map_uri[i.to_i] }, rea]
+    end
+  end
+
+  def self.move_data
+    return unless File.directory?(LogAnalysis::DATA_PATH)
+
+    system('mv', "*_#{Time.now.strftime('%Y%m%d')}.txt", LogAnalysis::DATA_PATH)
+  end
+end

data/lib/log_analysis/{preprocess.rb → loading_data.rb}

@@ -2,7 +2,7 @@ require 'log_analysis/model/record'
 require 'log_analysis/model/user_identity'
 require 'json'
 
-module PreProcess
+module LoadingData
   class Error < StandardError; end
   # Your code goes here...
 
@@ -13,15 +13,17 @@ module PreProcess
   CONVERT_RECORD = { 'nginx' => 'convert_nginx_logs', 'apache' => 'convert_apache_logs', 'default' => 'to_records' }.freeze
 
   def self.input(file_path, type)
-    @users = []
+    @users    = []
+    text_file = File.readlines(file_path)
 
-    File.readlines(file_path).each_with_object([]).with_index do |(line, arr), i|
+    text_file.each_with_object([]).with_index do |(line, arr), i|
       preprocessed_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
-      record           = Record.new(send(CONVERT_RECORD[type.nil? ? 'nginx' : type], preprocessed_log)) unless preprocessed_log.nil?
+      record_params    = send(CONVERT_RECORD[type.nil? ? 'nginx' : type], preprocessed_log)
+      record           = Record.new(record_params) if record_params && preprocessed_log
 
-      arr.push(record) if record.status_200? && record.method_get? && record.uri_without_data && !record.robot?
-
-      puts arr.size
+      system('clear')
+      puts "#{((i.to_f / text_file.size) * 100).round}/100"
+      arr.push(record) if record
     end
   end
 
@@ -35,6 +37,8 @@ module PreProcess
 
   def self.convert_nginx_logs(log)
     o = log.split(REGEX_NGINX)
+
+    return false if o.size <= 1
     o.delete('')
 
     {}.tap do |p|

data/lib/log_analysis/model/user_identity.rb

@@ -1,13 +1,15 @@
 require 'active_support/core_ext/module/delegation'
+require 'log_analysis/model/record'
 require 'useragent'
 
 class UserIdentity
-  attr_accessor :host, :user_agent
+  attr_accessor :host, :user_agent, :records
 
   delegate :browser, :version, :os, :platform, :mobile?, :application, :localization, to: :user_agent
 
   def initialize(params)
     @host       = params[:host]
     @user_agent = params[:user_agent]
+    @records    = params[:records]
   end
 end

data/lib/log_analysis/session_identification.rb

@@ -7,10 +7,12 @@ module SessionIdentification
   class Error < StandardError; end
   # Your code goes here...
 
-  def self.execute(cleaned_data)
-    cleaned_data.each_with_object([]) do |record, arr|
-      isession = arr.rindex { |s| s.user == record.user }
-      isession.present? && validate_time_session(arr[isession].records.last.time, record.time) ? arr[isession].records << record : arr << SessionIdentity.new(session_identity_params(record))
+  def self.execute(identified_user)
+    identified_user.each_with_object([]) do |user, arr|
+      isession = arr.rindex { |s| s.user == user }
+      user.records.each do |record|
+        isession.present? && validate_time_session(arr[isession].records.last.time, record.time) ? arr[isession].records << record : arr << SessionIdentity.new(session_identity_params(record))
+      end
     end
   end
 

data/lib/log_analysis/transformation.rb

@@ -3,13 +3,11 @@ require 'log_analysis/model/user_identity'
 require 'log_analysis/version'
 
 module Transformation
-  MAP_URI_FILE_PATH = File.expand_path("#{LogAnalysis::DATA_PATH}map_uri_#{Time.now.strftime('%Y%m%d')}.txt")
-
   class Error < StandardError; end
   # Your code goes here...
 
   def self.execute(identified_session)
-    map_uri = []
+    map_uri = ['-']
     transform = identified_session.each_with_object({}) do |v, hash|
       uries = v.records.map(&:uri)
       uries.each { |i| map_uri.push(i) unless map_uri.include?(i) }
@@ -20,7 +18,7 @@ module Transformation
       end
     end
 
-    File.open(MAP_URI_FILE_PATH, 'w+') { |f| f.write(map_uri.join(' ')) }
+    File.open(LogAnalysis::MAP_URI_FILE_PATH, 'w+') { |f| f.write(map_uri.join(' ')) }
     transform
   end
 end

data/lib/log_analysis/user_identification.rb

@@ -6,6 +6,14 @@ module UserIdentification
   # Your code goes here...
 
   def self.execute(cleaned_data)
-    cleaned_data.map(&:user).uniq
+    cleaned_data.each_with_object([]) do |record, arr|
+      user = arr.detect { |i| i == record.user }
+      if user
+        user.records.push(record)
+      else
+        record.user.records = [record]
+        arr << record.user
+      end
+    end
   end
 end

data/lib/log_analysis/version.rb

@@ -1,4 +1,8 @@
 class LogAnalysis
-  VERSION   = '0.1.1'.freeze
-  DATA_PATH = '~/data/waazabag/'.freeze
+  VERSION             = '0.1.6'.freeze
+  TRANSFORM_DATA_PATH = "transform_data_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  RULE_FILE_PATH      = "output_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  MAP_URI_FILE_PATH   = "map_uri_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  JAR_FILE_PATH       = File.join(File.dirname(__FILE__), './files/spmf.jar')
+  DATA_PATH           = File.expand_path('data/log_analysis', '~')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_analysis
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.6
 platform: ruby
 authors:
 - Michael Tran
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-12 00:00:00.000000000 Z
+date: 2020-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: useragent
@@ -57,24 +57,25 @@ files:
 - bin/console
 - bin/setup
 - lib/log_analysis.rb
+- lib/log_analysis/data_mining.rb
+- lib/log_analysis/files/spmf.jar
+- lib/log_analysis/intepretation.rb
+- lib/log_analysis/loading_data.rb
 - lib/log_analysis/model/record.rb
 - lib/log_analysis/model/session_identity.rb
 - lib/log_analysis/model/user_identity.rb
-- lib/log_analysis/preprocess.rb
-- lib/log_analysis/rule_generation.rb
 - lib/log_analysis/session_identification.rb
 - lib/log_analysis/transformation.rb
 - lib/log_analysis/user_identification.rb
 - lib/log_analysis/version.rb
 - log_analysis.gemspec
-- spmf.jar
 homepage: https://github.com/michaelt0520/log_analysis_thesis
 licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/michaelt0520/log_analysis_thesis
   source_code_uri: https://github.com/michaelt0520/log_analysis_thesis
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -89,8 +90,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Log Analysis for thesis Huflit
 test_files: []