log_analysis 0.1.4 → 0.1.5

data/lib/log_analysis.rb

@@ -1,52 +1,48 @@
 require 'log_analysis/version'
-require 'log_analysis/preprocess'
+require 'log_analysis/loading_data'
 require 'log_analysis/user_identification'
 require 'log_analysis/session_identification'
 require 'log_analysis/transformation'
-require 'log_analysis/rule_generation'
+require 'log_analysis/data_mining'
+require 'log_analysis/intepretation'
 require 'time'
 
 class LogAnalysis
   class Error < StandardError; end
   # Your code goes here...
 
-  attr_reader :path, :type, :cleaned_data
+  attr_accessor :path, :type, :match_uri, :conf, :sup, :origin_data
 
   def initialize(path, type = nil)
     @path         = path
     @type         = type
-    @cleaned_data = PreProcess.input(path, type) do |record|
-      if block_given?
-        yield(record) ? record : nil
-      else
-        record
-      end
-    end
+    @origin_data  = LoadingData.input(path, type)
   end
 
-  def identified_user
-    UserIdentification.execute(@cleaned_data)
+  def selecting_data
+    return @origin_data if @match_uri.nil?
+
+    @origin_data.select { |record| record.uri.match?(@match_uri) }
   end
 
-  def identified_session
-    SessionIdentification.execute(@cleaned_data)
+  def preprocessing_data
+    filter  = selecting_data.select { |record| record.status_200? && record.method_get? && record.uri_without_data && !record.robot? }
+    user    = UserIdentification.execute(filter)
+    session = SessionIdentification.execute(user)
+    session
   end
 
   def transformation
-    Transformation.execute(identified_session)
+    Transformation.execute(preprocessing_data)
   end
 
-  def rule_generation(conf = 0.5, sup = 60)
-    result = RuleGeneration.execute(transformation, conf, sup)
-    move_data
-    result
+  def data_mining
+    @conf ||= 0.5
+    @sup  ||= 60
+    DataMining.execute(transformation, @conf, @sup)
   end
 
-  def move_data
-    return unless File.directory?('/home/app/data/waazabag/')
-
-    system('mv', "transform_data_#{Time.now.strftime('%Y%m%d')}.txt", '/home/app/data/waazabag/')
-    system('mv', "output_#{Time.now.strftime('%Y%m%d')}.txt", '/home/app/data/waazabag/')
-    system('mv', "map_uri_#{Time.now.strftime('%Y%m%d')}.txt", '/home/app/data/waazabag/')
+  def intepretation
+    Intepretation.execute(data_mining)
   end
 end

data/lib/log_analysis/{rule_generation.rb → data_mining.rb}

@@ -1,25 +1,14 @@
 require 'time'
 require 'log_analysis/version'
 
-module RuleGeneration
-  JAR_FILE_PATH       = File.join(File.dirname(__FILE__), './files/spmf.jar')
-  TRANSFORM_DATA_PATH = "transform_data_#{Time.now.strftime('%Y%m%d')}.txt".freeze
-  RULE_FILE_PATH      = "output_#{Time.now.strftime('%Y%m%d')}.txt".freeze
-  MAP_URI_FILE_PATH   = "map_uri_#{Time.now.strftime('%Y%m%d')}.txt".freeze
-
+module DataMining
   class Error < StandardError; end
   # Your code goes here...
 
   def self.execute(transform_data, min_conf, min_sup)
-    File.open(TRANSFORM_DATA_PATH, 'w+') { |f| transform_data.keys.each { |e| f.puts(transform_data[e].map { |i| i.is_a?(Array) ? i.join(' ') : i }.join(' -1 ').concat(' -1 -2')) } }
-    system("java -jar #{JAR_FILE_PATH} run SPADE #{TRANSFORM_DATA_PATH} #{RULE_FILE_PATH} #{min_sup}%")
-    result = rule_gen(get_seq(File.read(RULE_FILE_PATH)), min_conf)
-    map_uri = File.read(MAP_URI_FILE_PATH).split(' ')
-
-    result.map do |rule|
-      seq, sub, rea = rule
-      [seq.map { |i| map_uri[i.to_i] }, sub.map { |i| map_uri[i.to_i] }, rea]
-    end
+    File.open(LogAnalysis::TRANSFORM_DATA_PATH, 'w+') { |f| transform_data.keys.each { |e| f.puts(transform_data[e].map { |i| i.is_a?(Array) ? i.join(' ') : i }.join(' -1 ').concat(' -1 -2')) } }
+    system("java -jar #{LogAnalysis::JAR_FILE_PATH} run SPADE #{LogAnalysis::TRANSFORM_DATA_PATH} #{LogAnalysis::RULE_FILE_PATH} #{min_sup}%")
+    rule_gen(get_seq(File.read(LogAnalysis::RULE_FILE_PATH)), min_conf)
   end
 
   def self.rule_gen(seqs, min_conf)

data/lib/log_analysis/intepretation.rb

@@ -0,0 +1,22 @@
+require 'log_analysis/version'
+
+module Intepretation
+  class Error < StandardError; end
+  # Your code goes here...
+
+  def self.execute(data_mining)
+    map_uri = File.read(LogAnalysis::MAP_URI_FILE_PATH).split(' ')
+    move_data
+
+    data_mining.map do |data|
+      seq, sub, rea = data
+      [seq.map { |i| map_uri[i.to_i] }, sub.map { |i| map_uri[i.to_i] }, rea]
+    end
+  end
+
+  def self.move_data
+    return unless File.directory?(LogAnalysis::DATA_PATH)
+
+    system('mv', "*_#{Time.now.strftime('%Y%m%d')}.txt", LogAnalysis::DATA_PATH)
+  end
+end

data/lib/log_analysis/{preprocess.rb → loading_data.rb}

@@ -2,7 +2,7 @@ require 'log_analysis/model/record'
 require 'log_analysis/model/user_identity'
 require 'json'
 
-module PreProcess
+module LoadingData
   class Error < StandardError; end
   # Your code goes here...
 
@@ -20,12 +20,10 @@ module PreProcess
       preprocessed_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
       record_params    = send(CONVERT_RECORD[type.nil? ? 'nginx' : type], preprocessed_log)
       record           = Record.new(record_params) if record_params && preprocessed_log
-      record           = yield(record) if block_given? && record
-
-      arr.push(record) if record && record.status_200? && record.method_get? && record.uri_without_data && !record.robot?
 
       system('clear')
       puts "#{((i.to_f / text_file.size) * 100).round}/100"
+      arr.push(record) if record
     end
   end
 

data/lib/log_analysis/model/user_identity.rb

@@ -1,13 +1,15 @@
 require 'active_support/core_ext/module/delegation'
+require 'log_analysis/model/record'
 require 'useragent'
 
 class UserIdentity
-  attr_accessor :host, :user_agent
+  attr_accessor :host, :user_agent, :records
 
   delegate :browser, :version, :os, :platform, :mobile?, :application, :localization, to: :user_agent
 
   def initialize(params)
     @host       = params[:host]
     @user_agent = params[:user_agent]
+    @records    = params[:records]
   end
 end

data/lib/log_analysis/session_identification.rb

@@ -7,10 +7,12 @@ module SessionIdentification
   class Error < StandardError; end
   # Your code goes here...
 
-  def self.execute(cleaned_data)
-    cleaned_data.each_with_object([]) do |record, arr|
-      isession = arr.rindex { |s| s.user == record.user }
-      isession.present? && validate_time_session(arr[isession].records.last.time, record.time) ? arr[isession].records << record : arr << SessionIdentity.new(session_identity_params(record))
+  def self.execute(identified_user)
+    identified_user.each_with_object([]) do |user, arr|
+      user.records.each do |record|
+        isession = arr.rindex { |s| s.user == user }
+        isession.present? && validate_time_session(arr[isession].records.last.time, record.time) ? arr[isession].records << record : arr << SessionIdentity.new(session_identity_params(record))
+      end
     end
   end
 

data/lib/log_analysis/transformation.rb

@@ -3,8 +3,6 @@ require 'log_analysis/model/user_identity'
 require 'log_analysis/version'
 
 module Transformation
-  MAP_URI_FILE_PATH = "map_uri_#{Time.now.strftime('%Y%m%d')}.txt".freeze
-
   class Error < StandardError; end
   # Your code goes here...
 
@@ -20,7 +18,7 @@ module Transformation
       end
     end
 
-    File.open(MAP_URI_FILE_PATH, 'w+') { |f| f.write(map_uri.join(' ')) }
+    File.open(LogAnalysis::MAP_URI_FILE_PATH, 'w+') { |f| f.write(map_uri.join(' ')) }
     transform
   end
 end

data/lib/log_analysis/user_identification.rb

@@ -6,6 +6,14 @@ module UserIdentification
   # Your code goes here...
 
   def self.execute(cleaned_data)
-    cleaned_data.map(&:user).uniq
+    cleaned_data.each_with_object([]) do |record, arr|
+      user = arr.detect { |i| i == record.user }
+      if user
+        user.records.push(record)
+      else
+        record.user.records = [record]
+        arr << record.user
+      end
+    end
   end
 end

data/lib/log_analysis/version.rb

@@ -1,3 +1,8 @@
 class LogAnalysis
-  VERSION   = '0.1.4'
+  VERSION             = '0.1.5'.freeze
+  TRANSFORM_DATA_PATH = "transform_data_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  RULE_FILE_PATH      = "output_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  MAP_URI_FILE_PATH   = "map_uri_#{Time.now.strftime('%Y%m%d')}.txt".freeze
+  JAR_FILE_PATH       = File.join(File.dirname(__FILE__), './files/spmf.jar')
+  DATA_PATH           = File.expand_path('data/log_analysis', '~')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_analysis
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Michael Tran
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2020-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: useragent
@@ -57,12 +57,13 @@ files:
 - bin/console
 - bin/setup
 - lib/log_analysis.rb
+- lib/log_analysis/data_mining.rb
 - lib/log_analysis/files/spmf.jar
+- lib/log_analysis/intepretation.rb
+- lib/log_analysis/loading_data.rb
 - lib/log_analysis/model/record.rb
 - lib/log_analysis/model/session_identity.rb
 - lib/log_analysis/model/user_identity.rb
-- lib/log_analysis/preprocess.rb
-- lib/log_analysis/rule_generation.rb
 - lib/log_analysis/session_identification.rb
 - lib/log_analysis/transformation.rb
 - lib/log_analysis/user_identification.rb