pmux-logview 0.3.4 → 0.3.10

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 kakine
+Copyright (c) 2013, Internet Initiative Japan Inc.
 
 MIT License
 

data/{conf → config/pmux-logview}/config.ru

@@ -11,5 +11,6 @@ Pmux::LogView::Controller.setup({ "default_user" => "pmux",
                                   "log_dir_path" => "/var/log/pmux-logview",
                                   "log_level" => "info",
                                   "use_syslog" => true,
-                                  "syslog_facility" => "user" })
+                                  "syslog_facility" => "user",
+                                  "cache_expire_time" => 86400 })
 run Pmux::LogView::Controller

data/{conf → config/pmux-logview}/pmux-logview.conf

@@ -2,11 +2,12 @@
 host: 0.0.0.0
 port: 28080
 pidfile: /var/run/pmux-logview.pid
-default_user: pmux
-use_basic_auth: true
+default_user: admin
+use_basic_auth: false
 password_file_path: /etc/pmux-logview/password
 cache_dir_path: /var/tmp/pmux-logview
 log_dir_path: /var/log/pmux-logview
 log_level: info
-use_syslog: true 
+use_syslog: false
 syslog_facility: user
+cache_expire_time: 86400

data/lib/pmux-logview/application.rb

@@ -51,6 +51,10 @@ module Pmux
       end
 
       def run
+
+#require 'ruby-prof'
+#RubyProf.start
+
         initialize()
         parse_args()
         daemonize()
@@ -62,6 +66,11 @@ module Pmux
         @port = @config["port"] if @config["port"]
         Controller.setup(@config)
         Controller.run! :bind => @host, :host => @host, :port => @port
+
+#result = RubyProf.stop
+#printer = RubyProf::FlatPrinter.new(result)
+#printer.print(STDOUT)
+
       end
     end
   end

data/lib/pmux-logview/controller.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 require 'sinatra'
 require 'sinatra/base'
 require 'json'
@@ -19,6 +21,7 @@ module Pmux
       @@use_syslog = false
       @@use_basic_auth = true
       @@default_user = "pmux"
+      @@cache_expire_time = 86400
       @@model = nil
       @@logger = nil
       @user = nil
@@ -48,13 +51,15 @@ module Pmux
               @@syslog_facility = args[key]
           when "default_user"
               @@default_user = args[key]
+          when "cache_expire_time"
+              @@cache_expire_time = args[key]
           end
         }
         log_file_path = File.expand_path([@@log_dir_path, @@log_filename].join(File::SEPARATOR))
         @@logger.foreground(@@foreground)
         @@logger.open(log_file_path, @@log_level, @@use_syslog, @@syslog_facility)
         AuthHelper.update(@@password_file_path)
-        @@model = Model.new(@@cache_dir_path)
+        @@model = Model.new(@@cache_dir_path, @@cache_expire_time)
       end
 
       configure do
@@ -64,7 +69,7 @@ module Pmux
         @@logger.open(log_file_path, @@log_level, @@use_syslog, @@syslog_facility)
         use Rack::CommonLogger, @@logger
         AuthHelper.init(@@password_file_path)
-        @@model = Model.new(@@cache_dir_path)
+        @@model = Model.new(@@cache_dir_path, @@cache_expire_time)
       end
 
       def logger
@@ -92,12 +97,11 @@ module Pmux
         @@logger.info("access user #{@user}")
         data = {}
         validations = {
-            "sort_key" => { "default" => "start_time", "type" => String, "values" => [ "job_id", "mapper", "start_time", "end_time", "elapsed_time"] },
             "sort_order" => { "default" => "desc", "type" => String, "values" => [ "asc", "desc" ] },
-            "type" => { "default" => "archive", "type" => String, "values" => [ "archive", "update" ] },
             "nitems" => { "default" => 20, "type" => Integer, "values" => nil  },
             "page" => { "default" => 0, "type" => Integer, "values" => nil },
-            "jobs_cookie" => { "default" => 0, "type" => Integer, "values" => nil },
+            "start_time_msec" => { "default" => nil, "type" => Integer, "values" => nil },
+            "end_time_msec" => { "default" => nil, "type" => Integer, "values" => nil },
         }
         validations.each_key{|key|
           if params.key?(key)

data/lib/pmux-logview/log_parser.rb

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 require 'fileutils'
 require 'date'
 require 'yaml'
@@ -16,59 +18,77 @@ module Pmux
       @@task_re = Regexp.new("  [a-z_]+:")
       @@futter_re = Regexp.new("^:[a-z_]+:")
       @@mapper_re = Regexp.new("^:mapper:")
-      @@job_started_at_re = Regexp.new("^:job_started_at:")
-      @@start_time_re = Regexp.new("^:start_time:")
-      @@invoked_at_re = Regexp.new("^:invoked_at:")
-      @@map_tasks_re = Regexp.new("^:map_tasks:")
+      @@start_at_re = Regexp.new("^(:job_started_at:)|(:start_time:)|(:invoked_at:)")
+      @@tasks_re = Regexp.new("^(:map_tasks:)|(:reduce_tasks:)")
       @@tasksize_re = Regexp.new("^:tasksize:")
-      @@reduce_tasks_re = Regexp.new("^:reduce_tasks:")
       @@params_re = Regexp.new("^:params:")
       @@task_id_re = Regexp.new("^[0-9]+:")
-      @@task_allocated_at_re = Regexp.new("^  allocated_at:")
-      @@task_welapse_re = Regexp.new("^  welapse:")
-      @@task_elapse_re = Regexp.new("^  elapse:")
-      @@job_finished_at_re = Regexp.new("^:job_finished_at:")
+      @@elapse_re = Regexp.new("^(  allocated_at:)|(  welapse:)|(  elapse:)")
+      @@finish_at_re = Regexp.new("^:job_finished_at:")
       @@error_status_re = Regexp.new("^:error_status:")
       @@error_message_re = Regexp.new("^:error_message:")
       @@quote_re = Regexp.new("^['\"]|['\"]$")
 
       @logger = nil
+      @cache_expire_time = 86400
 
-      def initialize cache_dir_path
-        @cache_dir_path = cache_dir_path
+      def initialize jobs_cache_dir_path, cache_expire
+        @jobs_cache_dir_path = jobs_cache_dir_path
+        @cache_expire_time = cache_expire
         @logger = LoggerWrapper.instance()
       end
 
-      def get_files user, log_path
-        return Dir.glob(File.expand_path(["~" + user, log_path, "*.yml"].join(File::SEPARATOR)))
+      def get_log_paths
+        return [@@pmux_log_path]
+      end
+
+      def get_files user, log_path, data, now
+        stime = Time.at(0) 
+        etime = now
+        need_slice = true
+        if !data["start_time_msec"].nil? || !data["end_time_msec"].nil?
+          need_slice = false
+          stime = Time.at(data["start_time_msec"] / 1000, (data["start_time_msec"] % 1000) * 1000) if !data["start_time_msec"].nil?
+          etime = Time.at(data["end_time_msec"] / 1000, (data["start_time_msec"] % 1000) * 1000) if !data["end_time_msec"].nil?
+        end
+        file_mtime_list = Dir.glob(File.expand_path(["~" + user, log_path, "*.yml"].join(File::SEPARATOR))).map{ |f| [f, File.mtime(f)] }
+        file_list = file_mtime_list.sort_by{ |fm| fm[1] }.reverse.map{ |fm| fm[0] if stime <= fm[1] && fm[1] <= etime }.compact
+        if need_slice
+          item_count = (data["page"] + 1) * data["nitems"]
+          file_list = file_list.slice(0, item_count)
+        end
+        return file_list
       end
 
-      def fast_parse file, job_id
+      def fast_parse file, job_id, parsed_time
         cachable = false
-        job = {"end_time" => nil, "elapsed_time" => nil, "finished_tasks" => 0}
-        job["job_id"] = job_id
+        job = {"end_time" => nil, "elapsed_time" => nil, "finished_tasks" => 0, "job_id" => job_id, "parsed_time" => parsed_time}
         task_cnt = 0
         start_time = nil
         end_time = nil
-        File.open(file) {|f|
-          doc_cnt = 0
-          f.each_line {|ln|
+        doc1_buffer = ""
+        doc_cnt = 0
+        File.open(file) do |f|
+          f.each_line do |ln|
             if @@document_re =~ ln
+              if doc_cnt == 1
+                new_doc = YAML.load(doc1_buffer)
+                job["mapper"] = new_doc[:mapper].encode("UTF-16BE", "UTF-8", :invalid => :replace, :undef => :replace, :replace => '?').encode("UTF-8")
+              end
               doc_cnt += 1
-            elsif doc_cnt == 1 && (@@job_started_at_re =~ ln || @@start_time_re =~ ln)
+            elsif doc_cnt == 1 && @@start_at_re =~ ln
               empty, key, value = ln.split(":", 3)
               start_time = DateTime::parse(value.strip())
               job["start_time_msec"] = start_time.strftime("%Q") 
               job["start_time"] = start_time.strftime("%Y-%m-%d %H:%M:%S")
-            elsif doc_cnt == 1 && (@@map_tasks_re =~ ln ||  @@reduce_tasks_re =~ ln)
+            elsif doc_cnt == 1 && @@tasks_re =~ ln
               empty, key, value = ln.split(":", 3)
               job[key] = value.strip().to_i()
-            elsif doc_cnt == 1 && @@mapper_re =~ ln
-              empty, key, value = ln.split(":", 3)
-              job[key] = value.strip()
+            elsif doc_cnt == 1
+              doc1_buffer += ln
             elsif doc_cnt == 2 && @@task_id_re =~ ln
               task_cnt += 1
-            elsif doc_cnt == 3 && @@job_finished_at_re =~ ln
+            elsif doc_cnt == 3 && @@finish_at_re =~ ln
               empty, key, value = ln.split(":", 3)
               end_time = DateTime::parse(value.strip())
               job["end_time_msec"] = end_time.strftime("%Q") 
@@ -80,10 +100,10 @@ module Pmux
               cachable = true
             elsif doc_cnt == 3 && @@error_message_re =~ ln
               empty, key, value = ln.split(":", 3)
-              job[key] = value.strip().gsub(@@quote_re, "")
+              job[key] = value.strip().gsub(@@quote_re, "").encode("UTF-16BE", "UTF-8", :invalid => :replace, :undef => :replace, :replace => '?').encode("UTF-8")
             end
-          }
-        }
+          end
+        end
         job["finished_tasks"] = task_cnt
         job["elapsed_time"] = ((end_time - start_time) * 86400).to_f if !start_time.nil? && !end_time.nil?
         if end_time.nil?
@@ -99,7 +119,7 @@ module Pmux
       end
 
       def load_cache file_path
-        cache = {} 
+        cache = { "jobs" => {} } 
         return cache if !File.exist?(file_path)
         begin
           File.open(file_path, "rb") {|f|
@@ -114,180 +134,106 @@ module Pmux
         end
       end
 
-      def save_cache file_path, jobs, cachable_ids
-        cache = {}
-        for job_id in cachable_ids
-            cache[job_id] = jobs[job_id]
-        end
-        FileUtils.mkdir_p(@cache_dir_path) if !File.exist?(@cache_dir_path)
+      def save_cache file_path, cache
+        FileUtils.mkdir_p(@jobs_cache_dir_path) if !File.exist?(@jobs_cache_dir_path)
         begin
-          File.open(file_path, File::RDWR|File::CREAT, 0644) {|f|
+          File.open(file_path, File::RDWR|File::CREAT, 0644) do |f|
             f.flock(File::LOCK_EX)
             f.rewind()
             Marshal.dump(cache, f)
             f.flush()
             f.truncate(f.pos)
             f.flock(File::LOCK_UN)
-          }
+          end
         rescue
           @logger.warn("cannot save cache file (#{file_path})")
         end
       end
- 
-      def add_cache_ids cache_ids, jobs, job_id, job, sort_key, sort_order
-        insert_idx = -1
-        for idx in 0..(cache_ids.length - 1)
-          id = cache_ids[idx] 
-          if sort_order == "desc"
-            case sort_key
-            when "start_time"
-              if job["start_time_msec"].to_i > jobs[id]["start_time_msec"].to_i
-                insert_idx = idx
-                break
-              end
-            when "job_id"
-              if job["job_id"].to_i > jobs[id]["job_id"].to_i
-                insert_idx = idx
-                break
-              end
-            when "mapper"
-              if job["mapper"] > jobs[id]["mapper"]
-                insert_idx = idx
-                break
-              end
-            when "end_time"
-              if !job["error_status"].nil?
-                insert_idx = idx
-              elsif !jobs[id]["error_status"].nil?
-                next
-              elsif job["end_time_msec"].to_i > jobs[id]["end_time_msec"].to_i
-                insert_idx = idx
-                break
-              end
-            when "elapsed_time"
-              if !job["error_status"].nil?
-                insert_idx = idx
-              elsif !jobs[id]["error_status"].nil?
-                next
-              elsif job["elapsed_time"].to_i > jobs[id]["elapsed_time"].to_i
-                insert_idx = idx
-                break
-              end
-            end
-          elsif sort_order == "asc"
-            case sort_key
-            when "start_time"
-              if job["start_time_msec"].to_i < jobs[id]["start_time_msec"].to_i
-                insert_idx = idx
-                break
-              end
-            when "job_id"
-              if job["job_id"].to_i < jobs[id]["job_id"].to_i
-                insert_idx = idx
-                break
-              end
-            when "mapper"
-              if job["mapper"] < jobs[id]["mapper"]
-                insert_idx = idx
-                break
-              end
-            when "end_time"
-              if !job["error_status"].nil?
-                next
-              elsif !jobs[id]["error_status"].nil?
-                insert_idx = idx
-              elsif job["end_time_msec"].to_i < jobs[id]["end_time_msec"].to_i
-                insert_idx = idx
-                break
-              end
-            when "elapsed_time"
-              if !job["elapsed_time"].nil?
-                next
-              elsif !jobs[id]["elapsed_time"].nil?
-                insert_idx = idx
-              elsif job["elapsed_time"].to_i < jobs[id]["elapsed_time"].to_i
-                insert_idx = idx
-                break
-              end
-            end
+
+      def expire_cache cache, now
+        new_cache = { "jobs" => {} }
+        need_save_cache = false
+        cache["jobs"].each do |job_id, job|
+          if job["parsed_time"] + @cache_expire_time < now
+            need_save_cache = true
+            next
           end
+          new_cache["jobs"][job_id] = cache["jobs"][job_id]
         end
-        if insert_idx != -1
-          cache_ids.insert(insert_idx, job_id);
-        else
-          cache_ids.push(job_id)
-        end
+        return [new_cache, need_save_cache]
       end
 
-      def parse_data_cache_ids parse_data, jobs, cache_ids, nitems, page 
-        start_idx = nitems * page
-        if start_idx >= (cache_ids.length - 1)
-           return
+      def pickup_job src, src_ids, data 
+        parse_data = { "jobs" => {} }
+        src_ids = src_ids.sort_by{|id_start| id_start[1] }.map{|id_start| id_start[0]}
+        src_ids = src_ids.reverse if data["sort_order"] == "desc"
+        start_idx = 0;
+        if data["start_time_msec"].nil? && data["end_time_msec"].nil?
+          start_idx = data["page"] * data["nitems"]
+          return parse_data if start_idx >= src_ids.length
         end
-        for idx in start_idx..cache_ids.length - 1
-          parse_data[cache_ids[idx]] = jobs[cache_ids[idx]]
-          nitems -= 1
-          if nitems == 0
-            break
-          end
+        for idx in start_idx..src_ids.length - 1
+          job_id = src_ids[idx]
+          parse_data["jobs"][job_id] = src["jobs"][job_id]
         end
+        return parse_data
       end
 
       def parse_log_job user, data
-        new_jobs_cookie = DateTime.now().strftime("%Q").to_i 
-        jobs = {}
-        new_cache_ids = []
-        update_ids = []
-        cache_file_path = [@cache_dir_path, user].join(File::SEPARATOR)
-        cache = load_cache(cache_file_path)
-        need_save_cache = false
-	for log_path in [@@pmux_log_path]
-          files = get_files(user, log_path)
-          for file in files
-            job_id = File::basename(file).sub(".yml", "")
-            if cache.key?(job_id)
-              jobs[job_id] = cache[job_id]
-              add_cache_ids(new_cache_ids, jobs, job_id, cache[job_id], data["sort_key"], data["sort_order"])
-              update_ids.push(job_id) if data["jobs_cookie"] > 0 && data["jobs_cookie"] <= cache[job_id]["end_time_msec"].to_i
-              next
-            else
-              job, cachable = fast_parse(file, job_id)
-              jobs[job_id] = job
-              update_ids.push(job_id)
-              if cachable
-                add_cache_ids(new_cache_ids, jobs, job_id, job, data["sort_key"], data["sort_order"])
-                need_save_cache = true
-              end
+        #  data structure
+        #  {
+        #    "jobs" => {
+        #                id1 => { <job> },
+        #                .
+        #                .
+        #              },
+        #  }
+        new = { "jobs" => {} }
+        ids = []
+        now = Time.now
+        jobs_cache_file_path = [@jobs_cache_dir_path, user].join(File::SEPARATOR)
+        cache = load_cache(jobs_cache_file_path)
+        new_cache, need_save_cache = expire_cache(cache, now.to_i)
+        files = get_files(user, get_log_paths, data, now)
+        files.each do |file|
+          job_id = File::basename(file).sub(".yml", "")
+          if new_cache["jobs"].key?(job_id)
+            new["jobs"][job_id] = new_cache["jobs"][job_id]
+            ids.push([job_id, new["jobs"][job_id]["start_time_msec"]])
+          else
+            job, cachable = fast_parse(file, job_id, now.to_i)
+            if cachable
+              new_cache["jobs"][job_id] = job
+              need_save_cache = true
             end
+            new["jobs"][job_id] = job
+            ids.push([job_id, new["jobs"][job_id]["start_time_msec"]])
           end
         end
-        save_cache(cache_file_path, jobs, new_cache_ids) if need_save_cache || cache.length != new_cache_ids.length
-        parse_data = { "jobs" => {}, "jobs_cookie" => new_jobs_cookie}
-        if data["type"] == "archive"
-          parse_data_cache_ids(parse_data["jobs"], jobs, new_cache_ids, data["nitems"], data["page"])
-        end
-        for job_id in update_ids
-          parse_data["jobs"][job_id] = jobs[job_id] if jobs.key?(job_id)
-        end
+        save_cache(jobs_cache_file_path, new_cache) if need_save_cache
+        parse_data = pickup_job(new, ids, data)
         return parse_data
       end
 
       def full_parse file_path
         documents = []
-        File.open(file_path) {|f|
+        File.open(file_path) do |f|
           doc1_buffer = ""
           doc1_param = {}
           doc_cnt = 0
           new_doc = nil
           task_id = nil
-          f.each_line {|ln|
+          f.each_line do |ln|
             if @@document_re =~ ln
               if doc_cnt == 1
-                  new_doc = YAML.load(doc1_buffer)
-                  new_doc[:job_started_at] = doc1_param["job_started_at"] if doc1_param["job_started_at"]
-                  new_doc[:invoked_at] = doc1_param["invoked_at"] if doc1_param["invoked_at"]
-                  new_doc[:start_time] = doc1_param["start_time"] if doc1_param["start_time"]
-                  print new_doc
+                new_doc = YAML.load(doc1_buffer)
+                new_doc[:job_started_at] = doc1_param["job_started_at"] if doc1_param["job_started_at"]
+                new_doc[:invoked_at] = doc1_param["invoked_at"] if doc1_param["invoked_at"]
+                new_doc[:start_time] = doc1_param["start_time"] if doc1_param["start_time"]
+                new_doc[:mapper] = new_doc[:mapper].to_s.encode("UTF-16BE", "UTF-8", :invalid => :replace, :undef => :replace, :replace => '?').encode("UTF-8") if new_doc[:mapper]
+                new_doc[:reducer] = new_doc[:reducer].to_s.encode("UTF-16BE", "UTF-8", :invalid => :replace, :undef => :replace, :replace => '?').encode("UTF-8") if new_doc[:reducer]
+                new_doc[:params][:mapper] = new_doc[:params][:mapper].to_s.encode("UTF-16BE", "UTF-8", :invalid => :replace, :undef => :replace, :replace => '?').encode("UTF-8") if new_doc.key?(:params) && new_doc[:params][:mapper]
+                new_doc[:params][:job_name] = new_doc[:params][:job_name].to_s.encode("UTF-16BE", "UTF-8", :invalid => :replace, :undef => :replace, :replace => '?').encode("UTF-8") if new_doc.key?(:params) && new_doc[:params][:job_name]
               end
               if !new_doc.nil?
                 new_doc.delete(task_id) if doc_cnt == 2 && !task_id.nil? && new_doc[task_id].length < 5
@@ -296,7 +242,7 @@ module Pmux
               doc_cnt += 1
               new_doc = {}
               task_id = nil
-            elsif doc_cnt == 1 && (@@job_started_at_re =~ ln || @@invoked_at_re =~ ln || @@start_time_re =~ ln)
+            elsif doc_cnt == 1 && @@start_at_re =~ ln
               empty, key, value = ln.split(":", 3)
               time = DateTime::parse(value.strip())
               doc1_param[key] = time
@@ -305,13 +251,13 @@ module Pmux
             elsif doc_cnt == 2 && @@task_id_re =~ ln
               task_id, empty = ln.split(":", 2)
               new_doc[task_id] = {}
-            elsif doc_cnt == 2 && (@@task_allocated_at_re =~ ln || @@task_welapse_re =~ ln || @@task_elapse_re =~ ln)
+            elsif doc_cnt == 2 && @@elapse_re =~ ln
               key, value = ln.split(":", 2)
               new_doc[task_id][key.strip()] = value.strip().to_f()
             elsif doc_cnt == 2 && @@task_re =~ ln
               key, value = ln.split(":", 2)
               new_doc[task_id][key.strip()] = value.strip().gsub(@@quote_re, "")
-            elsif doc_cnt == 3 && @@job_finished_at_re =~ ln
+            elsif doc_cnt == 3 && @@finish_at_re =~ ln
               empty, key, value = ln.split(":", 3)
               time = DateTime::parse(value.strip())
               new_doc[key] = time
@@ -319,7 +265,7 @@ module Pmux
               empty, key, value = ln.split(":", 3)
               new_doc[key] = value.strip().gsub(@@quote_re, "")
             end
-          }
+          end 
           if !new_doc.nil?
             new_doc.delete(task_id) if doc_cnt == 2 && new_doc[task_id].length < 5
             documents.push(new_doc)
@@ -330,7 +276,7 @@ module Pmux
           if documents.length == 2
             documents.push(nil)
           end
-        }
+        end 
         return documents
       end
 
@@ -378,8 +324,8 @@ module Pmux
         return parse_data
       end
 
-      def set_cache_dir_path cache_dir_path
-        @cache_dir_path = cache_dir_path
+      def set_jobs_cache_dir_path jobs_cache_dir_path
+        @jobs_cache_dir_path = jobs_cache_dir_path
       end
     end
   end