site24x7_apminsight 1.0

data/lib/agent/metrics/am_metricsparser.rb

@@ -0,0 +1,569 @@
+require 'agent/am_objectholder'
+
+module ManageEngine
+	class APMMetricsParser
+		def initialize
+			@obj = ManageEngine::APMObjectHolder.instance
+		end
+		def parse(data)
+			@obj = ManageEngine::APMObjectHolder.instance
+			parseddata = Hash.new
+			begin
+				data.each do |key,val|
+					@obj.log.debug "[Processing started for - #{key} ]"
+					#lastHash = val.last
+					#name = lastHash["name"]
+					keyForTrans = key
+#					if(name=="process_action.action_controller")
+						#Transaction completed
+						tdata = Hash.new
+						trdbdata = Array.new
+						trdata = Array.new
+						started = false
+						redirect=false
+						ended =false;
+						start_index = 0;
+						val.each do |arr|
+							#@obj.log.debug "Processing : #{arr["name"]}"
+							if(started || arr["name"]==("start_processing.action_controller") )
+
+								if(arr["name"]=="sql.active_record" && @obj.config.sql_capture)
+									dbd = getDBData(arr)
+									if tdata.has_key?("db")
+										tdata["db"].push(dbd)
+									else
+										temp = Array.new
+										temp.push(dbd)
+										tdata["db"]=temp
+									end
+									#trdbdata.push(updateDBTrace(arr,dbd))
+									#@obj.log.debug "TRANS DBBB TRACE : #{dbd}"
+								elsif (arr["name"]=="process_action.action_controller")
+
+									tdata["td"]=getTransData(arr,tdata)
+									#if redirect
+									#	started = false
+									#	redirect=false
+									#	ended =false;
+									#else
+										ended = true
+										started = false
+										redirect=false
+										parseddata = updateParsedData key,tdata.dup,parseddata
+										#@obj.log.info "TRACE #{@obj.config.trans_trace}"
+										if @obj.config.trans_trace
+												ret = tdata["td"]
+												#@obj.log.info "TRACE COMPARE : #{ret["rt"]} > #{(@obj.config.trans_trace_t.to_f*1000).to_i}"
+												if (ret["rt"]).to_i > (@obj.config.trans_trace_t.to_f* 1000).to_i
+													tval = val[(start_index)..val.index(arr)]
+													#@obj.log.debug "Gng To Trace : #{start_index} to #{val.index(arr)}"
+													parseddata =updateTraceData tval,parseddata
+												end
+										end
+										key = key + start_index.to_s
+										tdata=Hash.new
+										start_index = val.index(arr) 
+									#end
+									#trdata = updateTransTrace(arr,trdbdata,trdata);
+									#@obj.log.info "TRANS TRACE : #{trdata}"
+									#elsif(arr["name"]==("!render_template.action_view")
+									#Render Action 	
+									#elsif(arr["name"]==("render_template.action_view")
+									#Render Action data not in use
+								elsif(arr["name"]==("start_processing.action_controller"))
+									ended =false;
+									started=true
+									#trdata = updateStartTrace(arr,trdata);
+									#Render Action data not in use
+								elsif(arr["name"]==("redirect_to.action_controller"))
+									  redirect = true;
+								end
+							end
+
+						end
+					
+						if !ended
+							@obj.log.debug "Transactions in Thread Continues - so drop data in mem-store #{keyForTrans} -- #{start_index}"
+							if parseddata.has_key?(keyForTrans)
+								#@obj.log.info "BEFORE : #{parseddata.keys}"
+								value = parseddata.delete(keyForTrans)
+								parseddata[key+"_dup"] = value
+								
+								#@obj.log.info "AFTER : #{parseddata.keys}"
+							end
+						end
+						#@obj.log.info "BEFORE METRICS - #{@obj.store.metrics}"
+						@obj.store.removeData key,0,start_index
+						#@obj.log.info "AFTER METRICS - #{@obj.store.metrics}"
+					@obj.log.debug "[Processing END for - #{key} ]"
+					#end
+				end
+			rescue Exception=>e
+				@obj.log.info "Exception : #{e}"
+				@obj.log.logException "#{e.message}",e
+			end
+#			@obj.log.debug "[PARSER] End"
+			parseddata
+		end
+
+		def updateParsedData key , tdata, parseddata
+			begin
+				if parseddata.has_key?(key)
+					key =  @obj.util.currenttimemillis
+				end
+				#@obj.log.debug "Update parsed data  : #{key} = > #{tdata}"
+				parseddata[key]=tdata
+			rescue Exception=>e
+				@obj.log.info "Exception in updateParsedData: #{e}"
+				@obj.log.logException "#{e.message}",e
+			end
+				parseddata
+		end
+		def updateTraceData val,parseddata
+			if(parseddata.has_key?("trace-data"))
+				tData = parseddata["trace-data"];
+				if(tData.length == @obj.config.trace_overflow_t)
+					trac = getDummyTrace
+					tData.push(trac)
+					parseddata["trace-data"]=tData
+					@obj.log.debug "dummy trace added"
+					return parseddata
+				elsif tData.length > @obj.config.trace_overflow_t
+					@obj.log.debug "trace threshold exceeded"
+					return parseddata
+				end
+			end
+			begin
+			trdata = getTrace val
+						trac = updateTrace(trdata)
+						if(parseddata.has_key?("trace-data"))
+							traceData = parseddata["trace-data"];
+							traceData.push(trac);
+							parseddata["trace-data"] = traceData;
+						else
+							traceData = Array.new
+							traceData.push(trac)
+							parseddata["trace-data"] = traceData;
+						end
+			rescue Exception=>e
+				@obj.log.info "Exception in updateTraceData: #{e}"
+				@obj.log.logException "#{e.message}",e
+			end
+						parseddata
+		end
+
+		def getDummyTrace
+			top = Array.new
+			path = @obj.constants.mf_overflow
+			det = {"thread_name"=>"rorthread","s_time"=>0,"t_name"=>path,"r_time"=>0,"thread_id"=>141}
+			trData = Array.new
+			trData[0] = 0
+			trData[1] = path
+			trData[2] = ""
+			trData[3] = 0
+			trData[4] = 0
+			trData[5] = nil
+			trData[6] = Array.new
+			top[0] = det
+			top[1] = trData
+			return top
+		end
+
+		def getTransData(d,tdata)
+			pl = d["payload"]
+			ret = nil;
+			begin
+			if(tdata.has_key?("td"))
+				ret = tdata["td"]
+				ret["rt"] = ret["rt"] +  (d["end"] - d["start"]).to_i
+			else
+				rt = (d["end"] - d["start"]).to_i
+				path=pl[:path]
+				#path=pl["path"]
+				controller=pl[:controller]
+				#controller=pl["controller"]
+				action = pl[:action]
+				#action = pl["action"]
+				ret ={"rt"=>rt,"path"=>path,"name"=>controller+"#"+action}
+			end
+			rescue Exception=>e
+				@obj.log.info "Exception in getTranseData: #{e}"
+				@obj.log.logException "#{e.message}",e
+			end
+			ret
+		end
+
+
+		def getDBData(d)
+			pl = d["payload"]
+			rt = (d["end"] - d["start"])
+			sql = pl[:sql]
+			#sql = pl["sql"]
+			sql.strip!
+			sql.downcase!
+			sqlArr = sql.split(" ")
+			sqlStrip="";
+			tr_name="";
+			begin
+				sqlStrip = getSqlStrip sqlArr,false
+			rescue Exception=>e
+				@obj.log.logException "#{e.message}",e
+				sqlStrip = sqlArr[0]+"/-/dummydb"
+			end
+
+			name=pl[:name]
+			#name=pl["name"]
+			ret ={"rt"=>rt,"sql"=>format(sql),"sql-strip"=>sqlStrip,"name"=>name,"operation"=>sqlArr[0]}
+			ret
+		end
+
+		def getSqlStrip sqlArr ,trace
+			sqlStrip =""
+				begin
+				sqlStrip = case sqlArr[0]
+						   when "select" then sqlArr[sqlArr.index("from")+1]
+						   when "insert" then sqlArr[sqlArr.index("into")+1]
+						   when "update" then sqlArr[1]
+						   when "delete" then sqlArr[sqlArr.index("from")+1]
+						   when "create" then sqlArr[1] + sqlArr[2]
+						   when "alter" then sqlArr[1] + sqlArr[2]
+						   when "drop" then sqlArr[1] + sqlArr[2]
+						   when "show" then sqlArr[1] 
+						   when "describe" then sqlArr[1] 
+						   else "-"
+						   end
+				
+				if trace
+					sqlStrip = format(sqlArr[0]) +" - " + format(sqlStrip) 
+				else
+					sqlStrip = format(sqlArr[0]) +"/" + format(sqlStrip) +"/dummydb"
+				end
+
+			rescue Exception=>e
+				@obj.log.logException "#{e.message}",e
+					if trace					
+						sqlStrip = format(sqlArr[0])
+					else
+						sqlStrip = sqlArr[0]+"/-/dummydb"
+					end
+			end
+		end
+
+		def updateDBTrace(d)
+			pl = d["payload"]
+			rt = (d["end"] - d["start"])
+			sql = pl[:sql]
+			#sql = pl["sql"]
+			sql.strip!
+			sql.downcase!
+			sql = format(sql)
+			sqlArr = sql.split(" ")
+			
+			trData = Array.new
+			#trData[0] = (d["start"].to_f * 1000 ).to_i
+			trData[0] = d["ctime"]
+			trData[1] = getSqlStrip sqlArr,true
+			trData[2]  = ""
+			trData[3] = rt.to_i
+			trData[4] = rt.to_i
+			trData[5] = {"query"=>sql}
+			trData[6] = Array.new
+			if(!@obj.config.sql_capture_params)
+				trData[5] = updateQueryParams(d,sql,pl[:binds],rt)
+				#trData[5] = updateQueryParams(d,sql,pl["binds"],rt)
+			else
+				sql = @obj.util.parametrizeQuery sql
+				trData[5] = {"query"=>sql}
+			end
+			
+			
+
+			trData
+		end
+
+		def updateQueryParams d,sql,binds,rt
+			ret =  {"query"=>sql}
+			begin
+				args = "";
+				if binds!=nil && binds.size()>0
+					binds.each do |ar|
+						#args =  args + ar.last + "," 
+						if sql!=nil && sql.index("?")!=nil
+							sql["?"]=ar.last
+						end
+					end
+				end
+				if args.length>0
+					sql  = sql + "\\nPARAMS - ["+args+ "]"
+				end
+				ret =  {"query"=>sql}
+			rescue Exception=>exe
+				@obj.log.logException "Not severe -#{exe.message}",exe
+				ret =  {"query"=>sql}
+			end
+
+				begin
+					if(rt.to_i > (@obj.config.sql_trace_t.to_f * "1000".to_f ).to_i && d.has_key?("trace") )
+						t = updateSQLStackTrace(d["trace"])
+						ret["stacktrace"] =t
+					end
+				rescue Exception=>e
+					@obj.log.logException "#{e.message}",e
+				end
+
+			ret
+		end
+
+		def updateSQLStackTrace(d)
+			trace = Array.new;
+			started = false;
+			ended = false;
+			begin
+				#@obj.log.debug "STACKTRACE STARTS"
+				d.each do |arr|
+					
+					if started
+
+						end_ind = arr.index("`send_action'")
+						if  end_ind != nil &&  end_ind >0
+							ended = true
+							break;
+						end
+						#trArr = arr.split(/:[0-1,in]/);
+						#if	trArr[0].length>30
+					#		len = trArr.length
+					#		trArr[0] = "..."+trArr[(len-30),len]
+					#	end	
+						inx=arr.index("`instrument'") 
+						if inx!=nil && inx>0
+							#skip instrumet data
+						else
+							temp  = Array.new
+							temp[0]=arr
+							temp[1]=""
+							temp[2]=""
+							temp[3]=""
+							trace.push(temp)
+						end
+					end
+
+					if started==false
+						ind = arr.index("`instrument'")
+						if  ind != nil &&  ind >0
+							started = true
+						end
+					end
+
+					
+				end
+				#@obj.log.debug "STACKTRACE ENDS"
+
+			trace.shift
+			rescue Exception=>e
+				@obj.log.logException "#{e.message}",e
+			end
+			trace
+		end
+
+		def updateOtherTrace d
+			pl = d["payload"]
+			rt = (d["end"] - d["start"])
+			name = d["name"]
+			if pl.has_key?(:identifier)
+			#if pl.has_key?("identifier")
+				name = pl[:identifier]
+				#name = pl["identifier"]
+			end
+			trData = Array.new
+			#trData[0] = (d["start"].to_f * 1000 ).to_i
+			trData[0] = d["ctime"]
+			trData[1] = name
+			trData[2]  = ""
+			trData[3] = rt.to_i
+			trData[4] = rt.to_i
+			trData[5] = nil
+			trData[6] = Array.new
+			trData
+		end
+
+		def updateTransTrace(d,trData)
+
+			#trData = Array.new
+			#trData[0] = ((d["start"]).to_f * 1000).to_i
+			pl = d["payload"]
+			trData[1] = pl[:path] + " " +pl[:controller] + "#" + pl[:action]
+			#trData[1] = pl["path"] + " " +pl["controller"] + "#" + pl["action"]
+			trData[2] = ""
+			trData[3] = (d["end"] - trData[0]).to_i
+			trData[4] = (d["end"] - trData[0]).to_i
+			trData[5] = nil
+			#trData[6] = Array.new
+			#dbTime = 0
+			#trdbdata.each do |dbData|
+			#	dbTime = dbTime  + dbData[3]
+			#end
+			#trData[4] = ((trData[3]).to_f - (dbTime.to_f )).to_i
+#			if(pl[:db_runtime]!=nil)
+#				trData[4] = ((trData[3]).to_f - (pl[:db_runtime].to_f )).to_i
+#			end
+			trData
+		end
+
+		def updateMethodTrace(d,trData)
+
+			#trData = Array.new
+			#trData[0] = ((d["start"]).to_f * 1000).to_i
+			pl = d["payload"]
+			trData[1] = pl[:method] 
+			#trData[1] = pl["method"] 
+			trData[2] = ""
+			trData[3] = (d["end"] - trData[0]).to_i
+			trData[4] = (d["end"] - trData[0]).to_i
+			trData[5] = nil
+			#trData[6] = Array.new
+			#dbTime = 0
+			#trdbdata.each do |dbData|
+			#	dbTime = dbTime  + dbData[3]
+			#end
+			#trData[4] = ((trData[3]).to_f - (dbTime.to_f )).to_i
+#			if(pl[:db_runtime]!=nil)
+#				trData[4] = ((trData[3]).to_f - (pl[:db_runtime].to_f )).to_i
+#			end
+			trData
+		end
+
+		def updateStartTrace(d)
+			trData = Array.new
+			#trData[0] = ((d["start"]).to_f * 1000).to_i
+			trData[0] = d["ctime"]
+			trData[1] =""
+			trData[2] =""
+			trData[3] =0
+			trData[4] =0
+			trData[5] =nil
+			trData[6] =Array.new
+
+			trData
+		end
+
+		def updateTrace(trans)
+#			{"thread_name":"http-8080-6","s_time":1326276180289,"t_name":"transaction\/http\/Test-App\/login","r_time":18,"thread_id":141}
+			top =  Array.new
+			path = @obj.constants.mf_transaction + @obj.constants.mf_separator + trans[1]
+			det = {"thread_name"=>"rorthread","s_time"=>trans[0],"t_name"=>path,"r_time"=>trans[3],"thread_id"=>141}
+				#trans[6] = db;
+				top[0] = det
+				top[1] = trans
+				top
+		end
+
+		def format s
+			s.gsub!("\"", '')
+			s.gsub!("\n", '')
+			s
+		end
+
+		
+		def getTrace data
+			index = 0
+				data.each do |arr|
+
+					if(arr["name"]=="start_processing.action_controller")
+						index = data.index(arr)
+						break;
+					end
+				end
+				data = data.drop(index)
+				tdata =traceDetails data,Array.new,0
+				tdata = updateExclusiveTrace(tdata[0])
+				tdata
+		end
+
+		def traceDetails valo,stack,indx
+			if(valo.size>indx)
+			arr = valo[indx]
+			indx = indx+1
+					if(arr["name"]=="start_processing.action_controller")
+						traceArr = updateStartTrace arr
+						traceArr[6] = Array.new #childs
+						if stack.size()>0
+							cur = stack[stack.size()-1]							
+							c = cur[6]
+							c.push(traceArr)
+							cur[6] = c
+						end
+							stack.push(traceArr)
+
+						
+						traceDetails valo,stack,indx
+
+					elsif(arr["name"]=="sql.active_record" &&  @obj.config.sql_capture )
+
+						temp =  updateDBTrace arr
+
+						cur = stack[stack.size()-1]							
+						c = cur[6]
+						c.push(temp)
+						cur[6] = c
+						traceDetails valo,stack,indx
+					elsif(arr["name"]=="apm.methodstart")
+							temp = updateStartTrace arr
+							cur = stack[stack.size()-1]							
+							c = cur[6]
+							c.push(temp)
+							cur[6] = c
+							stack.push(temp)
+							traceDetails valo,stack,indx
+					elsif(arr["name"]=="apm.methodend")
+							updateMethodTrace(arr,stack[stack.size()-1])
+							stack.delete(stack[stack.size()-1])
+							traceDetails valo,stack,indx
+					elsif(arr["name"]=="process_action.action_controller")
+						updateTransTrace(arr,stack[stack.size()-1])
+						if(valo.size()>(indx+1))
+							#Redirection Data
+							arr = valo[indx]
+							while (arr["name"]!="start_processing.action_controller") do 
+								indx = indx + 1
+								arr = valo[indx]
+							end
+						end	
+						(stack[0])[3] = (arr["end"] - ((stack[0])[0])).to_i
+						(stack[0])[4] = (arr["end"] - ((stack[0])[0])).to_i
+						
+						
+						traceDetails valo,stack,indx
+						#parent[4] = parent[4] - traceArr[4]
+					elsif (arr["name"]=="render_template.action_view")
+						temp =  updateOtherTrace arr
+
+						cur = stack[stack.size()-1]							
+						c = cur[6]
+						c.push(temp)
+						cur[6] = c
+						#traceArr[6] = c
+						#parent[4] = parent[4] - temp[4]
+						traceDetails valo,stack,indx
+					else
+						traceDetails valo,stack,indx
+
+					end
+
+				stack
+			else
+			#Nothing todo
+			end
+		end
+
+
+		def updateExclusiveTrace data
+			childs = data[6]
+			childs.each do |arr|
+				data[4] = data[4] - (updateExclusiveTrace arr)[3]
+			end
+			data
+		end
+
+
+	end
+end

data/lib/agent/metrics/am_metricstore.rb

@@ -0,0 +1,29 @@
+require 'agent/am_objectholder'
+
+module ManageEngine
+	class APMMetricsStore
+		attr_accessor :metrics
+		def initialize
+			@metrics = Hash.new
+		end
+
+		def remove keys
+			if keys!=nil
+				keys.each {|key| @metrics.delete("#{key}")} 
+			end
+		end
+
+		def metrics_dup
+			@metrics.dup
+		end
+
+		def removeData key, strt_indx,end_indx
+			if @metrics.has_key?(key)
+				val = @metrics[key]
+				val = val.drop(end_indx)
+				@metrics[key]=val
+			end
+		end
+
+	end
+end

data/lib/agent/server/am_agent.rb

@@ -0,0 +1,116 @@
+require "agent/am_objectholder"
+require "agent/server/worker/am_worker"
+
+require 'socket'
+
+module ManageEngine
+	class APMAgent
+		def initialize
+			@obj = ManageEngine::APMObjectHolder.instance
+			@obj.log.debug "Agent Initialization - START"
+			doConnect
+
+			if !@obj.shutdown && @obj.agent_initialized
+				@obj.log.info "Agent Initialization - DONE"
+				@obj.instrumenter.doSubscribe
+				doDispatcherActions
+				doCollect
+				puts "APM Insight Ruby Agent Started"
+			else
+				@obj.log.info "Agent Initialization Failed - Going to shutdown"
+				@obj.instrumenter.doSubscribe
+				@obj.shutdownagent
+			end
+           	
+
+       end
+
+		def doConnect
+			begin	
+				if @obj.shutdown
+					@obj.log.info "[ Problem in Agent Startup ]"
+				else
+					agentInfo = @obj.config.getAgentInfo
+					resp = nil
+					if @obj.config.alreadyconnected
+						@obj.log.debug "[doConnect] Already Connected - Make Contact - Instance id = #{@obj.config.instance_id}"
+						if @obj.config.site24x7
+							resp = startConnect "?license.key="+@obj.config.license_key+"&instance_id="+@obj.config.instance_id,agentInfo
+						else
+							resp = startConnect "?instance_id="+@obj.config.instance_id,agentInfo
+						end
+					else
+						@obj.log.debug "[doConnect] Going to connect - New "
+						if @obj.config.site24x7
+							resp = startConnect "?license.key="+@obj.config.license_key,agentInfo
+						else
+							resp = startConnect "",agentInfo
+						end
+						if resp.has_key?("instance-info")
+							aData = resp["instance-info"]
+							aData["agent.id"]=aData.delete("instanceid")
+							aData["agent.enabled"]=true
+							@obj.config.updateAgentInfoFile(aData)
+							@obj.log.debug "[doConnect] Connected - InstanceID  : #{@obj.config.instance_id}"
+						else
+							@obj.log.info "[doConnect] [ Problem in connecting server] [ Going to shutdown ]"
+							@obj.shutdown=true
+						end
+					end
+
+					if resp==nil
+						@obj.log.info "[doConnect] [ Error in Response while connecting Server . [ Going to shutdown ]"
+						@obj.shutdown= true
+					end
+
+					if(!@obj.shutdown)
+						@obj.agent_initialized=true
+					end
+				end
+			rescue Exception=>e
+				@obj.shutdown = true
+				@obj.log.logException "[doConnect] Exception while connecting server. [ Going to shutdown ] ",e
+			end
+		end
+
+
+		def doCollect
+			@obj.log.info "[doCollect] Starts - Wait time : #{@obj.config.connect_interval} seconds "
+			begin
+				ManageEngine::APMWorker.getInstance.start
+			rescue Exception=>e
+				@obj.log.logException "[doCollect]  Exception during worker  startup  #{e.message}",e
+				@obj.shutdown=true
+			end
+		end
+
+
+		def startConnect uri,data
+			resp = @obj.connector.post @obj.constants.connect_uri+uri,data
+		end
+
+		def doDispatcherActions
+			case @obj.config.app_dispatcher
+				when 'passenger'
+					#starting a new process
+      				PhusionPassenger.on_event(:starting_worker_process) do |forked|
+        				if forked
+                			@obj.log.info "starting_worker_process : Process ID :#{Process.pid} : Creating new apm worker"
+                			doCollect
+        				else
+                			doCollect
+		                	@obj.log.info "starting_worker_process : Conservative Process ID :#{Process.pid} - No new worker"
+        				end
+     			 	end
+      				# shutting down a process.
+				    PhusionPassenger.on_event(:stopping_worker_process) do
+                		ManageEngine::APMWorker.getInstance.stop
+                		@obj.log.info "stopping_worker_process :Process ID :#{Process.pid} ---->  #$$ "
+     				end
+				else#case
+
+				end#case
+		end
+	end#c
+end#m
+