rbatch 1.6.3 → 1.6.4

data/sample/moto/src/main/java/jp/ossc/imortAuditlog/LogReader.java

@@ -0,0 +1,521 @@
+package jp.ossc.imortAuditlog;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.InputStreamReader;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.PropertyResourceBundle;
+import java.util.ResourceBundle;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+
+public class LogReader {
+	private static Logger logger = Logger.getLogger(LogReader.class);
+	MysqlController cont = new MysqlController();
+	private static final String not_available = "\"Not Available\"";
+
+	public boolean authLogReader(ResourceBundle sysResource){
+		logger.info("auth_log_file_read start");
+
+		int executeCount = Integer.valueOf(sysResource.getString("execute.count"));
+		String logPath = sysResource.getString("sso.authlog.path");
+		File f = new File(logPath);
+		if(f.isDirectory()){
+			String authLogName = sysResource.getString("sso.authlog.name");
+			String errLogName = sysResource.getString("sso.autherrlog.name");
+			
+			// 前方一致検索で一致するファイルリストを取得
+			File[] authLogFiles = f.listFiles(getFileExtensionFilter(authLogName));
+			File[] errLogFiles = f.listFiles(getFileExtensionFilter(errLogName));
+			// 更新日時が最新のファイル名を取得する
+			authLogName = getLatestFileName(authLogFiles);
+			errLogName = getLatestFileName(errLogFiles);
+			
+			if(!logPath.substring(logPath.length()-1).equals("/")){
+				logPath = logPath + "/";
+			}
+			if(authLogName == null){
+				logger.warn("auth_log_file does not exist!!");
+			}else{
+				String authLogFile = logPath + authLogName;
+				try {
+					InputStreamReader srAuth = new InputStreamReader(new FileInputStream(authLogFile), "UTF8");
+					BufferedReader brAuth = new BufferedReader(srAuth);
+					String strAuth;
+					cont.connect();
+					cont.setAuthQuery();
+					int dataCount = 0;
+					while ((strAuth = brAuth.readLine()) != null) {
+						Map authLog = editAuthLog(strAuth);
+						if(authLog != null){
+							dataCount++;
+							cont.insertAuth(authLog);
+							if(dataCount%executeCount == 0){
+								cont.executeAuthBatch();
+								logger.info("insert record:" + cont.excuteAuthCount);
+							}
+						}
+					}
+					cont.executeAuthBatch();
+					cont.commit();
+					logger.info("insert record:" + cont.excuteAuthCount);
+				} catch (FileNotFoundException e) {
+					logger.error("amAuthentication.access not file");
+					throw new IllegalStateException(e);
+				} catch (Exception e) {
+					logger.error("amAuthentication.access read error");
+					e.printStackTrace();
+				}
+				
+			}
+
+			if(errLogName == null){
+				logger.warn("auth_err_log_file does not exist!!");
+			}else{
+				String errLogFile = logPath + errLogName;
+				try {
+					InputStreamReader srErr = new InputStreamReader(new FileInputStream(errLogFile), "UTF8");
+					BufferedReader brErr = new BufferedReader(srErr);
+					String strErr;
+					cont.connect();
+					cont.setAuthQuery();
+					int dataCount = 0;
+					while ((strErr = brErr.readLine()) != null) {
+						Map errLog = editAuthLog(strErr);
+						if(errLog != null){
+							dataCount++;
+							cont.insertAuth(errLog);
+							if(dataCount%executeCount == 0){
+								cont.executeAuthBatch();
+								logger.info("insert record:" + cont.excuteAuthCount);
+							}
+						}
+					}
+					cont.executeAuthBatch();
+					cont.commit();
+					logger.info("insert record:" + cont.excuteAuthCount);
+				} catch (FileNotFoundException e) {
+					logger.error("amAuthentication.error not file " + errLogFile);
+					throw new IllegalStateException(e);
+				} catch (Exception e) {
+					logger.error("amAuthentication.error read error " + errLogFile);
+					e.printStackTrace();
+				}
+			}
+		}
+		logger.info("auth_log total insert record:" + cont.excuteAuthCount);
+		logger.info("auth_log_file_read end");
+
+		return true;
+	}
+
+	public static Map editAuthLog(String str){
+		Map<String, String> map = new HashMap<String, String>();
+		String strDate = null;
+		String strUserId = null;
+		String strClientIP = null;
+		String strStatus = null;
+		if(!str.startsWith("#")){
+			String[] strAry = str.split("\t");
+			if(strAry.length==12){
+				strDate = strAry[0].replace("\"", "");
+				String[] strLoginId = strAry[2].split(",");
+				for(int i =0; i < strLoginId.length; i++){
+					if(strLoginId[i].toString().startsWith("id=")){
+						String[] wUserId = strLoginId[i].split("=");
+						strUserId = wUserId[1];
+					// OpenLDAP 認証モジュールの場合の、ユーザIDは、"uid="で始まる為、それも取り込む
+					// また、パスワード未入力や、存在しないユーザIDの場合も、"uid="で出力されるように
+					// 認証モジュール側を修正したので、以下のコードで取り込まれる。
+					} else if(strLoginId[i].toString().startsWith("uid=")){
+						String[] wUserId = strLoginId[i].split("=");
+						strUserId = wUserId[1];
+					}
+				}
+				if(strUserId == null){
+					strUserId = "-";
+				}
+				strClientIP = strAry[4];
+				strStatus = getStatus(strAry[8]);
+			}
+		}
+		if(strDate != null && strUserId != null && strStatus != null){
+			map.put("DATE", strDate);
+			map.put("USERID", strUserId);
+			map.put("CLIENTIP", strClientIP);
+			map.put("STATUS", strStatus);
+			return map;
+		}else{
+			return null;
+		}
+	}
+
+	public static Map editErrLog(String str){
+		Map<String, String> map = new HashMap<String, String>();
+		String strDate = null;
+		String strUserId = null;
+		String strClientIP = null;
+		String strStatus = null;
+		if(!str.startsWith("#")){
+			String[] strAry = str.split("\t");
+			if(strAry.length==12){
+				strDate = strAry[0].replace("\"", "");
+				if(!strAry[2].equals(not_available)){
+						strUserId = strAry[2];
+				}
+				if(strUserId == null){
+					strUserId = "-";
+				}
+				strClientIP = strAry[4];
+				strStatus = getStatus(strAry[8]);
+			}
+		}
+		if(strDate != null && strUserId != null && strStatus != null){
+			map.put("DATE", strDate);
+			map.put("USERID", strUserId);
+			map.put("CLIENTIP", strClientIP);
+			map.put("STATUS", strStatus);
+			return map;
+		}else{
+			return null;
+		}
+	}
+
+	public static String getStatus(String strStatus){
+		String status = null;
+		ResourceBundle statusResource = PropertyResourceBundle.getBundle("status");
+		if(!strStatus.matches("-")){
+			if(strStatus.startsWith("AUTHENTICATION")){
+				String[] strStatusid = strStatus.split("-");
+				status = strStatusid[1];
+				Enumeration<String> test = statusResource.getKeys();
+				while (test.hasMoreElements()) {
+					if(test.nextElement().equals("auth."+strStatusid[1])){
+						status = statusResource.getString("auth."+ strStatusid[1]);
+					}
+				}
+			}
+		}
+		return status;
+	}
+	public boolean accessLogReader(ResourceBundle sysResource , String sw){
+		logger.info("access_log_file_read start");
+
+		int executeCount = Integer.valueOf(sysResource.getString("execute.count"));
+		String logPath = sysResource.getString(sw + ".accesslog.path");
+		File f = new File(logPath);
+		if(f.isDirectory()){
+			String accessLogName = sysResource.getString(sw + ".accesslog.name");
+			
+			// 前方一致検索で一致するファイルリストを取得
+			File[] accessLogFiles = f.listFiles(getFileExtensionFilter(accessLogName));
+			// 更新日時が最新のファイル名を取得する
+			accessLogName = getLatestFileName(accessLogFiles);
+			if(accessLogName == null){
+				logger.warn("access_log_file does not exist!!");
+			}else{
+				if(!logPath.substring(logPath.length()-1).equals("/")){
+					logPath = logPath + "/";
+				}
+				String accessLogFile = logPath + accessLogName;
+				try {
+					InputStreamReader srAccess = new InputStreamReader(new FileInputStream(accessLogFile), "UTF8");
+					BufferedReader brAuth = new BufferedReader(srAccess);
+					String strAccess;
+					cont.connect();
+					cont.setQuery();
+					int dataCount = 0;
+					while ((strAccess = brAuth.readLine()) != null) {
+						Map accessLog = editAccessLog(strAccess , sysResource ,sw);
+						// "#" がついた行を取り込もうとして、NPEが発生する為の対処
+						if(accessLog != null && !accessLog.isEmpty()){
+							dataCount++;
+							cont.insertAccessBatch(accessLog);
+							if(dataCount%executeCount == 0){
+								cont.executeBatch();
+								logger.info("insert record:" + cont.excuteCount);
+							}
+						}
+					}
+					cont.executeBatch();
+					cont.commit();
+					logger.info("access_log total insert record:" + cont.excuteCount);
+					cont.disconnect();
+				} catch (FileNotFoundException e) {
+					logger.error("access_log not file");
+					throw new IllegalStateException(e);
+				} catch (Exception e) {
+					logger.error("access_log read error");
+					e.printStackTrace();
+				}
+			}
+		}
+		logger.info("access_log_file_read end");
+		return true;
+	}
+
+	public static Map editAccessLog(String str ,ResourceBundle sysResource , String sw){
+		Map<String, String> map = new HashMap<String, String>();
+		String[] exclude = sysResource.getString("exclude.pattern").split(",");
+
+		int clientIp_p = Integer.valueOf(sysResource.getString(sw + ".clientip.point"));
+		int userId_p = Integer.valueOf(sysResource.getString(sw + ".userid.point"));
+		int date_p = Integer.valueOf(sysResource.getString(sw + ".date.point"));
+		int processingTime_p = Integer.valueOf(sysResource.getString(sw + ".processingtime.point"));
+		int request_p = Integer.valueOf(sysResource.getString(sw + ".request.point"));
+		int status_p = Integer.valueOf(sysResource.getString(sw + ".status.point"));
+		int agent_p = Integer.valueOf(sysResource.getString(sw + ".agent.point"));
+		int referer_p = 0;
+		if(sysResource.getString(sw + ".referer.point") != null && sysResource.getString(sw + ".referer.point").length() != 0){
+			referer_p = Integer.valueOf(sysResource.getString(sw + ".referer.point"));
+		}
+
+		String strClientIp = null;
+		String strUserId = null;
+		String strDate = null;
+		String strProcessingTime = null;
+		String strRequest = null;
+		String strStatus = null;
+		String strAgent = null;
+		String strReferer = null;
+		String str2 = new String();
+		if(!str.startsWith("#")){
+			str2 = logSplit(str);
+
+			String[] strAry = str2.split("\t");
+			String[] wkIp = strAry[clientIp_p -1].split(",");
+			strClientIp = wkIp[0];
+			strUserId = strAry[userId_p -1];
+			strDate = strAry[date_p -1];
+			strProcessingTime = strAry[processingTime_p -1];
+			Matcher m = Pattern.compile("^(.+\\s.+)(\\s.+)").matcher(strAry[request_p -1].replace("\"", ""));
+			if (m.find()) {
+				strRequest = m.group(1);
+			}else{
+				strRequest = strAry[request_p -1].replace("\"", "");
+			}
+			strStatus = strAry[status_p -1];
+			strAgent = strAry[agent_p -1].replace("\"", "");
+			
+			if(referer_p != 0 && strAry.length >= referer_p){
+				strReferer = strAry[referer_p -1].replace("\"", "");
+			}
+
+			for(int i = 0 ; i < exclude.length ; i++){
+				// 除外対象の値が空値であれば無視
+				if (exclude[i].toString().equals("")) {
+					continue;
+				}
+				String compare = ".*" + "."+exclude[i].toLowerCase() +".*";
+				if(strRequest.toLowerCase().matches(compare)){
+					return null;
+				}
+			}
+			
+			map.put("CLIENTIP", strClientIp);
+			map.put("USERID", strUserId);
+			map.put("DATE", strDate);
+			map.put("PROCESSINGTIME", strProcessingTime);
+			map.put("REQUEST", strRequest);
+			map.put("STATUS", strStatus);
+			map.put("AGENT", strAgent);
+			map.put("REFERER", strReferer);
+		}
+		return map;
+	}
+	
+	public boolean agentAuthLogReader(ResourceBundle sysResource){
+		logger.info("agent_auth_log_file_read start");
+
+		int executeCount = Integer.valueOf(sysResource.getString("execute.count"));
+		String logPath = sysResource.getString("sso.agent.authlog.path");
+		File f = new File(logPath);
+		if(f.isDirectory()){
+			String agentAuthLogName = sysResource.getString("sso.agent.authlog.name");
+			
+			// 前方一致検索で一致するファイルリストを取得
+			File[] agentAuthLogFiles = f.listFiles(getFileExtensionFilter(agentAuthLogName));
+			// 更新日時が最新のファイル名を取得する
+			agentAuthLogName = getLatestFileName(agentAuthLogFiles);
+			if(agentAuthLogName == null){
+				logger.warn("agent_auth_file does not exist!!");
+			}else{
+				if(!logPath.substring(logPath.length()-1).equals("/")){
+					logPath = logPath + "/";
+				}
+				String agetntAuthLogFile = logPath + agentAuthLogName;
+				try {
+					InputStreamReader srAgentAuth = new InputStreamReader(new FileInputStream(agetntAuthLogFile), "UTF8");
+					BufferedReader brAgentAuth = new BufferedReader(srAgentAuth);
+					String strAgentAuth;
+					cont.connect();
+					cont.setAgentAuthQuery();
+					int dataCount = 0;
+					while ((strAgentAuth = brAgentAuth.readLine()) != null) {
+						Map authLog = editAgentAuthLog(strAgentAuth, sysResource);
+						if(authLog != null){
+							dataCount++;
+							cont.insertAgentAuth(authLog);
+							if(dataCount%executeCount == 0){
+								cont.executeAgentAuthBatch();
+								logger.info("insert record:" + cont.excuteAgentAuthCount);
+							}
+						}
+					}
+					cont.executeAgentAuthBatch();
+					cont.commit();
+					logger.info("insert record:" + cont.excuteAgentAuthCount);
+				} catch (FileNotFoundException e) {
+					logger.error(agentAuthLogName + " not file");
+					throw new IllegalStateException(e);
+				} catch (Exception e) {
+					logger.error(agentAuthLogName + " read error");
+					e.printStackTrace();
+				}
+			}
+		}
+		logger.info("agent_auth_log total insert record:" + cont.excuteAgentAuthCount);
+		logger.info("agent_auth_log_file_read end");
+
+		return true;
+	}
+	
+	public static Map editAgentAuthLog(String str, ResourceBundle sysResource){
+		Map<String, String> map = new HashMap<String, String>();
+		String strDate = null;
+		String strUserId = null;	
+		String strRequest = null;
+		String strStatus = null;
+		String[] exclude = sysResource.getString("sso.agent.authlog.exclude.pattern").split(",");
+		String keyword = sysResource.getString("sso.agent.authlog.keyword");
+		int request_p = Integer.valueOf(sysResource.getString("sso.agent.authlog.request.point"));
+		
+		// ステータス取得
+		if(str.indexOf("allowed access") != -1) {
+			strStatus = "ALLOW";
+		}
+		else if (str.indexOf("denied access") != -1) {
+			strStatus = "DISALLOW";
+		}
+		
+		if(str.indexOf("\"") != -1) {
+			str=str.replaceAll("\"","");
+		}
+		if(str.indexOf("\t") != -1) {
+			str=str.replaceAll("\t"," ");
+		}
+		
+		if (str.indexOf(keyword) != -1 && strStatus != null) {
+			String[] strAry = str.split(" ");
+			if(strAry.length >= 12){
+				// 日付取得
+				strDate = strAry[0].toString() + " " + strAry[1].toString();
+				// ログに「User」の単語があれば次の配列値をユーザーIDとする
+				for(int i =2; i < strAry.length; i++){
+					if (strAry[i].toString().equals("User")) {
+						if ((i+1) < strAry.length) {
+							strUserId = strAry[i+1].toString();
+							break;
+						}
+					}
+				}
+				// リクエストURL取得
+//				strRequest = strAry[strAry.length-1].toString();
+				strRequest = strAry[request_p-1].toString();
+			}
+		}
+		
+		if(strDate != null && strUserId != null && strStatus != null && strRequest != null){
+			// リクエストURLが除外対象の拡張子の場合はDBへ登録しない
+			for(int i = 0 ; i < exclude.length ; i++){
+				// 除外対象の値が空値であれば無視
+				if (exclude[i].toString().equals("")) {
+					continue;
+				}
+				String compare = ".*" + "."+exclude[i].toLowerCase() +".*";
+				if(strRequest.toLowerCase().matches(compare)){
+					return null;
+				}
+			}
+			
+			map.put("DATE", strDate);
+			map.put("USERID", strUserId);
+			map.put("STATUS", strStatus);
+			map.put("REQUEST", strRequest);
+			return map;
+		}else{
+			return null;
+		}
+	}
+	
+	public static String logSplit(String var){
+		String strCahr = new String();
+		String strRep = new String();
+		boolean inSquareBracket = false;
+		boolean inDoubleQuotation = false;
+		var = var.replace(", ", ",");
+		for(int i = 0; i < var.length(); i++) {
+			strCahr = var.substring(i,i+1);
+			// "[" や '"' の内部のスペースでは、区切りタブ出力しない
+			if(strCahr.equals(" ") && !inDoubleQuotation && !inSquareBracket){
+				strRep = strRep + "\t";
+			}else{
+
+				// '"' に囲われていない場合にだけ、
+				// "[" と "]" の開始終了をチェックする
+				if(strCahr.equals("[") && !inSquareBracket && !inDoubleQuotation) {
+					inSquareBracket = true;
+				}else if (strCahr.equals("]") && inSquareBracket && !inDoubleQuotation){
+					inSquareBracket = false;
+				}
+
+				// '"' と '"' の開始終了をチェック
+				if((strCahr.equals("\"")) && !inDoubleQuotation) {
+					inDoubleQuotation = true;
+				}else if ((strCahr.equals("\"")) && inDoubleQuotation){
+					inDoubleQuotation = false;
+				}
+				strRep = strRep + strCahr;
+			}
+		}
+		return strRep;
+	}
+	private static FilenameFilter getFileExtensionFilter(String extension) {
+		// 末尾の*を削除し、前方一致でファイル検索する
+		final String _extension = extension.replace("*", "");
+		return new FilenameFilter() {
+			public boolean accept(File file, String name) {
+				boolean ret = name.startsWith(_extension);
+				return ret;  
+			}
+		};
+	}
+
+	/**
+	 * ファイルリストの中で、更新日時が最新のファイル名を取得する
+	 * @param files ファイルリスト
+	 * @return ファイル名(存在しない場合はnull)
+	 */
+	private String getLatestFileName(File[] files) {
+		File latestFile = null;
+		
+		for(File file : files) {
+			if(latestFile == null || latestFile.lastModified() < file.lastModified())
+				latestFile = file;
+		}
+
+		if(latestFile == null){
+			//ファイルなし
+			return null;
+		}
+		logger.info("file to read: " +latestFile.getName());
+		return latestFile.getName();
+	}
+}