rbatch 1.8.0 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/.gitignore +2 -0
  2. data/CHANGELOG +43 -0
  3. data/HOW_TO_TEST.md +9 -0
  4. data/README.ja.md +6 -5
  5. data/README.md +6 -5
  6. data/Rakefile +1 -38
  7. metadata +78 -112
  8. data/sample/bin/log_backup.rb +0 -20
  9. data/sample/bin/test.rb +0 -6
  10. data/sample/bin/test2.rb +0 -3
  11. data/sample/conf/log_backup.yaml +0 -4
  12. data/sample/log/20130120_apache_log_insert.log +0 -171
  13. data/sample/log/20130120_openam_log_insert.log +0 -143
  14. data/sample/log/20130121_apache_log_insert.log +0 -103
  15. data/sample/log/20130121_file_batch_copy.log +0 -15
  16. data/sample/log/20130121_openam_log_insert.log +0 -39
  17. data/sample/log/20130121_webagent_log_insert.log +0 -432
  18. data/sample/log/20130122_apache_log_insert.log +0 -153
  19. data/sample/log/20130122_openam_log_insert.log +0 -374
  20. data/sample/log/20130122_webagent_log_insert.log +0 -144
  21. data/sample/log/20130129_log_backup.log +0 -95
  22. data/sample/log/20130131_apache_log_insert.log +0 -23
  23. data/sample/log/20130209_test.log +0 -128
  24. data/sample/log/20130210_test.log +0 -68
  25. data/sample/moto/README +0 -16
  26. data/sample/moto/auditlog.src.zip +0 -0
  27. data/sample/moto/build.gradle +0 -106
  28. data/sample/moto/gradle/wrapper/gradle-wrapper.jar +0 -0
  29. data/sample/moto/gradle/wrapper/gradle-wrapper.properties +0 -7
  30. data/sample/moto/gradle.properties +0 -12
  31. data/sample/moto/gradlew +0 -164
  32. data/sample/moto/gradlew.bat +0 -90
  33. data/sample/moto/settings.gradle +0 -1
  34. data/sample/moto/src/main/bin/delete_auditlog.sh +0 -176
  35. data/sample/moto/src/main/bin/delete_workflowinstances.sh +0 -315
  36. data/sample/moto/src/main/bin/import_auth_log.sh +0 -23
  37. data/sample/moto/src/main/etc/delete_auditlog.param +0 -39
  38. data/sample/moto/src/main/etc/delete_workflowinstances.param +0 -30
  39. data/sample/moto/src/main/etc/log4j.xml +0 -28
  40. data/sample/moto/src/main/etc/status.properties +0 -27
  41. data/sample/moto/src/main/etc/sysparam.properties +0 -54
  42. data/sample/moto/src/main/java/jp/ossc/imortAuditlog/DateUtil.java +0 -48
  43. data/sample/moto/src/main/java/jp/ossc/imortAuditlog/ImportAuthLogMain.java +0 -36
  44. data/sample/moto/src/main/java/jp/ossc/imortAuditlog/LogReader.java +0 -521
  45. data/sample/moto/src/main/java/jp/ossc/imortAuditlog/MysqlController.java +0 -315
  46. data/sample/moto/src/main/logrotate/delete_auditlog +0 -6
  47. data/sample/moto/src/main/logrotate/delete_workflowinstanceslog +0 -6
  48. data/sample/moto/src/main/sql/tables.sql +0 -36
@@ -1,521 +0,0 @@
1
- package jp.ossc.imortAuditlog;
2
-
3
- import java.io.BufferedReader;
4
- import java.io.File;
5
- import java.io.FileInputStream;
6
- import java.io.FileNotFoundException;
7
- import java.io.FilenameFilter;
8
- import java.io.InputStreamReader;
9
- import java.util.Enumeration;
10
- import java.util.HashMap;
11
- import java.util.Map;
12
- import java.util.PropertyResourceBundle;
13
- import java.util.ResourceBundle;
14
- import java.util.regex.Matcher;
15
- import java.util.regex.Pattern;
16
-
17
- import org.apache.log4j.Logger;
18
-
19
- public class LogReader {
20
- private static Logger logger = Logger.getLogger(LogReader.class);
21
- MysqlController cont = new MysqlController();
22
- private static final String not_available = "\"Not Available\"";
23
-
24
- public boolean authLogReader(ResourceBundle sysResource){
25
- logger.info("auth_log_file_read start");
26
-
27
- int executeCount = Integer.valueOf(sysResource.getString("execute.count"));
28
- String logPath = sysResource.getString("sso.authlog.path");
29
- File f = new File(logPath);
30
- if(f.isDirectory()){
31
- String authLogName = sysResource.getString("sso.authlog.name");
32
- String errLogName = sysResource.getString("sso.autherrlog.name");
33
-
34
- // 前方一致検索で一致するファイルリストを取得
35
- File[] authLogFiles = f.listFiles(getFileExtensionFilter(authLogName));
36
- File[] errLogFiles = f.listFiles(getFileExtensionFilter(errLogName));
37
- // 更新日時が最新のファイル名を取得する
38
- authLogName = getLatestFileName(authLogFiles);
39
- errLogName = getLatestFileName(errLogFiles);
40
-
41
- if(!logPath.substring(logPath.length()-1).equals("/")){
42
- logPath = logPath + "/";
43
- }
44
- if(authLogName == null){
45
- logger.warn("auth_log_file does not exist!!");
46
- }else{
47
- String authLogFile = logPath + authLogName;
48
- try {
49
- InputStreamReader srAuth = new InputStreamReader(new FileInputStream(authLogFile), "UTF8");
50
- BufferedReader brAuth = new BufferedReader(srAuth);
51
- String strAuth;
52
- cont.connect();
53
- cont.setAuthQuery();
54
- int dataCount = 0;
55
- while ((strAuth = brAuth.readLine()) != null) {
56
- Map authLog = editAuthLog(strAuth);
57
- if(authLog != null){
58
- dataCount++;
59
- cont.insertAuth(authLog);
60
- if(dataCount%executeCount == 0){
61
- cont.executeAuthBatch();
62
- logger.info("insert record:" + cont.excuteAuthCount);
63
- }
64
- }
65
- }
66
- cont.executeAuthBatch();
67
- cont.commit();
68
- logger.info("insert record:" + cont.excuteAuthCount);
69
- } catch (FileNotFoundException e) {
70
- logger.error("amAuthentication.access not file");
71
- throw new IllegalStateException(e);
72
- } catch (Exception e) {
73
- logger.error("amAuthentication.access read error");
74
- e.printStackTrace();
75
- }
76
-
77
- }
78
-
79
- if(errLogName == null){
80
- logger.warn("auth_err_log_file does not exist!!");
81
- }else{
82
- String errLogFile = logPath + errLogName;
83
- try {
84
- InputStreamReader srErr = new InputStreamReader(new FileInputStream(errLogFile), "UTF8");
85
- BufferedReader brErr = new BufferedReader(srErr);
86
- String strErr;
87
- cont.connect();
88
- cont.setAuthQuery();
89
- int dataCount = 0;
90
- while ((strErr = brErr.readLine()) != null) {
91
- Map errLog = editAuthLog(strErr);
92
- if(errLog != null){
93
- dataCount++;
94
- cont.insertAuth(errLog);
95
- if(dataCount%executeCount == 0){
96
- cont.executeAuthBatch();
97
- logger.info("insert record:" + cont.excuteAuthCount);
98
- }
99
- }
100
- }
101
- cont.executeAuthBatch();
102
- cont.commit();
103
- logger.info("insert record:" + cont.excuteAuthCount);
104
- } catch (FileNotFoundException e) {
105
- logger.error("amAuthentication.error not file " + errLogFile);
106
- throw new IllegalStateException(e);
107
- } catch (Exception e) {
108
- logger.error("amAuthentication.error read error " + errLogFile);
109
- e.printStackTrace();
110
- }
111
- }
112
- }
113
- logger.info("auth_log total insert record:" + cont.excuteAuthCount);
114
- logger.info("auth_log_file_read end");
115
-
116
- return true;
117
- }
118
-
119
- public static Map editAuthLog(String str){
120
- Map<String, String> map = new HashMap<String, String>();
121
- String strDate = null;
122
- String strUserId = null;
123
- String strClientIP = null;
124
- String strStatus = null;
125
- if(!str.startsWith("#")){
126
- String[] strAry = str.split("\t");
127
- if(strAry.length==12){
128
- strDate = strAry[0].replace("\"", "");
129
- String[] strLoginId = strAry[2].split(",");
130
- for(int i =0; i < strLoginId.length; i++){
131
- if(strLoginId[i].toString().startsWith("id=")){
132
- String[] wUserId = strLoginId[i].split("=");
133
- strUserId = wUserId[1];
134
- // OpenLDAP 認証モジュールの場合の、ユーザIDは、"uid="で始まる為、それも取り込む
135
- // また、パスワード未入力や、存在しないユーザIDの場合も、"uid="で出力されるように
136
- // 認証モジュール側を修正したので、以下のコードで取り込まれる。
137
- } else if(strLoginId[i].toString().startsWith("uid=")){
138
- String[] wUserId = strLoginId[i].split("=");
139
- strUserId = wUserId[1];
140
- }
141
- }
142
- if(strUserId == null){
143
- strUserId = "-";
144
- }
145
- strClientIP = strAry[4];
146
- strStatus = getStatus(strAry[8]);
147
- }
148
- }
149
- if(strDate != null && strUserId != null && strStatus != null){
150
- map.put("DATE", strDate);
151
- map.put("USERID", strUserId);
152
- map.put("CLIENTIP", strClientIP);
153
- map.put("STATUS", strStatus);
154
- return map;
155
- }else{
156
- return null;
157
- }
158
- }
159
-
160
- public static Map editErrLog(String str){
161
- Map<String, String> map = new HashMap<String, String>();
162
- String strDate = null;
163
- String strUserId = null;
164
- String strClientIP = null;
165
- String strStatus = null;
166
- if(!str.startsWith("#")){
167
- String[] strAry = str.split("\t");
168
- if(strAry.length==12){
169
- strDate = strAry[0].replace("\"", "");
170
- if(!strAry[2].equals(not_available)){
171
- strUserId = strAry[2];
172
- }
173
- if(strUserId == null){
174
- strUserId = "-";
175
- }
176
- strClientIP = strAry[4];
177
- strStatus = getStatus(strAry[8]);
178
- }
179
- }
180
- if(strDate != null && strUserId != null && strStatus != null){
181
- map.put("DATE", strDate);
182
- map.put("USERID", strUserId);
183
- map.put("CLIENTIP", strClientIP);
184
- map.put("STATUS", strStatus);
185
- return map;
186
- }else{
187
- return null;
188
- }
189
- }
190
-
191
- public static String getStatus(String strStatus){
192
- String status = null;
193
- ResourceBundle statusResource = PropertyResourceBundle.getBundle("status");
194
- if(!strStatus.matches("-")){
195
- if(strStatus.startsWith("AUTHENTICATION")){
196
- String[] strStatusid = strStatus.split("-");
197
- status = strStatusid[1];
198
- Enumeration<String> test = statusResource.getKeys();
199
- while (test.hasMoreElements()) {
200
- if(test.nextElement().equals("auth."+strStatusid[1])){
201
- status = statusResource.getString("auth."+ strStatusid[1]);
202
- }
203
- }
204
- }
205
- }
206
- return status;
207
- }
208
- public boolean accessLogReader(ResourceBundle sysResource , String sw){
209
- logger.info("access_log_file_read start");
210
-
211
- int executeCount = Integer.valueOf(sysResource.getString("execute.count"));
212
- String logPath = sysResource.getString(sw + ".accesslog.path");
213
- File f = new File(logPath);
214
- if(f.isDirectory()){
215
- String accessLogName = sysResource.getString(sw + ".accesslog.name");
216
-
217
- // 前方一致検索で一致するファイルリストを取得
218
- File[] accessLogFiles = f.listFiles(getFileExtensionFilter(accessLogName));
219
- // 更新日時が最新のファイル名を取得する
220
- accessLogName = getLatestFileName(accessLogFiles);
221
- if(accessLogName == null){
222
- logger.warn("access_log_file does not exist!!");
223
- }else{
224
- if(!logPath.substring(logPath.length()-1).equals("/")){
225
- logPath = logPath + "/";
226
- }
227
- String accessLogFile = logPath + accessLogName;
228
- try {
229
- InputStreamReader srAccess = new InputStreamReader(new FileInputStream(accessLogFile), "UTF8");
230
- BufferedReader brAuth = new BufferedReader(srAccess);
231
- String strAccess;
232
- cont.connect();
233
- cont.setQuery();
234
- int dataCount = 0;
235
- while ((strAccess = brAuth.readLine()) != null) {
236
- Map accessLog = editAccessLog(strAccess , sysResource ,sw);
237
- // "#" がついた行を取り込もうとして、NPEが発生する為の対処
238
- if(accessLog != null && !accessLog.isEmpty()){
239
- dataCount++;
240
- cont.insertAccessBatch(accessLog);
241
- if(dataCount%executeCount == 0){
242
- cont.executeBatch();
243
- logger.info("insert record:" + cont.excuteCount);
244
- }
245
- }
246
- }
247
- cont.executeBatch();
248
- cont.commit();
249
- logger.info("access_log total insert record:" + cont.excuteCount);
250
- cont.disconnect();
251
- } catch (FileNotFoundException e) {
252
- logger.error("access_log not file");
253
- throw new IllegalStateException(e);
254
- } catch (Exception e) {
255
- logger.error("access_log read error");
256
- e.printStackTrace();
257
- }
258
- }
259
- }
260
- logger.info("access_log_file_read end");
261
- return true;
262
- }
263
-
264
- public static Map editAccessLog(String str ,ResourceBundle sysResource , String sw){
265
- Map<String, String> map = new HashMap<String, String>();
266
- String[] exclude = sysResource.getString("exclude.pattern").split(",");
267
-
268
- int clientIp_p = Integer.valueOf(sysResource.getString(sw + ".clientip.point"));
269
- int userId_p = Integer.valueOf(sysResource.getString(sw + ".userid.point"));
270
- int date_p = Integer.valueOf(sysResource.getString(sw + ".date.point"));
271
- int processingTime_p = Integer.valueOf(sysResource.getString(sw + ".processingtime.point"));
272
- int request_p = Integer.valueOf(sysResource.getString(sw + ".request.point"));
273
- int status_p = Integer.valueOf(sysResource.getString(sw + ".status.point"));
274
- int agent_p = Integer.valueOf(sysResource.getString(sw + ".agent.point"));
275
- int referer_p = 0;
276
- if(sysResource.getString(sw + ".referer.point") != null && sysResource.getString(sw + ".referer.point").length() != 0){
277
- referer_p = Integer.valueOf(sysResource.getString(sw + ".referer.point"));
278
- }
279
-
280
- String strClientIp = null;
281
- String strUserId = null;
282
- String strDate = null;
283
- String strProcessingTime = null;
284
- String strRequest = null;
285
- String strStatus = null;
286
- String strAgent = null;
287
- String strReferer = null;
288
- String str2 = new String();
289
- if(!str.startsWith("#")){
290
- str2 = logSplit(str);
291
-
292
- String[] strAry = str2.split("\t");
293
- String[] wkIp = strAry[clientIp_p -1].split(",");
294
- strClientIp = wkIp[0];
295
- strUserId = strAry[userId_p -1];
296
- strDate = strAry[date_p -1];
297
- strProcessingTime = strAry[processingTime_p -1];
298
- Matcher m = Pattern.compile("^(.+\\s.+)(\\s.+)").matcher(strAry[request_p -1].replace("\"", ""));
299
- if (m.find()) {
300
- strRequest = m.group(1);
301
- }else{
302
- strRequest = strAry[request_p -1].replace("\"", "");
303
- }
304
- strStatus = strAry[status_p -1];
305
- strAgent = strAry[agent_p -1].replace("\"", "");
306
-
307
- if(referer_p != 0 && strAry.length >= referer_p){
308
- strReferer = strAry[referer_p -1].replace("\"", "");
309
- }
310
-
311
- for(int i = 0 ; i < exclude.length ; i++){
312
- // 除外対象の値が空値であれば無視
313
- if (exclude[i].toString().equals("")) {
314
- continue;
315
- }
316
- String compare = ".*" + "."+exclude[i].toLowerCase() +".*";
317
- if(strRequest.toLowerCase().matches(compare)){
318
- return null;
319
- }
320
- }
321
-
322
- map.put("CLIENTIP", strClientIp);
323
- map.put("USERID", strUserId);
324
- map.put("DATE", strDate);
325
- map.put("PROCESSINGTIME", strProcessingTime);
326
- map.put("REQUEST", strRequest);
327
- map.put("STATUS", strStatus);
328
- map.put("AGENT", strAgent);
329
- map.put("REFERER", strReferer);
330
- }
331
- return map;
332
- }
333
-
334
- public boolean agentAuthLogReader(ResourceBundle sysResource){
335
- logger.info("agent_auth_log_file_read start");
336
-
337
- int executeCount = Integer.valueOf(sysResource.getString("execute.count"));
338
- String logPath = sysResource.getString("sso.agent.authlog.path");
339
- File f = new File(logPath);
340
- if(f.isDirectory()){
341
- String agentAuthLogName = sysResource.getString("sso.agent.authlog.name");
342
-
343
- // 前方一致検索で一致するファイルリストを取得
344
- File[] agentAuthLogFiles = f.listFiles(getFileExtensionFilter(agentAuthLogName));
345
- // 更新日時が最新のファイル名を取得する
346
- agentAuthLogName = getLatestFileName(agentAuthLogFiles);
347
- if(agentAuthLogName == null){
348
- logger.warn("agent_auth_file does not exist!!");
349
- }else{
350
- if(!logPath.substring(logPath.length()-1).equals("/")){
351
- logPath = logPath + "/";
352
- }
353
- String agetntAuthLogFile = logPath + agentAuthLogName;
354
- try {
355
- InputStreamReader srAgentAuth = new InputStreamReader(new FileInputStream(agetntAuthLogFile), "UTF8");
356
- BufferedReader brAgentAuth = new BufferedReader(srAgentAuth);
357
- String strAgentAuth;
358
- cont.connect();
359
- cont.setAgentAuthQuery();
360
- int dataCount = 0;
361
- while ((strAgentAuth = brAgentAuth.readLine()) != null) {
362
- Map authLog = editAgentAuthLog(strAgentAuth, sysResource);
363
- if(authLog != null){
364
- dataCount++;
365
- cont.insertAgentAuth(authLog);
366
- if(dataCount%executeCount == 0){
367
- cont.executeAgentAuthBatch();
368
- logger.info("insert record:" + cont.excuteAgentAuthCount);
369
- }
370
- }
371
- }
372
- cont.executeAgentAuthBatch();
373
- cont.commit();
374
- logger.info("insert record:" + cont.excuteAgentAuthCount);
375
- } catch (FileNotFoundException e) {
376
- logger.error(agentAuthLogName + " not file");
377
- throw new IllegalStateException(e);
378
- } catch (Exception e) {
379
- logger.error(agentAuthLogName + " read error");
380
- e.printStackTrace();
381
- }
382
- }
383
- }
384
- logger.info("agent_auth_log total insert record:" + cont.excuteAgentAuthCount);
385
- logger.info("agent_auth_log_file_read end");
386
-
387
- return true;
388
- }
389
-
390
- public static Map editAgentAuthLog(String str, ResourceBundle sysResource){
391
- Map<String, String> map = new HashMap<String, String>();
392
- String strDate = null;
393
- String strUserId = null;
394
- String strRequest = null;
395
- String strStatus = null;
396
- String[] exclude = sysResource.getString("sso.agent.authlog.exclude.pattern").split(",");
397
- String keyword = sysResource.getString("sso.agent.authlog.keyword");
398
- int request_p = Integer.valueOf(sysResource.getString("sso.agent.authlog.request.point"));
399
-
400
- // ステータス取得
401
- if(str.indexOf("allowed access") != -1) {
402
- strStatus = "ALLOW";
403
- }
404
- else if (str.indexOf("denied access") != -1) {
405
- strStatus = "DISALLOW";
406
- }
407
-
408
- if(str.indexOf("\"") != -1) {
409
- str=str.replaceAll("\"","");
410
- }
411
- if(str.indexOf("\t") != -1) {
412
- str=str.replaceAll("\t"," ");
413
- }
414
-
415
- if (str.indexOf(keyword) != -1 && strStatus != null) {
416
- String[] strAry = str.split(" ");
417
- if(strAry.length >= 12){
418
- // 日付取得
419
- strDate = strAry[0].toString() + " " + strAry[1].toString();
420
- // ログに「User」の単語があれば次の配列値をユーザーIDとする
421
- for(int i =2; i < strAry.length; i++){
422
- if (strAry[i].toString().equals("User")) {
423
- if ((i+1) < strAry.length) {
424
- strUserId = strAry[i+1].toString();
425
- break;
426
- }
427
- }
428
- }
429
- // リクエストURL取得
430
- // strRequest = strAry[strAry.length-1].toString();
431
- strRequest = strAry[request_p-1].toString();
432
- }
433
- }
434
-
435
- if(strDate != null && strUserId != null && strStatus != null && strRequest != null){
436
- // リクエストURLが除外対象の拡張子の場合はDBへ登録しない
437
- for(int i = 0 ; i < exclude.length ; i++){
438
- // 除外対象の値が空値であれば無視
439
- if (exclude[i].toString().equals("")) {
440
- continue;
441
- }
442
- String compare = ".*" + "."+exclude[i].toLowerCase() +".*";
443
- if(strRequest.toLowerCase().matches(compare)){
444
- return null;
445
- }
446
- }
447
-
448
- map.put("DATE", strDate);
449
- map.put("USERID", strUserId);
450
- map.put("STATUS", strStatus);
451
- map.put("REQUEST", strRequest);
452
- return map;
453
- }else{
454
- return null;
455
- }
456
- }
457
-
458
- public static String logSplit(String var){
459
- String strCahr = new String();
460
- String strRep = new String();
461
- boolean inSquareBracket = false;
462
- boolean inDoubleQuotation = false;
463
- var = var.replace(", ", ",");
464
- for(int i = 0; i < var.length(); i++) {
465
- strCahr = var.substring(i,i+1);
466
- // "[" や '"' の内部のスペースでは、区切りタブ出力しない
467
- if(strCahr.equals(" ") && !inDoubleQuotation && !inSquareBracket){
468
- strRep = strRep + "\t";
469
- }else{
470
-
471
- // '"' に囲われていない場合にだけ、
472
- // "[" と "]" の開始終了をチェックする
473
- if(strCahr.equals("[") && !inSquareBracket && !inDoubleQuotation) {
474
- inSquareBracket = true;
475
- }else if (strCahr.equals("]") && inSquareBracket && !inDoubleQuotation){
476
- inSquareBracket = false;
477
- }
478
-
479
- // '"' と '"' の開始終了をチェック
480
- if((strCahr.equals("\"")) && !inDoubleQuotation) {
481
- inDoubleQuotation = true;
482
- }else if ((strCahr.equals("\"")) && inDoubleQuotation){
483
- inDoubleQuotation = false;
484
- }
485
- strRep = strRep + strCahr;
486
- }
487
- }
488
- return strRep;
489
- }
490
- private static FilenameFilter getFileExtensionFilter(String extension) {
491
- // 末尾の*を削除し、前方一致でファイル検索する
492
- final String _extension = extension.replace("*", "");
493
- return new FilenameFilter() {
494
- public boolean accept(File file, String name) {
495
- boolean ret = name.startsWith(_extension);
496
- return ret;
497
- }
498
- };
499
- }
500
-
501
- /**
502
- * ファイルリストの中で、更新日時が最新のファイル名を取得する
503
- * @param files ファイルリスト
504
- * @return ファイル名(存在しない場合はnull)
505
- */
506
- private String getLatestFileName(File[] files) {
507
- File latestFile = null;
508
-
509
- for(File file : files) {
510
- if(latestFile == null || latestFile.lastModified() < file.lastModified())
511
- latestFile = file;
512
- }
513
-
514
- if(latestFile == null){
515
- //ファイルなし
516
- return null;
517
- }
518
- logger.info("file to read: " +latestFile.getName());
519
- return latestFile.getName();
520
- }
521
- }