rbatch 2.1.5 → 2.1.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -1
- data/README.ja.md +115 -35
- data/README.md +121 -40
- data/lib/rbatch/log.rb +9 -1
- data/lib/rbatch/version.rb +1 -1
- data/sample/bin/ec2_create_snapshot.rb +32 -0
- data/sample/bin/file_batch_copy.rb +3 -4
- data/sample/bin/log_backup.rb +3 -5
- data/sample/conf/common.yaml +0 -5
- data/sample/conf/ec2_create_snapshot.yaml +2 -0
- data/sample/log/.gitkeep +0 -0
- data/spec/rbatch/log_spec.rb +20 -0
- metadata +5 -7
- data/sample/bin/openam_log_insert.rb +0 -119
- data/sample/bin/webagent_log_insert.rb +0 -110
- data/sample/conf/log_backup.yaml +0 -4
- data/sample/conf/openam_log_insert.yaml +0 -35
- data/sample/conf/webagent_log_insert.yaml +0 -16
data/.gitignore
CHANGED
data/README.ja.md
CHANGED
@@ -79,14 +79,15 @@ RBatchでは、実行スクリプト、設定ファイルおよびログファ
|
|
79
79
|
|
80
80
|
スクリプト`${RB_HOME}/bin/sample1.rb` を作ります
|
81
81
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
82
|
+
```ruby
|
83
|
+
require 'rbatch'
|
84
|
+
|
85
|
+
RBatch::Log.new(){ |log| # ログブロック
|
86
|
+
log.info "info string"
|
87
|
+
log.error "error string"
|
88
|
+
raise "exception"
|
89
|
+
}
|
90
|
+
```
|
90
91
|
|
91
92
|
実行するとログファイル`${RB_HOME}/log/20121020_005953_sample1.log`が以下のように出力されます
|
92
93
|
|
@@ -149,12 +150,13 @@ RBatchは外部コマンド(たとえば"ls -l")を実行するラッパー
|
|
149
150
|
|
150
151
|
サンプル
|
151
152
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
153
|
+
```ruby
|
154
|
+
require 'rbatch'
|
155
|
+
r = RBatch.cmd("ls")
|
156
|
+
p r.stdout # => "fileA\nfileB\n"
|
157
|
+
p r.stderr # => ""
|
158
|
+
p r.status # => 0
|
159
|
+
```
|
158
160
|
|
159
161
|
外部コマンドにタイムアウトを設定したい場合は`cmd_timeout`をオプションを利用できます。
|
160
162
|
|
@@ -164,6 +166,80 @@ RBatchは外部コマンド(たとえば"ls -l")を実行するラッパー
|
|
164
166
|
|
165
167
|
`forbid_double_run`のオプションを利用すれば、RBatchを利用したプログラムの二重起動チェックができます。
|
166
168
|
|
169
|
+
|
170
|
+
サンプル
|
171
|
+
--------------
|
172
|
+
|
173
|
+
### AWS EC2 のボリュームバックアップスクリプト
|
174
|
+
|
175
|
+
最初に以下の設定ファイルを作ります
|
176
|
+
|
177
|
+
設定ファイル : `${RB_HOME}/conf/ec2_create_snapshot.yaml`
|
178
|
+
|
179
|
+
```
|
180
|
+
access_key : AKIAITHEXXXXXXXXX
|
181
|
+
secret_key : JoqJSdP8+tpdFYWljVbG0+XXXXXXXXXXXXXXX
|
182
|
+
|
183
|
+
```
|
184
|
+
|
185
|
+
次に、スクリプトを書きます。
|
186
|
+
|
187
|
+
スクリプト : `${RB_HOME}/bin/ec2_create_snapshot.rb`
|
188
|
+
|
189
|
+
```ruby
|
190
|
+
require 'rbatch' # <= rbatchをrequireします
|
191
|
+
require 'aws-sdk'
|
192
|
+
require 'net/http'
|
193
|
+
|
194
|
+
RBatch::Log.new do |log| # <= ログブロックを開始し、スクリプトはこの中に入れます
|
195
|
+
# get ec2 region
|
196
|
+
@ec2_region = "ec2." +
|
197
|
+
Net::HTTP.get("169.254.169.254", "/latest/meta-data/placement/availability-zone").chop +
|
198
|
+
".amazonaws.com"
|
199
|
+
log.info("ec2 region : #{@ec2_region}") # <= このようにログを出力できます
|
200
|
+
|
201
|
+
#create ec2 instance
|
202
|
+
@ec2 = AWS::EC2.new(:access_key_id => RBatch.config["access_key"], # <= 設定ファイルを読み込みます
|
203
|
+
:secret_access_key => RBatch.config["secret_key"],
|
204
|
+
:ec2_endpoint => @ec2_region)
|
205
|
+
|
206
|
+
|
207
|
+
# create instance
|
208
|
+
@instance_id = Net::HTTP.get("169.254.169.254", "/latest/meta-data/instance-id")
|
209
|
+
@instance = @ec2.instances[@instance_id]
|
210
|
+
log.info("instance_id : #{@instance_id}")
|
211
|
+
|
212
|
+
# create snapshots
|
213
|
+
@instance.block_devices.each do | dev |
|
214
|
+
desc = @instance_id + " " + dev[:device_name] + " " +
|
215
|
+
dev[:ebs][:volume_id] + " " +Time.now.strftime("%Y/%m/%d %H:%M").to_s
|
216
|
+
log.info("create snapshot : #{desc}")
|
217
|
+
@ec2.volumes[dev[:ebs][:volume_id]].create_snapshot(desc)
|
218
|
+
log.info("sucess")
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
```
|
223
|
+
|
224
|
+
最後にスクリプトを実行すると、以下のようなログが出力されます
|
225
|
+
|
226
|
+
ログファイル : `${RB_HOME}/log/20140121_123124_ec2_create_snapshot.log`
|
227
|
+
|
228
|
+
```
|
229
|
+
[2014-01-21 12:31:24 -0500] [INFO ] === START RBatch === (PID=10095)
|
230
|
+
[2014-01-21 12:31:24 -0500] [INFO ] RB_HOME : "/opt/MyProject"
|
231
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Load Run-Conf: "/opt/MyProject/.rbatchrc"
|
232
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Load Config : "/opt/MyProject/conf/ec2_create_snapshot.yaml"
|
233
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Start Script : "/opt/MyProject/bin/ec2_create_snapshot.rb"
|
234
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Logging Start: "/opt/MyProject/log/20140121_123124_ec2_create_snapshot.log"
|
235
|
+
[2014-01-21 12:31:24 -0500] [INFO ] ec2 region : ec2.ap-northeast-1.amazonaws.com
|
236
|
+
[2014-01-21 12:31:25 -0500] [INFO ] instance_id : i-cc25f1c9
|
237
|
+
[2014-01-21 12:31:25 -0500] [INFO ] create snapshot : i-cc25f1c9 /dev/sda1 vol-82483ea7 2014/01/21 12:31
|
238
|
+
[2014-01-21 12:31:25 -0500] [INFO ] sucess
|
239
|
+
```
|
240
|
+
|
241
|
+
これだけで、ログ出力と設定ファイル読み込みを兼ね備えたスクリプトを作成することができます。
|
242
|
+
|
167
243
|
カスタマイズ
|
168
244
|
--------------
|
169
245
|
|
@@ -347,31 +423,35 @@ Run-Conf(`${RB_HOME}/.rbatchrc`)のサンプルは以下の通り
|
|
347
423
|
|
348
424
|
#### RBatch::Logクラスのオプション
|
349
425
|
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
426
|
+
```ruby
|
427
|
+
opt = {
|
428
|
+
:name => "<date>_<time>_<prog>.log",
|
429
|
+
:dir => "/var/log/",
|
430
|
+
:append => true,
|
431
|
+
:level => "info",
|
432
|
+
:stdout => false,
|
433
|
+
:delete_old_log => false,
|
434
|
+
:delete_old_log_date => 7,
|
435
|
+
:send_mail => false,
|
436
|
+
:mail_to => nil,
|
437
|
+
:mail_from => "rbatch.localhost",
|
438
|
+
:mail_server_host => "localhost",
|
439
|
+
:mail_server_port => 25
|
440
|
+
}
|
441
|
+
|
442
|
+
RBatch::Log.new(opt)
|
443
|
+
```
|
366
444
|
|
367
445
|
#### RBatch::Cmdクラスのオプション
|
368
446
|
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
447
|
+
```ruby
|
448
|
+
opt = {
|
449
|
+
:raise => false,
|
450
|
+
:timeout => 0
|
451
|
+
}
|
373
452
|
|
374
|
-
|
453
|
+
RBatch::Cmd.new(cmd_str, opt).run
|
454
|
+
```
|
375
455
|
|
376
456
|
version 1 から 2 へのマイグレーション
|
377
457
|
--------------
|
data/README.md
CHANGED
@@ -79,13 +79,14 @@ sample
|
|
79
79
|
|
80
80
|
make script `${RB_HOME}/bin/sample1.rb`
|
81
81
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
82
|
+
```ruby
|
83
|
+
require 'rbatch'
|
84
|
+
RBatch::Log.new(){ |log| # Logging block
|
85
|
+
log.info "info string"
|
86
|
+
log.error "error string"
|
87
|
+
raise "exception"
|
88
|
+
}
|
89
|
+
```
|
89
90
|
|
90
91
|
Run script. Log file is `${RB_HOME}/log/20121020_005953_sample1.log`
|
91
92
|
|
@@ -123,15 +124,16 @@ make config file `${RB_HOME}/conf/sample2.yaml`.
|
|
123
124
|
|
124
125
|
In script `${RB_HOME}/bin/sample2.rb` , you can read config.
|
125
126
|
|
126
|
-
|
127
|
-
|
128
|
-
# You can read config value without loading file.
|
129
|
-
p RBatch.config["key"] # => "value"
|
130
|
-
p RBatch.config["array"] # => ["item1", "item2", "item3"]
|
127
|
+
```ruby
|
128
|
+
require 'rbatch'
|
131
129
|
|
132
|
-
|
133
|
-
|
130
|
+
# You can read config value without loading file.
|
131
|
+
p RBatch.config["key"] # => "value"
|
132
|
+
p RBatch.config["array"] # => ["item1", "item2", "item3"]
|
134
133
|
|
134
|
+
# If key does not exist, raise exception
|
135
|
+
p RBatch.config["not_exist"] # => Raise RBatch::ConfigException
|
136
|
+
```
|
135
137
|
|
136
138
|
#### Common Config
|
137
139
|
|
@@ -146,11 +148,14 @@ This function return a result object which contain command's "STDOUT", "STDERR"
|
|
146
148
|
|
147
149
|
sample
|
148
150
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
151
|
+
```ruby
|
152
|
+
require 'rbatch'
|
153
|
+
|
154
|
+
r = RBatch.cmd("ls")
|
155
|
+
p r.stdout # => "fileA\nfileB\n"
|
156
|
+
p r.stderr # => ""
|
157
|
+
p r.status # => 0
|
158
|
+
```
|
154
159
|
|
155
160
|
If you want to set a timeout of external command, you can use `cmd_timeout` option.
|
156
161
|
|
@@ -161,6 +166,79 @@ You have no use for an error handring.
|
|
161
166
|
|
162
167
|
Using `forbid_double_run` option, two same name scripts cannot run at the same time.
|
163
168
|
|
169
|
+
Sample
|
170
|
+
--------------
|
171
|
+
|
172
|
+
### AWS EC2 Volume Backup Script
|
173
|
+
|
174
|
+
First you make configuration file.
|
175
|
+
|
176
|
+
Config File : `${RB_HOME}/conf/ec2_create_snapshot.yaml`
|
177
|
+
|
178
|
+
```
|
179
|
+
access_key : AKIAITHEXXXXXXXXX
|
180
|
+
secret_key : JoqJSdP8+tpdFYWljVbG0+XXXXXXXXXXXXXXX
|
181
|
+
|
182
|
+
```
|
183
|
+
|
184
|
+
Next, you write script.
|
185
|
+
|
186
|
+
Script : `${RB_HOME}/bin/ec2_create_snapshot.rb`
|
187
|
+
|
188
|
+
```ruby
|
189
|
+
require 'rbatch' # <= require rbatch
|
190
|
+
require 'aws-sdk'
|
191
|
+
require 'net/http'
|
192
|
+
|
193
|
+
RBatch::Log.new do |log| # <= Start Log block. And write main logic in this block.
|
194
|
+
# get ec2 region
|
195
|
+
@ec2_region = "ec2." +
|
196
|
+
Net::HTTP.get("169.254.169.254", "/latest/meta-data/placement/availability-zone").chop +
|
197
|
+
".amazonaws.com"
|
198
|
+
log.info("ec2 region : #{@ec2_region}") # <= Output Log
|
199
|
+
|
200
|
+
#create ec2 instance
|
201
|
+
@ec2 = AWS::EC2.new(:access_key_id => RBatch.config["access_key"], # <= Read Config
|
202
|
+
:secret_access_key => RBatch.config["secret_key"],
|
203
|
+
:ec2_endpoint => @ec2_region)
|
204
|
+
|
205
|
+
|
206
|
+
# create instance
|
207
|
+
@instance_id = Net::HTTP.get("169.254.169.254", "/latest/meta-data/instance-id")
|
208
|
+
@instance = @ec2.instances[@instance_id]
|
209
|
+
log.info("instance_id : #{@instance_id}")
|
210
|
+
|
211
|
+
# create snapshots
|
212
|
+
@instance.block_devices.each do | dev |
|
213
|
+
desc = @instance_id + " " + dev[:device_name] + " " +
|
214
|
+
dev[:ebs][:volume_id] + " " +Time.now.strftime("%Y/%m/%d %H:%M").to_s
|
215
|
+
log.info("create snapshot : #{desc}")
|
216
|
+
@ec2.volumes[dev[:ebs][:volume_id]].create_snapshot(desc)
|
217
|
+
log.info("sucess")
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
```
|
222
|
+
|
223
|
+
Finally Run script , then following logfile is output.
|
224
|
+
|
225
|
+
Log file : `${RB_HOME}/log/20140121_123124_ec2_create_snapshot.log`
|
226
|
+
|
227
|
+
```
|
228
|
+
[2014-01-21 12:31:24 -0500] [INFO ] === START RBatch === (PID=10095)
|
229
|
+
[2014-01-21 12:31:24 -0500] [INFO ] RB_HOME : "/opt/MyProject"
|
230
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Load Run-Conf: "/opt/MyProject/.rbatchrc"
|
231
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Load Config : "/opt/MyProject/conf/ec2_create_snapshot.yaml"
|
232
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Start Script : "/opt/MyProject/bin/ec2_create_snapshot.rb"
|
233
|
+
[2014-01-21 12:31:24 -0500] [INFO ] Logging Start: "/opt/MyProject/log/20140121_123124_ec2_create_snapshot.log"
|
234
|
+
[2014-01-21 12:31:24 -0500] [INFO ] ec2 region : ec2.ap-northeast-1.amazonaws.com
|
235
|
+
[2014-01-21 12:31:25 -0500] [INFO ] instance_id : i-cc25f1c9
|
236
|
+
[2014-01-21 12:31:25 -0500] [INFO ] create snapshot : i-cc25f1c9 /dev/sda1 vol-82483ea7 2014/01/21 12:31
|
237
|
+
[2014-01-21 12:31:25 -0500] [INFO ] sucess
|
238
|
+
```
|
239
|
+
|
240
|
+
Only you write a small code, this batch script has logging and read-config function.
|
241
|
+
|
164
242
|
Customize
|
165
243
|
--------------
|
166
244
|
If you want to customize RBatch, you have two methods.
|
@@ -345,32 +423,35 @@ If you want to change options in a script, you pass an options object to the con
|
|
345
423
|
|
346
424
|
Sumple
|
347
425
|
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
426
|
+
```ruby
|
427
|
+
opt = {
|
428
|
+
:name => "<date>_<time>_<prog>.log",
|
429
|
+
:dir => "/var/log",
|
430
|
+
:append => true,
|
431
|
+
:level => "info",
|
432
|
+
:stdout => false,
|
433
|
+
:delete_old_log => false,
|
434
|
+
:delete_old_log_date => 7,
|
435
|
+
:send_mail => false,
|
436
|
+
:mail_to => nil,
|
437
|
+
:mail_from => "rbatch.localhost",
|
438
|
+
:mail_server_host => "localhost",
|
439
|
+
:mail_server_port => 25
|
440
|
+
}
|
441
|
+
RBatch::Log.new(opt)
|
442
|
+
```
|
363
443
|
|
364
444
|
#### option of RBatch::Cmd
|
365
445
|
|
366
446
|
Sample
|
367
447
|
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
448
|
+
```ruby
|
449
|
+
opt = {
|
450
|
+
:raise => false,
|
451
|
+
:timeout => 0
|
452
|
+
}
|
453
|
+
RBatch::Cmd.new("ls -l", opt).run
|
454
|
+
```
|
374
455
|
|
375
456
|
Migration from version 1 to version 2
|
376
457
|
--------------
|
data/lib/rbatch/log.rb
CHANGED
@@ -85,9 +85,17 @@ module RBatch
|
|
85
85
|
if block_given?
|
86
86
|
begin
|
87
87
|
yield self
|
88
|
+
rescue SystemExit => e
|
89
|
+
if e.status == 0
|
90
|
+
exit 0
|
91
|
+
else
|
92
|
+
self.fatal(e)
|
93
|
+
self.fatal("Caught SystemExit. RBatch Exit with status " + e.status.to_s)
|
94
|
+
exit e.status
|
95
|
+
end
|
88
96
|
rescue Exception => e
|
89
97
|
self.fatal(e)
|
90
|
-
self.fatal("Caught exception. Exit 1")
|
98
|
+
self.fatal("Caught exception. RBatch Exit with status 1")
|
91
99
|
exit 1
|
92
100
|
ensure
|
93
101
|
self.close
|
data/lib/rbatch/version.rb
CHANGED
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'rbatch'
|
2
|
+
|
3
|
+
RBatch::Log.new do |log|
|
4
|
+
require 'aws-sdk'
|
5
|
+
require 'net/http'
|
6
|
+
# get ec2 region
|
7
|
+
@ec2_region = "ec2." +
|
8
|
+
Net::HTTP.get("169.254.169.254", "/latest/meta-data/placement/availability-zone").chop +
|
9
|
+
".amazonaws.com"
|
10
|
+
log.info("ec2 region : #{@ec2_region}") # <= Output Log
|
11
|
+
|
12
|
+
#create ec2 instance
|
13
|
+
@ec2 = AWS::EC2.new(:access_key_id => RBatch.config["access_key"],
|
14
|
+
:secret_access_key => RBatch.config["secret_key"],
|
15
|
+
:ec2_endpoint => @ec2_region)
|
16
|
+
|
17
|
+
|
18
|
+
# create instance
|
19
|
+
@instance_id = Net::HTTP.get("169.254.169.254", "/latest/meta-data/instance-id")
|
20
|
+
@instance = @ec2.instances[@instance_id]
|
21
|
+
log.info("instance_id : #{@instance_id}")
|
22
|
+
|
23
|
+
# create snapshots
|
24
|
+
@instance.block_devices.each do | dev |
|
25
|
+
desc = @instance_id + " " + dev[:device_name] + " " +
|
26
|
+
dev[:ebs][:volume_id] + " " +Time.now.strftime("%Y/%m/%d %H:%M").to_s
|
27
|
+
log.info("create snapshot : #{desc}")
|
28
|
+
@ec2.volumes[dev[:ebs][:volume_id]].create_snapshot(desc)
|
29
|
+
log.info("sucess")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
@@ -1,12 +1,11 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
1
|
require 'rbatch'
|
3
|
-
require 'fileutils'
|
4
2
|
|
5
|
-
#
|
3
|
+
# File butch copy
|
6
4
|
RBatch::Log.new do |log|
|
5
|
+
require 'fileutils'
|
7
6
|
target_dir = RBatch::config["target_dir"]
|
8
7
|
RBatch::config["file_list"].each do | file |
|
9
|
-
log.info("copy " + file + "
|
8
|
+
log.info("copy " + file + " " + target_dir)
|
10
9
|
FileUtils.cp_r(file,target_dir)
|
11
10
|
end
|
12
11
|
end
|
data/sample/bin/log_backup.rb
CHANGED
@@ -1,15 +1,13 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
1
|
require 'rbatch'
|
3
|
-
require 'fileutils'
|
4
|
-
require 'date'
|
5
2
|
|
6
3
|
RBatch::Log.new do |log|
|
7
|
-
|
4
|
+
require 'fileutils'
|
5
|
+
require 'date'
|
8
6
|
target_dir = RBatch::config["target_dir"]
|
9
7
|
RBatch::config["file_list"].each do |file_wildcard|
|
10
8
|
Dir::glob(file_wildcard).each do |file|
|
11
9
|
if ! File.exists?(File.join(target_dir,File.basename(file)))
|
12
|
-
log.info("Copy " + file + "
|
10
|
+
log.info("Copy " + file + " " + target_dir)
|
13
11
|
FileUtils.cp(file,target_dir)
|
14
12
|
else
|
15
13
|
log.info("Skip " + file + " (already backuped)")
|
data/sample/conf/common.yaml
CHANGED
data/sample/log/.gitkeep
ADDED
File without changes
|
data/spec/rbatch/log_spec.rb
CHANGED
@@ -55,6 +55,26 @@ describe RBatch::Log do
|
|
55
55
|
}.to raise_error SystemExit
|
56
56
|
end
|
57
57
|
|
58
|
+
it "catch SystemExit 0 " do
|
59
|
+
expect {
|
60
|
+
RBatch::Log.new do | log |
|
61
|
+
exit 0
|
62
|
+
end
|
63
|
+
}.to raise_error SystemExit do |e|
|
64
|
+
expect(e.status).to eq 0
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
it "catch SystemExit 3" do
|
69
|
+
expect {
|
70
|
+
RBatch::Log.new do | log |
|
71
|
+
exit 3
|
72
|
+
end
|
73
|
+
}.to raise_error SystemExit do |e|
|
74
|
+
expect(e.status).to eq 3
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
58
78
|
it "raise error when log dir does not exist" do
|
59
79
|
Dir::rmdir(@log_dir)
|
60
80
|
expect{
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: rbatch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 2.1.
|
5
|
+
version: 2.1.6
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- fetaro
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2014-01-
|
13
|
+
date: 2014-01-24 00:00:00 Z
|
14
14
|
dependencies: []
|
15
15
|
|
16
16
|
description: Simple Framework of Batch Script
|
@@ -103,21 +103,19 @@ files:
|
|
103
103
|
- rbatch.gemspec
|
104
104
|
- sample/.rbatchrc
|
105
105
|
- sample/bin/apache_log_insert.rb
|
106
|
+
- sample/bin/ec2_create_snapshot.rb
|
106
107
|
- sample/bin/file_batch_copy.rb
|
107
108
|
- sample/bin/log_backup.rb
|
108
109
|
- sample/bin/mysql_data_backup.rb
|
109
|
-
- sample/bin/openam_log_insert.rb
|
110
110
|
- sample/bin/openldap_backup.rb
|
111
|
-
- sample/bin/webagent_log_insert.rb
|
112
111
|
- sample/conf/apache_log_insert.yaml
|
113
112
|
- sample/conf/common.yaml
|
113
|
+
- sample/conf/ec2_create_snapshot.yaml
|
114
114
|
- sample/conf/file_batch_copy.yaml
|
115
|
-
- sample/conf/log_backup.yaml
|
116
115
|
- sample/conf/mysql_data_backup.yaml
|
117
|
-
- sample/conf/openam_log_insert.yaml
|
118
116
|
- sample/conf/openldap_backup.yaml
|
119
|
-
- sample/conf/webagent_log_insert.yaml
|
120
117
|
- sample/lib/.gitkeep
|
118
|
+
- sample/log/.gitkeep
|
121
119
|
- spec/01_rbach_spec.rb
|
122
120
|
- spec/rbatch/cmd_spec.rb
|
123
121
|
- spec/rbatch/config_spec.rb
|
@@ -1,119 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
require 'rbatch'
|
3
|
-
require 'mysql'
|
4
|
-
|
5
|
-
#
|
6
|
-
# OpenAMの認証アクセスログと認証エラーログを解析してMySQLに挿入する
|
7
|
-
#
|
8
|
-
|
9
|
-
|
10
|
-
# ログの1行を表すクラス
|
11
|
-
class Entry
|
12
|
-
|
13
|
-
# 行をマッチさせる正規表現
|
14
|
-
@@reg=/^
|
15
|
-
\" # "
|
16
|
-
(\S+\s\S+) # 0 YYYY-MM-DD HH:MM:SS
|
17
|
-
\" # "
|
18
|
-
\t # タブ
|
19
|
-
.+ # 任意の文字列
|
20
|
-
\t # タブ
|
21
|
-
(.+) # 1 "DN"
|
22
|
-
\t # タブ
|
23
|
-
.+ # 任意の文字列
|
24
|
-
\t # タブ
|
25
|
-
(.+) # 2 IP
|
26
|
-
\t # タブ
|
27
|
-
.+ # 任意の文字列
|
28
|
-
\t # タブ
|
29
|
-
.+ # 任意の文字列
|
30
|
-
\t # タブ
|
31
|
-
.+ # 任意の文字列
|
32
|
-
\t # タブ
|
33
|
-
(.+) # 3 ステータス
|
34
|
-
\t # タブ
|
35
|
-
.+ # 任意の文字列
|
36
|
-
\t # タブ
|
37
|
-
.+ # 任意の文字列
|
38
|
-
\t # タブ
|
39
|
-
.+ # 任意の文字列
|
40
|
-
$/x
|
41
|
-
|
42
|
-
@entry
|
43
|
-
|
44
|
-
def initialize(line,status_map)
|
45
|
-
match = line.match(@@reg)
|
46
|
-
raise "parse error. line: <#{line}> " if match.nil?
|
47
|
-
captures = match.captures
|
48
|
-
if captures[1] =~ /uid=|id=/
|
49
|
-
login_id = captures[1].split(",")[0].split("=")[1]
|
50
|
-
else
|
51
|
-
login_id = "-"
|
52
|
-
end
|
53
|
-
if status_map.has_key?(captures[3])
|
54
|
-
status = status_map[captures[3]]
|
55
|
-
else
|
56
|
-
status = "-"
|
57
|
-
end
|
58
|
-
@entry = {
|
59
|
-
:date => DateTime.strptime(captures[0], '%Y-%m-%d %H:%M:%S'),
|
60
|
-
:login_id => login_id,
|
61
|
-
:access_ip => captures[2],
|
62
|
-
:status => status
|
63
|
-
}
|
64
|
-
end
|
65
|
-
|
66
|
-
# Insert用のSQLを返す
|
67
|
-
def insert_sql(table_name,host_name,companyId)
|
68
|
-
value = [ companyId ,
|
69
|
-
@entry[:date].strftime("%Y/%m/%d %H:%M:%S"),
|
70
|
-
@entry[:login_id],
|
71
|
-
@entry[:access_ip],
|
72
|
-
host_name,
|
73
|
-
@entry[:status]
|
74
|
-
].map{|s| "'#{s}'"}.join(",")
|
75
|
-
return "INSERT INTO #{table_name} (companyId,date,login_id,access_ip,host_name,status) VALUES (#{value})"
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
# メイン
|
80
|
-
|
81
|
-
RBatch::Log.new do |log|
|
82
|
-
log.info("Start -----------------");
|
83
|
-
entries = []
|
84
|
-
status_map = RBatch::config["auth_status"]
|
85
|
-
# 認証アクセスログ読み込み
|
86
|
-
File.foreach(RBatch::config["openam_access_log_path"]) do |line|
|
87
|
-
begin
|
88
|
-
entries << Entry.new(line,status_map)
|
89
|
-
rescue => e
|
90
|
-
# 解析に失敗した場合
|
91
|
-
log.warn("parse error: " + e.message )
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
# 認証エラーログ読み込み
|
96
|
-
File.foreach(RBatch::config["openam_error_log_path"]) do |line|
|
97
|
-
begin
|
98
|
-
entries << Entry.new(line,status_map)
|
99
|
-
rescue => e
|
100
|
-
# 解析に失敗した場合
|
101
|
-
log.warn("parse error: " + e.message )
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
# MySQLに接続
|
106
|
-
con = Mysql.new(RBatch::config["mysql_server"],
|
107
|
-
RBatch::config["mysql_user"],
|
108
|
-
RBatch::config["mysql_password"],
|
109
|
-
RBatch::config["mysql_db_name"])
|
110
|
-
# MySQLに挿入
|
111
|
-
entries.each do |entry|
|
112
|
-
sql = entry.insert_sql(RBatch::config["mysql_table_name"],
|
113
|
-
RBatch::config["host_name"],
|
114
|
-
RBatch::config["company_id"])
|
115
|
-
log.info("exec sql: " + sql);
|
116
|
-
con.query(sql)
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
@@ -1,110 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
require 'rbatch'
|
3
|
-
require 'mysql'
|
4
|
-
|
5
|
-
#
|
6
|
-
# WebAgentの認可ログを解析してMySQLに挿入する
|
7
|
-
#
|
8
|
-
|
9
|
-
# OpenAMエージェントの認可ログの一行
|
10
|
-
class Entry
|
11
|
-
# 行をマッチさせる正規表現
|
12
|
-
@@reg=/^
|
13
|
-
(\S+\s\S+) # 0 YYYY-MM-DD HH:MM:SS
|
14
|
-
\.\d\d\d # ミリ秒
|
15
|
-
\s # 半角スペース
|
16
|
-
\s # 半角スペース
|
17
|
-
\s # 半角スペース
|
18
|
-
\s # 半角スペース
|
19
|
-
Info # 固定文言
|
20
|
-
\s # 半角スペース
|
21
|
-
\S+ # xxxxxx:xxxxxx
|
22
|
-
\s # 半角スペース
|
23
|
-
LocalAuditLog: # 固定文言
|
24
|
-
\s # 半角スペース
|
25
|
-
User # 固定文言
|
26
|
-
\s # 半角スペース
|
27
|
-
(\S+) # 1 user_id
|
28
|
-
\s # 半角スペース
|
29
|
-
was # 固定文言
|
30
|
-
\s # 半角スペース
|
31
|
-
(\S+) # 2 allowed or denied
|
32
|
-
\s # 半角スペース
|
33
|
-
access # 固定文言
|
34
|
-
\s # 半角スペース
|
35
|
-
to # 固定文言
|
36
|
-
\s # 半角スペース
|
37
|
-
(\S+) # 3 request
|
38
|
-
$/x
|
39
|
-
|
40
|
-
@@status_map = {"allowed" => "ALLOW","denied" => "DISALLOW"}
|
41
|
-
@entry
|
42
|
-
|
43
|
-
# ログの1行を解析してEntryクラスを作る。
|
44
|
-
# 解析できない時は例外を発生させる。
|
45
|
-
#
|
46
|
-
# 期待しているフォーマットは以下の通り。
|
47
|
-
#
|
48
|
-
# 2012-07-20 11:04:45.059 Info 24006:16ed0460 LocalAuditLog: User amadmin was allowed access to http://www.fx.develop.jp:80/.
|
49
|
-
#
|
50
|
-
def initialize(line,url_ignore_keywords)
|
51
|
-
match = line.match(@@reg)
|
52
|
-
raise "parse error. line: <#{line}> " if match.nil?
|
53
|
-
captures = match.captures
|
54
|
-
url_ignore_keywords.each do | keyword |
|
55
|
-
raise "ignore keyword include in access_url: <#{line}}>" if captures[3].include?(keyword)
|
56
|
-
end
|
57
|
-
@entry = {
|
58
|
-
:datetime => DateTime.strptime( captures[0], '%Y-%m-%d %H:%M:%S'),
|
59
|
-
:login_id => captures[1],
|
60
|
-
:status => @@status_map[captures[2]],
|
61
|
-
:access_url => captures[3]
|
62
|
-
}
|
63
|
-
end
|
64
|
-
|
65
|
-
# Insert用のSQLを返す
|
66
|
-
def insert_sql(table_name,host_name,company_id)
|
67
|
-
value = [ company_id ,
|
68
|
-
@entry[:datetime].strftime("%Y/%m/%d %H:%M:%S"),
|
69
|
-
@entry[:login_id],
|
70
|
-
host_name,
|
71
|
-
@entry[:access_url],
|
72
|
-
@entry[:status]
|
73
|
-
].map{|s| "'#{s}'"}.join(",")
|
74
|
-
|
75
|
-
return "INSERT INTO #{table_name} (companyId,date,login_id,host_name,access_url,status) VALUES (#{value})"
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
|
80
|
-
# メイン
|
81
|
-
|
82
|
-
RBatch::Log.new do |log|
|
83
|
-
log.info("Start -----------------");
|
84
|
-
entries = []
|
85
|
-
url_ignore_keywords = RBatch::config["url_ignore_keywords"]
|
86
|
-
# ログ読み込み
|
87
|
-
File.foreach(RBatch::config["log_path"]) do |line|
|
88
|
-
begin
|
89
|
-
entries << Entry.new(line,url_ignore_keywords)
|
90
|
-
rescue => e
|
91
|
-
# 解析に失敗した場合
|
92
|
-
log.warn("parse error: " + e.message )
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
# MySQLに接続
|
97
|
-
con = Mysql.new(RBatch::config["mysql_server"],
|
98
|
-
RBatch::config["mysql_user"],
|
99
|
-
RBatch::config["mysql_password"],
|
100
|
-
RBatch::config["mysql_db_name"])
|
101
|
-
# MySQLに挿入
|
102
|
-
entries.each do |entry|
|
103
|
-
sql = entry.insert_sql(RBatch::config["mysql_table_name"],
|
104
|
-
RBatch::config["host_name"],
|
105
|
-
RBatch::config["company_id"])
|
106
|
-
log.info("exec sql: " + sql);
|
107
|
-
con.query(sql)
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
data/sample/conf/log_backup.yaml
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
mysql_server : "localhost"
|
2
|
-
mysql_user : "root"
|
3
|
-
mysql_password : ""
|
4
|
-
mysql_db_name : "test"
|
5
|
-
mysql_table_name : "authentication_log"
|
6
|
-
company_id : "sample.co"
|
7
|
-
host_name : "server1"
|
8
|
-
openam_access_log_path : "/root/rbatch/sample/data/amAuthentication.access.2012-07-10"
|
9
|
-
openam_error_log_path : "/root/rbatch/sample/data/amAuthentication.error.2012-07-10"
|
10
|
-
auth_status:
|
11
|
-
AUTHENTICATION-100 : Login Success
|
12
|
-
AUTHENTICATION-101 : Login Success
|
13
|
-
AUTHENTICATION-102 : Login Success
|
14
|
-
AUTHENTICATION-103 : Login Success
|
15
|
-
AUTHENTICATION-104 : Login Success
|
16
|
-
AUTHENTICATION-105 : Login Success
|
17
|
-
AUTHENTICATION-200 : Login Failed
|
18
|
-
AUTHENTICATION-201 : Invalid Password
|
19
|
-
AUTHENTICATION-202 : No user profile
|
20
|
-
AUTHENTICATION-203 : User Profile does not exist
|
21
|
-
AUTHENTICATION-204 : Not active
|
22
|
-
AUTHENTICATION-205 : User is Locked out
|
23
|
-
AUTHENTICATION-206 : User account has expired
|
24
|
-
AUTHENTICATION-207 : Login Timed Out
|
25
|
-
AUTHENTICATION-208 : Authentication module is denied
|
26
|
-
AUTHENTICATION-209 : maximum number of session
|
27
|
-
AUTHENTICATION-240 : Login Failed
|
28
|
-
AUTHENTICATION-241 : Invalid Password
|
29
|
-
AUTHENTICATION-245 : User is Locked out
|
30
|
-
AUTHENTICATION-247 : Login Timed Out
|
31
|
-
AUTHENTICATION-300 : Logout
|
32
|
-
AUTHENTICATION-301 : Logout
|
33
|
-
AUTHENTICATION-302 : Logout
|
34
|
-
AUTHENTICATION-303 : Logout
|
35
|
-
AUTHENTICATION-304 : Logout
|
@@ -1,16 +0,0 @@
|
|
1
|
-
mysql_server : "localhost"
|
2
|
-
mysql_user : "root"
|
3
|
-
mysql_password : ""
|
4
|
-
mysql_db_name : "test"
|
5
|
-
mysql_table_name : "authorization_log"
|
6
|
-
company_id : "sample.co"
|
7
|
-
host_name : "server1"
|
8
|
-
log_path : "/root/rbatch/sample/data/amAgent_localhost_80.log.2012-07-10"
|
9
|
-
url_ignore_keywords:
|
10
|
-
- ".jpg"
|
11
|
-
- ".gif"
|
12
|
-
- ".html"
|
13
|
-
- ".css"
|
14
|
-
- ".js"
|
15
|
-
- ".jpeg"
|
16
|
-
- ".ico"
|