mailcvt 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/lock-runmailcvt +10 -0
- data/bin/mailcvt +5 -3
- data/bin/runmailcvt +36 -0
- data/lib/big_mail_generator.rb +8 -2
- data/lib/grep_mail.rb +0 -2
- data/lib/mail_parser.rb +90 -54
- data/lib/mailcvt/version.rb +1 -1
- data/lib/output_helper.rb +27 -0
- metadata +7 -4
data/bin/lock-runmailcvt
ADDED
data/bin/mailcvt
CHANGED
@@ -26,7 +26,7 @@ command :write do |c|
|
|
26
26
|
c.default_value 'default'
|
27
27
|
c.flag :f
|
28
28
|
c.action do |global_options,options,args|
|
29
|
-
err = "write
|
29
|
+
err = "write usage: mailcvt write [input dir] [output dir] [output file number]"
|
30
30
|
raise err if args.length != 3
|
31
31
|
|
32
32
|
raise "Cannot find #{args[0]}" unless Dir.exists?(args[0])
|
@@ -44,7 +44,7 @@ desc 'Describe read here'
|
|
44
44
|
arg_name 'Describe arguments to read here'
|
45
45
|
command :read do |c|
|
46
46
|
c.action do |global_options,options,args|
|
47
|
-
err = "read
|
47
|
+
err = "read usage: mailcvt read [input dir] [output dir]"
|
48
48
|
raise err if args.length != 2
|
49
49
|
|
50
50
|
raise "Cannot find #{args[0]}" unless Dir.exists?(args[0])
|
@@ -80,7 +80,9 @@ end
|
|
80
80
|
on_error do |exception|
|
81
81
|
# Error logic here
|
82
82
|
# return false to skip default error handling
|
83
|
-
|
83
|
+
$stderr.puts "abort mailcvt. #{exception.message}"
|
84
|
+
$stderr.puts exception.backtrace.join("\n")
|
85
|
+
false
|
84
86
|
end
|
85
87
|
|
86
88
|
exit run(ARGV)
|
data/bin/runmailcvt
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
echo "*******************************************************************************"
|
4
|
+
|
5
|
+
function now {
|
6
|
+
echo $(date +'%Y/%m/%d %T')
|
7
|
+
}
|
8
|
+
|
9
|
+
set -e
|
10
|
+
timestamp=$(date +"%Y%m%d%H")
|
11
|
+
|
12
|
+
echo "[$(now)] runmailcvt started."
|
13
|
+
wdir=$(pwd)
|
14
|
+
isize=$(du -h $1)
|
15
|
+
echo "[$(now)] working dir = $wdir, input = $isize, output = $2"
|
16
|
+
|
17
|
+
if [ ! -d $2 ]; then
|
18
|
+
echo "[$(now)] create directory $2"
|
19
|
+
mkdir $2
|
20
|
+
fi
|
21
|
+
fngrep="$(cd $2 && pwd)/mout$timestamp.grep"
|
22
|
+
if [ -f $fngrep ]; then
|
23
|
+
echo "abort runmailcvt at $(now). $fngrep exists!!! "
|
24
|
+
exit
|
25
|
+
fi
|
26
|
+
|
27
|
+
allmail="$(cd $1 && pwd)/*"
|
28
|
+
echo "[$(now)] start grep output to $fngrep."
|
29
|
+
grep -E -A9 "^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;" $allmail > $fngrep
|
30
|
+
echo "[$(now)] grep finished."
|
31
|
+
|
32
|
+
bundle exec bin/mailcvt read $1 $2
|
33
|
+
|
34
|
+
|
35
|
+
echo "[$(now)] runmailcvt finished."
|
36
|
+
echo "-------------------------------------------------------------------------------"
|
data/lib/big_mail_generator.rb
CHANGED
@@ -21,10 +21,15 @@ class BigMailGenerator
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def cpall
|
24
|
-
|
24
|
+
if File.directory?(@input)
|
25
|
+
inputfiles = Dir.glob(File.join(@input, '*'))
|
26
|
+
else
|
27
|
+
inputfiles = [@input]
|
28
|
+
end
|
25
29
|
|
26
30
|
count = 0
|
27
31
|
copycounts = @number / inputfiles.length
|
32
|
+
@digit = copycounts.to_s.length + 1
|
28
33
|
inputfiles.each do |ifile|
|
29
34
|
size = File.size(ifile)
|
30
35
|
puts "copy #{ifile}(#{size / 1024}KB) #{copycounts} times"
|
@@ -47,7 +52,8 @@ class BigMailGenerator
|
|
47
52
|
|
48
53
|
def cpfile(ifile, i)
|
49
54
|
fn = File.basename(ifile)
|
50
|
-
|
55
|
+
pos = fn.rindex('.')
|
56
|
+
ofile = File.join(@output, "#{fn[0..(pos-1)]}#{(i+1).to_s.rjust(@digit, '0')}#{fn[pos..-1]}")
|
51
57
|
FileUtils.cp(ifile, ofile)
|
52
58
|
end
|
53
59
|
end
|
data/lib/grep_mail.rb
CHANGED
@@ -4,7 +4,6 @@ require 'log_helper'
|
|
4
4
|
|
5
5
|
class GrepMail
|
6
6
|
def initialize(file, lines)
|
7
|
-
puts "DEBUG> GrepMail.init> \n\t\tfile = #{file}\n"
|
8
7
|
@pcontent= file.length + 1
|
9
8
|
grouplines(lines)
|
10
9
|
end
|
@@ -61,7 +60,6 @@ class GrepMail
|
|
61
60
|
end
|
62
61
|
|
63
62
|
def tolog
|
64
|
-
puts "DEBUG> GrepMail.tolog > @groups = #{@groups}"
|
65
63
|
lh = LogHelper.new(@groups)
|
66
64
|
lh.tolog
|
67
65
|
end
|
data/lib/mail_parser.rb
CHANGED
@@ -1,60 +1,49 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require 'line_converter'
|
3
|
+
#require 'line_converter'
|
4
|
+
#require 'open3'
|
4
5
|
require 'grep_mail'
|
5
|
-
require '
|
6
|
+
require 'output_helper'
|
6
7
|
|
7
8
|
class MailParser
|
8
9
|
def initialize(input, output)
|
9
|
-
@
|
10
|
-
@output = output
|
10
|
+
@fh = OutputHelper.new(input, output)
|
11
11
|
end
|
12
12
|
|
13
13
|
def parse
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
else
|
21
|
-
puts "Elapsed time: #{(diff / 60).round(1)} minutes."
|
22
|
-
end
|
23
|
-
system("du -h #{@output}")
|
24
|
-
end
|
25
|
-
|
26
|
-
def parseall_grep
|
27
|
-
cmd = 'grep -E -A30 "^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;" ' + File.join(@input, '*')
|
14
|
+
grep = @fh.getmout('.grep')
|
15
|
+
raise "can't find #{grep}." unless File.exist?(grep)
|
16
|
+
mcvt = @fh.getmout('.mcvt')
|
17
|
+
raise "#{mcvt} exits." if File.exist?(mcvt)
|
18
|
+
del = @fh.getmout('.del')
|
19
|
+
raise "#{del} exits." if File.exist?(del)
|
28
20
|
|
29
|
-
|
30
|
-
|
31
|
-
|
21
|
+
lines = []
|
22
|
+
curfile = nil
|
23
|
+
@fh.output(mcvt, del) do |omcvt, odel|
|
24
|
+
File.foreach(grep) do |line|
|
25
|
+
curfile = getfile(line) unless curfile
|
26
|
+
if line != "--\n" and !line.start_with?(curfile)
|
27
|
+
convert curfile, lines, omcvt, odel
|
32
28
|
|
33
|
-
|
34
|
-
|
35
|
-
File.open(File.join(@output, "test_log.log"), 'w') do |out|
|
36
|
-
o.each do |line|
|
37
|
-
puts "DEBUG> MailParser> line = #{line}"
|
38
|
-
curfile = getfile(line) unless curfile
|
39
|
-
if line != "--\n" and !line.start_with?(curfile)
|
40
|
-
gm = GrepMail.new(curfile, lines)
|
41
|
-
out.puts gm.tolog
|
42
|
-
curfile = getfile(line)
|
43
|
-
lines = []
|
44
|
-
end
|
45
|
-
lines << line
|
46
|
-
end
|
47
|
-
|
48
|
-
if lines.length > 0
|
49
|
-
gm = GrepMail.new(curfile, lines)
|
50
|
-
out.puts gm.tolog
|
29
|
+
curfile = getfile(line)
|
30
|
+
lines = []
|
51
31
|
end
|
32
|
+
lines << line
|
52
33
|
end
|
53
34
|
|
54
|
-
|
35
|
+
convert curfile, lines, omcvt, odel
|
55
36
|
end
|
56
37
|
end
|
57
38
|
|
39
|
+
def convert(curfile, lines, omcvt, odel)
|
40
|
+
return if lines.length == 0
|
41
|
+
|
42
|
+
gm = GrepMail.new(curfile, lines)
|
43
|
+
omcvt.puts gm.tolog
|
44
|
+
odel.puts curfile
|
45
|
+
end
|
46
|
+
|
58
47
|
def getfile(line)
|
59
48
|
|
60
49
|
pos = line.index(':')
|
@@ -62,20 +51,67 @@ class MailParser
|
|
62
51
|
line[0..(pos-1)]
|
63
52
|
end
|
64
53
|
|
65
|
-
def parseall
|
66
|
-
File.open(File.join(@output, "test_log.log"), 'w') do |out|
|
67
|
-
Dir.glob(File.join(@input, '*')) do |file|
|
68
|
-
parsefile file, out
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
54
|
|
73
|
-
def
|
74
|
-
|
75
|
-
|
55
|
+
# def parse
|
56
|
+
# start = Time.now
|
57
|
+
# parseall_grep
|
58
|
+
# #parseall
|
59
|
+
# diff = Time.now - start
|
60
|
+
# if diff / 60 < 1
|
61
|
+
# puts "Elapsed time: #{diff.round(3)} seconds."
|
62
|
+
# else
|
63
|
+
# puts "Elapsed time: #{(diff / 60).round(1)} minutes."
|
64
|
+
# end
|
65
|
+
# system("du -h #{@output}")
|
66
|
+
# end
|
76
67
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
68
|
+
# def parseall_grep
|
69
|
+
# cmd = 'grep -E -A30 "^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;" ' + File.join(@input, '*')
|
70
|
+
#
|
71
|
+
# Open3.popen3(cmd) do |i, o, e, t|
|
72
|
+
# puts "Get keywords from mails by grep[#{t.pid}]."
|
73
|
+
# puts "#{cmd}"
|
74
|
+
#
|
75
|
+
# lines = []
|
76
|
+
# curfile = nil
|
77
|
+
# File.open(File.join(@output, "test_log.log"), 'w') do |out|
|
78
|
+
# o.each do |line|
|
79
|
+
# puts "DEBUG> MailParser> line = #{line}"
|
80
|
+
# curfile = getfile(line) unless curfile
|
81
|
+
# if line != "--\n" and !line.start_with?(curfile)
|
82
|
+
# gm = GrepMail.new(curfile, lines)
|
83
|
+
# out.puts gm.tolog
|
84
|
+
# curfile = getfile(line)
|
85
|
+
# lines = []
|
86
|
+
# end
|
87
|
+
# lines << line
|
88
|
+
# end
|
89
|
+
#
|
90
|
+
# if lines.length > 0
|
91
|
+
# gm = GrepMail.new(curfile, lines)
|
92
|
+
# out.puts gm.tolog
|
93
|
+
# end
|
94
|
+
# end
|
95
|
+
#
|
96
|
+
# puts "from grep[#{t.pid}]> #{t.value}."
|
97
|
+
# end
|
98
|
+
# end
|
99
|
+
#
|
100
|
+
#
|
101
|
+
# def parseall
|
102
|
+
# File.open(File.join(@output, "test_log.log"), 'w') do |out|
|
103
|
+
# Dir.glob(File.join(@input, '*')) do |file|
|
104
|
+
# parsefile file, out
|
105
|
+
# end
|
106
|
+
# end
|
107
|
+
# end
|
108
|
+
#
|
109
|
+
# def parsefile(file, out)
|
110
|
+
# cvt = LineConverter.new
|
111
|
+
# File.foreach(file) {|line| cvt.convert line}
|
112
|
+
#
|
113
|
+
# out.puts(cvt.log)
|
114
|
+
# out.flush
|
115
|
+
# #File.delete file
|
116
|
+
# end
|
81
117
|
end
|
data/lib/mailcvt/version.rb
CHANGED
@@ -0,0 +1,27 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
class OutputHelper
|
4
|
+
attr_reader :output, :input
|
5
|
+
|
6
|
+
def initialize(input, output)
|
7
|
+
@input = input
|
8
|
+
@output = output
|
9
|
+
@timestamp = Time.new.strftime("%Y%m%d%H")
|
10
|
+
end
|
11
|
+
|
12
|
+
def getmout(ext)
|
13
|
+
File.join(@output, "mout#{@timestamp}#{ext}")
|
14
|
+
end
|
15
|
+
|
16
|
+
def output(f1, f2)
|
17
|
+
tmp1 = f1 + '.tmp'
|
18
|
+
tmp2 = f2 + '.tmp'
|
19
|
+
File.open(tmp1, 'w') do |of1|
|
20
|
+
File.open(tmp2, 'w') do |of2|
|
21
|
+
yield of1, of2
|
22
|
+
end
|
23
|
+
end
|
24
|
+
File.rename(tmp1, f1)
|
25
|
+
File.rename(tmp2, f2)
|
26
|
+
end
|
27
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mailcvt
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-07-
|
12
|
+
date: 2013-07-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -85,6 +85,8 @@ extra_rdoc_files:
|
|
85
85
|
- mailcvt.rdoc
|
86
86
|
files:
|
87
87
|
- bin/mailcvt
|
88
|
+
- bin/lock-runmailcvt
|
89
|
+
- bin/runmailcvt
|
88
90
|
- lib/mailcvt/version.rb
|
89
91
|
- lib/mailcvt.rb
|
90
92
|
- lib/big_mail_generator.rb
|
@@ -92,6 +94,7 @@ files:
|
|
92
94
|
- lib/line_converter.rb
|
93
95
|
- lib/grep_mail.rb
|
94
96
|
- lib/log_helper.rb
|
97
|
+
- lib/output_helper.rb
|
95
98
|
- README.rdoc
|
96
99
|
- mailcvt.rdoc
|
97
100
|
homepage: http://github.com/ryu-kahou/mailcvt
|
@@ -114,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
114
117
|
version: '0'
|
115
118
|
segments:
|
116
119
|
- 0
|
117
|
-
hash:
|
120
|
+
hash: -3200601137855396094
|
118
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
119
122
|
none: false
|
120
123
|
requirements:
|
@@ -123,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
123
126
|
version: '0'
|
124
127
|
segments:
|
125
128
|
- 0
|
126
|
-
hash:
|
129
|
+
hash: -3200601137855396094
|
127
130
|
requirements: []
|
128
131
|
rubyforge_project:
|
129
132
|
rubygems_version: 1.8.25
|