mailcvt 0.1.18 → 0.1.19

data/bin/lock-runmailcvt

@@ -7,11 +7,7 @@ log=$outputdir/hourly-runmailcvt.log
 lock=$outputdir/runmailcvt.lock
 
 if mkdir $lock &> /dev/null; then
-    if [ ! -z $3 ] && [ $3 == "-D" ]; then
-        ./bin/runmailcvt $1 $2 $3 >> $log 2>&1
-    else
-        $bindir/runmailcvt $1 $2 >> $log 2>&1
-    fi
+    $bindir/runmailcvt $1 $2 >> $log 2>&1
     rmdir $lock
 else
     echo "lock failed. abort at $(date +'%Y/%m/%d %T')." >> $log

data/bin/mailcvt

@@ -40,32 +40,58 @@ command :write do |c|
   end
 end
 
-desc 'Describe read here'
-arg_name 'Describe arguments to read here'
-command :read do |c|
+desc 'Parse grep output to mail log.'
+command :parse do |c|
   c.action do |global_options,options,args|
-    err = "read usage: mailcvt read [input dir] [output dir]"
+    err = "parse usage: mailcvt parse [daily dir] [timestamp]"
     raise err if args.length != 2
     
-    raise "Cannot find #{args[0]}" unless Dir.exists?(args[0])
-    Dir.mkdir(args[1]) unless Dir.exists?(args[1])
+    raise "Cannot find daily dir #{args[0]}" unless Dir.exists?(args[0])
     mp = MailParser.new(args[0], args[1])
     mp.parse
   end
 end
 
-command :setup do |c|
+desc 'Parse long to or cc to mail log.'
+command :parsetocc do |c|
   c.action do |global_options,options,args|
-    mi = SetupMailcvt.new
-    mi.setup
+    err = "parsetocc usage: mailcvt parsetocc [daily dir] [timestamp]"
+    raise err if args.length != 2
+    
+    raise "Cannot find daily dir #{args[0]}" unless Dir.exists?(args[0])
+    tp = ToCcParser.new(args[0], args[1])
+    tp.parse
   end
 end
 
-desc 'Describe daemon here'
-arg_name 'Describe arguments to daemon here'
-command :daemon do |c|
+desc 'Remove parsed mails and move unprocessed files to special folder.'
+command :clean do |c|
   c.action do |global_options,options,args|
-    puts "daemon command ran"
+    err = "clean usage: mailcvt clean [daily dir] [timestamp]"
+    raise err if args.length != 2
+    
+    raise "Cannot find daily dir #{args[0]}" unless Dir.exists?(args[0])
+    mp = MailCleaner.new(args[0], args[1])
+    mp.clean
+  end
+end
+
+desc 'Move unprocessed tocc files to others folder.'
+command :cleantocc do |c|
+  c.action do |global_options,options,args|
+    err = "cleantocc usage: mailcvt cleantocc [daily dir] [timestamp]"
+    raise err if args.length != 2
+    
+    raise "Cannot find daily dir #{args[0]}" unless Dir.exists?(args[0])
+    mp = MailCleaner.new(args[0], args[1])
+    mp.cleantocc
+  end
+end
+
+command :setup do |c|
+  c.action do |global_options,options,args|
+    mi = SetupMailcvt.new
+    mi.setup
   end
 end
 

data/bin/runmailcvt

@@ -6,41 +6,75 @@ function now {
     echo $(date +'%Y/%m/%d %T')
 }
 
+function copyonly1 {
+    input=$(cd $1 && pwd)
+    filecount=$(ls -1 $input | wc -l)
+    if [ $filecount -eq 1 ]; then
+        for f in $input/*
+        do
+            echo "[$(now)] cp $f COPY_$f"
+            cp $f "COPY_$f"
+        done
+    fi
+}
+
 set -e 
 timestamp=$(date +"%Y%m%d%H")
+maildir=$(cd $1 && pwd)
+dailydir=$(cd $2 && pwd)/$(date +"%Y%m%d")
 
 echo "[$(now)] runmailcvt started."
 wdir=$(pwd)
-isize=$(du -h $1)
-echo "[$(now)] working dir = $wdir, input = $isize, output = $2"
+isize=$(du -h $maildir)
+echo "[$(now)] working dir = $wdir, input = $isize, output = $dailydir"
 
-if [ ! -d $2 ]; then
-    echo "[$(now)] create directory $2"
-    mkdir $2
+if [ ! -d $dailydir ]; then
+    echo "[$(now)] create daily output directory $dailydir"
+    mkdir $dailydir
 fi
-fngrep="$(cd $2 && pwd)/mout$timestamp.grep"
-if [ -f $fngrep ]; then
-    echo "abort runmailcvt at $(now). $fngrep exists!!! "
+grepout=$dailydir/mout$timestamp.grep
+if [ -f $grepout ]; then
+    echo "abort runmailcvt at $(now). $grepout exists!!! "
     exit
 fi
 
-allmail="$(cd $1 && pwd)/*"
-echo "[$(now)] grep -E -A9 \"^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;\" $allmail > $fngrep"
-grep -E -A9 "^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;" $allmail > $fngrep 
-echo "[$(now)] grep finished."
-
-
-if [ ! -z $3 ] && [ $3 == "-D" ]; then
-    bundle exec bin/mailcvt read $1 $2
-else
-    bindir=$(cd $(dirname $0) && pwd)
-    rubydir=$(cd $(dirname $(dirname $(dirname $bindir))) && pwd)
-    rvmdir=$(cd $(dirname $(dirname $rubydir)) && pwd)
-    source $rvmdir/environments/default
-    echo "[$(now)] mailcvt read $1 $2"
-    mailcvt read $1 $2
-    #$rubydir/bin/mailcvt read $1 $2
+copyonly1 $maildir
+allmail=$maildir/*
+grepexp="^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;"
+echo "[$(now)] grep -E -A9 \"$grepexp\" $allmail > $grepout"
+grep -E -A9 "$grepexp" $allmail > $grepout 
+
+bindir=$(cd $(dirname $0) && pwd)
+rubydir=$(cd $(dirname $(dirname $(dirname $bindir))) && pwd)
+rvmdir=$(cd $(dirname $(dirname $rubydir)) && pwd)
+source $rvmdir/environments/default
+echo "[$(now)] mailcvt parse $dailydir $timestamp"
+mailcvt parse $dailydir $timestamp
+
+echo "[$(now)] mailcvt clean $dailydir $timestamp"
+mailcvt clean $dailydir $timestamp
+
+dtocc="$dailydir/dtocc$timestamp"
+if [ -d $dtocc ]; then
+    copyonly1 $dtocc
+    alldtocc="$dtocc/*"
+    echo "[$(now)] grep -E -A500 \"^To: |^Cc: \" $alldtocc > $grepout.tocc"
+    grep -E -A500 "^To: |^Cc: " $alldtocc > "$grepout.tocc"
+
+    echo "[$(now)] mailcvt parsetocc $dailydir $timestamp"
+    mailcvt parsetocc $dailydir $timestamp
+
+    echo "[$(now)] mailcvt cleantocc $dailydir $timestamp"
+    mailcvt cleantocc $dailydir $timestamp
 fi
 
+echo "[$(now)] update reject size."
+recipient=/etc/postfix/recipient_access
+size=$({ du -sh $maildir; du -sh $dailydir; } | sed "s/\t.*$//g" | tr '\n' ' ')
+cursec=$(date +"%m%d%H%M%S")
+sed -i "s/REJECT\t.*$/REJECT\t$size$cursec/g" $recipient
+postmap $recipient
+
+
 echo "[$(now)] runmailcvt finished."
 echo "-------------------------------------------------------------------------------"

data/lib/grep_mail.rb

@@ -61,6 +61,24 @@ class GrepMail
 
     def tolog
         lh = LogHelper.new(@groups)
-        lh.tolog
+        mlog = lh.tolog
+
+        toccok = lh.tocc_complete?
+        othersok = lh.others_complete?
+        @iscomplete = false
+        @istocc = false
+        if toccok and othersok
+            @iscomplete = true
+        else if othersok
+            @istocc = true
+        end
+    end
+
+    def iscomplete?
+        @iscomplete
+    end
+
+    def istocc?
+        @istocc
     end
 end

data/lib/log_helper.rb

@@ -14,8 +14,18 @@ class LogHelper
         @groups = groups
     end
 
+    def tocc_complete?
+        @valends[4] && @valends[5]
+    end
+
+    def others_complete?
+        @valends[0] && @valends[1]
+        && @valends[2] && @valends[3] && @valends[6]
+    end
+
     def tolog
-        cols = Array.new(7)
+        @valends = Array.new(7, true)
+        cols = Array.new(6)
 
         @groups.each do |g|
             next if g[:key].nil? or g[:key].length < 2
@@ -24,8 +34,12 @@ class LogHelper
 
             if cap == 'C' and g[:key][1] == 'o'
                 cols << concatattach(g)
+                @valends[-1] &= false unless g[:valend]
             else
-                cols[@@ids[cap]] = concatval(g)
+                id = @@ids[cap]
+                next if cols[id]
+                cols[id] = concatval(g)
+                @valends[id] = false unless g[:valend]
             end
         end
 

data/lib/mail_cleaner.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+class MailRemover
+    def initialize(output, timestamp)
+        @fh = OutputHelper.new(output, timestamp)
+    end
+
+    def clean
+        removeok
+        move 'tocc'
+        move 'others', @fh.getdir("daily-leftovers", true)
+    end
+
+    def cleantocc
+        move 'tocc2', @fh.getdir("daily-leftovers", true)
+
+        d = @fh.getdir("dtocc")
+        FileUtils.rm_r(d) if Dir.exists?(d)
+    end
+
+    def removeok
+        fok = @fh.getmout('.fok')
+        raise "Can't find #{fok}." unless File.exist?(fok)
+
+        File.foreach(fok) do |fn|
+            begin
+                File.delete fn
+            rescue Exception
+                puts "Can't remove file #{fn}"
+            end
+        end
+    end
+
+    def move(suffix, d = nil)
+        f = @fh.getmout(".f#{suffix}")
+        return unless File.exist?(f)
+
+        unless d
+            d = @fh.getdir("d#{suffix}")
+            raise "#{d} exits." if Dir.exist?(d)
+        end
+
+        Dir.mkdir d
+        File.foreach(f) do |fn|
+            begin
+                filename = File.basename(fn)
+                File.rename fn, File.join(d, filename)
+            rescue Exception
+                puts "Can't move file #{fn} to #{d}"
+            end
+        end
+    end
+end

data/lib/mail_parser.rb

@@ -1,117 +1,53 @@
 #!/usr/bin/env ruby
 
-#require 'line_converter'
-#require 'open3'
 require 'grep_mail'
 require 'output_helper'
 
 class MailParser
-    def initialize(input, output)
-        @fh = OutputHelper.new(input, output)
+    def initialize(output, timestamp)
+        @fh = OutputHelper.new(output, timestamp)
     end
 
     def parse
         grep = @fh.getmout('.grep')
-        raise "can't find #{grep}." unless File.exist?(grep)
-        mcvt = @fh.getmout('.mcvt')
-        raise "#{mcvt} exits." if File.exist?(mcvt)
-        del = @fh.getmout('.del')
-        raise "#{del} exits." if File.exist?(del)
+        raise "Can't find #{grep}." unless File.exist?(grep)
+        mlog = @fh.getmout('.mlog')
+        raise "#{mlog} exits." if File.exist?(mlog)
+        fok = @fh.getmout('.fok')
+        raise "#{fok} exits." if File.exist?(fok)
 
         lines = []
         curfile = nil
-        @fh.output(mcvt, del) do |omcvt, odel|
+        @fh.output(mlog, fok) do |omlog, ofok|
             File.foreach(grep) do |line|
-                curfile = getfile(line) unless curfile
+                curfile = @fh.getfile(line) unless curfile
                 if line != "--\n" and !line.start_with?(curfile)
-                    convert curfile, lines, omcvt, odel
+                    convert curfile, lines, omlog, ofok
 
-                    curfile = getfile(line)
+                    curfile = @fh.getfile(line)
                     lines = []
                 end
                 lines << line
             end
 
-            convert curfile, lines, omcvt, odel
+            convert curfile, lines, omlog, ofok
         end
+        @fh.closetocc
+        @fh.closeothers
     end
 
-    def convert(curfile, lines, omcvt, odel)
+    def convert(curfile, lines, omlog, ofok)
         return if lines.length == 0
 
         gm = GrepMail.new(curfile, lines)
-        omcvt.puts gm.tolog
-        odel.puts curfile
-    end
-
-    def getfile(line)
-
-        pos = line.index(':')
-        return "" if pos.nil?
-        line[0..(pos-1)]
+        mlog = gm.tolog
+        if gm.iscomplete?
+            omlog.puts mlog
+            ofok.puts curfile
+        else if gm.istocc?
+            @fh.outputtocc(curfile, mlog)
+        else
+            @fh.outputothers(curfile)
+        end
     end
-
-
-#    def parse
-#        start = Time.now
-#        parseall_grep
-#        #parseall
-#        diff = Time.now - start
-#        if diff / 60 < 1
-#            puts "Elapsed time: #{diff.round(3)} seconds." 
-#        else
-#            puts "Elapsed time: #{(diff / 60).round(1)} minutes." 
-#        end
-#        system("du -h #{@output}")
-#    end
-
-#    def parseall_grep
-#        cmd = 'grep -E -A30 "^Date: |^Message-ID: |^Subject: |^From: |^To: |^Cc: |^Content-Disposition: attachment;" ' + File.join(@input, '*')
-#
-#        Open3.popen3(cmd) do |i, o, e, t|
-#            puts "Get keywords from mails by grep[#{t.pid}]."
-#            puts "#{cmd}"
-#
-#            lines = []
-#            curfile = nil
-#            File.open(File.join(@output, "test_log.log"), 'w') do |out|
-#                o.each do |line|
-#                    puts "DEBUG> MailParser> line = #{line}"
-#                    curfile = getfile(line) unless curfile
-#                    if line != "--\n" and !line.start_with?(curfile)
-#                        gm = GrepMail.new(curfile, lines)
-#                        out.puts gm.tolog
-#                        curfile = getfile(line)
-#                        lines = []
-#                    end
-#                    lines << line
-#                end
-#
-#                if lines.length > 0
-#                    gm = GrepMail.new(curfile, lines)
-#                    out.puts gm.tolog
-#                end
-#            end
-#
-#            puts "from grep[#{t.pid}]> #{t.value}."
-#        end
-#    end
-#
-#
-#    def parseall
-#        File.open(File.join(@output, "test_log.log"), 'w') do |out|
-#            Dir.glob(File.join(@input, '*')) do |file|
-#                parsefile file, out
-#            end
-#        end
-#    end
-#
-#    def parsefile(file, out)
-#        cvt = LineConverter.new
-#        File.foreach(file) {|line| cvt.convert line}
-#
-#        out.puts(cvt.log)
-#        out.flush
-#        #File.delete file
-#    end
 end

data/lib/mailcvt.rb

@@ -6,3 +6,5 @@ require 'fileutils'
 require 'big_mail_generator'
 require 'mail_parser'
 require 'setup_mailcvt'
+require 'to_cc_parser'
+require 'mail_cleaner'

data/lib/mailcvt/version.rb

@@ -1,3 +1,3 @@
 module Mailcvt
-  VERSION = '0.1.18'
+  VERSION = '0.1.19'
 end

data/lib/output_helper.rb

@@ -1,18 +1,30 @@
 #!/usr/bin/env ruby
 
 class OutputHelper
-    attr_reader :output, :input
-
-    def initialize(input, output)
-        @input = input
+    def initialize(output, timestamp)
         @output = output
-        @timestamp = Time.new.strftime("%Y%m%d%H")
+        @timestamp = timestamp
+    end
+
+    def getfile(line)
+        pos = line.index(':')
+        return "" if pos.nil?
+        line[0..(pos-1)]
     end
 
     def getmout(ext)
         File.join(@output, "mout#{@timestamp}#{ext}")
     end
 
+    def getdir(prefix, notimestamp = false)
+        if notimestamp
+            dirname = prefix
+        else
+            dirname = "#{prefix}#{@timestamp}"
+        end
+        File.join(@output, dirname)
+    end
+
     def output(f1, f2)
         tmp1 = f1 + '.tmp'
         tmp2 = f2 + '.tmp'
@@ -24,4 +36,47 @@ class OutputHelper
         File.rename(tmp1, f1)
         File.rename(tmp2, f2)
     end
+
+    def outputtocc(mailpath, mlog)
+        unless @oftocc
+            ftocc = @fh.getmout('.ftocc')
+            @oftocc = File.open(ftocc, 'w') 
+            mlogtocc = @fh.getmout('.mtocc')
+            @omlogtocc = File.open(mlogtocc, 'w') 
+        end
+
+        @oftocc.puts mailpath
+        @omlogtocc.puts "#{mailpath}\t#{mlog}"
+    end
+
+    def closetocc
+        @oftocc.close if @oftocc
+        @omlogtocc.close if @omlogtocc
+    end
+
+    def outputothers(mailpath)
+        unless @ofothers
+            fothers = @fh.getmout('.fothers')
+            @ofothers = File.open(fothers, 'w') 
+        end
+
+        @ofothers.puts mailpath
+    end
+
+    def closeothers
+        @ofothers.close if @ofothers
+    end
+
+    def outputtocc2(mailpath)
+        unless @oftocc2
+            ftocc2 = @fh.getmout('.ftocc2')
+            @oftocc2 = File.open(ftocc2, 'w') 
+        end
+
+        @oftocc2.puts mailpath
+    end
+
+    def closetocc2
+        @oftocc2.close if @oftocc2
+    end
 end

data/lib/to_cc_parser.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+require 'grep_mail'
+require 'output_helper'
+
+class ToCcParser
+    def initialize(output, timestamp)
+        @fh = OutputHelper.new(output, timestamp)
+    end
+
+    def parse
+        grep = @fh.getmout('.grep.tocc')
+        raise "Can't find #{grep}." unless File.exist?(grep)
+        mtocc = @fh.getmout('.mtocc')
+        raise "Can't find #{mtocc}." unless File.exist?(mtocc)
+        mlog = @fh.getmout('.mlog')
+        raise "Can't find #{mlog}." unless File.exist?(mlog)
+        
+        parsetocc grep
+        @fh.closetocc2
+
+        idTo = LogHelper.ids['T']
+        idCc = LogHelper.ids['C']
+        File.open(mlog, 'a') do |omlog|
+            File.foreach(mtocc) do |line|
+                next unless line
+                parts = line.split("\t")
+                next if parts.nil? or parts.length < 7 
+
+                fn = parts[0]
+                if @toccs.has_key?(fn)
+                    tocc = @toccs[fn]
+                    parts[idTo+1] = tocc[idTo]
+                    parts[idCc+1] = tocc[idCc]
+                end
+                omlog.puts parts[1..-1].join("\t")
+            end
+        end
+    end
+
+    def parsetocc(grep)
+        @toccs = {}
+        lines = []
+        curfile = nil
+        File.foreach(grep) do |line|
+            curfile = @fh.getfile(line) unless curfile
+            if line != "--\n" and !line.start_with?(curfile)
+                convert curfile, lines
+
+                curfile = @fh.getfile(line)
+                lines = []
+            end
+            lines << line
+        end
+
+        convert curfile, lines
+    end
+
+    def convert(curfile, lines)
+        return if lines.length == 0
+
+        gm = GrepMail.new(curfile, lines)
+        mlog = lh.tolog
+        if gm.istocc?
+            @fh.outputtocc2(mlog)
+        else
+            @toccs[curfile] = mlog unless @toccs.has_key?(curfile)
+        end
+    end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mailcvt
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.19
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-22 00:00:00.000000000 Z
+date: 2013-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -96,6 +96,8 @@ files:
 - lib/log_helper.rb
 - lib/output_helper.rb
 - lib/setup_mailcvt.rb
+- lib/mail_cleaner.rb
+- lib/to_cc_parser.rb
 - README.rdoc
 - mailcvt.rdoc
 homepage: http://github.com/ryu-kahou/mailcvt
@@ -118,7 +120,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 731630712782139179
+      hash: 3926626864906930717
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -127,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 731630712782139179
+      hash: 3926626864906930717
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25