mailcvt 0.2.13 → 0.2.15

data/bin/mailcvt

@@ -64,6 +64,18 @@ command :parsetocc do |c|
   end
 end
 
+desc 'Parse others to mail log.'
+command :parseothers do |c|
+  c.action do |global_options,options,args|
+    err = "parseothers usage: mailcvt parseothers [daily dir] [timestamp]"
+    raise err if args.length != 2
+    
+    raise "Cannot find daily dir #{args[0]}" unless Dir.exists?(args[0])
+    tp = OthersParser.new(args[0], args[1])
+    tp.parse
+  end
+end
+
 desc 'Remove parsed mails and move unprocessed files to special folder.'
 command :clean do |c|
   c.action do |global_options,options,args|
@@ -88,6 +100,18 @@ command :cleantocc do |c|
   end
 end
 
+desc 'Delete others folder.'
+command :cleanothers do |c|
+  c.action do |global_options,options,args|
+    err = "cleanothers usage: mailcvt cleanothers [daily dir] [timestamp]"
+    raise err if args.length != 2
+    
+    raise "Cannot find daily dir #{args[0]}" unless Dir.exists?(args[0])
+    mp = MailCleaner.new(args[0], args[1])
+    mp.cleanothers
+  end
+end
+
 command :setup do |c|
   c.action do |global_options,options,args|
     mi = SetupMailcvt.new

data/bin/runmailcvt

@@ -75,5 +75,18 @@ if [ -d $dtocc ]; then
     mailcvt cleantocc $dailydir $timestamp
 fi
 
+dothers="$dailydir/dothers$timestamp"
+if [ -d $dothers ]; then
+    copyonly1 $dothers
+    echo "[$(now)] find $dothers -type f -print0 | xargs -0 grep -i -E -A300 \"$grepexp\" > $grepout.others"
+    find $dothers -type f -print0 | xargs -0 grep -i -E -A300 "$grepexp" > "$grepout.others"
+
+    echo "[$(now)] mailcvt parseothers $dailydir $timestamp"
+    mailcvt parseothers $dailydir $timestamp
+
+    echo "[$(now)] mailcvt cleanothers $dailydir $timestamp"
+    mailcvt cleanothers $dailydir $timestamp
+fi
+
 echo "[$(now)] runmailcvt finished."
 echo "-------------------------------------------------------------------------------"

data/lib/base_parser.rb

@@ -60,4 +60,12 @@ class BaseParser
 
     def convert(curfile, lines)
     end
+
+    def isduplicate(msgid)
+        return false unless msgid and msgid.length > 0
+        return true if @prevmsgid and msgid == @prevmsgid
+
+        @prevmsgid = msgid
+        return false
+    end
 end

data/lib/mail_cleaner.rb

@@ -19,14 +19,18 @@ class MailCleaner
     end
 
     def cleantocc
-        move 'tocc2'
-
         d = @oh.getdir("dtocc")
         FileUtils.rm_r(d) if Dir.exists?(d)
 
         delmout '.grep.tocc'
         delmout '.mtocc'
-        delmout '.ftocc2'
+    end
+
+    def cleanothers
+        d = @oh.getdir("dothers")
+        FileUtils.rm_r(d) if Dir.exists?(d)
+
+        delmout '.grep.others'
     end
 
     def removeok

data/lib/mail_parser.rb

@@ -18,10 +18,9 @@ class MailParser < BaseParser
 
         gm = GrepMail.new(curfile, lines, 9)
         msgid, mlog = gm.tolog
-        if msgid and msgid.length > 0 and msgid == @prevmsgid
+        if isduplicate(msgid)
             @oh.outputmlog curfile, nil
         else
-            @prevmsgid = msgid
             if gm.iscomplete?
                 @oh.outputmlog curfile, mlog
             elsif gm.istocc?

data/lib/mailcvt.rb

@@ -8,3 +8,4 @@ require 'mail_parser'
 require 'setup_mailcvt'
 require 'to_cc_parser'
 require 'mail_cleaner'
+require 'others_parser.rb'

data/lib/mailcvt/version.rb

@@ -1,3 +1,3 @@
 module Mailcvt
-  VERSION = '0.2.13'
+  VERSION = '0.2.15'
 end

data/lib/others_parser.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'grep_mail'
+require 'output_helper'
+require 'base_parser'
+
+class OthersParser < BaseParser
+    def parse_with_summary
+        grep = shouldexist('.grep.others')
+
+        parsegrep grep
+        @oh.finishoutput
+    end
+
+    def convert(curfile, lines)
+        return unless lines and lines.length > 0
+
+        gm = GrepMail.new(curfile, lines, 300)
+        msgid, mlog = gm.tolog
+        if isduplicate(msgid)
+            @oh.outputmlog_from_others nil
+        else
+#output tocc or others mails anyway, better than disk overflow.
+            @oh.outputmlog_from_others mlog
+        end
+    end
+end

data/lib/output_helper.rb

@@ -44,7 +44,6 @@ class OutputHelper
         summary << "\n  duplicate = #{countmsg(@duplicount, totalcount)}"
         summary << ", tocc = #{countmsg(@tocccount, totalcount)}"
         summary << ", others = #{countmsg(@otherscount, totalcount)}"
-        summary
     end
 
     def countmsg(count, total)
@@ -96,16 +95,16 @@ class OutputHelper
         @otherscount += 1
     end
 
-    def outputtocc2(mailpath)
-        unless @oftocc2
-            ftocc2 = getmout('.ftocc2')
-            @oftocc2 = File.open(ftocc2, 'w') 
+    def outputmlog_from_others(line)
+        unless @omlog
+            mlog = getmout('.mlog')
+            @omlog = File.open(mlog, 'w') 
+        end
+        if line
+            @omlog.puts line
+            @mlogcount += 1
+        else
+            @duplicount += 1
         end
-
-        @oftocc2.puts mailpath
-    end
-
-    def closetocc2
-        @oftocc2.close if @oftocc2
     end
 end

data/lib/to_cc_parser.rb

@@ -8,6 +8,7 @@ class ToCcParser < BaseParser
     def parse_with_summary
         grep, mtocc = shouldexist('.grep.tocc', '.mtocc')
         
+        @tocc2count = 0
         parsetocc grep
         replace_tocc mtocc
     end
@@ -15,7 +16,6 @@ class ToCcParser < BaseParser
     def replace_tocc(mtocc)
         inputcount = 0
         replacecount = 0 
-        tocc2count = 0 
         mlog = @oh.getmout('.mlog')
         File.open(mlog, 'a') do |omlog|
             File.foreach(mtocc) do |line|
@@ -30,15 +30,13 @@ class ToCcParser < BaseParser
                     cols[4+1] = tocc[4]
                     cols[5+1] = tocc[5]
                     omlog.puts cols[1..-1].join("\t")
-                else
-                    tocc2count += 1
                 end
             end
         end
 
-        msg = "#{formatcount(replacecount, inputcount)} replaced. "
-        msg << "#{formatcount(tocc2count, inputcount)} to others dir. "
-        msg << "#{formatcount(inputcount - replacecount - tocc2count, inputcount)} ignored. "
+        msg = "#{formatcount(replacecount, inputcount)} replaced "
+        msg << "(include #{formatcount(@tocc2count, inputcount)} tocc cut). "
+        msg << "#{formatcount(inputcount - replacecount, inputcount)} ignored. "
         msg << "Totally #{inputcount} row#{inputcount > 1 ? 's' : ''} from .mtocc file."
     end
 
@@ -57,7 +55,6 @@ class ToCcParser < BaseParser
     def parsetocc(grep)
         @toccs = {}
         parsegrep grep
-        @oh.closetocc2
     end
 
     def convert(curfile, lines)
@@ -65,12 +62,12 @@ class ToCcParser < BaseParser
 
         gm = GrepMail.new(curfile, lines, 300)
         cols = gm.tocols
-        if gm.istocc?
-            @oh.outputtocc2(curfile)
-        else
-            key = File.basename(curfile)
-            @toccs[key] = cols unless @toccs.has_key?(key)
-        end
+#no need to check duplicate here, cause no COPY_ file in mlog
+#output tocc or others mails anyway, better than disk overflow.
+        key = File.basename(curfile)
+        @toccs[key] = cols unless @toccs.has_key?(key)
+
+        @tocc2count += 1 if gm.istocc?
     end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mailcvt
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.2.15
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-30 00:00:00.000000000 Z
+date: 2013-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -96,6 +96,7 @@ files:
 - lib/mail_parser.rb
 - lib/mailcvt/version.rb
 - lib/mailcvt.rb
+- lib/others_parser.rb
 - lib/output_helper.rb
 - lib/setup_mailcvt.rb
 - lib/to_cc_parser.rb
@@ -121,7 +122,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1463266947147316996
+      hash: -4026955693453475253
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1463266947147316996
+      hash: -4026955693453475253
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25