embulk-parser-sisimai 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca1f55c7eb54c8e1beb658a51830320073f073bd
-  data.tar.gz: 85e0b1f14856907b7862e70eb7c6a6d8245f434b
+  metadata.gz: 4a754752d0081a9a705dc7a2a51579601cfe29f5
+  data.tar.gz: c66a713d3a714fa899edba49a4df910589aa10ac
 SHA512:
-  metadata.gz: 6acd75c73bd36b988f35ead1be20c9059bb30834ad8a5aa74c82c4ff922affda987af8f29f8e998fa7b4d73da55066c92ddd296c1bcc959c31d99e423a3ab9b4
-  data.tar.gz: d86b40e1fde84d4e87202fc47fe9e3e114cdb9e2da779dd8ebba430e4d962b5fc6654eb510158af21836c6f900d7c17e8405e827ac1c303000482bbe168c00d5
+  metadata.gz: 1b9b7b433242f5e3b40670c8cdf93be4c751a9b37e5aba3deeefadbf85aabd8c016541c62a34c45740fd40804ab45f5bd9b49dc1770e41fa6de379d43552c032
+  data.tar.gz: dd4383b276f956c32b0727c63539923ac3004f4021cb0d6c8c3bf71f0d62450bfa4743283813f4ef0eaf6e4719c939d16d622505ded89d359defa58f89c3b3d0

data/.ruby-version

@@ -1 +1 @@
-jruby-9.0.4.0
+jruby-9.1.5.0

data/CHANGES.md

@@ -1,3 +1,8 @@
+0.1.1 2017-09-20
+----------------
+
+* NEW sisito format support
+
 0.1.0 2016-02-18
 ----------------
 

data/README.md

@@ -10,14 +10,16 @@ bounce mail analyzing interface(A successor to bounceHammer).
 
 ## Configuration
 
-- **format**: output format (`column` or `json`, default: `column`)
+- **format**: output format (`column`,`json` or `sisito`, default: `column`)
 - **extract_mail_address**: extract mail_address into user, host and verp parts(bool, default: false).
 - **include_delivered**: include delivered mail Status: 2.X.Y, (boolean, default: `false`)
 
-The ``extract_mail_address`` parameter is column format mode only.
+The ``extract_mail_address`` parameter is `column` format mode only.
+And the `include_delivered` parameter can't use format `sisito`.
+
 ## Example
 
-## column format
+### format: column
 
 ```yaml
 in:
@@ -92,7 +94,7 @@ recipient_vrep (   string) :
 ```
 
 
-## json format
+### format: json
 
 ```yaml
 in:
@@ -135,6 +137,32 @@ result (json) : { "token": "d059e55e074333fe59001b1d30d27da85a1a9c1d", "lhost":
 }
 ```
 
+### format: sisito
+
+[sisito](https://github.com/winebarrel/sisito) is the GUI interface for libsisimai.
+The following cofiguration import bounce mails into sisito database.
+
+```yaml
+in:
+  path_prefix: path/to/maildir
+  type: file
+  parser:
+    type: sisimai
+    format: sisito
+out:
+  type: mysql
+  host: localhost
+  user: user
+  password: password
+  mode: replace
+  database: maillog_test
+  table: bounce_mails # Don't change. sisito use this table name
+  column_options:
+    created_at: { type: datetime }
+    updated_at: { type: datetime }
+```
+
+
 ## Install
 
 ```

data/embulk-parser-sisimai.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-sisimai"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["Hiroyuki Sato"]
   spec.summary       = "Sisimai Analyzer parser plugin for Embulk"
   spec.description   = "Parses Sisimai Analyzer files read by other file input plugins."
@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
-  spec.add_dependency 'sisimai', ['~> 4.16.0']
-  spec.add_development_dependency 'embulk', ['>= 0.8.1']
+  spec.add_dependency 'sisimai', ['~> 4.16']
+  spec.add_development_dependency 'embulk', ['>= 0.8.30']
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']
 end

data/lib/embulk/parser/sisimai.rb

@@ -1,6 +1,7 @@
 require 'sisimai'
 require 'sisimai/message'
 require 'sisimai/data'
+require 'digest/sha1'
 
 module Embulk
   module Parser
@@ -56,10 +57,41 @@ module Embulk
             ]
           end
           c
+        when "sisito"
+          c = [
+            Column.new(0, "timestamp", :timestamp),
+            Column.new(1, "lhost", :string),
+            Column.new(2, "rhost", :string),
+            Column.new(3, "alias", :string),
+            Column.new(4, "listid", :string),
+            Column.new(5, "reason", :string),
+            Column.new(6, "action", :string),
+            Column.new(7, "subject", :string),
+            Column.new(8, "messageid", :string),
+            Column.new(9, "smtpagent", :string),
+            Column.new(10, "softbounce", :long),
+            Column.new(11, "smtpcommand", :string),
+            Column.new(12, "destination", :string),
+            Column.new(13, "senderdomain", :string),
+            Column.new(14, "feedbacktype", :string),
+            Column.new(15, "diagnostictype", :string),
+            Column.new(16, "deliverystatus", :string),
+            Column.new(17, "timezoneoffset", :string),
+            Column.new(18, "addresser",:string),
+            Column.new(19, "recipient", :string),
+            Column.new(20, "addresseralias",:string),
+            Column.new(21, "digest",:string),
+            Column.new(22, "created_at",:timestamp),
+            Column.new(23, "updated_at",:timestamp),
+          ]
         else
           raise ArgumentError,"Unkown format type: #{format}"
         end
 
+        format = task["format"]
+        inc_delivered = task["include_delivered"]
+        extract_mail_address = task["extract_mail_address"]
+        Embulk.logger.info "sisimai format: #{format} include_delivered: #{inc_delivered}, extract_mail_address: #{extract_mail_address}"
         yield(task, columns)
       end
 
@@ -68,13 +100,19 @@ module Embulk
         @format = task["format"]
         @inc_delivered = task["include_delivered"]
         @extract_mail_address = task["extract_mail_address"]
-        Embulk.logger.info "sisimai format: #{@format} include_delivered: #{@inc_delivered}, extract_mail_address: #{@extract_mail_address}"
       end
 
       def run(file_input)
         while file = file_input.next_file
-          mesg = ::Sisimai::Message.new( data: file.read )
-          datas = ::Sisimai::Data.make( data: mesg, delivered: @inc_delivered )
+          begin
+            # Sisimai expects input data is UTF-8 string.
+            src = file.read.force_encoding(Encoding::UTF_8)
+            mesg = ::Sisimai::Message.new( data: src )
+            datas = ::Sisimai::Data.make( data: mesg, delivered: @inc_delivered )
+          rescue
+            Embulk.logger.error "Error #{$!} #{src}"
+            raise
+          end
           if datas.nil?
             Embulk.logger.info "This file does not contaion bounce mail. skip."
             next
@@ -86,6 +124,9 @@ module Embulk
             when "column"
               column_data = make_column_array(data)
 
+              page_builder.add(column_data)
+            when "sisito"
+              column_data = make_sisito_array(data)
               page_builder.add(column_data)
             else
               raise RuntimeError,"Invalid format #{@format}"
@@ -133,6 +174,43 @@ module Embulk
         end
         row
       end
+
+      def make_sisito_array(data)
+        #data.diagnosticcode,
+        #data.replycode,
+        #data.token,
+
+        addresseralias = data.addresser.alias
+        addresseralias = data.addresser.to_s if addresseralias.empty?
+
+        now = Time.now
+        row = [
+          data.timestamp.to_time.utc,
+          data.lhost,
+          data.rhost,
+          data.alias,
+          data.listid,
+          data.reason,
+          data.action,
+          data.subject,
+          data.messageid,
+          data.smtpagent,
+          data.softbounce,
+          data.smtpcommand,
+          data.destination,
+          data.senderdomain,
+          data.feedbacktype,
+          "", # data.diagnostictype,
+          data.deliverystatus,
+          data.timezoneoffset,
+          data.addresser.to_json,
+          data.recipient.to_json,
+          addresseralias,
+          Digest::SHA1.hexdigest(data.recipient.to_s),
+          now,
+          now
+        ]
+      end
     end
   end
 end

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-sisimai
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Hiroyuki Sato
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-18 00:00:00.000000000 Z
+date: 2017-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.16.0
+        version: '4.16'
   name: sisimai
   prerelease: false
   type: :runtime
@@ -23,13 +23,13 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.16.0
+        version: '4.16'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 0.8.30
   name: embulk
   prerelease: false
   type: :development
@@ -37,7 +37,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 0.8.30
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -110,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Sisimai Analyzer parser plugin for Embulk