embulk-parser-sisimai 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ca1f55c7eb54c8e1beb658a51830320073f073bd
4
- data.tar.gz: 85e0b1f14856907b7862e70eb7c6a6d8245f434b
3
+ metadata.gz: 4a754752d0081a9a705dc7a2a51579601cfe29f5
4
+ data.tar.gz: c66a713d3a714fa899edba49a4df910589aa10ac
5
5
  SHA512:
6
- metadata.gz: 6acd75c73bd36b988f35ead1be20c9059bb30834ad8a5aa74c82c4ff922affda987af8f29f8e998fa7b4d73da55066c92ddd296c1bcc959c31d99e423a3ab9b4
7
- data.tar.gz: d86b40e1fde84d4e87202fc47fe9e3e114cdb9e2da779dd8ebba430e4d962b5fc6654eb510158af21836c6f900d7c17e8405e827ac1c303000482bbe168c00d5
6
+ metadata.gz: 1b9b7b433242f5e3b40670c8cdf93be4c751a9b37e5aba3deeefadbf85aabd8c016541c62a34c45740fd40804ab45f5bd9b49dc1770e41fa6de379d43552c032
7
+ data.tar.gz: dd4383b276f956c32b0727c63539923ac3004f4021cb0d6c8c3bf71f0d62450bfa4743283813f4ef0eaf6e4719c939d16d622505ded89d359defa58f89c3b3d0
@@ -1 +1 @@
1
- jruby-9.0.4.0
1
+ jruby-9.1.5.0
data/CHANGES.md CHANGED
@@ -1,3 +1,8 @@
1
+ 0.1.1 2017-09-20
2
+ ----------------
3
+
4
+ * NEW sisito format support
5
+
1
6
  0.1.0 2016-02-18
2
7
  ----------------
3
8
 
data/README.md CHANGED
@@ -10,14 +10,16 @@ bounce mail analyzing interface(A successor to bounceHammer).
10
10
 
11
11
  ## Configuration
12
12
 
13
- - **format**: output format (`column` or `json`, default: `column`)
13
+ - **format**: output format (`column`,`json` or `sisito`, default: `column`)
14
14
  - **extract_mail_address**: extract mail_address into user, host and verp parts(bool, default: false).
15
15
  - **include_delivered**: include delivered mail Status: 2.X.Y, (boolean, default: `false`)
16
16
 
17
- The ``extract_mail_address`` parameter is column format mode only.
17
+ The ``extract_mail_address`` parameter is `column` format mode only.
18
+ And the `include_delivered` parameter can't use format `sisito`.
19
+
18
20
  ## Example
19
21
 
20
- ## column format
22
+ ### format: column
21
23
 
22
24
  ```yaml
23
25
  in:
@@ -92,7 +94,7 @@ recipient_vrep ( string) :
92
94
  ```
93
95
 
94
96
 
95
- ## json format
97
+ ### format: json
96
98
 
97
99
  ```yaml
98
100
  in:
@@ -135,6 +137,32 @@ result (json) : { "token": "d059e55e074333fe59001b1d30d27da85a1a9c1d", "lhost":
135
137
  }
136
138
  ```
137
139
 
140
+ ### format: sisito
141
+
142
+ [sisito](https://github.com/winebarrel/sisito) is the GUI interface for libsisimai.
143
+ The following cofiguration import bounce mails into sisito database.
144
+
145
+ ```yaml
146
+ in:
147
+ path_prefix: path/to/maildir
148
+ type: file
149
+ parser:
150
+ type: sisimai
151
+ format: sisito
152
+ out:
153
+ type: mysql
154
+ host: localhost
155
+ user: user
156
+ password: password
157
+ mode: replace
158
+ database: maillog_test
159
+ table: bounce_mails # Don't change. sisito use this table name
160
+ column_options:
161
+ created_at: { type: datetime }
162
+ updated_at: { type: datetime }
163
+ ```
164
+
165
+
138
166
  ## Install
139
167
 
140
168
  ```
@@ -1,7 +1,7 @@
1
1
 
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "embulk-parser-sisimai"
4
- spec.version = "0.1.0"
4
+ spec.version = "0.1.1"
5
5
  spec.authors = ["Hiroyuki Sato"]
6
6
  spec.summary = "Sisimai Analyzer parser plugin for Embulk"
7
7
  spec.description = "Parses Sisimai Analyzer files read by other file input plugins."
@@ -14,8 +14,8 @@ Gem::Specification.new do |spec|
14
14
  spec.require_paths = ["lib"]
15
15
 
16
16
  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
17
- spec.add_dependency 'sisimai', ['~> 4.16.0']
18
- spec.add_development_dependency 'embulk', ['>= 0.8.1']
17
+ spec.add_dependency 'sisimai', ['~> 4.16']
18
+ spec.add_development_dependency 'embulk', ['>= 0.8.30']
19
19
  spec.add_development_dependency 'bundler', ['>= 1.10.6']
20
20
  spec.add_development_dependency 'rake', ['>= 10.0']
21
21
  end
@@ -1,6 +1,7 @@
1
1
  require 'sisimai'
2
2
  require 'sisimai/message'
3
3
  require 'sisimai/data'
4
+ require 'digest/sha1'
4
5
 
5
6
  module Embulk
6
7
  module Parser
@@ -56,10 +57,41 @@ module Embulk
56
57
  ]
57
58
  end
58
59
  c
60
+ when "sisito"
61
+ c = [
62
+ Column.new(0, "timestamp", :timestamp),
63
+ Column.new(1, "lhost", :string),
64
+ Column.new(2, "rhost", :string),
65
+ Column.new(3, "alias", :string),
66
+ Column.new(4, "listid", :string),
67
+ Column.new(5, "reason", :string),
68
+ Column.new(6, "action", :string),
69
+ Column.new(7, "subject", :string),
70
+ Column.new(8, "messageid", :string),
71
+ Column.new(9, "smtpagent", :string),
72
+ Column.new(10, "softbounce", :long),
73
+ Column.new(11, "smtpcommand", :string),
74
+ Column.new(12, "destination", :string),
75
+ Column.new(13, "senderdomain", :string),
76
+ Column.new(14, "feedbacktype", :string),
77
+ Column.new(15, "diagnostictype", :string),
78
+ Column.new(16, "deliverystatus", :string),
79
+ Column.new(17, "timezoneoffset", :string),
80
+ Column.new(18, "addresser",:string),
81
+ Column.new(19, "recipient", :string),
82
+ Column.new(20, "addresseralias",:string),
83
+ Column.new(21, "digest",:string),
84
+ Column.new(22, "created_at",:timestamp),
85
+ Column.new(23, "updated_at",:timestamp),
86
+ ]
59
87
  else
60
88
  raise ArgumentError,"Unkown format type: #{format}"
61
89
  end
62
90
 
91
+ format = task["format"]
92
+ inc_delivered = task["include_delivered"]
93
+ extract_mail_address = task["extract_mail_address"]
94
+ Embulk.logger.info "sisimai format: #{format} include_delivered: #{inc_delivered}, extract_mail_address: #{extract_mail_address}"
63
95
  yield(task, columns)
64
96
  end
65
97
 
@@ -68,13 +100,19 @@ module Embulk
68
100
  @format = task["format"]
69
101
  @inc_delivered = task["include_delivered"]
70
102
  @extract_mail_address = task["extract_mail_address"]
71
- Embulk.logger.info "sisimai format: #{@format} include_delivered: #{@inc_delivered}, extract_mail_address: #{@extract_mail_address}"
72
103
  end
73
104
 
74
105
  def run(file_input)
75
106
  while file = file_input.next_file
76
- mesg = ::Sisimai::Message.new( data: file.read )
77
- datas = ::Sisimai::Data.make( data: mesg, delivered: @inc_delivered )
107
+ begin
108
+ # Sisimai expects input data is UTF-8 string.
109
+ src = file.read.force_encoding(Encoding::UTF_8)
110
+ mesg = ::Sisimai::Message.new( data: src )
111
+ datas = ::Sisimai::Data.make( data: mesg, delivered: @inc_delivered )
112
+ rescue
113
+ Embulk.logger.error "Error #{$!} #{src}"
114
+ raise
115
+ end
78
116
  if datas.nil?
79
117
  Embulk.logger.info "This file does not contaion bounce mail. skip."
80
118
  next
@@ -86,6 +124,9 @@ module Embulk
86
124
  when "column"
87
125
  column_data = make_column_array(data)
88
126
 
127
+ page_builder.add(column_data)
128
+ when "sisito"
129
+ column_data = make_sisito_array(data)
89
130
  page_builder.add(column_data)
90
131
  else
91
132
  raise RuntimeError,"Invalid format #{@format}"
@@ -133,6 +174,43 @@ module Embulk
133
174
  end
134
175
  row
135
176
  end
177
+
178
+ def make_sisito_array(data)
179
+ #data.diagnosticcode,
180
+ #data.replycode,
181
+ #data.token,
182
+
183
+ addresseralias = data.addresser.alias
184
+ addresseralias = data.addresser.to_s if addresseralias.empty?
185
+
186
+ now = Time.now
187
+ row = [
188
+ data.timestamp.to_time.utc,
189
+ data.lhost,
190
+ data.rhost,
191
+ data.alias,
192
+ data.listid,
193
+ data.reason,
194
+ data.action,
195
+ data.subject,
196
+ data.messageid,
197
+ data.smtpagent,
198
+ data.softbounce,
199
+ data.smtpcommand,
200
+ data.destination,
201
+ data.senderdomain,
202
+ data.feedbacktype,
203
+ "", # data.diagnostictype,
204
+ data.deliverystatus,
205
+ data.timezoneoffset,
206
+ data.addresser.to_json,
207
+ data.recipient.to_json,
208
+ addresseralias,
209
+ Digest::SHA1.hexdigest(data.recipient.to_s),
210
+ now,
211
+ now
212
+ ]
213
+ end
136
214
  end
137
215
  end
138
216
  end
metadata CHANGED
@@ -1,21 +1,21 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-parser-sisimai
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiroyuki Sato
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-18 00:00:00.000000000 Z
11
+ date: 2017-09-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - "~>"
17
17
  - !ruby/object:Gem::Version
18
- version: 4.16.0
18
+ version: '4.16'
19
19
  name: sisimai
20
20
  prerelease: false
21
21
  type: :runtime
@@ -23,13 +23,13 @@ dependencies:
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: 4.16.0
26
+ version: '4.16'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
30
  - - ">="
31
31
  - !ruby/object:Gem::Version
32
- version: 0.8.1
32
+ version: 0.8.30
33
33
  name: embulk
34
34
  prerelease: false
35
35
  type: :development
@@ -37,7 +37,7 @@ dependencies:
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: 0.8.1
40
+ version: 0.8.30
41
41
  - !ruby/object:Gem::Dependency
42
42
  requirement: !ruby/object:Gem::Requirement
43
43
  requirements:
@@ -110,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
110
110
  version: '0'
111
111
  requirements: []
112
112
  rubyforge_project:
113
- rubygems_version: 2.5.2
113
+ rubygems_version: 2.6.13
114
114
  signing_key:
115
115
  specification_version: 4
116
116
  summary: Sisimai Analyzer parser plugin for Embulk