glima 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,32 @@
1
+ require "fileutils"
2
+ require "pathname"
3
+
4
+ module Glima
5
+ class Context
6
+
7
+ def initialize(basedir)
8
+ unless basedir and File.directory?(File.expand_path(basedir.to_s))
9
+ raise Glima::ConfigurationError, "datastore directory '#{basedir}' not found"
10
+ end
11
+ @basedir = Pathname.new(File.expand_path(basedir))
12
+ end
13
+
14
+ def save_page_token(token)
15
+ File.open(page_token_path, "w") do |f|
16
+ f.write(token)
17
+ end
18
+ end
19
+
20
+ def load_page_token
21
+ File.open(page_token_path).read.to_s
22
+ end
23
+
24
+ ################################################################
25
+ private
26
+
27
+ def page_token_path
28
+ File.expand_path("page_token.context", @basedir)
29
+ end
30
+
31
+ end # class Context
32
+ end # module Glima
@@ -0,0 +1,70 @@
1
+ require "fileutils"
2
+ require "pathname"
3
+
4
+ module Glima
5
+ class DataStore
6
+
7
+ def initialize(basedir)
8
+ unless basedir and File.directory?(File.expand_path(basedir.to_s))
9
+ raise Glima::ConfigurationError, "datastore directory '#{basedir}' not found"
10
+ end
11
+ @basedir = Pathname.new(File.expand_path(basedir))
12
+ end
13
+
14
+ def update(message)
15
+ if message.raw
16
+ save(message)
17
+ else
18
+ message.raw = load(message)
19
+ end
20
+ return message
21
+ end
22
+
23
+ def save(message)
24
+ path = folder_message_to_path("+all", message.id)
25
+ File.open(path, "w") do |f|
26
+ f.write(message.raw.gsub("\r\n", "\n"))
27
+ end
28
+ end
29
+
30
+ def load(message)
31
+ path = folder_message_to_path("+all", message.id)
32
+ return File.open(path).read
33
+ end
34
+
35
+ def exist?(message_id)
36
+ File.exist?(folder_message_to_path("+all", message_id))
37
+ end
38
+
39
+ ################################################################
40
+ private
41
+
42
+ def folder_to_directory(folder)
43
+ folder = folder.sub(/^\+/, "")
44
+ File.expand_path(folder, @basedir)
45
+ end
46
+
47
+ def save_message_in_id(message, folder)
48
+ directory = folder_to_directory(folder)
49
+
50
+ raise "Error: #{directory} not exist" unless File.exist?(directory)
51
+
52
+ # name = message.thread_id + "-" + message.id
53
+ name = message.id + ".eml"
54
+ filename = File.expand_path(name, directory)
55
+
56
+ File.open(filename, "w") do |f|
57
+ f.write(message.raw.gsub("\r\n", "\n"))
58
+ end
59
+ return message.id
60
+ end
61
+
62
+ def folder_message_to_path(folder, message_id = nil)
63
+ folder = folder.sub(/^\+/, "")
64
+ directory = File.expand_path(folder, @basedir)
65
+ return directory unless message_id
66
+ return File.expand_path(message_id + ".eml", directory)
67
+ end
68
+
69
+ end # class DataStore
70
+ end # module Glima
@@ -0,0 +1,270 @@
1
+ require 'google/apis/gmail_v1'
2
+ require 'googleauth'
3
+ require 'googleauth/stores/file_token_store'
4
+ require 'launchy'
5
+ require 'forwardable'
6
+
7
+ # Retry if rate-limit.
8
+ Google::Apis::RequestOptions.default.retries = 5
9
+
10
+ module Glima
11
+ class Authorizer
12
+ def initialize(client_id, client_secret, scope, token_store_path)
13
+ @authorizer = Google::Auth::UserAuthorizer.new(
14
+ Google::Auth::ClientId.new(client_id, client_secret),
15
+ scope,
16
+ Google::Auth::Stores::FileTokenStore.new(file: token_store_path)
17
+ )
18
+ end
19
+
20
+ def credentials(user_id)
21
+ @authorizer.get_credentials(user_id)
22
+ end
23
+
24
+ def auth_interactively(user_id)
25
+ shell = Thor.new.shell
26
+ oob_uri = "urn:ietf:wg:oauth:2.0:oob"
27
+
28
+ url = @authorizer.get_authorization_url(base_url: oob_uri)
29
+ begin
30
+ Launchy.open(url)
31
+ rescue
32
+ puts "Open URL in your browser:\n #{url}"
33
+ end
34
+
35
+ code = shell.ask "Enter the resulting code:"
36
+
37
+ @authorizer.get_and_store_credentials_from_code(
38
+ user_id: user_id,
39
+ code: code,
40
+ base_url: oob_uri
41
+ )
42
+ end
43
+ end # Authorizer
44
+
45
+ class GmailClient
46
+ def self.logger
47
+ Google::Apis.logger
48
+ end
49
+
50
+ def self.logger=(logger)
51
+ Google::Apis.logger = logger
52
+ end
53
+
54
+ extend Forwardable
55
+
56
+ def_delegators :@client,
57
+ # Users.histoy
58
+ :list_user_histories,
59
+
60
+ # Users.labels
61
+ :list_user_labels,
62
+ # :get_user_label,
63
+ :patch_user_label,
64
+
65
+ # Users.messages
66
+ :get_user_message,
67
+ :insert_user_message,
68
+ :list_user_messages,
69
+ :modify_message,
70
+ # :trash_user_message,
71
+
72
+ # Users.threads
73
+ :get_user_thread,
74
+
75
+ # Users getProfile
76
+ :get_user_profile
77
+
78
+
79
+ # Find nearby messages from pivot_message
80
+ # `Nearby' message:
81
+ # + has same From: address
82
+ # + has near Date: field (+-1day)
83
+ # with the pivot_message.
84
+ def nearby_mails(pivot_mail)
85
+ from = "from:#{pivot_mail.from}"
86
+ date1 = (pivot_mail.date.to_date - 1).strftime("after:%Y/%m/%d")
87
+ date2 = (pivot_mail.date.to_date + 1).strftime("before:%Y/%m/%d")
88
+ query = "#{from} -in:trash #{date1} #{date2}"
89
+ scan_batch("+all", query) do |mail|
90
+ next if pivot_mail.id == mail.id
91
+ yield mail
92
+ end
93
+ end
94
+
95
+ # * message types by format:
96
+ # | field/fromat: | list | minimal | raw | value type |
97
+ # |-----------------+------+---------+-----+-----------------|
98
+ # | id | ○ | ○ | ○ | string |
99
+ # | threadId | ○ | ○ | ○ | string |
100
+ # | labelIds | | ○ | ○ | string[] |
101
+ # | snippet | | ○ | ○ | string |
102
+ # | historyId | | ○ | ○ | unsinged long |
103
+ # | internalDate | | ○ | ○ | long |
104
+ # | sizeEstimate | | ○ | ○ | int |
105
+ # |-----------------+------+---------+-----+-----------------|
106
+ # | payload | | | | object |
107
+ # | payload.headers | | | | key/value pairs |
108
+ # | raw | | | ○ | bytes |
109
+ #
110
+ def get_user_smart_message(id)
111
+ fmt = if @datastore.exist?(id) then "minimal" else "raw" end
112
+
113
+ mail = nil
114
+ @client.get_user_message(me, id, format: fmt) do |m, err|
115
+ mail = Glima::Resource::Mail.new(@datastore.update(m)) if m
116
+ yield(mail, err)
117
+ end
118
+ return mail
119
+ end
120
+
121
+ def online?
122
+ Socket.getifaddrs.select {|i|
123
+ i.addr.ipv4? and ! i.addr.ipv4_loopback?
124
+ }.map(&:addr).map(&:ip_address).length > 0
125
+ end
126
+
127
+ def initialize(config, datastore)
128
+ authorizer = Authorizer.new(config.client_id,
129
+ config.client_secret,
130
+ Google::Apis::GmailV1::AUTH_SCOPE,
131
+ config.token_store)
132
+
133
+ credentials = authorizer.credentials(config.default_user) ||
134
+ authorizer.auth_interactively(config.default_user)
135
+ @datastore = datastore
136
+ @client = Google::Apis::GmailV1::GmailService.new
137
+ @client.client_options.application_name = 'glima'
138
+ @client.authorization = credentials
139
+ @client.authorization.username = config.default_user # for IMAP
140
+
141
+ return @client
142
+ end
143
+
144
+ # label == nil means "[Gmail]/All Mail"
145
+ def wait(label = nil)
146
+ @imap ||= Glima::ImapWatch.new("imap.gmail.com", @client.authorization)
147
+ @imap.wait(label&.name)
148
+ end
149
+
150
+ def watch(label = nil, &block)
151
+ loop do
152
+ puts "tick"
153
+
154
+ curr_hid = get_user_profile(me).history_id.to_i
155
+ last_hid ||= curr_hid
156
+
157
+ # FIXME: if server is changed at this point, we will miss the event.
158
+ wait(label) if last_hid == curr_hid
159
+
160
+ each_events(since: last_hid) do |ev|
161
+ yield ev
162
+ last_hid = ev.history_id.to_i
163
+ end
164
+ end
165
+ end
166
+
167
+ def each_events(since:)
168
+ options, response = {start_history_id: since}, nil
169
+
170
+ loop do
171
+ client.list_user_histories(me, options) do |res, err|
172
+ raise err if err
173
+ response = res
174
+ end
175
+
176
+ break unless response.history
177
+
178
+ response.history.each do |h|
179
+ Glima::Resource::History.new(h).to_events.each do |ev|
180
+ yield ev
181
+ end
182
+ end
183
+
184
+ break unless options[:page_token] = response.next_page_token
185
+ end
186
+ end
187
+
188
+ def get_user_label(id, fields: nil, options: nil, &block)
189
+ client.get_user_label(me, id, fields: fields, options: options, &block)
190
+ end
191
+
192
+ def trash_user_message(id, fields: nil, options: nil, &block)
193
+ client.trash_user_message(me, id, fields: fields, options: options, &block)
194
+ end
195
+
196
+ def batch(options = nil)
197
+ @client.batch(options) do |batch_client|
198
+ begin
199
+ Thread.current[:glima_api_batch] = batch_client
200
+ yield self
201
+ ensure
202
+ Thread.current[:glima_api_batch] = nil
203
+ end
204
+ end
205
+ end
206
+
207
+ def scan_batch(folder, search_or_range = nil, &block)
208
+ qp = Glima::QueryParameter.new(folder, search_or_range)
209
+ list_user_messages(me, qp.to_hash) do |res, error|
210
+ fail "#{error}" if error
211
+ ids = (res.messages || []).map(&:id)
212
+ unless ids.empty?
213
+ batch_on_messages(ids) do |message|
214
+ yield Glima::Resource::Mail.new(message) if block
215
+ end
216
+ # context.save_page_token(res.next_page_token)
217
+ end
218
+ end
219
+ rescue Glima::QueryParameter::FormatError => e
220
+ STDERR.print "Error: " + e.message + "\n"
221
+ end
222
+
223
+ def find_messages(query)
224
+ qp = Glima::QueryParameter.new("+all", query)
225
+ list_user_messages(me, qp.to_hash) do |res, error|
226
+ STDERR.print "#{error}" if error
227
+ return (res.messages || []).map(&:id)
228
+ end
229
+ rescue Glima::QueryParameter::FormatError => e
230
+ STDERR.print "Error: " + e.message + "\n"
231
+ end
232
+
233
+ def labels
234
+ @labels ||= client.list_user_labels(me).labels
235
+ end
236
+
237
+ def label_by_name(label_name)
238
+ labels.find {|label| label.name == label_name}
239
+ end
240
+
241
+ def label_by_id(label_id)
242
+ labels.find {|label| label.id == label_id}
243
+ end
244
+
245
+ private
246
+
247
+ def me
248
+ 'me'
249
+ end
250
+
251
+ def client
252
+ Thread.current[:glima_api_batch] || @client
253
+ end
254
+
255
+ def batch_on_messages(ids, &block)
256
+ @client.batch do |batch_client|
257
+ ids.each do |id|
258
+ fmt = if @datastore.exist?(id) then "minimal" else "raw" end
259
+
260
+ batch_client.get_user_message(me, id, format: fmt) do |m, err|
261
+ fail "#{err}" if err
262
+ message = @datastore.update(m)
263
+ yield message
264
+ end
265
+ end
266
+ end
267
+ end
268
+
269
+ end # class GmailClient
270
+ end # module Glima
@@ -0,0 +1,219 @@
1
+ require 'forwardable'
2
+ require "net/imap"
3
+ require "pp"
4
+
5
+ module Glima
6
+ class IMAP < Net::IMAP
7
+ def initialize(host, port_or_options = {},
8
+ usessl = false, certs = nil, verify = true)
9
+ super
10
+ @parser = Glima::IMAP::ResponseParser.new
11
+ end
12
+
13
+ class Xoauth2Authenticator
14
+ def initialize(user, oauth2_token)
15
+ @user = user
16
+ @oauth2_token = oauth2_token
17
+ end
18
+
19
+ def process(data)
20
+ build_oauth2_string(@user, @oauth2_token)
21
+ end
22
+
23
+ private
24
+ # https://developers.google.com/google-apps/gmail/xoauth2_protocol
25
+ def build_oauth2_string(user, oauth2_token)
26
+ str = "user=%s\1auth=Bearer %s\1\1".encode("us-ascii") % [user, oauth2_token]
27
+ return str
28
+ end
29
+ end
30
+ private_constant :Xoauth2Authenticator
31
+ add_authenticator('XOAUTH2', Xoauth2Authenticator)
32
+
33
+ class ResponseParser < Net::IMAP::ResponseParser
34
+ def msg_att(n = 0)
35
+ match(T_LPAR)
36
+ attr = {}
37
+ while true
38
+ token = lookahead
39
+ case token.symbol
40
+ when T_RPAR
41
+ shift_token
42
+ break
43
+ when T_SPACE
44
+ shift_token
45
+ next
46
+ end
47
+ case token.value
48
+ when /\A(?:ENVELOPE)\z/ni
49
+ name, val = envelope_data
50
+ when /\A(?:FLAGS)\z/ni
51
+ name, val = flags_data
52
+ when /\A(?:INTERNALDATE)\z/ni
53
+ name, val = internaldate_data
54
+ when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
55
+ name, val = rfc822_text
56
+ when /\A(?:RFC822\.SIZE)\z/ni
57
+ name, val = rfc822_size
58
+ when /\A(?:BODY(?:STRUCTURE)?)\z/ni
59
+ name, val = body_data
60
+ when /\A(?:UID)\z/ni
61
+ name, val = uid_data
62
+
63
+ # Gmail extension additions.
64
+ # https://gist.github.com/WojtekKruszewski/1404434
65
+ when /\A(?:X-GM-LABELS)\z/ni
66
+ name, val = flags_data
67
+ when /\A(?:X-GM-MSGID)\z/ni
68
+ name, val = uid_data
69
+ when /\A(?:X-GM-THRID)\z/ni
70
+ name, val = uid_data
71
+ else
72
+ parse_error("unknown attribute `%s' for {%d}", token.value, n)
73
+ end
74
+ attr[name] = val
75
+ end
76
+ return attr
77
+ end
78
+ end # class ResponseParser
79
+ end # class IMAP
80
+ end # module Glima
81
+
82
+ module Glima
83
+ class ImapWatch
84
+ extend Forwardable
85
+
86
+ def_delegators :@imap,
87
+ :fetch,
88
+ :select,
89
+ :disconnect
90
+
91
+ def initialize(imap_server, authorization)
92
+ connect(imap_server, authorization)
93
+ end
94
+
95
+ def watch(folder, &block)
96
+ @imap.select(Net::IMAP.encode_utf7(folder))
97
+
98
+ @thread = Thread.new do
99
+ while true
100
+ begin
101
+ @imap.idle do |resp|
102
+ yield resp if resp.name == "EXISTS"
103
+ end
104
+ rescue Net::IMAP::Error => e
105
+ if e.inspect.include? "connection closed"
106
+ connect
107
+ else
108
+ raise
109
+ end
110
+ end
111
+ end
112
+ end
113
+ end
114
+
115
+ def wait(folder = nil)
116
+ if folder
117
+ folder = Net::IMAP.encode_utf7(folder)
118
+ else
119
+ # select "[Gmail]/All Mail" or localized one like "[Gmail]/すべてのメール"
120
+ folder = @imap.list("", "[Gmail]/*").find {|f| f.attr.include?(:All)}.name
121
+ end
122
+
123
+ @imap.select(folder)
124
+ begin
125
+ @imap.idle do |resp|
126
+ puts "#{resp.name}"
127
+ @imap.idle_done # if resp.name == "EXISTS"
128
+ end
129
+ rescue Net::IMAP::Error => e
130
+ if e.inspect.include? "connection closed"
131
+ connect
132
+ else
133
+ raise
134
+ end
135
+ end
136
+ end
137
+
138
+ # Ruby IMAP IDLE concurrency - how to tackle? - Stack Overflow
139
+ # http://stackoverflow.com/questions/5604480/ruby-imap-idle-concurrency-how-to-tackle
140
+ # How bad is IMAP IDLE? Joshua Tauberer's Archived Blog
141
+ # https://joshdata.wordpress.com/2014/08/09/how-bad-is-imap-idle/
142
+ #
143
+ def watch_and_fetch(folder, &block)
144
+ @imap.select(folder)
145
+ num = @imap.responses["EXISTS"].last
146
+
147
+ last_uid = @imap.fetch(num, "UID").first.attr['UID']
148
+ puts "Last_UID: #{last_uid}"
149
+
150
+ while true
151
+ id = waitfor()
152
+
153
+ puts "*********** GET EXISTS: #{id}"
154
+ mails = @imap.uid_fetch(last_uid..-1, %w(UID X-GM-MSGID X-GM-THRID X-GM-LABELS))
155
+
156
+ mails.each do |mail|
157
+ next if mail.attr['UID'] <= last_uid
158
+ resp = yield mail
159
+ return unless resp
160
+ end
161
+
162
+ last_uid = mails.last.attr['UID']
163
+ end
164
+ end
165
+
166
+ private
167
+
168
+ def waitfor
169
+ id = -1
170
+ begin
171
+ @imap.idle do |resp|
172
+ pp resp
173
+ if resp.name == "EXISTS"
174
+ @imap.idle_done
175
+ id = resp.data.to_i
176
+ else
177
+ # pp resp
178
+ end
179
+ end
180
+ rescue Net::IMAP::Error => e
181
+ if e.inspect.include? "connection closed"
182
+ connect
183
+ else
184
+ raise
185
+ end
186
+ end
187
+ return id
188
+ end
189
+
190
+ def connect(imap_server, authorization)
191
+ retry_count = 0
192
+
193
+ begin
194
+ username = authorization.username
195
+ access_token = authorization.access_token
196
+
197
+ @imap = Glima::IMAP.new(imap_server, 993, true)
198
+ @imap.authenticate('XOAUTH2', username, access_token)
199
+ puts "connected to imap server #{imap_server}."
200
+
201
+ rescue Net::IMAP::NoResponseError => e
202
+ if e.inspect.include? "Invalid credentials" && retry_count < 2
203
+ authorization.refresh!
204
+ retry_count += 1
205
+ retry
206
+ else
207
+ raise
208
+ end
209
+ end
210
+ end
211
+
212
+ # def connect(imap_server, user_name, access_token)
213
+ # @imap = Glima::IMAP.new(imap_server, 993, true)
214
+ # # @imap.class.debug = true
215
+ # @imap.authenticate('XOAUTH2', user_name, access_token)
216
+ # puts "connected to imap server #{imap_server}."
217
+ # end
218
+ end # class ImapWatch
219
+ end # module Glima