glima 0.2.1

data/lib/glima/context.rb

@@ -0,0 +1,32 @@
+require "fileutils"
+require "pathname"
+
+module Glima
+  class Context
+
+    def initialize(basedir)
+      unless basedir and File.directory?(File.expand_path(basedir.to_s))
+        raise Glima::ConfigurationError, "datastore directory '#{basedir}' not found"
+      end
+      @basedir = Pathname.new(File.expand_path(basedir))
+    end
+
+    def save_page_token(token)
+      File.open(page_token_path, "w") do |f|
+        f.write(token)
+      end
+    end
+
+    def load_page_token
+      File.open(page_token_path).read.to_s
+    end
+
+    ################################################################
+    private
+
+    def page_token_path
+      File.expand_path("page_token.context", @basedir)
+    end
+
+  end # class Context
+end # module Glima

data/lib/glima/datastore.rb

@@ -0,0 +1,70 @@
+require "fileutils"
+require "pathname"
+
+module Glima
+  class DataStore
+
+    def initialize(basedir)
+      unless basedir and File.directory?(File.expand_path(basedir.to_s))
+        raise Glima::ConfigurationError, "datastore directory '#{basedir}' not found"
+      end
+      @basedir = Pathname.new(File.expand_path(basedir))
+    end
+
+    def update(message)
+      if message.raw
+        save(message)
+      else
+        message.raw = load(message)
+      end
+      return message
+    end
+
+    def save(message)
+      path = folder_message_to_path("+all", message.id)
+      File.open(path, "w") do |f|
+        f.write(message.raw.gsub("\r\n", "\n"))
+      end
+    end
+
+    def load(message)
+      path = folder_message_to_path("+all", message.id)
+      return File.open(path).read
+    end
+
+    def exist?(message_id)
+      File.exist?(folder_message_to_path("+all", message_id))
+    end
+
+    ################################################################
+    private
+
+    def folder_to_directory(folder)
+      folder = folder.sub(/^\+/, "")
+      File.expand_path(folder, @basedir)
+    end
+
+    def save_message_in_id(message, folder)
+      directory = folder_to_directory(folder)
+
+      raise "Error: #{directory} not exist" unless File.exist?(directory)
+
+      # name =  message.thread_id + "-" + message.id
+      name =  message.id + ".eml"
+      filename = File.expand_path(name, directory)
+
+      File.open(filename, "w") do |f|
+        f.write(message.raw.gsub("\r\n", "\n"))
+      end
+      return message.id
+    end
+
+    def folder_message_to_path(folder, message_id = nil)
+      folder = folder.sub(/^\+/, "")
+      directory = File.expand_path(folder, @basedir)
+      return directory unless message_id
+      return File.expand_path(message_id + ".eml", directory)
+    end
+
+  end # class DataStore
+end # module Glima

data/lib/glima/gmail_client.rb

@@ -0,0 +1,270 @@
+require 'google/apis/gmail_v1'
+require 'googleauth'
+require 'googleauth/stores/file_token_store'
+require 'launchy'
+require 'forwardable'
+
+# Retry if rate-limit.
+Google::Apis::RequestOptions.default.retries = 5
+
+module Glima
+  class Authorizer
+    def initialize(client_id, client_secret, scope, token_store_path)
+      @authorizer = Google::Auth::UserAuthorizer.new(
+        Google::Auth::ClientId.new(client_id, client_secret),
+        scope,
+        Google::Auth::Stores::FileTokenStore.new(file: token_store_path)
+      )
+    end
+
+    def credentials(user_id)
+      @authorizer.get_credentials(user_id)
+    end
+
+    def auth_interactively(user_id)
+      shell   = Thor.new.shell
+      oob_uri = "urn:ietf:wg:oauth:2.0:oob"
+
+      url = @authorizer.get_authorization_url(base_url: oob_uri)
+      begin
+        Launchy.open(url)
+      rescue
+        puts "Open URL in your browser:\n #{url}"
+      end
+
+      code = shell.ask "Enter the resulting code:"
+
+      @authorizer.get_and_store_credentials_from_code(
+        user_id:  user_id,
+        code:     code,
+        base_url: oob_uri
+      )
+    end
+  end # Authorizer
+
+  class GmailClient
+    def self.logger
+      Google::Apis.logger
+    end
+
+    def self.logger=(logger)
+      Google::Apis.logger = logger
+    end
+
+    extend Forwardable
+
+    def_delegators :@client,
+    # Users.histoy
+    :list_user_histories,
+
+    # Users.labels
+    :list_user_labels,
+    # :get_user_label,
+    :patch_user_label,
+
+    # Users.messages
+    :get_user_message,
+    :insert_user_message,
+    :list_user_messages,
+    :modify_message,
+    # :trash_user_message,
+
+    # Users.threads
+    :get_user_thread,
+
+    # Users getProfile
+    :get_user_profile
+
+
+    # Find nearby messages from pivot_message
+    # `Nearby' message:
+    #   + has same From: address
+    #   + has near Date: field (+-1day)
+    # with the pivot_message.
+    def nearby_mails(pivot_mail)
+      from  = "from:#{pivot_mail.from}"
+      date1 = (pivot_mail.date.to_date - 1).strftime("after:%Y/%m/%d")
+      date2 = (pivot_mail.date.to_date + 1).strftime("before:%Y/%m/%d")
+      query = "#{from} -in:trash #{date1} #{date2}"
+      scan_batch("+all", query) do |mail|
+        next if pivot_mail.id == mail.id
+        yield mail
+      end
+    end
+
+    # * message types by format:
+    #   | field/fromat:   | list | minimal | raw | value type      |
+    #   |-----------------+------+---------+-----+-----------------|
+    #   | id              | ○   | ○      | ○  | string          |
+    #   | threadId        | ○   | ○      | ○  | string          |
+    #   | labelIds        |      | ○      | ○  | string[]        |
+    #   | snippet         |      | ○      | ○  | string          |
+    #   | historyId       |      | ○      | ○  | unsinged long   |
+    #   | internalDate    |      | ○      | ○  | long            |
+    #   | sizeEstimate    |      | ○      | ○  | int             |
+    #   |-----------------+------+---------+-----+-----------------|
+    #   | payload         |      |         |     | object          |
+    #   | payload.headers |      |         |     | key/value pairs |
+    #   | raw             |      |         | ○  | bytes           |
+    #
+    def get_user_smart_message(id)
+      fmt = if @datastore.exist?(id) then "minimal" else "raw" end
+
+      mail = nil
+      @client.get_user_message(me, id, format: fmt) do |m, err|
+        mail = Glima::Resource::Mail.new(@datastore.update(m)) if m
+        yield(mail, err)
+      end
+      return mail
+    end
+
+    def online?
+      Socket.getifaddrs.select {|i|
+        i.addr.ipv4? and ! i.addr.ipv4_loopback?
+      }.map(&:addr).map(&:ip_address).length > 0
+    end
+
+    def initialize(config, datastore)
+      authorizer = Authorizer.new(config.client_id,
+                                  config.client_secret,
+                                  Google::Apis::GmailV1::AUTH_SCOPE,
+                                  config.token_store)
+
+      credentials = authorizer.credentials(config.default_user) ||
+                    authorizer.auth_interactively(config.default_user)
+      @datastore = datastore
+      @client = Google::Apis::GmailV1::GmailService.new
+      @client.client_options.application_name = 'glima'
+      @client.authorization = credentials
+      @client.authorization.username = config.default_user # for IMAP
+
+      return @client
+    end
+
+    # label == nil means "[Gmail]/All Mail"
+    def wait(label = nil)
+      @imap ||= Glima::ImapWatch.new("imap.gmail.com", @client.authorization)
+      @imap.wait(label&.name)
+    end
+
+    def watch(label = nil, &block)
+      loop do
+        puts "tick"
+
+        curr_hid = get_user_profile(me).history_id.to_i
+        last_hid ||= curr_hid
+
+        # FIXME: if server is changed at this point, we will miss the event.
+        wait(label) if last_hid == curr_hid
+
+        each_events(since: last_hid) do |ev|
+          yield ev
+          last_hid = ev.history_id.to_i
+        end
+      end
+    end
+
+    def each_events(since:)
+      options, response = {start_history_id: since}, nil
+
+      loop do
+        client.list_user_histories(me, options) do |res, err|
+          raise err if err
+          response = res
+        end
+
+        break unless response.history
+
+        response.history.each do |h|
+          Glima::Resource::History.new(h).to_events.each do |ev|
+            yield ev
+          end
+        end
+
+        break unless options[:page_token] = response.next_page_token
+      end
+    end
+
+    def get_user_label(id, fields: nil, options: nil, &block)
+      client.get_user_label(me, id, fields: fields, options: options, &block)
+    end
+
+    def trash_user_message(id, fields: nil, options: nil, &block)
+      client.trash_user_message(me, id, fields: fields, options: options, &block)
+    end
+
+    def batch(options = nil)
+      @client.batch(options) do |batch_client|
+        begin
+          Thread.current[:glima_api_batch] = batch_client
+          yield self
+        ensure
+          Thread.current[:glima_api_batch] = nil
+        end
+      end
+    end
+
+    def scan_batch(folder, search_or_range = nil, &block)
+      qp = Glima::QueryParameter.new(folder, search_or_range)
+      list_user_messages(me, qp.to_hash) do |res, error|
+        fail "#{error}" if error
+        ids = (res.messages || []).map(&:id)
+        unless ids.empty?
+          batch_on_messages(ids) do |message|
+            yield Glima::Resource::Mail.new(message) if block
+          end
+          # context.save_page_token(res.next_page_token)
+        end
+      end
+    rescue Glima::QueryParameter::FormatError => e
+      STDERR.print "Error: " + e.message + "\n"
+    end
+
+    def find_messages(query)
+      qp = Glima::QueryParameter.new("+all", query)
+      list_user_messages(me, qp.to_hash) do |res, error|
+        STDERR.print "#{error}" if error
+        return (res.messages || []).map(&:id)
+      end
+    rescue Glima::QueryParameter::FormatError => e
+      STDERR.print "Error: " + e.message + "\n"
+    end
+
+    def labels
+      @labels ||= client.list_user_labels(me).labels
+    end
+
+    def label_by_name(label_name)
+      labels.find {|label| label.name == label_name}
+    end
+
+    def label_by_id(label_id)
+      labels.find {|label| label.id == label_id}
+    end
+
+    private
+
+    def me
+      'me'
+    end
+
+    def client
+      Thread.current[:glima_api_batch] || @client
+    end
+
+    def batch_on_messages(ids, &block)
+      @client.batch do |batch_client|
+        ids.each do |id|
+          fmt = if @datastore.exist?(id) then "minimal" else "raw" end
+
+          batch_client.get_user_message(me, id, format: fmt) do |m, err|
+            fail "#{err}" if err
+            message = @datastore.update(m)
+            yield message
+          end
+        end
+      end
+    end
+
+  end # class GmailClient
+end # module Glima

data/lib/glima/imap.rb

@@ -0,0 +1,219 @@
+require 'forwardable'
+require "net/imap"
+require "pp"
+
+module Glima
+  class IMAP < Net::IMAP
+    def initialize(host, port_or_options = {},
+                   usessl = false, certs = nil, verify = true)
+      super
+      @parser = Glima::IMAP::ResponseParser.new
+    end
+
+    class Xoauth2Authenticator
+      def initialize(user, oauth2_token)
+        @user = user
+        @oauth2_token = oauth2_token
+      end
+
+      def process(data)
+        build_oauth2_string(@user, @oauth2_token)
+      end
+
+      private
+      # https://developers.google.com/google-apps/gmail/xoauth2_protocol
+      def build_oauth2_string(user, oauth2_token)
+        str = "user=%s\1auth=Bearer %s\1\1".encode("us-ascii") % [user, oauth2_token]
+        return str
+      end
+    end
+    private_constant :Xoauth2Authenticator
+    add_authenticator('XOAUTH2', Xoauth2Authenticator)
+
+    class ResponseParser < Net::IMAP::ResponseParser
+      def msg_att(n = 0)
+        match(T_LPAR)
+        attr = {}
+        while true
+          token = lookahead
+          case token.symbol
+          when T_RPAR
+            shift_token
+            break
+          when T_SPACE
+            shift_token
+            next
+          end
+          case token.value
+          when /\A(?:ENVELOPE)\z/ni
+            name, val = envelope_data
+          when /\A(?:FLAGS)\z/ni
+            name, val = flags_data
+          when /\A(?:INTERNALDATE)\z/ni
+            name, val = internaldate_data
+          when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
+            name, val = rfc822_text
+          when /\A(?:RFC822\.SIZE)\z/ni
+            name, val = rfc822_size
+          when /\A(?:BODY(?:STRUCTURE)?)\z/ni
+            name, val = body_data
+          when /\A(?:UID)\z/ni
+            name, val = uid_data
+
+          # Gmail extension additions.
+          # https://gist.github.com/WojtekKruszewski/1404434
+          when /\A(?:X-GM-LABELS)\z/ni
+            name, val = flags_data
+          when /\A(?:X-GM-MSGID)\z/ni
+            name, val = uid_data
+          when /\A(?:X-GM-THRID)\z/ni
+            name, val = uid_data
+          else
+            parse_error("unknown attribute `%s' for {%d}", token.value, n)
+          end
+          attr[name] = val
+        end
+        return attr
+      end
+    end # class ResponseParser
+  end # class IMAP
+end # module Glima
+
+module Glima
+  class ImapWatch
+    extend Forwardable
+
+    def_delegators :@imap,
+    :fetch,
+    :select,
+    :disconnect
+
+    def initialize(imap_server, authorization)
+      connect(imap_server, authorization)
+    end
+
+    def watch(folder, &block)
+      @imap.select(Net::IMAP.encode_utf7(folder))
+
+      @thread = Thread.new do
+        while true
+          begin
+            @imap.idle do |resp|
+              yield resp if resp.name == "EXISTS"
+            end
+          rescue Net::IMAP::Error => e
+            if e.inspect.include? "connection closed"
+              connect
+            else
+              raise
+            end
+          end
+        end
+      end
+    end
+
+    def wait(folder = nil)
+      if folder
+        folder = Net::IMAP.encode_utf7(folder)
+      else
+        # select "[Gmail]/All Mail" or localized one like "[Gmail]/すべてのメール"
+        folder = @imap.list("", "[Gmail]/*").find {|f| f.attr.include?(:All)}.name
+      end
+
+      @imap.select(folder)
+      begin
+        @imap.idle do |resp|
+          puts "#{resp.name}"
+          @imap.idle_done # if resp.name == "EXISTS"
+        end
+      rescue Net::IMAP::Error => e
+        if e.inspect.include? "connection closed"
+          connect
+        else
+          raise
+        end
+      end
+    end
+
+    # Ruby IMAP IDLE concurrency - how to tackle? - Stack Overflow
+    # http://stackoverflow.com/questions/5604480/ruby-imap-idle-concurrency-how-to-tackle
+    # How bad is IMAP IDLE? Joshua Tauberer's Archived Blog
+    # https://joshdata.wordpress.com/2014/08/09/how-bad-is-imap-idle/
+    #
+    def watch_and_fetch(folder, &block)
+      @imap.select(folder)
+      num = @imap.responses["EXISTS"].last
+
+      last_uid = @imap.fetch(num, "UID").first.attr['UID']
+      puts "Last_UID: #{last_uid}"
+
+      while true
+        id = waitfor()
+
+        puts "*********** GET EXISTS: #{id}"
+        mails = @imap.uid_fetch(last_uid..-1, %w(UID X-GM-MSGID X-GM-THRID X-GM-LABELS))
+
+        mails.each do |mail|
+          next if mail.attr['UID'] <= last_uid
+          resp  = yield mail
+          return unless resp
+        end
+
+        last_uid = mails.last.attr['UID']
+      end
+    end
+
+    private
+
+    def waitfor
+      id = -1
+      begin
+        @imap.idle do |resp|
+          pp resp
+          if resp.name == "EXISTS"
+            @imap.idle_done
+            id = resp.data.to_i
+          else
+            # pp resp
+          end
+        end
+      rescue Net::IMAP::Error => e
+        if e.inspect.include? "connection closed"
+          connect
+        else
+          raise
+        end
+      end
+      return id
+    end
+
+    def connect(imap_server, authorization)
+      retry_count = 0
+
+      begin
+        username     = authorization.username
+        access_token = authorization.access_token
+
+        @imap = Glima::IMAP.new(imap_server, 993, true)
+        @imap.authenticate('XOAUTH2', username, access_token)
+        puts "connected to imap server #{imap_server}."
+
+      rescue Net::IMAP::NoResponseError => e
+        if e.inspect.include? "Invalid credentials" && retry_count < 2
+          authorization.refresh!
+          retry_count += 1
+          retry
+        else
+          raise
+        end
+      end
+    end
+
+    # def connect(imap_server, user_name, access_token)
+    #   @imap = Glima::IMAP.new(imap_server, 993, true)
+    #   # @imap.class.debug = true
+    #   @imap.authenticate('XOAUTH2', user_name, access_token)
+    #   puts "connected to imap server #{imap_server}."
+    # end
+  end # class ImapWatch
+end # module Glima