chronicle-email 0.2.0 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9096dfd9082ccbbf59651fbeb3f7d8bbd6b00a246b1cff5094ce196f2a201ec
-  data.tar.gz: f3a702a0f56ce3301345c7be20cf2fdff5722354ef4e6f34aa83c1e0fb73a928
+  metadata.gz: a910606f4987580ac86be55c02a6e3003b7cab21e93476dc1bf6fa4cc4512355
+  data.tar.gz: ef1dbfa473c66feb93c730a5bdd4e426203519a8d94da6afccf227bfa1fa084a
 SHA512:
-  metadata.gz: f522c605c2277ac3552670aa67eafbd8ffe258717ac81bfafb64f4d22db0ec63891ce47faa85e761216ba7d4a2d2cb10138c67aaf28228a0cd3f5c53d8f3f8a5
-  data.tar.gz: 700aa4b3162b5b566f83354232303873cda6377c0ead42d57c96d7db7fec41b620ed153a0d4af81ac29e5c1a0e253a6e74bb99ccfca25ab824db39238d36d5ed
+  metadata.gz: f1ddc8cabf1b9fbb319ad36406e9a99e585eaad8abd772aea378438f838e11be3e75a9c3fe2a1c4805ee6c84d9dfe256ba9cff3ac205406c899ee33a71d1122f
+  data.tar.gz: 00c745bf81b65f59c7e514e735bd3319548f747503907646ca619fbfb1634dd2bb5e199753141a9c651bf210326e803dbb3d4bdc4331bdaeba0ecb70425dd265

data/README.md

@@ -9,16 +9,55 @@ Extract and work with your email using the command line with this plugin for [ch
 # Install chronicle-etl and this plugin
 $ gem install chronicle-etl
 $ chronicle-etl plugins:install email
+```
+
+### Extracting email from IMAP
+
+For Gmail accounts, you can create an [app password](https://myaccount.google.com/apppasswords); your email address is your username.
+
+```sh
+# Save username and password
+$ chronicle-etl secrets:set imap username foo@gmail.com
+$ chronicle-etl secrets:set imap password APPPASSWORD
+
+# Then, retrieve your email from the last five days
+$ chronicle-etl --extractor email:imap --transformer email --since 5d --loader json
+
+# If you don't want to save your credentials as a secret, you can just pass
+# them to the extractor directly
+$ chronicle-etl --extractor email:imap --transformer email --since 5d --loader json \
+    --extractor-opts username:foo@gmail.com --password:APPPASSWORD
+```
 
-# Process emails from an mbox file
-$ chronicle-etl --extractor email:mbox -i test.mbox  --transformer email --fields subject
+### Processing email from an .mbox file
+The MBOX format is used to archive an email mailbox. [Google Takeout](https://takeout.google.com/settings/takeout) exports emails from gmail in this format.
+
+```sh
+# Retrieve the subject lines of all emails in test.mbox
+$ chronicle-etl --extractor email:mbox --input inbox.mbox --transformer email --fields subject
 ```
 
 ## Available Connectors
 ### Extractors
 
+#### `imap`
+Extractor for importing recent emails from an IMAP server.
+
+##### Settings
+
+- `since`: Retrieve emails since this date
+- `until`: Retrieve emails until this date
+- `username`
+- `password`
+- `host`: (default: imap.gmail.com)
+- `port`: (default: 993) Use 143 for unencrypted connections
+- `mailbox`: (default: "[Gmail]/All Mail")
+- `search_query`: When using Gmail, you can pass in a search query (`from:foo has:attachment`) to filter messages by
+
+For accessing Gmail, you can create a one-time [app password](https://myaccount.google.com/apppasswords). Your email address is your username.
+
 #### `mbox`
-Extractor for importing emails from an mbox file
+Extractor for importing emails from an MBOX file
 
 ##### Settings
 - `input`: A path to an .mbox file
@@ -31,6 +70,3 @@ Transform an email (in the form of a string) into Chronicle Schema
 ##### Settings
 - `body_as_markdown`: (default: false) Whether to convert the email body into markdown
 - `remove_signature`: (default: true) Whether to attempt to strip out the email signature (using the [`email_reply_parser`](https://github.com/github/email_reply_parser) gem)
-
-## Roadmap
-- Add an IMAP (and gmail) extractor #1

data/chronicle-email.gemspec

@@ -36,10 +36,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "chronicle-etl", "~> 0.4.4"
-  spec.add_dependency "mail", "~> 2.7"
   spec.add_dependency 'email_reply_parser', '~> 0.5'
   spec.add_dependency 'reverse_markdown', '~> 2.0'
+  spec.add_dependency "chronicle-etl", "~> 0.5"
+  spec.add_dependency "mail", "~> 2.7"
+  # Needed for Ruby 3.1 compatibility (https://github.com/mikel/mail/pull/1439#issuecomment-1002801221)
+  # TODO: check back for new version of `mail`
+  spec.add_dependency "net-imap"
+  spec.add_dependency "net-smtp"
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0"

data/lib/chronicle/email/email_transformer.rb

@@ -85,8 +85,7 @@ module Chronicle
       end
 
       def clean_body message
-        # FIXME: this all needs to be refactored
-
+        # FIXME: this all needs to be refactored        
         if message.multipart?
           body = message.text_part&.decoded rescue Mail::UnknownEncodingType
         else
@@ -94,6 +93,9 @@ module Chronicle
           body = body_to_markdown if @config.body_as_markdown
         end
 
+        return if body == Mail::UnknownEncodingType
+        return unless body && body != ""
+
         body = body_without_signature(body) if @config.remove_signature
 
         # Force UTF-8 encoding
@@ -107,8 +109,6 @@ module Chronicle
       end
 
       def body_without_signature(body)
-        return unless body && body != ""
-
         # FIXME: regex in EmailReplyParse gem seems to get into infinite loops
         #   with certain long bodies that have binary data
         parsed_body = Timeout::timeout(5) do

data/lib/chronicle/email/imap_extractor.rb

@@ -0,0 +1,91 @@
+require 'net/imap'
+
+module Chronicle
+  module Email
+    class IMAPExtractor < Chronicle::ETL::Extractor
+      register_connector do |r|
+        r.provider = 'email'
+        r.description = 'imap server'
+        r.identifier = 'imap'
+      end
+
+      setting :host, required: true, default: 'imap.gmail.com'
+      setting :port, type: :numeric, required: true, default: 993
+      setting :mailbox, required: true, default: '[Gmail]/All Mail'
+      setting :username, required: true
+      setting :password, required: true
+      setting :search_query
+
+      def prepare
+        @connection = create_connection
+        @message_ids = fetch_message_ids
+      end
+
+      def results_count
+        @message_ids.count
+      end
+
+      def extract
+        @message_ids.each do |message_id|
+          message = fetch_message(message_id)
+          yield Chronicle::ETL::Extraction.new(data: { email: message.attr["BODY[]"]} )
+        end
+      end
+
+      private
+
+      def create_connection
+        connection = Net::IMAP.new(@config.host, @config.port, true)
+        connection.login(@config.username, @config.password)
+        connection.select(@config.mailbox)
+        connection
+      rescue Net::IMAP::NoResponseError => e
+        raise(Chronicle::ETL::ExtractionError, "Error connecting to IMAP server. Please check username and password")
+      end
+
+      def fetch_message_ids
+        keys = gmail_mode? ? search_keys_gmail : search_keys_default
+        message_ids = @connection.search(keys)
+        message_ids = message_ids.first(@config.limit) if @config.limit
+        message_ids
+      rescue Net::IMAP::BadResponseError => e
+        raise(Chronicle::ETL::ExtractionError, "Error searching IMAP server for messages")
+      end
+
+      def fetch_message(message_id)
+        response = @connection.fetch(message_id, "BODY.PEEK[]")
+        raise(Chronicle::ETL::ExtractionError, "Error loading message") unless response
+
+        return response[0]
+      end
+
+      def search_keys_gmail
+        # Gmail offers an extension to IMAP that lets us use gmail queries
+        q = ""
+
+        # First, we ignore drafts beacuse they break a lot of assumptions we
+        # make when when processing emails (lack of timestamps, ids, etc)
+        q = "-label:draft"
+
+        # We use UNIX timestamps in gmail filters which let us do more precise
+        # since/until compared with date-based imap filters
+        q += " after:#{@config.since.to_i}" if @config.since
+        q += " before:#{@config.until.to_i}" if @config.until
+        q += " #{@config.search_query}" if @config.search_query
+
+        ["X-GM-RAW", q]
+      end
+
+      def search_keys_default
+        keys = []
+        # TODO: test out non-gmail IMAP searching (for @config.search_query)
+        keys += ['SINCE', Net::IMAP.format_date(@config.since)] if @config.since
+        keys += ['BEFORE', Net::IMAP.format_date(@config.until)] if @config.until
+      end
+
+      def gmail_mode?
+        @config.host == 'imap.gmail.com'
+      end
+    end
+  end
+end

data/lib/chronicle/email/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module Email
-    VERSION = "0.2.0"
+    VERSION = "0.2.3"
   end
 end

data/lib/chronicle/email.rb

@@ -1,6 +1,7 @@
 require "chronicle/email/version"
 require "chronicle/email/email_transformer"
 require "chronicle/email/mbox_extractor"
+require "chronicle/email/imap_extractor"
 
 module Chronicle
   module Email

metadata

@@ -1,45 +1,45 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-email
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-20 00:00:00.000000000 Z
+date: 2022-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: chronicle-etl
+  name: email_reply_parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.4
+        version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: mail
+  name: reverse_markdown
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: email_reply_parser
+  name: chronicle-etl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -53,19 +53,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.5'
 - !ruby/object:Gem::Dependency
-  name: reverse_markdown
+  name: mail
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: net-imap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-smtp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +156,7 @@ files:
 - chronicle-email.gemspec
 - lib/chronicle/email.rb
 - lib/chronicle/email/email_transformer.rb
+- lib/chronicle/email/imap_extractor.rb
 - lib/chronicle/email/mbox_extractor.rb
 - lib/chronicle/email/version.rb
 homepage: https://github.com/chronicle-app/chronicle-email