chronicle-email 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38fae4ff87eef33cef28435958b5d2ee599ab32f318b505d63949ac4a70a066b
-  data.tar.gz: 694d121dbc9698d435f533edb5e74ed5338d37efe381c3bfb395f1c8591ba89f
+  metadata.gz: e37ee5bdabd254e03a06962503f5875ab75d2aa0b271aff0b946c73f673d5cc3
+  data.tar.gz: c351c35f1b5545f0d0f9e6b4375076b159dab88eb454cf3e946fdab84ddc03a4
 SHA512:
-  metadata.gz: c32fdef034cc65e3bf6139a9b311132ec0330acac4fd8c146248ad53da5f51a7cb517b130cd9337f4f76a84251e958667c269e43b6885c4f4db8df389ba35ec3
-  data.tar.gz: 5b06bcc3f6f3ef0d9b524e3904614f2fc7cf933eb322c61ef46bdf5212b6413a354660aa29c771f03e7ed2d34c5e0ca20aab385c1501bfa20d07a8abeba0c6e2
+  metadata.gz: fedb95cf29c21d0dac17d6cbeca81da0a5318ab41ea8a5eaf355e8996606b03398ba70c4233e4591a78ceaef5ce5242b59c5e227e79a6310ba4474c245476e57
+  data.tar.gz: 7a06834ea16425d456da704b3272c96d6e0f427044e46e60a477dbce1cab7f9621056f4f18f7b90a964f83e3e85281a981fdf155155ebf4cb8b06871cd9884ab

data/README.md

@@ -9,16 +9,55 @@ Extract and work with your email using the command line with this plugin for [ch
 # Install chronicle-etl and this plugin
 $ gem install chronicle-etl
 $ chronicle-etl plugins:install email
+```
+
+### Extracting email from IMAP
+
+For Gmail accounts, you can create an [app password](https://myaccount.google.com/apppasswords); your email address is your username.
+
+```sh
+# Save username and password
+$ chronicle-etl secrets:set imap username foo@gmail.com
+$ chronicle-etl secrets:set imap password APPPASSWORD
+
+# Then, retrieve your email from the last five days
+$ chronicle-etl --extractor email:imap --transformer email --since 5d --loader json
+
+# If you don't want to save your credentials as a secret, you can just pass
+# them to the extractor directly
+$ chronicle-etl --extractor email:imap --transformer email --since 5d --loader json \
+    --extractor-opts username:foo@gmail.com --password:APPPASSWORD
+```
 
-# Process emails from an mbox file
-$ chronicle-etl --extractor email:mbox -i test.mbox  --transformer email --fields subject
+### Processing email from an .mbox file
+The MBOX format is used to archive an email mailbox. [Google Takeout](https://takeout.google.com/settings/takeout) exports emails from gmail in this format.
+
+```sh
+# Retrieve the subject lines of all emails in test.mbox
+$ chronicle-etl --extractor email:mbox --input inbox.mbox --transformer email --fields subject
 ```
 
 ## Available Connectors
 ### Extractors
 
+#### `imap`
+Extractor for importing recent emails from an IMAP server.
+
+##### Settings
+
+- `since`: Retrieve emails since this date
+- `until`: Retrieve emails until this date
+- `username`
+- `password`
+- `host`: (default: imap.gmail.com)
+- `port`: (default: 993) Use 143 for unencrypted connections
+- `mailbox`: (default: "[Gmail]/All Mail")
+- `search_query`: When using Gmail, you can pass in a search query (`from:foo has:attachment`) to filter messages by
+
+For accessing Gmail, you can create a one-time [app password](https://myaccount.google.com/apppasswords). Your email address is your username.
+
 #### `mbox`
-Extractor for importing emails from an mbox file
+Extractor for importing emails from an MBOX file
 
 ##### Settings
 - `input`: A path to an .mbox file
@@ -31,6 +70,3 @@ Transform an email (in the form of a string) into Chronicle Schema
 ##### Settings
 - `body_as_markdown`: (default: false) Whether to convert the email body into markdown
 - `remove_signature`: (default: true) Whether to attempt to strip out the email signature (using the [`email_reply_parser`](https://github.com/github/email_reply_parser) gem)
-
-## Roadmap
-- Add an IMAP (and gmail) extractor #1

data/lib/chronicle/email/email_transformer.rb

@@ -85,8 +85,7 @@ module Chronicle
       end
 
       def clean_body message
-        # FIXME: this all needs to be refactored
-
+        # FIXME: this all needs to be refactored        
         if message.multipart?
           body = message.text_part&.decoded rescue Mail::UnknownEncodingType
         else
@@ -94,6 +93,9 @@ module Chronicle
           body = body_to_markdown if @config.body_as_markdown
         end
 
+        return if body == Mail::UnknownEncodingType
+        return unless body && body != ""
+
         body = body_without_signature(body) if @config.remove_signature
 
         # Force UTF-8 encoding
@@ -107,8 +109,6 @@ module Chronicle
       end
 
       def body_without_signature(body)
-        return unless body && body != ""
-
         # FIXME: regex in EmailReplyParse gem seems to get into infinite loops
         #   with certain long bodies that have binary data
         parsed_body = Timeout::timeout(5) do

data/lib/chronicle/email/imap_extractor.rb

@@ -0,0 +1,89 @@
+require 'net/imap'
+
+module Chronicle
+  module Email
+    class IMAPExtractor < Chronicle::ETL::Extractor
+      register_connector do |r|
+        r.provider = 'email'
+        r.description = 'imap server'
+        r.identifier = 'imap'
+      end
+
+      setting :host, required: true, default: 'imap.gmail.com'
+      setting :port, type: :numeric, required: true, default: 993
+      setting :mailbox, required: true, default: '[Gmail]/All Mail'
+      setting :username, required: true
+      setting :password, required: true
+      setting :search_query
+
+      def prepare
+        @connection = create_connection
+        @message_ids = fetch_message_ids
+      end
+
+      def results_count
+        @message_ids.count
+      end
+
+      def extract
+        @message_ids.each do |message_id|
+          message = fetch_message(message_id)
+          yield Chronicle::ETL::Extraction.new(data: { email: message.attr["BODY[]"]} )
+        end
+      end
+
+      private
+
+      def create_connection
+        connection = Net::IMAP.new(@config.host, @config.port, true)
+        connection.login(@config.username, @config.password)
+        connection.select(@config.mailbox)
+        connection
+      rescue Net::IMAP::NoResponseError => e
+        raise(Chronicle::ETL::ExtractionError, "Error connecting to IMAP server. Please check username and password")
+      end
+
+      def fetch_message_ids
+        keys = gmail_mode? ? search_keys_gmail : search_keys_default
+        @connection.search(keys)
+      rescue Net::IMAP::BadResponseError => e
+        raise(Chronicle::ETL::ExtractionError, "Error searching IMAP server for messages")
+      end
+
+      def fetch_message(message_id)
+        response = @connection.fetch(3100020, "BODY.PEEK[]")
+        raise(Chronicle::ETL::ExtractionError, "Error loading message") unless response
+
+        return response[0]
+      end
+
+      def search_keys_gmail
+        # Gmail offers an extension to IMAP that lets us use gmail queries
+        q = ""
+
+        # First, we ignore drafts beacuse they break a lot of assumptions we
+        # make when when processing emails (lack of timestamps, ids, etc)
+        q = "-label:draft"
+
+        # We use UNIX timestamps in gmail filters which let us do more precise
+        # since/until compared with date-based imap filters
+        q += " after:#{@config.since.to_i}" if @config.since
+        q += " before:#{@config.until.to_i}" if @config.until
+        q += " #{@config.search_query}" if @config.search_query
+
+        ["X-GM-RAW", q]
+      end
+
+      def search_keys_default
+        keys = []
+        # TODO: test out non-gmail IMAP searching (for @config.search_query)
+        keys += ['SINCE', Net::IMAP.format_date(@config.since)] if @config.since
+        keys += ['BEFORE', Net::IMAP.format_date(@config.until)] if @config.until
+      end
+
+      def gmail_mode?
+        @config.host == 'imap.gmail.com'
+      end
+    end
+  end
+end

data/lib/chronicle/email/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module Email
-    VERSION = "0.2.1"
+    VERSION = "0.2.2"
   end
 end

data/lib/chronicle/email.rb

@@ -1,6 +1,7 @@
 require "chronicle/email/version"
 require "chronicle/email/email_transformer"
 require "chronicle/email/mbox_extractor"
+require "chronicle/email/imap_extractor"
 
 module Chronicle
   module Email

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-email
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-25 00:00:00.000000000 Z
+date: 2022-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chronicle-etl
@@ -128,6 +128,7 @@ files:
 - chronicle-email.gemspec
 - lib/chronicle/email.rb
 - lib/chronicle/email/email_transformer.rb
+- lib/chronicle/email/imap_extractor.rb
 - lib/chronicle/email/mbox_extractor.rb
 - lib/chronicle/email/version.rb
 homepage: https://github.com/chronicle-app/chronicle-email