chronicle-email 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a910606f4987580ac86be55c02a6e3003b7cab21e93476dc1bf6fa4cc4512355
-  data.tar.gz: ef1dbfa473c66feb93c730a5bdd4e426203519a8d94da6afccf227bfa1fa084a
+  metadata.gz: e0d66a6d2109f4f46f23fdf0c157085aed6f757f962339a422e4e38ca86ed129
+  data.tar.gz: e4136e05cf2827f64a07657e9442f9a8cdcac1ab88ed589c52ea76f62fd0d315
 SHA512:
-  metadata.gz: f1ddc8cabf1b9fbb319ad36406e9a99e585eaad8abd772aea378438f838e11be3e75a9c3fe2a1c4805ee6c84d9dfe256ba9cff3ac205406c899ee33a71d1122f
-  data.tar.gz: 00c745bf81b65f59c7e514e735bd3319548f747503907646ca619fbfb1634dd2bb5e199753141a9c651bf210326e803dbb3d4bdc4331bdaeba0ecb70425dd265
+  metadata.gz: 5d58d42a6db25951b46a08ad44302dab836fc3063fae2b9074e5156b4db59ab147b8401b60860665a0a27394ddca98a48c58a6abdf87ce474439e4d7b179604f
+  data.tar.gz: b11bea8171510544c35742cb7ab72c162b7774d0bd5603a7453f065d270a230fa8b2a17d77d5bfc1205a60710f1b7d076ec4f6521630754006a5efe4e6b18fcc

data/.rubocop.yml

@@ -0,0 +1,2 @@
+inherit_gem:
+  chronicle-core: .rubocop-plugin.yml

data/Gemfile

@@ -1,6 +1,6 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in chronicle-email.gemspec
 gemspec

data/README.md

@@ -21,11 +21,11 @@ $ chronicle-etl secrets:set imap username foo@gmail.com
 $ chronicle-etl secrets:set imap password APPPASSWORD
 
 # Then, retrieve your email from the last five days
-$ chronicle-etl --extractor email:imap --transformer email --since 5d --loader json
+$ chronicle-etl --extractor email:imap --schema chronicle --since 5d
 
 # If you don't want to save your credentials as a secret, you can just pass
 # them to the extractor directly
-$ chronicle-etl --extractor email:imap --transformer email --since 5d --loader json \
+$ chronicle-etl --extractor email:imap --schema chronicle --since 5d --loader json \
     --extractor-opts username:foo@gmail.com --password:APPPASSWORD
 ```
 
@@ -34,7 +34,7 @@ The MBOX format is used to archive an email mailbox. [Google Takeout](https://ta
 
 ```sh
 # Retrieve the subject lines of all emails in test.mbox
-$ chronicle-etl --extractor email:mbox --input inbox.mbox --transformer email --fields subject
+$ chronicle-etl --extractor email:mbox --input ~/Downloads/inbox.mbox --fields subject
 ```
 
 ## Available Connectors

data/Rakefile

@@ -1,6 +1,2 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
-
-RSpec::Core::RakeTask.new(:spec)
-
-task :default => :spec
+require 'bundler/gem_tasks'
+task default: %i[]

data/bin/console

@@ -1,14 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "chronicle/email"
+require 'bundler/setup'
+require 'chronicle/email'
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)
+require 'pry'
+Pry.start

data/chronicle-email.gemspec

@@ -1,51 +1,54 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "chronicle/email/version"
+require 'chronicle/email/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "chronicle-email"
+  spec.name          = 'chronicle-email'
   spec.version       = Chronicle::Email::VERSION
-  spec.authors       = ["Andrew Louis"]
-  spec.email         = ["andrew@hyfen.net"]
+  spec.authors       = ['Andrew Louis']
+  spec.email         = ['andrew@hyfen.net']
 
-  spec.summary       = "Email importer for Chronicle"
-  spec.description   = "Various email importers for chronicle-etl"
-  spec.homepage      = "https://github.com/chronicle-app/chronicle-email"
-  spec.license       = "MIT"
+  spec.summary       = 'Email importer for Chronicle'
+  spec.description   = 'Various email importers for chronicle-etl'
+  spec.homepage      = 'https://github.com/chronicle-app/chronicle-email'
+  spec.license       = 'MIT'
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.
   if spec.respond_to?(:metadata)
     # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
-    spec.metadata["homepage_uri"] = spec.homepage
-    spec.metadata["source_code_uri"] = "https://github.com/chronicle-app/chronicle-email"
-    spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-email"
+    spec.metadata['homepage_uri'] = spec.homepage
+    spec.metadata['source_code_uri'] = 'https://github.com/chronicle-app/chronicle-email'
+    spec.metadata['changelog_uri'] = 'https://github.com/chronicle-app/chronicle-email'
   else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
+    raise 'RubyGems 2.0 or newer is required to protect against ' \
+          'public gem pushes.'
   end
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
+  spec.metadata['rubygems_mfa_required'] = 'true'
+  spec.required_ruby_version = '>= 3.1'
 
   spec.add_dependency 'email_reply_parser', '~> 0.5'
+  spec.add_dependency 'mail', '~> 2.7'
   spec.add_dependency 'reverse_markdown', '~> 2.0'
-  spec.add_dependency "chronicle-etl", "~> 0.5"
-  spec.add_dependency "mail", "~> 2.7"
   # Needed for Ruby 3.1 compatibility (https://github.com/mikel/mail/pull/1439#issuecomment-1002801221)
   # TODO: check back for new version of `mail`
-  spec.add_dependency "net-imap"
-  spec.add_dependency "net-smtp"
-
-  spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.9"
+  spec.add_dependency 'chronicle-core', '~> 0.3'
+  spec.add_dependency 'chronicle-etl', '~> 0.6'
+  spec.add_dependency 'net-imap'
+  spec.add_dependency 'net-smtp'
+
+  spec.add_development_dependency 'bundler', '~> 2.3'
+  spec.add_development_dependency 'pry-byebug', '~> 3.10'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '~> 1.63'
 end

data/lib/chronicle/email/email_transformer.rb

@@ -1,5 +1,5 @@
 require 'chronicle/etl'
-require 'mail'
+require 'chronicle/models'
 require 'timeout'
 require 'email_reply_parser'
 require 'reverse_markdown'
@@ -8,98 +8,97 @@ module Chronicle
   module Email
     class EmailTransformer < Chronicle::ETL::Transformer
       register_connector do |r|
+        r.source = :email
+        r.type = :message
         r.description = 'an email object'
-        r.provider = 'email'
-        r.identifier = 'email'
+        r.from_schema = :extraction
+        r.to_schema = :chronicle
       end
 
       setting :body_as_markdown, default: false
       setting :remove_signature, default: true
 
-      def transform
-        build_messaged
+      def transform(record)
+        build_messaged(record.data[:raw])
       end
 
-      def id
-        message.message_id || raise(Chronicle::ETL::UntransformableRecordError, "Email doesn't have an ID")
-      end
+      private
 
-      def timestamp
-        message.date&.to_time || raise(Chronicle::ETL::UntransformableRecordError, "Email doesn't have a timestamp")
-      end
+      def build_messaged(email)
+        timestamp = email.date&.to_time || raise(Chronicle::ETL::UntransformableRecordError,
+          "Email doesn't have a timestamp")
 
-      private
+        email.message_id || raise(Chronicle::ETL::UntransformableRecordError, "Email doesn't have an ID")
 
-      def message
-        @message ||= Mail.new(@extraction.data[:email])
+        Chronicle::Models::CommunicateAction.new do |r|
+          r.end_time = timestamp
+          r.agent = build_agent(email[:from])
+          r.source = 'email'
+          r.source_id = email.message_id
+          r.object = build_message(email)
+        end
       end
 
-      def build_messaged
-        record = ::Chronicle::ETL::Models::Activity.new
-        record.verb = 'messaged'
-        record.provider = 'email'
-        record.provider_id = id
-        record.end_at = timestamp
+      def build_agent(from)
+        raise(Chronicle::ETL::UntransformableRecordError, "Can't determine email sender") unless from&.addrs&.any?
 
-        record.dedupe_on << [:verb, :provider, :provider_id]
-
-        record.actor = build_actor
-        record.involved = build_message
-        record
+        build_person(from.addrs.first)
       end
 
-      def build_actor
-        # sometimes From: fields are malformed and we can't build an
-        # actor out of it.
-        raise(Chronicle::ETL::UntransformableRecordError, "Can't determine email sender") unless message[:from]&.addrs&.any?
+      def build_message(email)
+        Chronicle::Models::Message.new do |r|
+          r.name = clean_subject(email.subject)
+          r.text = clean_body(email)
+          r.source = 'email'
+          r.source_id = email.message_id
 
-        record = ::Chronicle::ETL::Models::Entity.new
-        record.represents = 'identity'
-        record.provider = 'email'
-        record.slug = message[:from].addrs.first.address
-        record.title = message[:from].addrs.first.display_name
+          r.recipient = email[:to]&.addrs&.map { |addr| build_person(addr) }
 
-        record.dedupe_on << [:represents, :provider, :slug]
+          # TODO: handle email references
+          # TODO: handle email account owner
+          # TODO: handle attachments
 
-        record
+          r.dedupe_on << %i[source source_id type]
+        end
       end
 
-      def build_message
-        record = ::Chronicle::ETL::Models::Entity.new
-        record.represents = 'message'
-        record.title = clean_subject(message.subject)
-        record.body = clean_body(message)
-        record.provider = 'email'
-        record.provider_id = id
-
-        # TODO: handle consumer
-        # TODO: handle email references
-        # TODO: handle email account owner
-        # TODO: handle attachments
-
-        record
+      def build_person(addr)
+        Chronicle::Models::Person.new do |r|
+          r.source = 'email'
+          r.slug = addr.address
+          r.name = addr.display_name
+          r.dedupe_on << %i[represents provider slug]
+        end
       end
 
       def clean_subject(subject)
-        subject&.encode("UTF-8", invalid: :replace, undef: :replace)
+        subject&.encode('UTF-8', invalid: :replace, undef: :replace)
       end
 
-      def clean_body message
-        # FIXME: this all needs to be refactored        
+      def clean_body(message)
+        # FIXME: this all needs to be refactored
         if message.multipart?
-          body = message.text_part&.decoded rescue Mail::UnknownEncodingType
+          body = begin
+            message.text_part&.decoded
+          rescue StandardError
+            Mail::UnknownEncodingType
+          end
         else
-          body = message.body&.decoded rescue Mail::UnknownEncodingType
+          body = begin
+            message.body&.decoded
+          rescue StandardError
+            Mail::UnknownEncodingType
+          end
           body = body_to_markdown if @config.body_as_markdown
         end
 
         return if body == Mail::UnknownEncodingType
-        return unless body && body != ""
+        return unless body && body != ''
 
         body = body_without_signature(body) if @config.remove_signature
 
         # Force UTF-8 encoding
-        body.encode("UTF-8", invalid: :replace, undef: :replace)
+        body.encode('UTF-8', invalid: :replace, undef: :replace)
       end
 
       def body_to_markdown(body)
@@ -111,11 +110,11 @@ module Chronicle
       def body_without_signature(body)
         # FIXME: regex in EmailReplyParse gem seems to get into infinite loops
         #   with certain long bodies that have binary data
-        parsed_body = Timeout::timeout(5) do
+        Timeout.timeout(5) do
           EmailReplyParser.parse_reply(body)
         end
-      rescue Timeout::Error, StandardError => e
-        return body
+      rescue Timeout::Error, StandardError
+        body
       end
     end
   end

data/lib/chronicle/email/imap_extractor.rb

@@ -1,12 +1,14 @@
 require 'net/imap'
+require 'mail'
 
 module Chronicle
   module Email
     class IMAPExtractor < Chronicle::ETL::Extractor
       register_connector do |r|
-        r.provider = 'email'
-        r.description = 'imap server'
-        r.identifier = 'imap'
+        r.source = :email
+        r.type = :message
+        r.strategy = :imap
+        r.description = 'IMAP server'
       end
 
       setting :host, required: true, default: 'imap.gmail.com'
@@ -28,7 +30,15 @@ module Chronicle
       def extract
         @message_ids.each do |message_id|
           message = fetch_message(message_id)
-          yield Chronicle::ETL::Extraction.new(data: { email: message.attr["BODY[]"]} )
+          email = Mail.new(message.attr['BODY[]'])
+          data = {
+            raw: email,
+            time: email.date&.to_time,
+            subject: email.subject,
+            from: email&.from&.join(', '),
+            to: email&.to&.join(', ')
+          }
+          yield build_extraction(data:)
         end
       end
 
@@ -39,8 +49,8 @@ module Chronicle
         connection.login(@config.username, @config.password)
         connection.select(@config.mailbox)
         connection
-      rescue Net::IMAP::NoResponseError => e
-        raise(Chronicle::ETL::ExtractionError, "Error connecting to IMAP server. Please check username and password")
+      rescue Net::IMAP::NoResponseError
+        raise(Chronicle::ETL::ExtractionError, 'Error connecting to IMAP server. Please check username and password')
       end
 
       def fetch_message_ids
@@ -48,24 +58,23 @@ module Chronicle
         message_ids = @connection.search(keys)
         message_ids = message_ids.first(@config.limit) if @config.limit
         message_ids
-      rescue Net::IMAP::BadResponseError => e
-        raise(Chronicle::ETL::ExtractionError, "Error searching IMAP server for messages")
+      rescue Net::IMAP::BadResponseError
+        raise(Chronicle::ETL::ExtractionError, 'Error searching IMAP server for messages')
       end
 
       def fetch_message(message_id)
-        response = @connection.fetch(message_id, "BODY.PEEK[]")
-        raise(Chronicle::ETL::ExtractionError, "Error loading message") unless response
+        response = @connection.fetch(message_id, 'BODY.PEEK[]')
+        raise(Chronicle::ETL::ExtractionError, 'Error loading message') unless response
 
-        return response[0]
+        response[0]
       end
 
       def search_keys_gmail
         # Gmail offers an extension to IMAP that lets us use gmail queries
-        q = ""
 
         # First, we ignore drafts beacuse they break a lot of assumptions we
         # make when when processing emails (lack of timestamps, ids, etc)
-        q = "-label:draft"
+        q = '-label:draft'
 
         # We use UNIX timestamps in gmail filters which let us do more precise
         # since/until compared with date-based imap filters
@@ -73,14 +82,14 @@ module Chronicle
         q += " before:#{@config.until.to_i}" if @config.until
         q += " #{@config.search_query}" if @config.search_query
 
-        ["X-GM-RAW", q]
+        ['X-GM-RAW', q]
       end
 
       def search_keys_default
         keys = []
         # TODO: test out non-gmail IMAP searching (for @config.search_query)
         keys += ['SINCE', Net::IMAP.format_date(@config.since)] if @config.since
-        keys += ['BEFORE', Net::IMAP.format_date(@config.until)] if @config.until
+        keys + ['BEFORE', Net::IMAP.format_date(@config.until)] if @config.until
       end
 
       def gmail_mode?
@@ -88,4 +97,4 @@ module Chronicle
       end
     end
   end
-end
+end

data/lib/chronicle/email/mbox_extractor.rb

@@ -6,9 +6,10 @@ module Chronicle
   module Email
     class MboxExtractor < Chronicle::ETL::Extractor
       register_connector do |r|
-        r.provider = 'email'
+        r.source = :email
+        r.type = :message
+        r.strategy = :mbox
         r.description = 'an .mbox file'
-        r.identifier = 'mbox'
       end
 
       setting :input, required: true
@@ -40,14 +41,20 @@ module Chronicle
         #
         # TODO: make this thread-safe (one tmp file per email?)
         file.each do |line|
-          if line =~ NEW_EMAIL_REGEX
-            if File.size(tmp) > 0
-              tmp.rewind
-              email = tmp.read
-              yield Chronicle::ETL::Extraction.new(data: { email: email} )
-              tmp.truncate(0)
-              tmp.rewind
-            end
+          if line =~ (NEW_EMAIL_REGEX) && File.size(tmp).positive?
+            tmp.rewind
+
+            email = Mail.new(tmp.read)
+            data = {
+              raw: email,
+              time: email.date&.to_time,
+              subject: email.subject,
+              from: email&.from&.join(', '),
+              to: email&.to&.join(', ')
+            }
+            yield build_extraction(data:)
+            tmp.truncate(0)
+            tmp.rewind
           end
           tmp.write(line)
         end

data/lib/chronicle/email/version.rb

@@ -1,5 +1,5 @@
 module Chronicle
   module Email
-    VERSION = "0.2.3"
+    VERSION = '0.3.0'.freeze
   end
 end

data/lib/chronicle/email.rb

@@ -1,7 +1,7 @@
-require "chronicle/email/version"
-require "chronicle/email/email_transformer"
-require "chronicle/email/mbox_extractor"
-require "chronicle/email/imap_extractor"
+require 'chronicle/email/version'
+require 'chronicle/email/email_transformer'
+require 'chronicle/email/mbox_extractor'
+require 'chronicle/email/imap_extractor'
 
 module Chronicle
   module Email

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chronicle-email
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Louis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-31 00:00:00.000000000 Z
+date: 2024-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: email_reply_parser
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: reverse_markdown
   requirement: !ruby/object:Gem::Requirement
@@ -39,33 +53,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: chronicle-etl
+  name: chronicle-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: mail
+  name: chronicle-etl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: net-imap
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -123,19 +151,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '1.63'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '1.63'
 description: Various email importers for chronicle-etl
 email:
 - andrew@hyfen.net
@@ -145,6 +173,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -166,6 +195,7 @@ metadata:
   homepage_uri: https://github.com/chronicle-app/chronicle-email
   source_code_uri: https://github.com/chronicle-app/chronicle-email
   changelog_uri: https://github.com/chronicle-app/chronicle-email
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -174,14 +204,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Email importer for Chronicle