teems 0.3.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +67 -0
- data/README.md +77 -0
- data/bin/teems +8 -1
- data/lib/teems/api/calendar.rb +45 -14
- data/lib/teems/api/client.rb +19 -4
- data/lib/teems/api/mail.rb +132 -0
- data/lib/teems/api/mail_actions.rb +22 -0
- data/lib/teems/api/mail_rules.rb +24 -0
- data/lib/teems/cli.rb +2 -0
- data/lib/teems/commands/base.rb +31 -7
- data/lib/teems/commands/cal.rb +109 -28
- data/lib/teems/commands/cal_ics.rb +47 -0
- data/lib/teems/commands/confirmation.rb +19 -0
- data/lib/teems/commands/help.rb +4 -0
- data/lib/teems/commands/mail.rb +207 -0
- data/lib/teems/commands/mail_decrypting.rb +46 -0
- data/lib/teems/commands/mail_help.rb +85 -0
- data/lib/teems/commands/mail_rules.rb +201 -0
- data/lib/teems/commands/mail_sync.rb +92 -0
- data/lib/teems/commands/mail_thread.rb +87 -0
- data/lib/teems/commands/mail_triage.rb +241 -0
- data/lib/teems/commands/meet.rb +159 -0
- data/lib/teems/commands/sync.rb +15 -8
- data/lib/teems/commands/sync_folders.rb +118 -0
- data/lib/teems/formatters/format_utils.rb +23 -2
- data/lib/teems/formatters/html_entities.rb +49 -0
- data/lib/teems/formatters/html_text.rb +181 -0
- data/lib/teems/formatters/ics_format.rb +112 -0
- data/lib/teems/formatters/link_cleaner.rb +91 -0
- data/lib/teems/formatters/link_collector.rb +70 -0
- data/lib/teems/formatters/mail_formatter.rb +58 -0
- data/lib/teems/formatters/mail_markdown.rb +70 -0
- data/lib/teems/formatters/mail_rule_formatter.rb +45 -0
- data/lib/teems/formatters/mail_rule_text.rb +67 -0
- data/lib/teems/formatters/output.rb +45 -8
- data/lib/teems/models/chat.rb +19 -5
- data/lib/teems/models/event.rb +1 -23
- data/lib/teems/models/event_display.rb +40 -0
- data/lib/teems/models/ics_event.rb +60 -0
- data/lib/teems/models/mail_conversation.rb +62 -0
- data/lib/teems/models/mail_encryption.rb +41 -0
- data/lib/teems/models/mail_folder.rb +44 -0
- data/lib/teems/models/mail_message.rb +152 -0
- data/lib/teems/models/mail_rule.rb +38 -0
- data/lib/teems/runner.rb +6 -0
- data/lib/teems/services/api_client.rb +23 -11
- data/lib/teems/services/browser.rb +17 -0
- data/lib/teems/services/clipboard.rb +17 -0
- data/lib/teems/services/mail_folders.rb +76 -0
- data/lib/teems/services/mail_id_cache.rb +51 -0
- data/lib/teems/services/mail_legacy_files.rb +96 -0
- data/lib/teems/services/mail_sync_attachments.rb +103 -0
- data/lib/teems/services/mail_sync_engine.rb +98 -0
- data/lib/teems/services/mail_sync_folder.rb +142 -0
- data/lib/teems/services/mail_sync_store.rb +116 -0
- data/lib/teems/services/smime_content.rb +36 -0
- data/lib/teems/services/smime_decryptor.rb +36 -0
- data/lib/teems/services/sync_dir_naming.rb +16 -10
- data/lib/teems/services/sync_dir_ownership.rb +111 -0
- data/lib/teems/services/sync_dir_planning.rb +130 -0
- data/lib/teems/services/sync_store.rb +20 -43
- data/lib/teems/services/token_extractor.rb +4 -0
- data/lib/teems/support/control_chars.rb +19 -0
- data/lib/teems/support/mime_header.rb +61 -0
- data/lib/teems/support/mime_part.rb +97 -0
- data/lib/teems/support/signed_data.rb +23 -0
- data/lib/teems/support/time_parsing.rb +9 -1
- data/lib/teems/version.rb +1 -1
- data/lib/teems.rb +63 -2
- metadata +48 -1
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# Remembers full Graph message IDs for the short IDs `teems mail list` and `search` print,
|
|
6
|
+
# so `teems mail read <short id>` works afterwards. Kept in the XDG cache dir.
|
|
7
|
+
class MailIdCache
|
|
8
|
+
MAX_ENTRIES = 2000
|
|
9
|
+
|
|
10
|
+
def initialize(paths: Support::XdgPaths.new)
|
|
11
|
+
@path = paths.cache_file('mail-ids.json')
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def remember(messages)
|
|
15
|
+
return if messages.empty?
|
|
16
|
+
|
|
17
|
+
# Listed IDs move to the end, so the ones in use are the last evicted
|
|
18
|
+
listed = messages.to_h { |message| [message.short_id, message.id] }
|
|
19
|
+
ids = load.except(*listed.keys).merge(listed)
|
|
20
|
+
FileUtils.mkdir_p(File.dirname(@path), mode: 0o700)
|
|
21
|
+
File.write(@path, JSON.generate(ids.to_a.last(MAX_ENTRIES).to_h))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Full ID for a short ID or unique prefix of one (4+ hex digits); full IDs pass through
|
|
25
|
+
def resolve(token)
|
|
26
|
+
prefix = self.class.short_prefix(token)
|
|
27
|
+
return token.to_s.strip unless prefix
|
|
28
|
+
|
|
29
|
+
matches = ids_starting_with(prefix)
|
|
30
|
+
matches.one? ? matches.first : nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Lowercased short ID (or prefix) for a token that looks like one, else nil
|
|
34
|
+
def self.short_prefix(token)
|
|
35
|
+
stripped = token.to_s.strip
|
|
36
|
+
stripped.match?(/\A\h{4,16}\z/) ? stripped.downcase : nil
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def ids_starting_with(prefix) = load.select { |short, _id| short.start_with?(prefix) }.values.uniq
|
|
42
|
+
|
|
43
|
+
def load
|
|
44
|
+
data = File.file?(@path) ? JSON.parse(File.read(@path)) : {}
|
|
45
|
+
data.is_a?(Hash) ? data : {}
|
|
46
|
+
rescue JSON::ParserError
|
|
47
|
+
{}
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# Finds Markdown files an older outlook-cli sync wrote for the same message, so syncing
|
|
6
|
+
# into an existing ~/Mail adopts them instead of writing duplicates. outlook-cli named
|
|
7
|
+
# messages by AppleScript integer IDs, so files are matched on received time (to the
|
|
8
|
+
# second, in local time, the exact second first), subject (ignoring case and
|
|
9
|
+
# punctuation) and sender address. The sender matters for bursts such as a dozen
|
|
10
|
+
# "Automatic reply" messages arriving in the same couple of seconds.
|
|
11
|
+
class MailLegacyFiles
|
|
12
|
+
DATE_FORMATS = ['%A, %B %d, %Y at %I:%M:%S %p', '%B %d, %Y at %I:%M:%S %p', '%m/%d/%Y %I:%M:%S %p'].freeze
|
|
13
|
+
HEADER_BYTES = 4096
|
|
14
|
+
# Long recipient lists push the closing --- past HEADER_BYTES; give up beyond this
|
|
15
|
+
MAX_HEADER_BYTES = 65_536
|
|
16
|
+
|
|
17
|
+
def initialize(dir, known_files: [])
|
|
18
|
+
@dir = dir
|
|
19
|
+
@known = known_files.to_set
|
|
20
|
+
@entries = nil
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# File name of an unclaimed legacy file for this message, or nil. Each file matches once.
|
|
24
|
+
def match(message)
|
|
25
|
+
received = message.received_at
|
|
26
|
+
return unless received
|
|
27
|
+
|
|
28
|
+
wanted = { subject: subject_key(message.subject), sender: message.from[:address].downcase }
|
|
29
|
+
candidates = [0, -1, 1].flat_map { |offset| entries.fetch(received.to_i + offset, []) }
|
|
30
|
+
found = candidates.find { |candidate| same_message?(candidate, wanted) }
|
|
31
|
+
found&.store(:claimed, true)
|
|
32
|
+
found&.fetch(:file)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def count = entries.sum { |_time, list| list.length }
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# Unclaimed, same subject, and the same sender unless one side has no address
|
|
40
|
+
def same_message?(candidate, wanted)
|
|
41
|
+
return false if candidate[:claimed] || candidate[:subject] != wanted[:subject]
|
|
42
|
+
|
|
43
|
+
senders = [candidate[:sender], wanted[:sender]]
|
|
44
|
+
senders.include?('') || senders.uniq.one?
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def entries
|
|
48
|
+
@entries ||= Dir.glob(File.join(@dir, '*.md')).filter_map { |path| legacy_entry(path) }
|
|
49
|
+
.group_by { |entry| entry[:time] }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def legacy_entry(path)
|
|
53
|
+
file = File.basename(path)
|
|
54
|
+
return if @known.include?(file)
|
|
55
|
+
|
|
56
|
+
fields = frontmatter(path)
|
|
57
|
+
return unless fields['id'].to_s.match?(/\A\d+\z/)
|
|
58
|
+
|
|
59
|
+
time = parse_date(fields['date'].to_s)
|
|
60
|
+
time && { time: time.to_i, subject: subject_key(fields['subject'].to_s), sender: sender_key(fields['from']),
|
|
61
|
+
file: file }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def frontmatter(path)
|
|
65
|
+
block = frontmatter_block(path, HEADER_BYTES) || frontmatter_block(path, MAX_HEADER_BYTES)
|
|
66
|
+
block.to_s.scan(/^(\w+):[ \t]*(.*)$/).to_h.transform_values { |value| unquote(value) }
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def frontmatter_block(path, bytes)
|
|
70
|
+
head = File.read(path, bytes).to_s.force_encoding(Encoding::UTF_8).scrub('')
|
|
71
|
+
head[/\A---\n(.*?)\n---/m, 1]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def unquote(value) = value.strip.delete_prefix('"').delete_suffix('"').gsub('\\"', '"')
|
|
75
|
+
|
|
76
|
+
def parse_date(text)
|
|
77
|
+
text = text.tr("\u202f\u00a0", ' ').squeeze(' ').strip
|
|
78
|
+
DATE_FORMATS.each do |format|
|
|
79
|
+
return Time.strptime(text, format)
|
|
80
|
+
rescue ArgumentError
|
|
81
|
+
next
|
|
82
|
+
end
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def subject_key(subject) = subject.downcase.gsub(/[^[:alnum:]]/, '')
|
|
87
|
+
|
|
88
|
+
# Address from "Name <address>" (or a bare address), lower case; '' when there is none
|
|
89
|
+
def sender_key(from)
|
|
90
|
+
text = from.to_s
|
|
91
|
+
address = text[/<([^<>@\s]+@[^<>\s]+)>/, 1] || text[/\A[^<>@\s]+@[^<>\s]+\z/]
|
|
92
|
+
address.to_s.downcase
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# A message's attachments during mail sync: lists them for its Markdown file and, with
|
|
6
|
+
# the attachments option, saves file attachments under <folder>/attachments/<short id>/.
|
|
7
|
+
# The listing is also where S/MIME encrypted mail shows itself: Exchange wraps it as TNEF,
|
|
8
|
+
# so only an empty body and an smime.p7m attachment give it away.
|
|
9
|
+
# One that fails the same way every time is skipped with a warning. Raising would fail
|
|
10
|
+
# the page before its checkpoint, so every later run would stop there too.
|
|
11
|
+
class MailSyncAttachments
|
|
12
|
+
FILE_ATTACHMENT = '#microsoft.graph.fileAttachment'
|
|
13
|
+
# 4xx errors that can succeed on a later run, unlike a missing or forbidden attachment
|
|
14
|
+
RETRYABLE_STATUSES = [401, 408, 429].freeze
|
|
15
|
+
# Local failures that can clear up (disk full, I/O), unlike a name the disk refuses
|
|
16
|
+
RETRYABLE_ERRNOS = [Errno::ENOSPC, Errno::EIO, Errno::EDQUOT].freeze
|
|
17
|
+
|
|
18
|
+
# names: attachment names for the Markdown; encrypted: an S/MIME envelope stood in for the body
|
|
19
|
+
Listing = Data.define(:names, :encrypted)
|
|
20
|
+
|
|
21
|
+
# options as for MailSyncEngine: attachments and on_warning
|
|
22
|
+
def initialize(api:, store:, options:)
|
|
23
|
+
@api = api
|
|
24
|
+
@store = store
|
|
25
|
+
@options = options
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The message's attachments other than inline images and, for a message with no body, its
|
|
29
|
+
# S/MIME envelope: ciphertext, so neither listed nor saved as a file
|
|
30
|
+
def listing(message, folder_path)
|
|
31
|
+
envelopes, files = list(message).partition { |item| message.bodyless? && Models::MailEncryption.envelope?(item) }
|
|
32
|
+
download_all(message, files, File.join(@store.folder_dir(folder_path), 'attachments')) if @options[:attachments]
|
|
33
|
+
Listing.new(names: Models::MailMessage.attachment_names(files), encrypted: envelopes.any?)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.permanent?(error)
|
|
37
|
+
return RETRYABLE_ERRNOS.none? { |errno| error.is_a?(errno) } unless error.is_a?(ApiError)
|
|
38
|
+
|
|
39
|
+
status = error.status_code.to_i
|
|
40
|
+
status.between?(400, 499) && !RETRYABLE_STATUSES.include?(status)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The attachment's safe file name, or "name (2).ext" and so on when an earlier one in
|
|
44
|
+
# the message took it (compared without case, as on macOS). The first keeps the plain
|
|
45
|
+
# name, so files saved before stay where they are.
|
|
46
|
+
def self.unique_name(name, taken)
|
|
47
|
+
base = Formatters::FormatUtils.safe_filename(name.to_s)
|
|
48
|
+
ext = File.extname(base)
|
|
49
|
+
candidates = (2..).lazy.map { |number| "#{base.delete_suffix(ext)} (#{number})#{ext}" }
|
|
50
|
+
unique = [base].lazy.chain(candidates).find { |candidate| !taken.include?(candidate.downcase) }
|
|
51
|
+
taken << unique.downcase
|
|
52
|
+
unique
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# A listing Graph refuses for good leaves the message without attachment names
|
|
58
|
+
def list(message)
|
|
59
|
+
@api.attachments(message.id).reject { |attachment| attachment['isInline'] }
|
|
60
|
+
rescue ApiError => e
|
|
61
|
+
skip(e, "attachments of message #{message.short_id}")
|
|
62
|
+
[]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Item and reference attachments (attached emails, cloud links) have no file to save
|
|
66
|
+
def download_all(message, files, attachments_dir)
|
|
67
|
+
dir = File.join(attachments_dir, message.short_id)
|
|
68
|
+
taken = Set.new
|
|
69
|
+
files.each do |attachment|
|
|
70
|
+
next unless attachment.fetch('@odata.type', FILE_ATTACHMENT) == FILE_ATTACHMENT
|
|
71
|
+
|
|
72
|
+
download_or_skip(message, attachment, File.join(dir, self.class.unique_name(attachment['name'], taken)))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def download_or_skip(message, attachment, path)
|
|
77
|
+
download(message, attachment, path)
|
|
78
|
+
rescue SystemCallError, ApiError => e
|
|
79
|
+
skip(e, "attachment #{attachment['name'].to_s.inspect} of message #{message.short_id}")
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def skip(error, what)
|
|
83
|
+
raise error unless self.class.permanent?(error)
|
|
84
|
+
|
|
85
|
+
@options[:on_warning]&.call("Skipped #{what}: #{error.message}")
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Skipped when a file of the same size is already there
|
|
89
|
+
def download(message, attachment, path)
|
|
90
|
+
attachment_id, size = attachment.values_at('id', 'size')
|
|
91
|
+
return if File.size?(path) == size
|
|
92
|
+
|
|
93
|
+
bytes = @api.attachment(message.id, attachment_id)['contentBytes']
|
|
94
|
+
save(path, bytes) if bytes
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def save(path, encoded)
|
|
98
|
+
FileUtils.mkdir_p(File.dirname(path), mode: 0o700)
|
|
99
|
+
File.binwrite(path, encoded.unpack1('m'))
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# Incremental mail sync for one folder at a time using Graph delta queries. Each page is
|
|
6
|
+
# written and checkpointed before the next is fetched, so an interrupted run resumes where
|
|
7
|
+
# it stopped. Messages that leave a folder keep their file; the index records removed_at.
|
|
8
|
+
class MailSyncEngine
|
|
9
|
+
# Graph's answers to a delta link it can no longer use: 410 Gone, or 400 with one of
|
|
10
|
+
# these codes (compared without case; the last is Exchange's, for an old or malformed
|
|
11
|
+
# delta token)
|
|
12
|
+
RESYNC_CODES = %w[syncstatenotfound syncstateinvalid resyncrequired errorinvalidsyncstatedata].freeze
|
|
13
|
+
|
|
14
|
+
# Raised by fetch_page when Graph rejects the delta link itself (cause: the ApiError).
|
|
15
|
+
# A 400 anywhere else, such as one message's attachments, must not restart the folder.
|
|
16
|
+
class ResyncNeeded < StandardError; end
|
|
17
|
+
|
|
18
|
+
# options: since (Time limiting the first sync of a folder; nil syncs everything),
|
|
19
|
+
# attachments (also download file attachments) and on_warning (called with problems
|
|
20
|
+
# that are skipped rather than raised)
|
|
21
|
+
def initialize(api:, store:, options: {})
|
|
22
|
+
@api = api
|
|
23
|
+
@store = store
|
|
24
|
+
@options = options
|
|
25
|
+
@folder = nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Syncs one Models::MailFolder; returns counts keyed by MailSyncFolder::STAT_KEYS.
|
|
29
|
+
# A delta link Graph rejects starts the folder over once; a second rejection raises
|
|
30
|
+
# Graph's error.
|
|
31
|
+
def sync(folder)
|
|
32
|
+
attempts = 0
|
|
33
|
+
begin
|
|
34
|
+
@folder = MailSyncFolder.new(folder, @store)
|
|
35
|
+
run_delta
|
|
36
|
+
rescue ResyncNeeded => e
|
|
37
|
+
raise e.cause unless (attempts += 1) == 1
|
|
38
|
+
|
|
39
|
+
@folder.reset_state
|
|
40
|
+
retry
|
|
41
|
+
end
|
|
42
|
+
@folder.stats
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.resync?(error)
|
|
46
|
+
case error.status_code
|
|
47
|
+
when 410 then true
|
|
48
|
+
when 400 then RESYNC_CODES.include?(error.error_code.to_s.downcase)
|
|
49
|
+
else false
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
# Each page is checkpointed, so an interrupted run resumes from its last page
|
|
56
|
+
def run_delta
|
|
57
|
+
link = @folder.resume_link
|
|
58
|
+
loop do
|
|
59
|
+
page = fetch_page(link)
|
|
60
|
+
page['value'].to_a.each { |item| process(item) }
|
|
61
|
+
@folder.checkpoint(page)
|
|
62
|
+
link = page['@odata.nextLink']
|
|
63
|
+
break unless link
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def fetch_page(link)
|
|
68
|
+
link ? @api.delta_page(link) : @api.delta(folder: @folder.id, since: @options[:since])
|
|
69
|
+
rescue ApiError => e
|
|
70
|
+
raise ResyncNeeded if self.class.resync?(e)
|
|
71
|
+
|
|
72
|
+
raise
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def process(item)
|
|
76
|
+
return @folder.mark_removed(item['id']) if item.key?('@removed')
|
|
77
|
+
|
|
78
|
+
message = Models::MailMessage.from_api(item)
|
|
79
|
+
legacy_file = @folder.legacy_file(message)
|
|
80
|
+
return @folder.adopt(message, legacy_file) if legacy_file
|
|
81
|
+
|
|
82
|
+
write(message)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def attachments = MailSyncAttachments.new(api: @api, store: @store, options: @options)
|
|
86
|
+
|
|
87
|
+
def write(message)
|
|
88
|
+
path = @folder.path
|
|
89
|
+
file = @folder.file_for(message)
|
|
90
|
+
listing = attachments.listing(message, path) if message.attachments_to_list?
|
|
91
|
+
message = message.with(protection: 'smime') if listing&.encrypted
|
|
92
|
+
content = Formatters::MailMarkdown.render(message, folder: path, attachments: listing ? listing.names : [])
|
|
93
|
+
@folder.count(@store.write_message(path, file, content))
|
|
94
|
+
@folder.record(message, file)
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# The index of one synced mail folder: Graph message ID -> file name and summary, plus
|
|
6
|
+
# the matcher for files an older outlook-cli sync left in the same directory
|
|
7
|
+
class MailSyncIndex
|
|
8
|
+
def initialize(messages, legacy:)
|
|
9
|
+
@messages = messages
|
|
10
|
+
@snapshot = JSON.generate(messages)
|
|
11
|
+
@legacy = legacy
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def to_h = @messages
|
|
15
|
+
|
|
16
|
+
def entry(message_id) = @messages[message_id]
|
|
17
|
+
|
|
18
|
+
def include?(message_id) = @messages.key?(message_id)
|
|
19
|
+
|
|
20
|
+
# Legacy file already recorded for this message, or an unclaimed one that matches it
|
|
21
|
+
def legacy_file(message)
|
|
22
|
+
entry = @messages[message.id]
|
|
23
|
+
return entry['legacy'] ? entry['file'] : nil if entry
|
|
24
|
+
|
|
25
|
+
@legacy.match(message)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def self.entry_for(message, file)
|
|
29
|
+
{ 'file' => file, 'short_id' => message.short_id, 'subject' => message.subject,
|
|
30
|
+
'from' => message.from[:address], 'received' => message.received_at&.utc&.iso8601,
|
|
31
|
+
'read' => message.read? }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def record(message, file)
|
|
35
|
+
@messages[message.id] = MailSyncIndex.entry_for(message, file)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Records a legacy file (left as it is) as this message's file
|
|
39
|
+
def adopt(message, file)
|
|
40
|
+
record(message, file)['legacy'] = true
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Marks a known message removed; true unless it was unknown or already marked
|
|
44
|
+
def mark_removed?(message_id)
|
|
45
|
+
return false unless removable?(message_id)
|
|
46
|
+
|
|
47
|
+
@messages[message_id]['removed_at'] = Time.now.utc.iso8601
|
|
48
|
+
true
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def changed? = JSON.generate(@messages) != @snapshot
|
|
52
|
+
|
|
53
|
+
def mark_saved
|
|
54
|
+
@snapshot = JSON.generate(@messages)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Unknown IDs fall back to an entry that already counts as removed
|
|
60
|
+
def removable?(message_id) = !@messages.fetch(message_id, { 'removed_at' => nil }).key?('removed_at')
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Index, delta state, and counts for the folder a MailSyncEngine is syncing
|
|
64
|
+
class MailSyncFolder
|
|
65
|
+
STAT_KEYS = %i[created updated unchanged adopted removed].freeze
|
|
66
|
+
|
|
67
|
+
attr_reader :stats
|
|
68
|
+
|
|
69
|
+
def initialize(folder, store)
|
|
70
|
+
@folder = folder
|
|
71
|
+
@store = store
|
|
72
|
+
@index = load_index
|
|
73
|
+
@stats = STAT_KEYS.to_h { |key| [key, 0] }
|
|
74
|
+
# Without an index the files the delta link already covered are gone: start over
|
|
75
|
+
reset_state unless store.index_exists?(path)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def id = @folder.id
|
|
79
|
+
def path = @folder.path
|
|
80
|
+
|
|
81
|
+
def resume_link
|
|
82
|
+
state = @store.load_state[id] || {}
|
|
83
|
+
state['next_link'] || state['delta_link']
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def reset_state
|
|
87
|
+
states = @store.load_state
|
|
88
|
+
states.delete(id) && @store.save_state(states)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def legacy_file(message) = @index.legacy_file(message)
|
|
92
|
+
|
|
93
|
+
def file_for(message) = @index.entry(message.id)&.fetch('file') || Formatters::MailMarkdown.filename(message)
|
|
94
|
+
|
|
95
|
+
def count(result) = @stats[result] += 1
|
|
96
|
+
|
|
97
|
+
def record(message, file) = @index.record(message, file)
|
|
98
|
+
|
|
99
|
+
# Legacy files are never rewritten, so seeing one again leaves it unchanged
|
|
100
|
+
def adopt(message, file)
|
|
101
|
+
count(@index.include?(message.id) ? :unchanged : :adopted)
|
|
102
|
+
@index.adopt(message, file)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def mark_removed(message_id)
|
|
106
|
+
count(:removed) if @index.mark_removed?(message_id)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Saves the index (when anything changed) and the link to continue from
|
|
110
|
+
def checkpoint(page)
|
|
111
|
+
save_index if @index.changed? || !@store.index_exists?(path)
|
|
112
|
+
states = @store.load_state
|
|
113
|
+
states[id] = next_state(states[id] || {}, page)
|
|
114
|
+
@store.save_state(states)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
|
|
119
|
+
def load_index
|
|
120
|
+
messages = @store.load_index(path)
|
|
121
|
+
known = messages.values.map { |entry| entry['file'] }
|
|
122
|
+
legacy = MailLegacyFiles.new(@store.folder_dir(path), known_files: known)
|
|
123
|
+
MailSyncIndex.new(messages, legacy: legacy)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def save_index
|
|
127
|
+
@store.save_index(path, @index.to_h)
|
|
128
|
+
@index.mark_saved
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def next_state(state, page)
|
|
132
|
+
next_link = page['@odata.nextLink']
|
|
133
|
+
update = next_link ? { 'next_link' => next_link } : finished_state(page)
|
|
134
|
+
state.merge('name' => path, **update)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def finished_state(page)
|
|
138
|
+
{ 'next_link' => nil, 'delta_link' => page['@odata.deltaLink'], 'last_synced_at' => Time.now.utc.iso8601 }
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# On-disk layout for `teems mail sync`:
|
|
6
|
+
#
|
|
7
|
+
# <root>/<Folder>/<YYYY-MM-DD>-<subject>-<short id>.md one file per message
|
|
8
|
+
# <root>/<Folder>/index.json message ID -> file and summary
|
|
9
|
+
# <root>/<Folder>/attachments/<short id>/ with --attachments
|
|
10
|
+
#
|
|
11
|
+
# <Folder> is the folder path with "/" as "-", as outlook-cli named them, so its files
|
|
12
|
+
# are adopted. A folder whose name another folder's index already claims (Projects/Reports
|
|
13
|
+
# and a top-level Projects-Reports), or that is no usable directory name ("." or ".."),
|
|
14
|
+
# gets "<Folder> (<8 hex digits>)" instead.
|
|
15
|
+
#
|
|
16
|
+
# Delta links and the run lock live in the XDG state dir, outside the root, so a run that
|
|
17
|
+
# changes no mail changes nothing under the root (watchers keyed on mtimes stay quiet).
|
|
18
|
+
class MailSyncStore
|
|
19
|
+
include SyncFileOps
|
|
20
|
+
|
|
21
|
+
INDEX_FILE = 'index.json'
|
|
22
|
+
STATE_FILE = 'mail-sync.json'
|
|
23
|
+
UNUSABLE_NAMES = ['', '.', '..'].freeze
|
|
24
|
+
|
|
25
|
+
attr_reader :root
|
|
26
|
+
|
|
27
|
+
def self.default_root = File.join(Support::XdgPaths.new.data_dir, SyncStore::SYNC_DIR, 'mail')
|
|
28
|
+
|
|
29
|
+
def self.default_state_dir
|
|
30
|
+
File.join(ENV.fetch('XDG_STATE_HOME') { File.join(Dir.home, '.local', 'state') }, 'teems')
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(root: MailSyncStore.default_root, state_dir: MailSyncStore.default_state_dir)
|
|
34
|
+
@root = File.expand_path(root)
|
|
35
|
+
@state_dir = state_dir
|
|
36
|
+
@folder_dirs = {}
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def folder_dir(folder_path) = @folder_dirs[folder_path] ||= File.join(root, dir_name(folder_path))
|
|
40
|
+
|
|
41
|
+
def index_exists?(folder_path) = File.file?(index_path(folder_path))
|
|
42
|
+
|
|
43
|
+
def load_index(folder_path)
|
|
44
|
+
read_json(index_path(folder_path)).fetch('messages', {})
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def save_index(folder_path, messages)
|
|
48
|
+
dir = ensure_dir(folder_dir(folder_path))
|
|
49
|
+
atomic_write(File.join(dir, INDEX_FILE),
|
|
50
|
+
JSON.pretty_generate('version' => 1, 'folder' => folder_path, 'messages' => messages))
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# :created, :updated, or :unchanged (identical content is never rewritten)
|
|
54
|
+
def write_message(folder_path, filename, content)
|
|
55
|
+
path = File.join(ensure_dir(folder_dir(folder_path)), filename)
|
|
56
|
+
existed = File.exist?(path)
|
|
57
|
+
return :unchanged if existed && File.read(path) == content
|
|
58
|
+
|
|
59
|
+
atomic_write(path, content)
|
|
60
|
+
existed ? :updated : :created
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Delta state for this root, keyed by Graph folder ID
|
|
64
|
+
def load_state
|
|
65
|
+
folders = read_json(state_path).dig('roots', root, 'folders')
|
|
66
|
+
folders.is_a?(Hash) ? folders : {}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def save_state(folders)
|
|
70
|
+
data = read_json(state_path)
|
|
71
|
+
(data['roots'] ||= {})[root] = { 'folders' => folders }
|
|
72
|
+
ensure_dir(@state_dir)
|
|
73
|
+
atomic_write(state_path, JSON.pretty_generate(data))
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Yields while holding the mail sync lock; returns :locked without yielding if another run holds it
|
|
77
|
+
def with_lock
|
|
78
|
+
ensure_dir(@state_dir)
|
|
79
|
+
File.open(File.join(@state_dir, 'mail-sync.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
|
80
|
+
return :locked unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
|
81
|
+
|
|
82
|
+
yield
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def index_path(folder_path) = File.join(folder_dir(folder_path), INDEX_FILE)
|
|
89
|
+
|
|
90
|
+
def dir_name(folder_path)
|
|
91
|
+
name = folder_path.tr('/', '-')
|
|
92
|
+
return name unless UNUSABLE_NAMES.include?(name) || claimed_by_another?(name, folder_path)
|
|
93
|
+
|
|
94
|
+
"#{name} (#{Digest::SHA256.hexdigest(folder_path)[0, 8]})"
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# An index records the folder path it was saved for
|
|
98
|
+
def claimed_by_another?(name, folder_path)
|
|
99
|
+
read_json(File.join(root, name, INDEX_FILE)).fetch('folder', folder_path) != folder_path
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def state_path = File.join(@state_dir, STATE_FILE)
|
|
103
|
+
|
|
104
|
+
def ensure_dir(dir)
|
|
105
|
+
FileUtils.mkdir_p(dir, mode: 0o700)
|
|
106
|
+
dir
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# A JSON object from the file, or {} for anything else
|
|
110
|
+
def read_json(path)
|
|
111
|
+
data = load_json_or_default(path, {})
|
|
112
|
+
data.is_a?(Hash) ? data : {}
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# What a decrypted S/MIME message says: its body (HTML when there is one, as Outlook
|
|
6
|
+
# shows it), the names of its attachments, and whether it was also signed. A signature
|
|
7
|
+
# is unwrapped to reach the content but not checked.
|
|
8
|
+
class SmimeContent
|
|
9
|
+
# body: text in body_type ('html' or 'text'); attachments: file names; signed: true/false
|
|
10
|
+
Result = Data.define(:body, :body_type, :attachments, :signed) do
|
|
11
|
+
# The bodyless message Graph returned, with the decrypted body in its place
|
|
12
|
+
def apply_to(message) = message.with(body: Models::MailMessage.clean(body), body_type: body_type)
|
|
13
|
+
|
|
14
|
+
def attachment_names = attachments.map { |name| Models::MailMessage.clean(name) }
|
|
15
|
+
end
|
|
16
|
+
BODY_TYPES = %w[text/html text/plain].freeze
|
|
17
|
+
|
|
18
|
+
def self.parse(mime) = new.parse(Support::MimePart.parse(mime))
|
|
19
|
+
|
|
20
|
+
def parse(entity)
|
|
21
|
+
content, signed = entity.unsigned
|
|
22
|
+
leaves = content.leaves
|
|
23
|
+
body = body_part(leaves)
|
|
24
|
+
Result.new(body: body ? body.text : '', body_type: body&.content_type == 'text/html' ? 'html' : 'text',
|
|
25
|
+
attachments: leaves.select(&:attachment?).map { |part| part.filename.to_s }, signed: signed)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def body_part(leaves)
|
|
31
|
+
leaves.select { |part| part.text? && !part.attachment? }
|
|
32
|
+
.min_by { |part| BODY_TYPES.index(part.content_type) }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Teems
|
|
4
|
+
module Services
|
|
5
|
+
# Decrypts an S/MIME envelope with macOS's `security cms -D`, which finds the matching
|
|
6
|
+
# private key among keychain identities, PIV card keys (current and retired key management
|
|
7
|
+
# keys) included, and shows the system PIN dialog when the card needs it. The ciphertext
|
|
8
|
+
# goes in on stdin and the plaintext comes back on stdout, so neither touches the disk.
|
|
9
|
+
class SmimeDecryptor
|
|
10
|
+
# SEC_ERROR_NOT_A_RECIPIENT: no key here matches any recipient of the message
|
|
11
|
+
NOT_A_RECIPIENT = '-8147'
|
|
12
|
+
COMMAND = %w[security cms -D].freeze
|
|
13
|
+
|
|
14
|
+
def decrypt(envelope)
|
|
15
|
+
plain, errors, status = Support::Subprocess.capture3(*COMMAND, stdin_data: envelope, binmode: true)
|
|
16
|
+
return plain if status.success?
|
|
17
|
+
|
|
18
|
+
raise Error, failure(errors)
|
|
19
|
+
rescue Errno::ENOENT
|
|
20
|
+
raise Error, 'Decrypting needs the macOS security command'
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def failure(errors)
|
|
26
|
+
if errors.include?(NOT_A_RECIPIENT)
|
|
27
|
+
return 'None of your keys can open this message: it was encrypted for other certificates. ' \
|
|
28
|
+
'Is your PIV card inserted?'
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
detail = errors.lines.map(&:strip).reject(&:empty?).first
|
|
32
|
+
"Could not decrypt the message#{": #{detail.delete_prefix('security: ')}" if detail}"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|