teems 0.3.3 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +67 -0
  3. data/README.md +77 -0
  4. data/bin/teems +8 -1
  5. data/lib/teems/api/calendar.rb +45 -14
  6. data/lib/teems/api/client.rb +19 -4
  7. data/lib/teems/api/mail.rb +132 -0
  8. data/lib/teems/api/mail_actions.rb +22 -0
  9. data/lib/teems/api/mail_rules.rb +24 -0
  10. data/lib/teems/cli.rb +2 -0
  11. data/lib/teems/commands/base.rb +31 -7
  12. data/lib/teems/commands/cal.rb +109 -28
  13. data/lib/teems/commands/cal_ics.rb +47 -0
  14. data/lib/teems/commands/confirmation.rb +19 -0
  15. data/lib/teems/commands/help.rb +4 -0
  16. data/lib/teems/commands/mail.rb +207 -0
  17. data/lib/teems/commands/mail_decrypting.rb +46 -0
  18. data/lib/teems/commands/mail_help.rb +85 -0
  19. data/lib/teems/commands/mail_rules.rb +201 -0
  20. data/lib/teems/commands/mail_sync.rb +92 -0
  21. data/lib/teems/commands/mail_thread.rb +87 -0
  22. data/lib/teems/commands/mail_triage.rb +241 -0
  23. data/lib/teems/commands/meet.rb +159 -0
  24. data/lib/teems/commands/sync.rb +15 -8
  25. data/lib/teems/commands/sync_folders.rb +118 -0
  26. data/lib/teems/formatters/format_utils.rb +23 -2
  27. data/lib/teems/formatters/html_entities.rb +49 -0
  28. data/lib/teems/formatters/html_text.rb +181 -0
  29. data/lib/teems/formatters/ics_format.rb +112 -0
  30. data/lib/teems/formatters/link_cleaner.rb +91 -0
  31. data/lib/teems/formatters/link_collector.rb +70 -0
  32. data/lib/teems/formatters/mail_formatter.rb +58 -0
  33. data/lib/teems/formatters/mail_markdown.rb +70 -0
  34. data/lib/teems/formatters/mail_rule_formatter.rb +45 -0
  35. data/lib/teems/formatters/mail_rule_text.rb +67 -0
  36. data/lib/teems/formatters/output.rb +45 -8
  37. data/lib/teems/models/chat.rb +19 -5
  38. data/lib/teems/models/event.rb +1 -23
  39. data/lib/teems/models/event_display.rb +40 -0
  40. data/lib/teems/models/ics_event.rb +60 -0
  41. data/lib/teems/models/mail_conversation.rb +62 -0
  42. data/lib/teems/models/mail_encryption.rb +41 -0
  43. data/lib/teems/models/mail_folder.rb +44 -0
  44. data/lib/teems/models/mail_message.rb +152 -0
  45. data/lib/teems/models/mail_rule.rb +38 -0
  46. data/lib/teems/runner.rb +6 -0
  47. data/lib/teems/services/api_client.rb +23 -11
  48. data/lib/teems/services/browser.rb +17 -0
  49. data/lib/teems/services/clipboard.rb +17 -0
  50. data/lib/teems/services/mail_folders.rb +76 -0
  51. data/lib/teems/services/mail_id_cache.rb +51 -0
  52. data/lib/teems/services/mail_legacy_files.rb +96 -0
  53. data/lib/teems/services/mail_sync_attachments.rb +103 -0
  54. data/lib/teems/services/mail_sync_engine.rb +98 -0
  55. data/lib/teems/services/mail_sync_folder.rb +142 -0
  56. data/lib/teems/services/mail_sync_store.rb +116 -0
  57. data/lib/teems/services/smime_content.rb +36 -0
  58. data/lib/teems/services/smime_decryptor.rb +36 -0
  59. data/lib/teems/services/sync_dir_naming.rb +16 -10
  60. data/lib/teems/services/sync_dir_ownership.rb +111 -0
  61. data/lib/teems/services/sync_dir_planning.rb +130 -0
  62. data/lib/teems/services/sync_store.rb +20 -43
  63. data/lib/teems/services/token_extractor.rb +4 -0
  64. data/lib/teems/support/control_chars.rb +19 -0
  65. data/lib/teems/support/mime_header.rb +61 -0
  66. data/lib/teems/support/mime_part.rb +97 -0
  67. data/lib/teems/support/signed_data.rb +23 -0
  68. data/lib/teems/support/time_parsing.rb +9 -1
  69. data/lib/teems/version.rb +1 -1
  70. data/lib/teems.rb +63 -2
  71. metadata +48 -1
@@ -0,0 +1,159 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'securerandom'
4
+
5
+ module Teems
6
+ module Commands
7
+ MEET_HELP = <<~HELP
8
+ teems meet - Start a Teams meeting and print its join link
9
+
10
+ USAGE:
11
+ teems meet [minutes] [options]
12
+
13
+ Puts a Teams meeting on your calendar, starting now for 30 minutes unless told
14
+ otherwise, prints its join link (alone on stdout, so it pipes) and opens it here
15
+ to join. With nobody invited, nothing is sent.
16
+
17
+ Joining here matters: people from organizations yours does not trust wait in the
18
+ lobby until a member of yours joins, even with "Everyone" set to bypass the lobby
19
+ in Meeting options, and even when invited. Once you are in you can admit them;
20
+ the meeting usually carries on after you leave.
21
+
22
+ OPTIONS:
23
+ --at TIME Start: now (default), HH:MM, "today HH:MM",
24
+ "tomorrow HH:MM", or "YYYY-MM-DD HH:MM"
25
+ --title TEXT Event title (default: Meet now)
26
+ --with EMAILS Comma-separated people to invite. Exchange emails
27
+ each an invitation, so this asks first
28
+ -y, --yes Invite without asking (needed without a terminal)
29
+ --copy Also copy the link to the clipboard
30
+ --no-open Do not open the link here (just print it)
31
+ -q, --quiet Just the link
32
+ --json Event ID, title, start, end and link as JSON
33
+
34
+ EXAMPLES:
35
+ teems meet # Now, 30 minutes, and join here
36
+ teems meet 60 --title "Pairing" # Now, an hour
37
+ teems meet --copy # Also copy the link
38
+ teems meet --no-open | pbcopy # Just the link, for someone else
39
+ teems meet --at 14:30 --with alice@example.com
40
+ HELP
41
+
42
+ # `teems meet`: a calendar event with a Teams meeting, starting now by default, and
43
+ # its join link. Calendars.ReadWrite is enough for that, where creating a bare online
44
+ # meeting would need OnlineMeetings.ReadWrite.
45
+ class Meet < Base
46
+ include Support::Timezone
47
+ include Support::TimeParsing
48
+ include Confirmation
49
+
50
+ DEFAULT_MINUTES = 30
51
+ MAX_MINUTES = 24 * 60
52
+ VALUE_OPTIONS = { '--at' => :at, '--title' => :title, '--with' => :with }.freeze
53
+ # --open is the default; it is still accepted, so scripts that pass it keep working
54
+ FLAGS = { '--copy' => :copy, '--open' => :open, '--no-open' => :no_open, '--yes' => :yes, '-y' => :yes }.freeze
55
+
56
+ def initialize(args, runner:)
57
+ @options = {}
58
+ super
59
+ end
60
+
61
+ def execute
62
+ validation = validate_options
63
+ return validation if validation
64
+
65
+ auth = require_auth
66
+ return auth if auth
67
+
68
+ with_token_refresh { create_meeting }
69
+ rescue ApiError => e
70
+ api_error_result('Could not create the meeting', e)
71
+ rescue Teems::Error => e
72
+ error(e.message)
73
+ end
74
+
75
+ protected
76
+
77
+ def handle_option(arg, pending)
78
+ return @options[FLAGS[arg]] = true if FLAGS.key?(arg)
79
+ return super unless VALUE_OPTIONS.key?(arg)
80
+
81
+ @options[VALUE_OPTIONS[arg]] = pending.shift
82
+ end
83
+
84
+ def help_text = MEET_HELP
85
+
86
+ private
87
+
88
+ def create_meeting
89
+ start = start_time
90
+ finish = start + (minutes * 60)
91
+ return error('Cancelled; no meeting created') unless invite_confirmed?
92
+
93
+ event = runner.calendar_api.create_event(event_body(start, finish))
94
+ link = event.online_meeting_url or raise Error, 'Graph created the event but gave no Teams link'
95
+ report(event, link, start..finish)
96
+ end
97
+
98
+ def minutes
99
+ given = positional_args.first
100
+ return DEFAULT_MINUTES unless given
101
+
102
+ count = Integer(given, 10, exception: false)
103
+ raise Error, "Minutes must be a whole number from 1 to #{MAX_MINUTES}" unless count&.between?(1, MAX_MINUTES)
104
+
105
+ count
106
+ end
107
+
108
+ def start_time
109
+ given = @options[:at] || 'now'
110
+ parse_time_input(given) or raise Error, "Unknown time: #{given} (use now, HH:MM, \"tomorrow HH:MM\", ...)"
111
+ end
112
+
113
+ def invitees = @options[:with].to_s.split(',').map(&:strip).reject(&:empty?)
114
+
115
+ # Inviting people sends them mail, so it needs a yes (at the prompt, or --yes)
116
+ def invite_confirmed?
117
+ people = invitees
118
+ return true if people.empty? || @options[:yes]
119
+
120
+ confirm?("Invite #{people.join(', ')} (Exchange emails each an invitation)",
121
+ refusal: 'Inviting people sends them email; pass --yes to go ahead')
122
+ end
123
+
124
+ def event_body(start, finish)
125
+ zone = detect_timezone
126
+ { subject: @options[:title] || 'Meet now', isOnlineMeeting: true, onlineMeetingProvider: 'teamsForBusiness',
127
+ start: { dateTime: graph_time(start), timeZone: zone }, end: { dateTime: graph_time(finish), timeZone: zone },
128
+ attendees: invitees.map { |address| { emailAddress: { address: address }, type: 'required' } },
129
+ transactionId: SecureRandom.uuid }
130
+ end
131
+
132
+ # On stderr, beside the link on stdout; -q leaves just the link
133
+ def note(message) = @options[:quiet] || output.note(message)
134
+
135
+ def graph_time(time) = time.strftime('%Y-%m-%dT%H:%M:%S')
136
+
137
+ def report(event, link, times)
138
+ return json_report(event, link, times) if @options[:json]
139
+
140
+ output.puts(link)
141
+ note("#{event.subject}, #{times.begin.strftime('%-I:%M')}-#{times.end.strftime('%-I:%M %p')}")
142
+ hand_off(link)
143
+ 0
144
+ end
145
+
146
+ def json_report(event, link, times)
147
+ hand_off(link)
148
+ output_json(id: event.id, title: event.subject, start: times.begin.iso8601, end: times.end.iso8601,
149
+ join_url: link)
150
+ 0
151
+ end
152
+
153
+ def hand_off(link)
154
+ runner.clipboard.copy(link) && note('Copied the link') if @options[:copy]
155
+ runner.browser.open(link) && note('Opened it to join') unless @options[:no_open]
156
+ end
157
+ end
158
+ end
159
+ end
@@ -13,7 +13,9 @@ module Teems
13
13
  --chat CHAT_ID Sync only this chat
14
14
  --images Also download inline (pasted) images into each chat's images/ dir
15
15
  --auth Authenticate via Safari before syncing
16
- --dry-run Show what would be synced without writing files
16
+ --dry-run Show what would be synced and every folder change, without writing anything
17
+ --migrate-dirs Also rename existing chat folders to "<name> (<chat ID>)" and write
18
+ a map of old to new paths (sync/dir-maps/)
17
19
  -v, --verbose Show debug output
18
20
  -q, --quiet Suppress output
19
21
 
@@ -23,11 +25,13 @@ module Teems
23
25
  teems sync --chat 19:abc@thread.v2 # Sync a single chat
24
26
  teems sync --images # Include pasted screenshots
25
27
  teems sync --dry-run # Preview what would be synced
28
+ teems sync --dry-run --migrate-dirs # Preview renaming every folder
26
29
 
27
30
  OUTPUT:
28
31
  Files are stored in ~/.local/share/teems/sync/chats/
29
32
  Each chat gets: messages.md, messages.json, chat_metadata.json
30
33
  (plus images/ with --images; messages.md links saved images, otherwise [image: ...])
34
+ New chats get a "<name> (<chat ID>)" folder; existing folders keep their names.
31
35
  HELP
32
36
 
33
37
  # Handles syncing individual chats: fetch, merge, retry on 404
@@ -62,10 +66,7 @@ module Teems
62
66
  end
63
67
 
64
68
  def sync_single_chat(chat)
65
- @sync_store.ensure_dir_name(
66
- @state, chat_info: { chat_id: chat.id, display_name: chat.display_name,
67
- chat_type: chat.chat_type }
68
- )
69
+ report_dir_move(@sync_store.ensure_chat_dir(@state, chat_info: dir_info(chat)))
69
70
  new_messages = fetch_new_messages(chat)
70
71
  debug(" Fetched #{new_messages.length} new message(s)")
71
72
  process_fetched_messages(chat, new_messages)
@@ -82,7 +83,7 @@ module Teems
82
83
 
83
84
  def fetch_new_messages(chat)
84
85
  chat_id = chat.id
85
- start_time = @sync_store.last_synced_time(@state, chat_id) || since_time
86
+ start_time = @sync_store.last_synced_time(@state, chat_id) || resync_start(@state, chat_id)
86
87
  with_token_refresh { sync_engine.fetch_all_messages(chat_id, start_time) }
87
88
  end
88
89
 
@@ -198,12 +199,13 @@ module Teems
198
199
  def show_dry_run(chats)
199
200
  syncable = chats.reject { |chat| skip_reason(chat['id']) }
200
201
  display_dry_run_list(chats.length - syncable.length, syncable)
202
+ show_dir_plan(syncable)
203
+ 0
201
204
  end
202
205
 
203
206
  def display_dry_run_list(skipped_count, syncable)
204
207
  display_dry_run_header(skipped_count, syncable.length)
205
208
  syncable.each { |chat| format_dry_run_chat(chat) }
206
- 0
207
209
  end
208
210
 
209
211
  def display_dry_run_header(skipped, syncable_count)
@@ -228,6 +230,7 @@ module Teems
228
230
  success('Sync complete!')
229
231
  show_summary_stats
230
232
  info(" Output: #{@sync_store.sync_dir}")
233
+ show_folder_summary
231
234
  end
232
235
 
233
236
  def show_summary_stats
@@ -315,6 +318,7 @@ module Teems
315
318
  include SyncImageHandler
316
319
  include SyncDisplay
317
320
  include SyncAuth
321
+ include SyncFolders
318
322
 
319
323
  DEFAULT_SINCE_DAYS = 180
320
324
  SKIP_PREFIXES = %w[48:].freeze
@@ -345,6 +349,7 @@ module Teems
345
349
  '--chat' => ->(opts, args) { opts[:chat_id] = args.shift },
346
350
  '--dry-run' => ->(opts, _args) { opts[:dry_run] = true },
347
351
  '--images' => ->(opts, _args) { opts[:images] = true },
352
+ '--migrate-dirs' => ->(opts, _args) { opts[:migrate_dirs] = true },
348
353
  '--auth' => ->(opts, _args) { opts[:auth] = true }
349
354
  }.freeze
350
355
 
@@ -369,6 +374,7 @@ module Teems
369
374
  end
370
375
 
371
376
  def sync_all_chats(chats)
377
+ detach_shared_dirs
372
378
  chats.each_with_index { |chat_data, index| sync_or_skip_chat(chat_data, "[#{index + 1}/#{chats.length}]") }
373
379
  save_state_safely
374
380
  show_summary
@@ -404,7 +410,8 @@ module Teems
404
410
  error("Warning: Failed to save sync state: #{e.message}")
405
411
  end
406
412
 
407
- def since_time = Time.now - ((@options[:since_days] || DEFAULT_SINCE_DAYS) * 86_400)
413
+ def since_days = @options[:since_days] || DEFAULT_SINCE_DAYS
414
+ def since_time = Time.now - (since_days * 86_400)
408
415
 
409
416
  def setup_api_logging
410
417
  out = output
@@ -0,0 +1,118 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Teems
4
+ module Commands
5
+ # Chat folder changes made by sync: detaching shared folders, moving folders, listing old
6
+ # shared folders still on disk, and previewing all of it for --dry-run
7
+ module SyncFolders
8
+ # How each kind of folder change is described; :split is described by split_note, and :new
9
+ # and :kept aren't reported
10
+ MOVE_NOTES = {
11
+ move: '',
12
+ retarget: ' (the old folder is already gone)',
13
+ repoint: ' (the new folder already exists: switching to it, the old folder is left in place)',
14
+ release: " (the old folder also holds another chat's messages and is left in place)"
15
+ }.freeze
16
+
17
+ private
18
+
19
+ def dir_info(chat) = { **chat.sync_identity, migrate: @options[:migrate_dirs] }
20
+
21
+ # sync --chat X only detaches X, so the other chats keep their cursors until they sync
22
+ def detach_shared_dirs
23
+ detached = @sync_store.detach_shared_dirs(@state, only: @options[:chat_id])
24
+ @detached_dirs = detached.keys
25
+ warn_shared_dirs(detached)
26
+ end
27
+
28
+ def warn_shared_dirs(shared)
29
+ return if shared.empty?
30
+
31
+ warn([shared_dirs_intro(shared), *dir_lines(shared), shared_dirs_plan].join("\n"))
32
+ end
33
+
34
+ def shared_dirs_intro(shared)
35
+ "#{shared.length} folder(s) in #{File.join(@sync_store.sync_dir, 'chats')} mix messages from " \
36
+ 'several chats (an older teems gave those chats the same folder name):'
37
+ end
38
+
39
+ def shared_dirs_plan
40
+ 'These folders are left as they are. Each affected chat now syncs to its own folder, re-fetching ' \
41
+ 'history back to the oldest message in the old folder. Older history may still exist only in the ' \
42
+ "old folders, so keep them until you've checked the new ones."
43
+ end
44
+
45
+ def dir_lines(dirs) = dirs.map { |path, ids| " #{path} (#{ids.length} chat#{'s' unless ids.one?})" }
46
+
47
+ def report_dir_move(move)
48
+ line = dir_move_line(move, @state)
49
+ return unless line
50
+
51
+ (@dir_moves ||= []) << move
52
+ move.kind == :repoint ? warn(" Folder: #{line}") : info(" Folder: #{line}")
53
+ end
54
+
55
+ def dir_move_line(move, state)
56
+ note = move_note(move.to_h, state)
57
+ "#{move}#{note}" if note
58
+ end
59
+
60
+ def move_note(move_attrs, state)
61
+ kind, chat_id = move_attrs.values_at(:kind, :chat_id)
62
+ return MOVE_NOTES[kind] unless kind == :split
63
+
64
+ start = resync_start(state, chat_id).strftime('%Y-%m-%d')
65
+ " (split from a shared folder, which is left in place; re-fetching from #{start})"
66
+ end
67
+
68
+ # A detached chat re-syncs from the oldest message in its old folder, or --since if earlier
69
+ def resync_start(state, chat_id) = [@sync_store.resync_from(state, chat_id), since_time].compact.min
70
+
71
+ def show_folder_summary
72
+ write_dir_map
73
+ show_legacy_dirs(@detached_dirs)
74
+ end
75
+
76
+ def write_dir_map
77
+ moves = @dir_moves
78
+ return unless @options[:migrate_dirs] && moves
79
+
80
+ info(" Folder map: #{@sync_store.write_dir_map(moves)}")
81
+ end
82
+
83
+ def show_legacy_dirs(exclude)
84
+ legacy = @sync_store.legacy_shared_dirs(@state).except(*exclude)
85
+ return if legacy.empty?
86
+
87
+ info("Old shared folders still on disk (they may hold history the new folders don't):\n" \
88
+ "#{dir_lines(legacy).join("\n")}")
89
+ end
90
+
91
+ # Plans on a copy of the state, so nothing changes on disk or in sync_state.json
92
+ def show_dir_plan(syncable)
93
+ plan = JSON.parse(JSON.generate(@state))
94
+ detached = @sync_store.detach_shared_dirs(plan, only: @options[:chat_id])
95
+ lines = syncable.filter_map do |chat_data|
96
+ dir_move_line(@sync_store.plan_chat_dir(plan, dir_info(Models::Chat.from_api(chat_data))), plan)
97
+ end
98
+ show_planned_moves(lines)
99
+ show_planned_detaches(detached)
100
+ show_legacy_dirs([])
101
+ end
102
+
103
+ def show_planned_moves(lines)
104
+ puts
105
+ return info('Folder changes: none') if lines.empty?
106
+
107
+ info("Folder changes (#{lines.length}):\n#{lines.map { |line| " #{line}" }.join("\n")}")
108
+ end
109
+
110
+ def show_planned_detaches(detached)
111
+ return if detached.empty?
112
+
113
+ info("Shared folders that would be detached (#{detached.length}):\n" \
114
+ "#{dir_lines(detached).join("\n")}\n#{shared_dirs_plan}")
115
+ end
116
+ end
117
+ end
118
+ end
@@ -4,6 +4,13 @@ module Teems
4
4
  module Formatters
5
5
  # Shared stateless formatting utilities used across commands and formatters
6
6
  module FormatUtils
7
+ # Longest file name most filesystems allow, in bytes
8
+ NAME_MAX = 255
9
+ # Names File.basename can return that are not a file in the directory
10
+ UNSAFE_NAMES = ['', '.', '..', '/'].freeze
11
+ # Longer "extensions" are just dots in a long name, and are cut with the rest of it
12
+ MAX_EXTENSION = 16
13
+
7
14
  module_function
8
15
 
9
16
  def truncate(text, max = 120)
@@ -17,9 +24,23 @@ module Teems
17
24
  end
18
25
  end
19
26
 
27
+ # A name safe to join onto a directory: no path or control characters, not "." or "..",
28
+ # and short enough for the filesystem (keeping the extension when it has to be cut)
20
29
  def safe_filename(name)
21
- base = File.basename(name)
22
- base.empty? ? 'file' : base
30
+ base = File.basename(name.to_s.scrub('').gsub(/[[:cntrl:]]/, ''))
31
+ UNSAFE_NAMES.include?(base) ? 'file' : fit_filename(base)
32
+ end
33
+
34
+ def fit_filename(base)
35
+ return base if base.bytesize <= NAME_MAX
36
+
37
+ ext = kept_extension(base)
38
+ "#{base.delete_suffix(ext).byteslice(0, NAME_MAX - ext.bytesize).scrub('')}#{ext}"
39
+ end
40
+
41
+ def kept_extension(base)
42
+ ext = File.extname(base)
43
+ ext.bytesize > MAX_EXTENSION ? '' : ext
23
44
  end
24
45
 
25
46
  def attachment_name(att)
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Teems
4
+ module Formatters
5
+ # Decodes the HTML character references found in mail bodies
6
+ module HtmlEntities
7
+ ENTITY = /&(?:#(\d{1,7})|#[xX](\h{1,6})|([a-zA-Z][a-zA-Z0-9]{1,31}));?/
8
+ NAMED = {
9
+ 'amp' => '&', 'lt' => '<', 'gt' => '>', 'quot' => '"', 'apos' => "'", 'nbsp' => "\u00a0",
10
+ 'ensp' => "\u2002", 'emsp' => "\u2003", 'thinsp' => "\u2009", 'zwnj' => "\u200c", 'zwj' => "\u200d",
11
+ 'shy' => "\u00ad", 'ndash' => "\u2013", 'mdash' => "\u2014", 'lsquo' => "\u2018", 'rsquo' => "\u2019",
12
+ 'sbquo' => "\u201a", 'ldquo' => "\u201c", 'rdquo' => "\u201d", 'bdquo' => "\u201e",
13
+ 'hellip' => "\u2026", 'bull' => "\u2022", 'middot' => "\u00b7", 'copy' => "\u00a9", 'reg' => "\u00ae",
14
+ 'trade' => "\u2122", 'deg' => "\u00b0", 'times' => "\u00d7", 'divide' => "\u00f7",
15
+ 'laquo' => "\u00ab", 'raquo' => "\u00bb", 'euro' => "\u20ac", 'pound' => "\u00a3", 'cent' => "\u00a2",
16
+ 'yen' => "\u00a5", 'sect' => "\u00a7", 'para' => "\u00b6", 'plusmn' => "\u00b1", 'micro' => "\u00b5",
17
+ 'frac12' => "\u00bd", 'frac14' => "\u00bc", 'frac34' => "\u00be", 'rarr' => "\u2192",
18
+ 'larr' => "\u2190", 'uarr' => "\u2191", 'darr' => "\u2193", 'iexcl' => "\u00a1", 'iquest' => "\u00bf"
19
+ }.freeze
20
+
21
+ module_function
22
+
23
+ def decode(text)
24
+ return text unless text.include?('&')
25
+
26
+ text.gsub(ENTITY) do
27
+ dec, hex, name = Regexp.last_match.captures
28
+ name ? NAMED.fetch(name, Regexp.last_match(0)) : codepoint((dec || hex.to_i(16)).to_i)
29
+ end
30
+ end
31
+
32
+ # Like browsers, 0x80-0x9F mean their Windows-1252 characters (&#146; is a right quote).
33
+ # Other control characters decode to nothing (see Support::ControlChars).
34
+ def codepoint(number)
35
+ return windows_char(number) if number.between?(0x80, 0x9F)
36
+ return '' if number.between?(0xD800, 0xDFFF) || number > 0x10FFFF
37
+
38
+ Support::ControlChars.strip([number].pack('U'))
39
+ end
40
+
41
+ # '' for the five bytes Windows-1252 leaves undefined
42
+ def windows_char(number)
43
+ number.chr.force_encoding(Encoding::Windows_1252).encode(Encoding::UTF_8)
44
+ rescue EncodingError
45
+ ''
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,181 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Teems
4
+ module Formatters
5
+ # Tag groups used by HtmlText
6
+ module HtmlTags
7
+ # Removed with their contents before conversion
8
+ DROPPED = %r{<(head|style|script|title|xml)\b[^>]*>.*?</\1\s*>}im
9
+ COMMENTS = /<!--.*?-->/m
10
+ TOKENS = /<[^>]*>|<|[^<]+/m
11
+ BLOCKS = %w[address article blockquote center dd div dl dt footer form h1 h2 h3 h4 h5 h6 header
12
+ li ol section table tbody thead tfoot tr ul].freeze
13
+
14
+ module_function
15
+
16
+ # [name, closing?] for a tag token, or nil for anything that is not a tag
17
+ def parse(token)
18
+ match = token.match(%r{\A<\s*(/?)\s*([a-zA-Z][a-zA-Z0-9:-]*)})
19
+ match && [match[2].downcase, !match[1].empty?]
20
+ end
21
+
22
+ def attribute(token, name)
23
+ match = token.match(/\s#{name}\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i)
24
+ match && HtmlEntities.decode(match.captures.compact.first)
25
+ end
26
+ end
27
+
28
+ # Accumulates converted text, tracking line starts so block boundaries never stack up
29
+ # more newlines than asked for and indentation from source HTML never leaks in
30
+ class TextBuffer
31
+ ZERO_WIDTH = /[\u200b\u200c\u200d\u2060\ufeff\u034f\u00ad]/
32
+
33
+ def initialize
34
+ @out = +''
35
+ end
36
+
37
+ def <<(text)
38
+ @out << text
39
+ self
40
+ end
41
+
42
+ # Inline text with source whitespace collapsed; << keeps it as-is
43
+ def add_text(text)
44
+ collapsed = text.gsub(/[ \t\r\n\f]+/, ' ')
45
+ self << (line_start? ? collapsed.lstrip : collapsed)
46
+ end
47
+
48
+ def line_break
49
+ trim_line_end
50
+ self << "\n"
51
+ end
52
+
53
+ # At least count newlines at the end, without adding to ones already there
54
+ def ensure_newlines(count)
55
+ return self if @out.empty?
56
+
57
+ trim_line_end
58
+ missing = count - trailing_newlines(count)
59
+ missing.positive? ? self << ("\n" * missing) : self
60
+ end
61
+
62
+ def space
63
+ @out.end_with?(' ', "\n") || @out.empty? ? self : self << ' '
64
+ end
65
+
66
+ def line_start? = @out.empty? || @out.end_with?("\n")
67
+
68
+ def to_s
69
+ text = @out.tr("\u00a0", ' ').gsub(ZERO_WIDTH, '').gsub(/[ \t]+$/, '')
70
+ text.gsub(/\n{3,}/, "\n\n").sub(/\A\n+/, '').rstrip
71
+ end
72
+
73
+ private
74
+
75
+ # Both helpers look only at the end of the buffer. An end-anchored regex such as
76
+ # /[ \t]+\z/ is scanned from the start, which made long messages quadratic.
77
+ def trim_line_end
78
+ @out.chop! while @out.end_with?(' ', "\t")
79
+ end
80
+
81
+ # Newlines already at the end, counting no further than limit
82
+ def trailing_newlines(limit) = (1..limit).count { |run| @out.end_with?("\n" * run) }
83
+ end
84
+
85
+ # Converts an HTML mail body to plain text. Each anchor's text stays inline followed by
86
+ # a [N] marker, and the URLs are listed once each in a numbered "Links:" footer
87
+ # (SafeLinks unwrapped, click tracking dropped, mailto:/tel:/cid: skipped).
88
+ class HtmlText
89
+ # Converted text and its "Links:" footer ('' when there are no links)
90
+ Result = Data.define(:text, :footer) do
91
+ # Text with the footer appended after a blank line, or the text alone when there are no links
92
+ def to_s = footer.empty? ? text : "#{text}\n\n#{footer}"
93
+ end
94
+
95
+ OPEN_TAGS = {
96
+ 'a' => :open_anchor, 'br' => :line_break, 'p' => :open_paragraph, 'li' => :open_list_item,
97
+ 'img' => :image, 'hr' => :rule, 'pre' => :open_pre, 'td' => :cell, 'th' => :cell
98
+ }.freeze
99
+ CLOSE_TAGS = { 'a' => :close_anchor, 'br' => :line_break, 'p' => :close_paragraph, 'pre' => :close_pre }.freeze
100
+
101
+ def self.convert(html) = new.convert(html.to_s)
102
+
103
+ def initialize
104
+ @buffer = TextBuffer.new
105
+ @links = LinkCollector.new
106
+ # Preformatted text, and whether the open <p> is one of Outlook's one-line paragraphs
107
+ @state = { pre: false, outlook_paragraph: false }
108
+ end
109
+
110
+ def convert(html)
111
+ tokens(html).each { |token| token.start_with?('<') ? tag(token) : text(token) }
112
+ close_anchor
113
+ Result.new(text: @buffer.to_s, footer: @links.footer)
114
+ end
115
+
116
+ private
117
+
118
+ def tokens(html)
119
+ Support::ControlChars.strip(html.scrub('')).gsub(HtmlTags::COMMENTS, '').gsub(HtmlTags::DROPPED, '').scan(HtmlTags::TOKENS)
120
+ end
121
+
122
+ def tag(token)
123
+ name, closing = HtmlTags.parse(token)
124
+ return text(token) unless name
125
+
126
+ handler = (closing ? CLOSE_TAGS : OPEN_TAGS)[name]
127
+ return send(handler, token) if handler
128
+
129
+ @buffer.ensure_newlines(1) if HtmlTags::BLOCKS.include?(name)
130
+ end
131
+
132
+ def text(token)
133
+ chunk = HtmlEntities.decode(token)
134
+ @state[:pre] ? @buffer << chunk : @buffer.add_text(chunk)
135
+ @links.add_text(chunk)
136
+ end
137
+
138
+ def open_anchor(token)
139
+ href = HtmlTags.attribute(token, 'href')
140
+ href ? @links.start(href) : close_anchor
141
+ end
142
+
143
+ def close_anchor(_token = nil)
144
+ number = @links.finish
145
+ @buffer.space << "[#{number}]" if number
146
+ end
147
+
148
+ def line_break(_token) = @buffer.line_break
149
+
150
+ # Outlook writes every line as its own <p class=MsoNormal>, with &nbsp; paragraphs for
151
+ # blank lines, so those end with one newline; other paragraphs get a blank line.
152
+ def open_paragraph(token)
153
+ @state[:outlook_paragraph] = HtmlTags.attribute(token, 'class').to_s.match?(/\bMso/i)
154
+ @buffer.ensure_newlines(1)
155
+ end
156
+
157
+ def close_paragraph(_token) = @buffer.ensure_newlines(@state[:outlook_paragraph] ? 1 : 2)
158
+
159
+ def open_list_item(_token) = @buffer.ensure_newlines(1) << '- '
160
+
161
+ def image(token)
162
+ @links.add_alt(HtmlTags.attribute(token, 'alt').to_s) if @links.capturing?
163
+ end
164
+
165
+ def rule(_token) = @buffer.ensure_newlines(1) << "#{'_' * 32}\n"
166
+
167
+ def open_pre(_token) = pre(true)
168
+
169
+ def close_pre(_token) = pre(false)
170
+
171
+ def pre(enabled)
172
+ @buffer.ensure_newlines(1)
173
+ @state[:pre] = enabled
174
+ end
175
+
176
+ def cell(_token)
177
+ @buffer.space unless @buffer.line_start?
178
+ end
179
+ end
180
+ end
181
+ end