mailmate 1.4.0 → 1.5.0

data/lib/mailmate/index_reader.rb

@@ -35,23 +35,34 @@
 #                            or [] if the message has no flags / isn't indexed.
 #
 # IndexReader instances cache both files in memory and build a hash from
-# id → [[start,end], …] for O(1) lookup. Construction cost ≈ 5–20 ms for
-# 50–200k records; memory ≈ a few MB. For a CLI invocation that's fine; the
-# evaluator instantiates one lazily when first needed.
+# id → [packed_range, …] (start << 32 | end, one Integer per record) for
+# O(1) lookup. Construction cost ≈ 5–20 ms for 50–200k records (one bulk
+# unpack("V*") pass); memory ≈ a few MB. For a CLI invocation that's fine;
+# the evaluator instantiates one lazily when first needed.
 
 module Mailmate
   # @api public
   class IndexReader
     RECORD_SIZE = 12
 
+    # Re-stat the underlying files at most this often per reader (seconds).
+    # Short-lived CLI processes never hit the recheck; the persistent MCP
+    # server picks up MailMate's continuous index rewrites within this window
+    # instead of serving a snapshot from its first request forever.
+    FRESHNESS_INTERVAL = 1.0
+
     class << self
       # Per-process cache of readers keyed by [name, db_headers]. Including
       # db_headers means a Mailmate.config swap (e.g. a test pointing at a
       # different tmpdir) doesn't return stale readers built from the old
-      # path.
+      # path. Cached readers are re-validated against the on-disk files'
+      # mtime+size (throttled; see FRESHNESS_INTERVAL) so long-lived
+      # processes don't serve stale data after MailMate rewrites an index.
       def for(name)
         @cache ||= {}
-        @cache[cache_key(name)] ||= new(name)
+        key = cache_key(name)
+        @cache.delete(key) if @cache[key]&.stale?
+        @cache[key] ||= new(name)
       end
 
       # Invalidate cached readers. With no argument, drops the entire cache
@@ -78,13 +89,33 @@ module Mailmate
 
     def initialize(name)
       @name = name
-      base = "#{Mailmate.config.db_headers}/#{name}"
-      raise ArgumentError, "Index not found: #{name} (looked at #{base}.{cache,offsets})" \
-        unless File.exist?("#{base}.cache") && File.exist?("#{base}.offsets")
+      @base = "#{Mailmate.config.db_headers}/#{name}"
+      raise ArgumentError, "Index not found: #{name} (looked at #{@base}.{cache,offsets})" \
+        unless File.exist?("#{@base}.cache") && File.exist?("#{@base}.offsets")
+
+      @cache_bytes   = File.binread("#{@base}.cache")
+      @offsets_bytes = File.binread("#{@base}.offsets")
+      @cache_sig     = file_sig("#{@base}.cache")
+      @offsets_sig   = file_sig("#{@base}.offsets")
+      @checked_at    = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      # The id→ranges hash builds lazily (see index): ids_matching-only
+      # consumers (the inverted body search) never need it, and skipping it
+      # saves ~250 ms of construction on the big body indexes.
+      @index = nil
+    end
 
-      @cache_bytes   = File.binread("#{base}.cache")
-      @offsets_bytes = File.binread("#{base}.offsets")
-      build_index!
+    # True when the on-disk files no longer match what this reader was built
+    # from. Throttled to one stat-pair per FRESHNESS_INTERVAL; a vanished
+    # file (mid-swap while MailMate rewrites) counts as not-stale so we keep
+    # serving the last good snapshot rather than racing the writer.
+    def stale?
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      return false if now - @checked_at < FRESHNESS_INTERVAL
+      @checked_at = now
+      cache_sig   = file_sig("#{@base}.cache")
+      offsets_sig = file_sig("#{@base}.offsets")
+      return false if cache_sig.nil? || offsets_sig.nil?
+      cache_sig != @cache_sig || offsets_sig != @offsets_sig
     end
 
     # Returns the raw cached value for a given .eml body-part ID, or nil if
@@ -94,19 +125,19 @@ module Mailmate
     # body indexes (`#unquoted#lc`, `#quoted#lc`) last-alone is meaningless
     # — use values_for to read every segment.
     def value_for(eml_id)
-      pairs = @index[eml_id.to_i]
-      return nil if pairs.nil? || pairs.empty?
-      s, e = pairs[-1]
-      @cache_bytes[s...e]
+      packs = index[eml_id.to_i]
+      return nil if packs.nil? || packs.empty?
+      v = packs[-1]
+      @cache_bytes[(v >> 32)...(v & 0xFFFFFFFF)]
     end
 
     # Returns every recorded value for an id, in offsets-file order. Returns
     # [] if the id isn't in the index. Use this for body indexes
     # (#unquoted#lc, #quoted#lc), which store one record per text segment.
     def values_for(eml_id)
-      pairs = @index[eml_id.to_i]
-      return [] if pairs.nil?
-      pairs.map { |(s, e)| @cache_bytes[s...e] }
+      packs = index[eml_id.to_i]
+      return [] if packs.nil?
+      packs.map { |v| @cache_bytes[(v >> 32)...(v & 0xFFFFFFFF)] }
     end
 
     # `#flags.flag` semantics: the cache stores a space-separated list of IMAP
@@ -117,15 +148,45 @@ module Mailmate
       v.split(/\s+/).reject(&:empty?)
     end
 
+    # True when the index has at least one record for this id. Cheaper than
+    # values_for(id).empty? — no substring slicing.
+    def key?(eml_id)
+      index.key?(eml_id.to_i)
+    end
+
+    # Inverted substring search: returns a Hash whose keys are every id with
+    # at least one record containing `needle` (byte-wise; pass pre-downcased
+    # bytes when querying an #lc index). One memchr-fast String#index scan
+    # of the whole cache instead of one substring test per record — for a
+    # 77 MB body cache that's ~75 ms versus seconds of per-message lookups.
+    #
+    # A raw cache hit can span two adjacent records' ranges; interval
+    # stabbing keeps only hits that fall entirely inside a single record
+    # (per-segment semantics, matching MailMate's own body search). Records
+    # sharing a byte range (deduped values) all report their ids.
+    def ids_matching(needle)
+      needle = needle.b
+      found = {}
+      return found if needle.empty? || @cache_bytes.empty?
+      ensure_stab_table!
+      nlen = needle.bytesize
+      pos = 0
+      while (pos = @cache_bytes.index(needle, pos))
+        stab(pos, pos + nlen) { |id| found[id] = true }
+        pos += 1
+      end
+      found
+    end
+
     # Number of distinct ids in the index. For multi-record indexes this is
     # smaller than the on-disk record count (use record_count for that).
     def size
-      @index.size
+      index.size
     end
 
     # Total number of on-disk records (sum across all ids). Diagnostics.
     def record_count
-      @index.values.sum(&:size)
+      index.values.sum(&:size)
     end
 
     # Iterate every recorded eml-id. Yields just the id; callers that also
@@ -133,7 +194,7 @@ module Mailmate
     # modules don't have to reach into `@index` directly.
     def each_eml_id(&block)
       return enum_for(:each_eml_id) unless block
-      @index.each_key(&block)
+      index.each_key(&block)
     end
 
     # Iterate every (eml_id, raw_value) pair, once per on-disk record.
@@ -142,22 +203,92 @@ module Mailmate
     # should massage it themselves.
     def each_record
       return enum_for(:each_record) unless block_given?
-      @index.each do |eml_id, pairs|
-        pairs.each { |(s, e)| yield eml_id, @cache_bytes[s...e] }
+      index.each do |eml_id, packs|
+        packs.each { |v| yield eml_id, @cache_bytes[(v >> 32)...(v & 0xFFFFFFFF)] }
       end
     end
 
     private
 
-    def build_index!
-      @index = Hash.new { |h, k| h[k] = [] }
-      n = @offsets_bytes.bytesize / RECORD_SIZE
+    # Lazy tables for ids_matching's interval stabbing, built once per
+    # reader snapshot (a rebuilt reader starts fresh, so staleness handling
+    # comes for free). @stab_flat is the raw [id, start, end, …] triple
+    # stream; @stab_order holds record numbers sorted by start;
+    # @stab_prefix_max_end[i] is the max end among @stab_order[0..i], which
+    # lets stab() stop walking left as soon as no earlier-starting record
+    # could still reach the queried range. Integers only — no per-record
+    # object allocations.
+    def ensure_stab_table!
+      return if @stab_order
+      flat = @offsets_bytes.unpack("V*")
+      recs = (flat.size - (flat.size % 3)) / 3
+      # Pack (start, recnum) into one Integer and sort! with native compare —
+      # a sort_by block is ~3× slower at this record count. start sorts as
+      # the high bits; recnum keeps the low bits unique.
+      packed = Array.new(recs) { |k| (flat[k * 3 + 1] << 32) | k }
+      packed.sort!
+      order = packed
+      order.map! { |p| p & 0xFFFFFFFF }
+      prefix = Array.new(recs)
+      max_end = -1
+      order.each_with_index do |k, i|
+        e = flat[k * 3 + 2]
+        max_end = e if e > max_end
+        prefix[i] = max_end
+      end
+      @stab_flat = flat
+      @stab_order = order
+      @stab_prefix_max_end = prefix
+    end
+
+    # Yields the id of every record whose [start, end) range fully contains
+    # [lo, hi). Classic stabbing query over ranges sorted by start: binary
+    # search to the last range starting at or before lo, then walk left
+    # while the prefix-max end says a covering range is still possible —
+    # O(log n + overlap depth), and body-index ranges rarely overlap.
+    def stab(lo, hi)
+      order = @stab_order
+      flat = @stab_flat
+      i = order.bsearch_index { |k| flat[k * 3 + 1] > lo }
+      i = i.nil? ? order.size - 1 : i - 1
+      while i >= 0 && @stab_prefix_max_end[i] >= hi
+        k = order[i]
+        yield flat[k * 3] if flat[k * 3 + 1] <= lo && flat[k * 3 + 2] >= hi
+        i -= 1
+      end
+    end
+
+    def file_sig(path)
+      st = File.stat(path)
+      [st.mtime, st.size]
+    rescue SystemCallError
+      nil
+    end
+
+    # Decode the offsets file in one bulk unpack — one C call instead of one
+    # String#[] + unpack per record (2× faster on the 730k-record body
+    # indexes). Each (start, end) range is packed into a single Integer
+    # (start << 32 | end) so a record costs an immediate value, not a
+    # two-element Array; accessors decode with shift/mask. Caches are tens
+    # of MB, so both halves fit 32 bits with room to spare. unpack("V*")
+    # silently drops trailing bytes that don't fill a uint32; the % 3 guard
+    # drops a trailing partial record.
+    def build_index
+      h = {}
+      flat = @offsets_bytes.unpack("V*")
+      n = flat.size - (flat.size % 3)
       i = 0
       while i < n
-        rec = @offsets_bytes[i * RECORD_SIZE, RECORD_SIZE].unpack("V3")
-        @index[rec[0]] << [rec[1], rec[2]]
-        i += 1
+        (h[flat[i]] ||= []) << ((flat[i + 1] << 32) | flat[i + 2])
+        i += 3
       end
+      h
+    end
+
+    # Lazy id→ranges hash: built on first keyed access, skipped entirely by
+    # ids_matching-only consumers (the inverted body search).
+    def index
+      @index ||= build_index
     end
   end
 end

data/lib/mailmate/mcp.rb

@@ -7,6 +7,7 @@ require "mailmate"
 require "mailmate/cli/search"
 require "mailmate/cli/message"
 require "mailmate/cli/modify"
+require "mailmate/cli/verify"
 require "mailmate/cli/send"
 require "mailmate/cli/draft"
 require "mailmate/cli/open"
@@ -127,13 +128,14 @@ module Mailmate
       },
       {
         name: "message",
-        description: "Read one MailMate message. Accepts either local eml-id (digits) or RFC Message-ID (with or without angle brackets). Default output: headers block + plain-text body.",
+        description: "Read one MailMate message. Accepts either local eml-id (digits) or RFC Message-ID (with or without angle brackets). Default output: headers block (incl. any user tags) + plain-text body. For HTML-only mail (most newsletters), pass markdown:true to get clean readable markdown instead of raw HTML — strongly preferred for reading and far more token-efficient; it's a no-op on plain-text messages.",
         inputSchema: {
           type: "object",
           properties: {
             id: { type: "string", description: "eml-id (e.g. '183715') or RFC Message-ID (e.g. '<abc@example.com>')." },
             raw: { type: "boolean", description: "Return raw .eml bytes." },
             text_only: { type: "boolean", description: "Body only, no headers block." },
+            markdown: { type: "boolean", description: "Render an HTML-only body as clean markdown (drops <style>/<script>, strips newsletter spacer chars). No-op for plain-text messages." },
           },
           required: ["id"],
           additionalProperties: false,
@@ -155,6 +157,20 @@ module Mailmate
 
           Valid actions: read unread flag unflag tag untag clear-tags archive
           junk not-junk mute delete move
+
+          Verifying the action landed (it can't be read back from MailMate, so
+          this re-reads the target eml-id's #flags index — the only way to catch
+          a Message-ID that resolved to a different duplicate copy):
+            check:"inline"  confirm now, before returning (failed → isError).
+                            Costs a few seconds — MailMate flushes #flags ~5s
+                            after acting. Use for a high-stakes single mutation.
+            check:"defer"   don't wait; return a JSON check-ticket instead.
+                            Collect tickets across a batch, then pass them to the
+                            `verify` tool to confirm them ALL with one flush-wait
+                            (the efficient choice for bulk work — 50 modifies pay
+                            the ~5s latency once, not 50 times).
+            check:"none"    (default) fire-and-forget.
+          Location-changing chains (move/archive/delete) aren't flag-verifiable.
         DESC
         inputSchema: {
           type: "object",
@@ -166,13 +182,40 @@ module Mailmate
               description: "Flat list of action tokens; arg-taking actions consume the following item.",
             },
             dry_run: { type: "boolean", description: "Print plan, don't execute." },
-            verify: { type: "boolean", description: "Re-read flags after acting to confirm." },
+            verify: { type: "boolean", description: "Print the message's current flags after acting (raw probe)." },
+            check: { type: "string", enum: %w[none inline defer], description: "Effect-verification mode (default none). 'inline' confirms now (slow, returns isError on failure); 'defer' returns a JSON ticket for batched verification via the `verify` tool." },
             keep_window: { type: "boolean", description: "Skip the close-window keystroke at the end." },
           },
           required: %w[id actions],
           additionalProperties: false,
         },
       },
+      {
+        name: "verify",
+        description: <<~DESC.strip,
+          Batch-confirm `modify` check-tickets (from check:"defer") against the
+          #flags index in ONE flush-wait. Pass the tickets you collected from a
+          run of deferred modifies; this polls the index until every ticket's
+          expected flag/tag/read state holds (or check_timeout elapses) and
+          returns a JSON summary {checked, passed, failed, waited_seconds,
+          results:[{eml_id, ok, flags, unmet}]}. isError if any ticket failed —
+          a failure means that action didn't land on that eml-id (wrong
+          duplicate copy, or it never registered).
+        DESC
+        inputSchema: {
+          type: "object",
+          properties: {
+            tickets: {
+              type: "array",
+              items: { type: "object" },
+              description: "The check-ticket objects returned by modify calls made with check:\"defer\".",
+            },
+            check_timeout: { type: "number", description: "Max seconds to wait for #flags to reflect the batch (default 8)." },
+          },
+          required: ["tickets"],
+          additionalProperties: false,
+        },
+      },
       {
         name: "send",
         description: "Send mail via MailMate's `emate` (markdown body). Recipients and subject via fields; body is the markdown source. For replies, set `in_reply_to` and `references` so recipients' clients thread the message — without them a `Re:` subject alone is not enough. MailMate generates the outgoing Message-ID automatically.",
@@ -326,6 +369,7 @@ module Mailmate
       when "search"         then call_search(args)
       when "message"        then call_message(args)
       when "modify"         then call_modify(args)
+      when "verify"         then call_verify(args)
       when "send"           then call_send(args)
       when "draft"          then call_draft(args)
       when "open"           then call_open(args)
@@ -336,6 +380,12 @@ module Mailmate
       end
     rescue StandardError => e
       text_error("#{e.class}: #{e.message}\n#{e.backtrace.first(8).join("\n")}")
+    rescue SystemExit => e
+      # A CLI path that calls exit/abort (e.g. a missing optional gem) must
+      # not take down the persistent server — surface it as a tool error and
+      # keep the loop alive. SystemExit isn't a StandardError, so it needs
+      # its own clause.
+      text_error("Tool '#{name}' called exit(#{e.status}) — treated as failure, server still running.")
     end
 
     # ---- tool handlers ----------------------------------------------------
@@ -359,6 +409,7 @@ module Mailmate
       argv = [args["id"].to_s]
       argv << "--raw"       if args["raw"]
       argv << "--text-only" if args["text_only"]
+      argv << "--markdown"  if args["markdown"]
       run_cli(Mailmate::CLI::Message, argv)
     end
 
@@ -366,10 +417,24 @@ module Mailmate
       argv = [args["id"].to_s] + Array(args["actions"]).map(&:to_s)
       argv << "--dry-run"     if args["dry_run"]
       argv << "--verify"      if args["verify"]
+      case args["check"]
+      when "inline" then argv << "--check"
+      when "defer"  then argv << "--emit-check"
+      end
       argv << "--keep-window" if args["keep_window"]
       run_cli(Mailmate::CLI::Modify, argv)
     end
 
+    # Batch-verify deferred check-tickets. Tickets arrive as JSON objects;
+    # pipe them to mm-verify on stdin (the same array-or-NDJSON it reads
+    # from the CLI).
+    def call_verify(args)
+      argv = []
+      argv.push("--check-timeout", args["check_timeout"].to_s) if args["check_timeout"]
+      payload = JSON.generate(Array(args["tickets"]))
+      with_stdin(payload) { run_cli(Mailmate::CLI::Verify, argv) }
+    end
+
     def call_send(args)
       argv = compose_argv(args)
       argv << "--send-now" if args["send_now"]

data/lib/mailmate/part_lookup.rb

@@ -39,14 +39,23 @@ module Mailmate
 
       private
 
+      # The cached inversion is keyed by db_headers AND pinned to the exact
+      # IndexReader object it was built from — when IndexReader.for returns a
+      # rebuilt reader (file changed on disk; see IndexReader#stale?), the
+      # identity check fails and the inversion rebuilds with it. Keeps the
+      # persistent MCP server's part map in sync without explicit resets.
       def inversion
+        reader = Mailmate::IndexReader.for("#root-body-part")
         @inversions ||= {}
-        @inversions[Mailmate.config.db_headers] ||= build_inversion
+        entry = @inversions[Mailmate.config.db_headers]
+        return entry[:inv] if entry && entry[:reader].equal?(reader)
+        inv = build_inversion(reader)
+        @inversions[Mailmate.config.db_headers] = { reader: reader, inv: inv }
+        inv
       end
 
-      def build_inversion
+      def build_inversion(reader)
         inv = Hash.new { |h, k| h[k] = [] }
-        reader = Mailmate::IndexReader.for("#root-body-part")
         reader.each_eml_id do |part_id|
           root_str = reader.value_for(part_id)
           # Skip deleted parts: MailMate appends an empty trailing record to

data/lib/mailmate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Mailmate
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mailmate
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Brian Murphy-Dye
@@ -98,6 +98,7 @@ executables:
 - mm-mailboxes
 - mm-modify
 - mm-send
+- mm-verify
 - mmdiscover
 - mmmessage
 - mmopen
@@ -114,6 +115,7 @@ files:
 - exe/mm-mailboxes
 - exe/mm-modify
 - exe/mm-send
+- exe/mm-verify
 - exe/mmdiscover
 - exe/mmmessage
 - exe/mmopen
@@ -132,11 +134,13 @@ files:
 - lib/mailmate/cli/search.rb
 - lib/mailmate/cli/send.rb
 - lib/mailmate/cli/tags.rb
+- lib/mailmate/cli/verify.rb
 - lib/mailmate/config.rb
 - lib/mailmate/duplicate_scanner.rb
 - lib/mailmate/eml_lookup.rb
 - lib/mailmate/evaluator.rb
 - lib/mailmate/filter_classifier.rb
+- lib/mailmate/flag_check.rb
 - lib/mailmate/header_reader.rb
 - lib/mailmate/identity.rb
 - lib/mailmate/index_reader.rb