msg_extractor 0.1.0

data/lib/msg_extractor/mapi/decoders.rb

@@ -0,0 +1,59 @@
+module MsgExtractor
+  module Mapi
+    # Converts raw MAPI property bytes into Ruby values. All text becomes
+    # UTF-8 with invalid sequences replaced; binary stays ASCII-8BIT.
+    module Decoders
+      CODE_PAGES = {
+        437 => "IBM437", 850 => "IBM850", 932 => "Windows-31J", 936 => "GBK",
+        949 => "EUC-KR", 950 => "Big5",
+        1250 => "Windows-1250", 1251 => "Windows-1251", 1252 => "Windows-1252",
+        1253 => "Windows-1253", 1254 => "Windows-1254", 1255 => "Windows-1255",
+        1256 => "Windows-1256", 1257 => "Windows-1257", 1258 => "Windows-1258",
+        20127 => "US-ASCII", 28591 => "ISO-8859-1", 28592 => "ISO-8859-2",
+        28605 => "ISO8859-15", 65001 => "UTF-8"
+      }.freeze
+
+      # Seconds between 1601-01-01 (FILETIME epoch) and 1970-01-01 (Unix).
+      EPOCH_DELTA = 11_644_473_600
+
+      module_function
+
+      # For fixed-width types, +bytes+ may be the full 8-byte record value
+      # field; unpack reads only the leading bytes it needs.
+      def decode(type, bytes, codepage: 1252)
+        case type
+        when PT_UNICODE then utf16(bytes)
+        when PT_STRING8 then string8(bytes, codepage)
+        when PT_SYSTIME then filetime(bytes.unpack1("Q<"))
+        when PT_LONG    then bytes.unpack1("l<")
+        when PT_SHORT   then bytes.unpack1("s<")
+        when PT_I8      then bytes.unpack1("q<")
+        when PT_DOUBLE  then bytes.unpack1("E")
+        when PT_BOOLEAN then (bytes.unpack1("v") || 0) != 0
+        else bytes # PT_BINARY, PT_OBJECT, PT_CLSID and anything unknown: raw
+        end
+      end
+
+      def utf16(bytes)
+        bytes.dup.force_encoding(Encoding::UTF_16LE)
+             .encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+             .sub(/\0+\z/, "")
+      end
+
+      def string8(bytes, codepage)
+        encoding = CODE_PAGES.fetch(codepage, "Windows-1252")
+        bytes.dup.force_encoding(encoding)
+             .encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+             .sub(/\0+\z/, "")
+      end
+
+      # PR_HTML bytes -> UTF-8 string using PR_INTERNET_CPID.
+      def bytes_to_utf8(bytes, codepage) = string8(bytes, codepage)
+
+      def filetime(ticks)
+        return nil if ticks.nil? || ticks.zero?
+        Time.at(Rational(ticks, 10_000_000) - EPOCH_DELTA).utc
+      end
+    end
+  end
+end

data/lib/msg_extractor/mapi/named_property_map.rb

@@ -0,0 +1,74 @@
+module MsgExtractor
+  module Mapi
+    # Parses the __nameid_version1.0 storage: maps (property-set GUID, numeric
+    # LID or string name) pairs to the 0x8000+ property ids used in this file.
+    class NamedPropertyMap
+      PS_MAPI = "00020328-0000-0000-c000-000000000046"
+      PS_PUBLIC_STRINGS = "00020329-0000-0000-c000-000000000046"
+
+      def self.read(cfbf, storage)
+        nameid = storage.children["__NAMEID_VERSION1.0"]
+        return new({}) unless nameid&.storage?
+
+        guid_stream = read_child(cfbf, nameid, "__SUBSTG1.0_00020102")
+        entry_stream = read_child(cfbf, nameid, "__SUBSTG1.0_00030102")
+        string_stream = read_child(cfbf, nameid, "__SUBSTG1.0_00040102")
+        parse(guid_stream, entry_stream, string_stream)
+      end
+
+      def self.read_child(cfbf, storage, name)
+        entry = storage.children[name]
+        entry&.stream? ? cfbf.read_stream(entry) : "".b
+      end
+      private_class_method :read_child
+
+      def self.parse(guid_stream, entry_stream, string_stream)
+        map = {}
+        (entry_stream.bytesize / 8).times do |i|
+          name_id, info, prop_index = entry_stream.byteslice(i * 8, 8).unpack("Vvv")
+          guid_index = info >> 1
+          guid =
+            case guid_index
+            when 1 then PS_MAPI
+            when 2 then PS_PUBLIC_STRINGS
+            else
+              if guid_index >= 3
+                format_guid(guid_stream.byteslice((guid_index - 3) * 16, 16))
+              else
+                "00000000-0000-0000-0000-000000000000"
+              end
+            end
+          key =
+            if (info & 1) == 1
+              length = string_stream.byteslice(name_id, 4)&.unpack1("V")
+              next if length.nil?
+              raw = string_stream.byteslice(name_id + 4, length) || "".b
+              raw.force_encoding(Encoding::UTF_16LE)
+                 .encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+                 .downcase
+            else
+              name_id
+            end
+          map[[guid, key]] = 0x8000 + prop_index
+        end
+        new(map)
+      end
+
+      def self.format_guid(bytes)
+        return "00000000-0000-0000-0000-000000000000" if bytes.nil? || bytes.bytesize < 16
+        d1, d2, d3 = bytes.unpack("Vvv")
+        format("%08x-%04x-%04x-%s-%s", d1, d2, d3,
+               bytes.byteslice(8, 2).unpack1("H4"), bytes.byteslice(10, 6).unpack1("H12"))
+      end
+
+      def initialize(map)
+        @map = map
+      end
+
+      def resolve(guid, name_or_lid)
+        key = name_or_lid.is_a?(::String) ? name_or_lid.downcase : name_or_lid
+        @map[[guid.downcase, key]]
+      end
+    end
+  end
+end

data/lib/msg_extractor/mapi/property_store.rb

@@ -0,0 +1,106 @@
+module MsgExtractor
+  module Mapi
+    # Reads the MAPI properties of one storage: fixed-width values from the
+    # __properties_version1.0 stream, variable-width values from companion
+    # __substg1.0_XXXXYYYY streams. The properties stream header length
+    # depends on what kind of storage this is.
+    class PropertyStore
+      HEADER_SIZES = { root: 32, embedded: 24, attachment: 8, recipient: 8 }.freeze
+
+      # These counts come directly from the file and are untrusted; the model
+      # layer iterates actual storages rather than relying on them.
+      attr_reader :recipient_count, :attachment_count
+
+      SUBSTG_RE = /\A__SUBSTG1\.0_([0-9A-F]{4})([0-9A-F]{4})\z/
+
+      def initialize(cfbf, storage, kind)
+        @cfbf = cfbf
+        @storage = storage
+        @kind = kind
+        @records = {} # id => [type, 8-byte value field]
+        @streams = {} # id => [type, Cfbf::Entry]
+        parse
+      end
+
+      def key?(id) = @streams.key?(id) || @records.key?(id)
+
+      # Variable-width types (PT_UNICODE, PT_STRING8, PT_BINARY, PT_OBJECT,
+      # PT_CLSID) must be sourced from a substg stream. If one appears only in
+      # @records the 8-byte value field is not the actual payload, so return nil
+      # rather than decoding garbage.
+      VARIABLE_WIDTH_TYPES = [
+        MsgExtractor::Mapi::PT_UNICODE,
+        MsgExtractor::Mapi::PT_STRING8,
+        MsgExtractor::Mapi::PT_BINARY,
+        MsgExtractor::Mapi::PT_OBJECT,
+        MsgExtractor::Mapi::PT_CLSID
+      ].freeze
+
+      def [](id)
+        if (type_entry = @streams[id])
+          type, entry = type_entry
+          Decoders.decode(type, @cfbf.read_stream(entry), codepage: codepage)
+        elsif (type_value = @records[id])
+          type, value = type_value
+          return nil if VARIABLE_WIDTH_TYPES.include?(type)
+          Decoders.decode(type, value, codepage: codepage)
+        end
+      end
+
+      # Raw bytes without decoding (binary props, or the 8-byte record field).
+      def raw(id)
+        if (type_entry = @streams[id])
+          @cfbf.read_stream(type_entry[1])
+        elsif (type_value = @records[id])
+          type_value[1]
+        end
+      end
+
+      def type_of(id) = (@streams[id] || @records[id])&.first
+
+      def codepage
+        @codepage ||=
+          if (record = @records[PR_MESSAGE_CODEPAGE])
+            record[1].unpack1("l<")
+          else
+            1252
+          end
+      end
+
+      def internet_codepage
+        if (record = @records[PR_INTERNET_CPID])
+          record[1].unpack1("l<")
+        else
+          codepage
+        end
+      end
+
+      private
+
+      def parse
+        @storage.children.each_value do |entry|
+          next unless entry.stream?
+          match = SUBSTG_RE.match(entry.name.upcase) or next
+          type = match[2].to_i(16)
+          next unless (type & MV_FLAG).zero? # multi-valued props unsupported in v1
+          @streams[match[1].to_i(16)] = [type, entry]
+        end
+
+        properties_entry = @storage.children["__PROPERTIES_VERSION1.0"] or return
+        data = @cfbf.read_stream(properties_entry)
+        if %i[root embedded].include?(@kind) && data.bytesize >= 24
+          @recipient_count, @attachment_count = data.byteslice(16, 8).unpack("V2")
+        end
+        position = HEADER_SIZES.fetch(@kind)
+        while position + 16 <= data.bytesize
+          tag = data.byteslice(position, 4).unpack1("V")
+          value = data.byteslice(position + 8, 8)
+          id = tag >> 16
+          type = tag & 0xFFFF
+          @records[id] = [type, value] unless @streams.key?(id)
+          position += 16
+        end
+      end
+    end
+  end
+end

data/lib/msg_extractor/mapi/ptag.rb

@@ -0,0 +1,55 @@
+module MsgExtractor
+  module Mapi
+    # Property types ([MS-OXCDATA] 2.11.1)
+    PT_SHORT   = 0x0002
+    PT_LONG    = 0x0003
+    PT_DOUBLE  = 0x0005
+    PT_BOOLEAN = 0x000B
+    PT_OBJECT  = 0x000D
+    PT_I8      = 0x0014
+    PT_STRING8 = 0x001E
+    PT_UNICODE = 0x001F
+    PT_SYSTIME = 0x0040
+    PT_CLSID   = 0x0048
+    PT_BINARY  = 0x0102
+    MV_FLAG    = 0x1000
+
+    # Property IDs (MAPI PR_* convention, 16-bit id without the type word)
+    PR_MESSAGE_CLASS         = 0x001A
+    PR_SUBJECT               = 0x0037
+    PR_CLIENT_SUBMIT_TIME    = 0x0039
+    PR_TRANSPORT_HEADERS     = 0x007D
+    PR_RECIPIENT_TYPE        = 0x0C15
+    PR_SENDER_NAME           = 0x0C1A
+    PR_SENDER_EMAIL          = 0x0C1F
+    PR_DISPLAY_BCC           = 0x0E02
+    PR_DISPLAY_CC            = 0x0E03
+    PR_DISPLAY_TO            = 0x0E04
+    PR_MESSAGE_DELIVERY_TIME = 0x0E06
+    PR_BODY                  = 0x1000
+    PR_RTF_COMPRESSED        = 0x1009
+    PR_HTML                  = 0x1013
+    PR_DISPLAY_NAME          = 0x3001
+    PR_ADDRTYPE              = 0x3002
+    PR_EMAIL_ADDRESS         = 0x3003
+    PR_ATTACH_DATA           = 0x3701
+    PR_ATTACH_FILENAME       = 0x3704
+    PR_ATTACH_METHOD         = 0x3705
+    PR_ATTACH_LONG_FILENAME  = 0x3707
+    PR_ATTACH_MIME_TAG       = 0x370E
+    PR_ATTACH_CONTENT_ID     = 0x3712
+    PR_SMTP_ADDRESS          = 0x39FE
+    PR_POSTAL_ADDRESS        = 0x3A15
+    PR_GIVEN_NAME            = 0x3A06
+    PR_BUSINESS_PHONE        = 0x3A08
+    PR_HOME_PHONE            = 0x3A09
+    PR_SURNAME               = 0x3A11
+    PR_COMPANY_NAME          = 0x3A16
+    PR_JOB_TITLE             = 0x3A17
+    PR_MOBILE_PHONE          = 0x3A1C
+    PR_INTERNET_CPID         = 0x3FDE
+    PR_MESSAGE_CODEPAGE      = 0x3FFD
+    PR_SENDER_SMTP           = 0x5D01
+    PR_ATTACHMENT_HIDDEN     = 0x7FFE
+  end
+end

data/lib/msg_extractor/message.rb

@@ -0,0 +1,6 @@
+module MsgExtractor
+  # A regular email (IPM.Note and reports). All behavior lives on
+  # MessageObject; this class exists so dispatch results are introspectable.
+  class Message < MessageObject
+  end
+end

data/lib/msg_extractor/message_object.rb

@@ -0,0 +1,100 @@
+require "fileutils"
+
+module MsgExtractor
+  # Shared base for every MSG item type: property access, bodies, recipients,
+  # attachments. Works for the root message and for embedded messages.
+  class MessageObject
+    attr_reader :properties
+
+    attr_reader :named  # :nodoc: internal reuse by MsgExtractor.from_storage
+
+    def initialize(cfbf, storage: nil, named: nil, kind: :root, properties: nil)
+      @cfbf = cfbf
+      @storage = storage || cfbf.root
+      @kind = kind
+      @properties = properties || Mapi::PropertyStore.new(cfbf, @storage, kind)
+      @named = named || Mapi::NamedPropertyMap.read(cfbf, @storage)
+    end
+
+    def message_class = properties[Mapi::PR_MESSAGE_CLASS]
+    def subject = properties[Mapi::PR_SUBJECT]
+
+    def date
+      properties[Mapi::PR_CLIENT_SUBMIT_TIME] || properties[Mapi::PR_MESSAGE_DELIVERY_TIME]
+    end
+
+    def body
+      return @body if defined?(@body)
+      @body = properties[Mapi::PR_BODY] || (html_body && Util.html_to_text(html_body))
+    end
+
+    def rtf_body
+      return @rtf_body if defined?(@rtf_body)
+      raw = properties.raw(Mapi::PR_RTF_COMPRESSED)
+      @rtf_body = raw && Rtf::CompressedRtf.decompress(raw)
+    end
+
+    def html_body
+      return @html_body if defined?(@html_body)
+      @html_body =
+        if properties.type_of(Mapi::PR_HTML) == Mapi::PT_UNICODE
+          properties[Mapi::PR_HTML]
+        elsif (raw = properties.raw(Mapi::PR_HTML))
+          Mapi::Decoders.bytes_to_utf8(raw, properties.internet_codepage)
+        elsif rtf_body
+          Rtf::Decapsulator.html_from(rtf_body)
+        end
+    end
+
+    def headers
+      @headers ||= Headers.parse(properties[Mapi::PR_TRANSPORT_HEADERS])
+    end
+
+    def sender
+      return @sender if defined?(@sender)
+      name = properties[Mapi::PR_SENDER_NAME]
+      email = properties[Mapi::PR_SENDER_SMTP] || properties[Mapi::PR_SENDER_EMAIL]
+      email = nil unless email&.include?("@")
+      @sender = (name || email) && Recipient.new(name: name, email: email, type: nil)
+    end
+
+    def recipients
+      @recipients ||= child_storages("__RECIP_VERSION1.0_#")
+                      .map { |e| Recipient.from_storage(@cfbf, e) }
+    end
+
+    def to = recipients.select { |r| r.type == Recipient::TO }
+    def cc = recipients.select { |r| r.type == Recipient::CC }
+    def bcc = recipients.select { |r| r.type == Recipient::BCC }
+
+    def attachments
+      @attachments ||= child_storages("__ATTACH_VERSION1.0_#")
+                       .map { |e| Attachment.new(@cfbf, e, named: @named) }
+    end
+
+    def named_value(guid, lid_or_name)
+      id = @named.resolve(guid, lid_or_name)
+      id && properties[id]
+    end
+
+    def save(dir: ".")
+      name = Util.sanitize_filename(subject || "message")
+      base = Util.dedupe_path(::File.join(dir, name))
+      FileUtils.mkdir_p(base)
+      ::File.write(::File.join(base, "message.txt"), body, encoding: Encoding::UTF_8) if body
+      ::File.binwrite(::File.join(base, "message.html"), html_body.b) if html_body
+      attachments.each do |attachment|
+        attachment.save(dir: base) unless attachment.embedded_message?
+      end
+      base
+    end
+
+    private
+
+    def child_storages(prefix)
+      @storage.children.values
+              .select { |e| e.storage? && e.name.upcase.start_with?(prefix) }
+              .sort_by(&:name)
+    end
+  end
+end

data/lib/msg_extractor/recipient.rb

@@ -0,0 +1,34 @@
+module MsgExtractor
+  class Recipient
+    TO = 1
+    CC = 2
+    BCC = 3
+
+    attr_reader :name, :email, :type
+
+    def initialize(name:, email:, type:)
+      @name = name
+      @email = email
+      @type = type
+    end
+
+    def self.from_storage(cfbf, storage)
+      props = Mapi::PropertyStore.new(cfbf, storage, :recipient)
+      email = props[Mapi::PR_SMTP_ADDRESS]
+      if email.nil?
+        address = props[Mapi::PR_EMAIL_ADDRESS]
+        email = address if address&.include?("@")
+      end
+      new(name: props[Mapi::PR_DISPLAY_NAME], email: email,
+          type: props[Mapi::PR_RECIPIENT_TYPE] || TO)
+    end
+
+    def to_s
+      if name && email && name != email
+        "#{name} <#{email}>"
+      else
+        (email || name).to_s
+      end
+    end
+  end
+end

data/lib/msg_extractor/rtf/compressed_rtf.rb

@@ -0,0 +1,88 @@
+module MsgExtractor
+  module Rtf
+    # LZFu decompression for PR_RTF_COMPRESSED per [MS-OXRTFCP].
+    module CompressedRtf
+      MAGIC_COMPRESSED = 0x75465A4C   # "LZFu"
+      MAGIC_UNCOMPRESSED = 0x414C454D # "MELA"
+
+      # The fixed 207-byte initial dictionary defined by the spec.
+      INITIAL_DICTIONARY =
+        ("{\\rtf1\\ansi\\mac\\deff0\\deftab720{\\fonttbl;}" \
+         "{\\f0\\fnil \\froman \\fswiss \\fmodern \\fscript " \
+         "\\fdecor MS Sans SerifSymbolArialTimes New RomanCourier" \
+         "{\\colortbl\\red0\\green0\\blue0\r\n\\par " \
+         "\\pard\\plain\\f0\\fs20\\b\\i\\u\\tab\\tx").b.freeze
+
+      # CRC32 table (polynomial 0xEDB88320), init 0, no final XOR — per spec.
+      CRC_TABLE = (0...256).map { |i|
+        crc = i
+        8.times { crc = crc.odd? ? (0xEDB88320 ^ (crc >> 1)) : (crc >> 1) }
+        crc
+      }.freeze
+
+      module_function
+
+      def crc32(bytes)
+        bytes.each_byte.reduce(0) { |crc, b| CRC_TABLE[(crc ^ b) & 0xFF] ^ (crc >> 8) }
+      end
+
+      def decompress(data)
+        raise CorruptFileError, "compressed RTF too short" if data.nil? || data.bytesize < 16
+        comp_size, raw_size, magic, crc = data.unpack("V4")
+        case magic
+        when MAGIC_UNCOMPRESSED
+          raise CorruptFileError, "MELA RTF truncated" if data.bytesize - 16 < raw_size
+          data.byteslice(16, raw_size)
+        when MAGIC_COMPRESSED
+          payload = data.byteslice(16, comp_size - 12) if comp_size >= 12
+          raise CorruptFileError, "compressed RTF truncated header" if payload.nil?
+          unless crc32(payload) == crc
+            raise CorruptFileError, "compressed RTF CRC mismatch"
+          end
+          lzfu(payload)
+        else
+          raise CorruptFileError, format("bad compressed RTF magic 0x%08x", magic)
+        end
+      end
+
+      def lzfu(payload)
+        dictionary = INITIAL_DICTIONARY.dup
+        dictionary << ("\0".b * (4096 - dictionary.bytesize))
+        write_pos = INITIAL_DICTIONARY.bytesize # 207
+        out = +"".b
+        pos = 0
+        while pos < payload.bytesize
+          control = payload.getbyte(pos)
+          pos += 1
+          8.times do |bit|
+            if ((control >> bit) & 1) == 1
+              high = payload.getbyte(pos)
+              low = payload.getbyte(pos + 1)
+              return out if high.nil? || low.nil?
+              pos += 2
+              reference = (high << 8) | low
+              offset = reference >> 4
+              length = (reference & 0x0F) + 2
+              return out if offset == write_pos # end-of-stream marker
+              length.times do
+                byte = dictionary.getbyte(offset)
+                offset = (offset + 1) % 4096
+                out << byte
+                dictionary.setbyte(write_pos, byte)
+                write_pos = (write_pos + 1) % 4096
+              end
+            else
+              byte = payload.getbyte(pos)
+              return out if byte.nil?
+              pos += 1
+              out << byte
+              dictionary.setbyte(write_pos, byte)
+              write_pos = (write_pos + 1) % 4096
+            end
+          end
+        end
+        out
+      end
+    end
+  end
+end