ruby-msg-nx 0.3.0

data/lib/mapi/convert/note-tmail.rb

@@ -0,0 +1,287 @@
+require 'rubygems'
+require 'tmail'
+
+# these will be removed later
+require 'time'
+require 'mapi/mime'
+
+# there is some Msg specific stuff in here.
+
+class TMail::Mail
+	def quoted_body= str
+		body_port.wopen { |f| f.write str }
+		str
+	end
+end
+
+module Mapi
+	class Message
+		def mime
+			return @mime if @mime
+			# if these headers exist at all, they can be helpful. we may however get a
+			# application/ms-tnef mime root, which means there will be little other than
+			# headers. we may get nothing.
+			# and other times, when received from external, we get the full cigar, boundaries
+			# etc and all.
+			# sometimes its multipart, with no boundaries. that throws an error. so we'll be more
+			# forgiving here
+			@mime = Mime.new props.transport_message_headers.to_s, true
+			populate_headers
+			@mime
+		end
+
+		def headers
+			mime.headers
+		end
+
+		# copy data from msg properties storage to standard mime. headers
+		# i've now seen it where the existing headers had heaps on stuff, and the msg#props had
+		# practically nothing. think it was because it was a tnef - msg conversion done by exchange.
+		def populate_headers
+			# construct a From value
+			# should this kind of thing only be done when headers don't exist already? maybe not. if its
+			# sent, then modified and saved, the headers could be wrong?
+			# hmmm. i just had an example where a mail is sent, from an internal user, but it has transport
+			# headers, i think because one recipient was external. the only place the senders email address
+			# exists is in the transport headers. so its maybe not good to overwrite from.
+			# recipients however usually have smtp address available.
+			# maybe we'll do it for all addresses that are smtp? (is that equivalent to 
+			# sender_email_address !~ /^\//
+			name, email = props.sender_name, props.sender_email_address
+			if props.sender_addrtype == 'SMTP'
+				headers['From'] = if name and email and name != email
+					[%{"#{name}" <#{email}>}]
+				else
+					[email || name]
+				end
+			elsif !headers.has_key?('From')
+				# some messages were never sent, so that sender stuff isn't filled out. need to find another
+				# way to get something
+				# what about marking whether we thing the email was sent or not? or draft?
+				# for partition into an eventual Inbox, Sent, Draft mbox set?
+				# i've now seen cases where this stuff is missing, but exists in transport message headers,
+				# so maybe i should inhibit this in that case.
+				if email
+					# disabling this warning for now
+					#Log.warn "* no smtp sender email address available (only X.400). creating fake one"
+					# this is crap. though i've specially picked the logic so that it generates the correct
+					# email addresses in my case (for my organisation).
+					# this user stuff will give valid email i think, based on alias.
+					user = name ? name.sub(/(.*), (.*)/, "\\2.\\1") : email[/\w+$/].downcase
+					domain = (email[%r{^/O=([^/]+)}i, 1].downcase + '.com' rescue email)
+					headers['From'] = [name ? %{"#{name}" <#{user}@#{domain}>} : "<#{user}@#{domain}>" ]
+				elsif name
+					# we only have a name? thats screwed up.
+					# disabling this warning for now
+					#Log.warn "* no smtp sender email address available (only name). creating fake one"
+					headers['From'] = [%{"#{name}"}]
+				else
+					# disabling this warning for now
+					#Log.warn "* no sender email address available at all. FIXME"
+				end
+			# else we leave the transport message header version
+			end
+
+			# for all of this stuff, i'm assigning in utf8 strings.
+			# thats ok i suppose, maybe i can say its the job of the mime class to handle that.
+			# but a lot of the headers are overloaded in different ways. plain string, many strings
+			# other stuff. what happens to a person who has a " in their name etc etc. encoded words
+			# i suppose. but that then happens before assignment. and can't be automatically undone
+			# until the header is decomposed into recipients.
+			recips_by_type = recipients.group_by { |r| r.type }
+			# i want to the the types in a specific order.
+			[:to, :cc, :bcc].each do |type|
+				# don't know why i bother, but if we can, we try to sort recipients by the numerical part
+				# of the ole name, or just leave it if we can't
+				recips = recips_by_type[type]
+				recips = (recips.sort_by { |r| r.obj.name[/\d{8}$/].hex } rescue recips)
+				# switched to using , for separation, not ;. see issue #4
+				# recips.empty? is strange. i wouldn't have thought it possible, but it was right?
+				headers[type.to_s.sub(/^(.)/) { $1.upcase }] = [recips.join(', ')] unless recips.empty?
+			end
+			headers['Subject'] = [props.subject] if props.subject
+
+			# fill in a date value. by default, we won't mess with existing value hear
+			if !headers.has_key?('Date')
+				# we want to get a received date, as i understand it.
+				# use this preference order, or pull the most recent?
+				keys = %w[message_delivery_time client_submit_time last_modification_time creation_time]
+				time = keys.each { |key| break time if time = props.send(key) }
+				time = nil unless Date === time
+
+				# now convert and store
+				# this is a little funky. not sure about time zone stuff either?
+				# actually seems ok. maybe its always UTC and interpreted anyway. or can be timezoneless.
+				# i have no timezone info anyway.
+				# in gmail, i see stuff like 15 Jan 2007 00:48:19 -0000, and it displays as 11:48.
+				# can also add .localtime here if desired. but that feels wrong.
+				headers['Date'] = [Time.iso8601(time.to_s).rfc2822] if time
+			end
+
+			# some very simplistic mapping between internet message headers and the
+			# mapi properties
+			# any of these could be causing duplicates due to case issues. the hack in #to_mime
+			# just stops re-duplication at that point. need to move some smarts into the mime
+			# code to handle it.
+			mapi_header_map = [
+				[:internet_message_id, 'Message-ID'],
+				[:in_reply_to_id, 'In-Reply-To'],
+				# don't set these values if they're equal to the defaults anyway
+				[:importance, 'Importance', proc { |val| val.to_s == '1' ? nil : val }],
+				[:priority, 'Priority', proc { |val| val.to_s == '1' ? nil : val }],
+				[:sensitivity, 'Sensitivity', proc { |val| val.to_s == '0' ? nil : val }],
+				# yeah?
+				[:conversation_topic, 'Thread-Topic'],
+				# not sure of the distinction here
+				# :originator_delivery_report_requested ??
+				[:read_receipt_requested, 'Disposition-Notification-To', proc { |val| from }]
+			]
+			mapi_header_map.each do |mapi, mime, *f|
+				next unless q = val = props.send(mapi) or headers.has_key?(mime)
+				next if f[0] and !(val = f[0].call(val))
+				headers[mime] = [val.to_s]
+			end
+		end
+
+		# redundant?
+		def type
+			props.message_class[/IPM\.(.*)/, 1].downcase rescue nil
+		end
+
+		# shortcuts to some things from the headers
+		%w[From To Cc Bcc Subject].each do |key|
+			define_method(key.downcase) { headers[key].join(' ') if headers.has_key?(key) }
+		end
+
+		def body_to_tmail
+			# to create the body
+			# should have some options about serializing rtf. and possibly options to check the rtf
+			# for rtf2html conversion, stripping those html tags or other similar stuff. maybe want to
+			# ignore it in the cases where it is generated from incoming html. but keep it if it was the
+			# source for html and plaintext.
+			if props.body_rtf or props.body_html
+				# should plain come first?
+				part = TMail::Mail.new
+				# its actually possible for plain body to be empty, but the others not.
+				# if i can get an html version, then maybe a callout to lynx can be made...
+				part.parts << TMail::Mail.parse("Content-Type: text/plain\r\n\r\n" + props.body) if props.body
+				# this may be automatically unwrapped from the rtf if the rtf includes the html
+				part.parts << TMail::Mail.parse("Content-Type: text/html\r\n\r\n"  + props.body_html) if props.body_html
+				# temporarily disabled the rtf. its just showing up as an attachment anyway.
+				#mime.parts << Mime.new("Content-Type: text/rtf\r\n\r\n"   + props.body_rtf)  if props.body_rtf
+				# its thus currently possible to get no body at all if the only body is rtf. that is not
+				# really acceptable FIXME
+				part['Content-Type'] = 'multipart/alternative'
+				part
+			else
+				# check no header case. content type? etc?. not sure if my Mime class will accept
+				Log.debug "taking that other path"
+				# body can be nil, hence the to_s
+				TMail::Mail.parse "Content-Type: text/plain\r\n\r\n" + props.body.to_s
+			end
+		end
+
+		def to_tmail
+			# intended to be used for IPM.note, which is the email type. can use it for others if desired,
+			# YMMV
+			Log.warn "to_mime used on a #{props.message_class}" unless props.message_class == 'IPM.Note'
+			# we always have a body
+			mail = body = body_to_tmail
+
+			# If we have attachments, we take the current mime root (body), and make it the first child
+			# of a new tree that will contain body and attachments.
+			unless attachments.empty?
+				raise NotImplementedError
+				mime = Mime.new "Content-Type: multipart/mixed\r\n\r\n"
+				mime.parts << body
+				# i don't know any better way to do this. need multipart/related for inline images
+				# referenced by cid: urls to work, but don't want to use it otherwise...
+				related = false
+				attachments.each do |attach|
+					part = attach.to_mime
+					related = true if part.headers.has_key?('Content-ID') or part.headers.has_key?('Content-Location')
+					mime.parts << part
+				end
+				mime.headers['Content-Type'] = ['multipart/related'] if related
+			end
+
+			# at this point, mime is either
+			# - a single text/plain, consisting of the body ('taking that other path' above. rare)
+			# - a multipart/alternative, consiting of a few bodies (plain and html body. common)
+			# - a multipart/mixed, consisting of 1 of the above 2 types of bodies, and attachments.
+			# we add this standard preamble if its multipart
+			# FIXME preamble.replace, and body.replace both suck.
+			# preamble= is doable. body= wasn't being done because body will get rewritten from parts
+			# if multipart, and is only there readonly. can do that, or do a reparse...
+			# The way i do this means that only the first preamble will say it, not preambles of nested
+			# multipart chunks.
+			mail.quoted_body = "This is a multi-part message in MIME format.\r\n" if mail.multipart?
+
+			# now that we have a root, we can mix in all our headers
+			headers.each do |key, vals|
+				# don't overwrite the content-type, encoding style stuff
+				next if mail[key]
+				# some new temporary hacks
+				next if key =~ /content-type/i and vals[0] =~ /base64/
+				#next if mime.headers.keys.map(&:downcase).include? key.downcase
+				mail[key] = vals.first
+			end
+			# just a stupid hack to make the content-type header last
+			#mime.headers['Content-Type'] = mime.headers.delete 'Content-Type'
+
+			mail
+		end
+	end
+
+	class Attachment
+		def to_tmail
+			# TODO: smarter mime typing.
+			mimetype = props.attach_mime_tag || 'application/octet-stream'
+			part = TMail::Mail.parse "Content-Type: #{mimetype}\r\n\r\n"
+			part['Content-Disposition'] = %{attachment; filename="#{filename}"}
+			part['Content-Transfer-Encoding'] = 'base64'
+			part['Content-Location'] = props.attach_content_location if props.attach_content_location
+			part['Content-ID'] = props.attach_content_id if props.attach_content_id
+			# data.to_s for now. data was nil for some reason.
+			# perhaps it was a data object not correctly handled?
+			# hmmm, have to use read here. that assumes that the data isa stream.
+			# but if the attachment data is a string, then it won't work. possible?
+			data_str = if @embedded_msg
+				raise NotImplementedError
+				mime.headers['Content-Type'] = 'message/rfc822'
+				# lets try making it not base64 for now
+				mime.headers.delete 'Content-Transfer-Encoding'
+				# not filename. rather name, or something else right?
+				# maybe it should be inline?? i forget attach_method / access meaning
+				mime.headers['Content-Disposition'] = [%{attachment; filename="#{@embedded_msg.subject}"}]
+				@embedded_msg.to_mime.to_s
+			elsif @embedded_ole
+				raise NotImplementedError
+				# kind of hacky
+				io = StringIO.new
+				Ole::Storage.new io do |ole|
+					ole.root.type = :dir
+					Ole::Storage::Dirent.copy @embedded_ole, ole.root
+				end
+				io.string
+			else
+				data.read.to_s
+			end
+			part.body = @embedded_msg ? data_str : Base64.encode64(data_str).gsub(/\n/, "\r\n")
+			part
+		end
+	end
+
+	class Msg < Message
+		def populate_headers
+			super
+			if !headers.has_key?('Date')
+				# can employ other methods for getting a time. heres one in a similar vein to msgconvert.pl,
+				# ie taking the time from an ole object
+				time = @root.ole.dirents.map { |dirent| dirent.modify_time || dirent.create_time }.compact.sort.last
+				headers['Date'] = [Time.iso8601(time.to_s).rfc2822] if time
+			end
+		end
+	end
+end
+

data/lib/mapi/convert.rb

@@ -0,0 +1,69 @@
+# we have two different "backends" for note conversion. we're sticking with
+# the current (home grown) mime one until the tmail version is suitably
+# polished.
+require 'mapi/convert/note-mime'
+require 'mapi/convert/contact'
+
+module Mapi
+	class Message
+		CONVERSION_MAP = {
+			'text/x-vcard'   => [:to_vcard, 'vcf'],
+			'message/rfc822' => [:to_mime, 'eml'],
+			'text/plain'     => [:to_post, 'txt']
+			# ...
+		}
+
+		# get the mime type of the message. 
+		#
+		# @return [String] `message/rfc822` and so on
+		# @return [nil] Not known converion way
+		def mime_type
+			case props.message_class #.downcase <- have a feeling i saw other cased versions
+			when 'IPM.Contact'
+				# apparently "text/directory; profile=vcard" is what you're supposed to use
+				'text/x-vcard'
+			when 'IPM.Note'
+				'message/rfc822'
+			when 'IPM.Post'
+				'text/plain'
+			when 'IPM.StickyNote'
+				'text/plain' # hmmm....
+			else
+				Mapi::Log.warn 'unknown message_class - %p' % props.message_class
+				nil
+			end
+		end	
+
+		# @return [Object] Use to_s
+		def convert
+			type = mime_type
+			unless pair = CONVERSION_MAP[type]
+				raise 'unable to convert message with mime type - %p' % type
+			end
+			send pair.first
+		end
+
+		# should probably be moved to mapi/convert/post
+		class Post
+			# not really sure what the pertinent properties are. we just do nothing for now...
+			#
+			# @param message [Message]
+			def initialize message
+				@message = message
+			end
+
+			# @return [String]
+			def to_s
+				# should maybe handle other types, like html body. need a better format for post
+				# probably anyway, cause a lot of meta data is getting chucked.
+				@message.props.body
+			end
+		end
+
+		# @return [Post]
+		def to_post
+			Post.new self
+		end
+	end
+end
+

data/lib/mapi/helper.rb

@@ -0,0 +1,45 @@
+module Mapi
+	# This is a helper class for {Pst} and {Msg}.
+	class Helper
+		# @return [String, nil] Encoding name of ANSI string we assume
+		attr_reader :ansi_encoding
+
+		# @return [Boolean] Convert all ANSI string to UTF-8
+		attr_reader :to_unicode
+
+		# @param ansi_encoding [String]
+		# @param to_unicode [Boolean]
+		def initialize ansi_encoding=nil, to_unicode=false
+			@ansi_encoding = ansi_encoding || "BINARY"
+			@to_unicode = to_unicode
+		end
+
+		# Convert `ASCII_8BIT` string. Maybe produce UTF_8 string, or arbitrary object
+		#
+		# Use cases:
+		# 
+		# - Decode PT_STRING8 in {Pst}
+		# - Decode `0x001e` in {Msg}
+		# - Decode body (rtf, text) in {PropertySet}
+		#
+		# @param str [String]
+		# @return [Object]
+		def convert_ansi_str str
+			if @ansi_encoding
+				if @to_unicode
+					# assume we can convert this text to UTF-8
+					begin
+						str.force_encoding(@ansi_encoding).encode("UTF-8")
+					rescue Encoding::UndefinedConversionError => ex
+						# some text are already UTF-8 due to unknown reason
+						str.force_encoding("UTF-8").encode("UTF-8")
+					end
+				else
+					str.force_encoding(@ansi_encoding)
+				end
+			else
+				str
+			end
+		end
+	end
+end

data/lib/mapi/mime.rb

@@ -0,0 +1,227 @@
+#
+# = Introduction
+#
+# A *basic* mime class for _really_ _basic_ and probably non-standard parsing
+# and construction of MIME messages.
+#
+# Intended for two main purposes in this project:
+# 1. As the container that is used to build up the message for eventual
+#    serialization as an eml.
+# 2. For assistance in parsing the +transport_message_headers+ provided in .msg files,
+#    which are then kept through to the final eml.
+#
+# = TODO
+#
+# * Better streaming support, rather than an all-in-string approach.
+# * A fair bit remains to be done for this class, its fairly immature. But generally I'd like
+#   to see it be more generally useful.
+# * All sorts of correctness issues, encoding particular.
+# * Duplication of work in net/http.rb's +HTTPHeader+? Don't know if the overlap is sufficient.
+#   I don't want to lower case things, just for starters.
+# * Mime was the original place I wrote #to_tree, intended as a quick debug hack.
+#
+
+require 'base64'
+
+module Mapi
+  class Mime
+	# @return [Hash{String => Array}]
+  	attr_reader :headers
+	# @return [String]
+	attr_reader :body
+	# @return [Mime]
+	attr_reader :parts
+	# @return [String]
+	attr_reader :content_type
+	# @return [String]
+	attr_reader :preamble
+	# @return [String]
+	attr_reader :epilogue
+
+  	# Create a Mime object using +str+ as an initial serialization, which must contain headers
+  	# and a body (even if empty). Needs work.
+	#
+	# @param str [String]
+	# @param ignore_body [Boolean]
+  	def initialize str, ignore_body=false
+  		headers, @body = $~[1..-1] if str[/(.*?\r?\n)(?:\r?\n(.*))?\Z/m]
+
+  		@headers = Hash.new { |hash, key| hash[key] = [] }
+  		@body ||= ''
+  		headers.to_s.scan(/^\S+:\s*.*(?:\n\t.*)*/).each do |header|
+  			@headers[header[/(\S+):/, 1]] << header[/\S+:\s*(.*)/m, 1].gsub(/\s+/m, ' ').strip # this is kind of wrong
+  		end
+
+  		# don't have to have content type i suppose
+  		@content_type, attrs = nil, {}
+  		if content_type = @headers['Content-Type'][0]
+  			@content_type, attrs = Mime.split_header content_type
+  		end
+
+  		return if ignore_body
+
+  		if multipart?
+  			if body.empty?
+  				@preamble = ''
+  				@epilogue = ''
+  				@parts = []
+  			else
+  				# we need to split the message at the boundary
+  				boundary = attrs['boundary'] or raise "no boundary for multipart message"
+
+  				# splitting the body:
+  				parts = body.split(/--#{Regexp.quote boundary}/m)
+  				unless parts[-1] =~ /^--/; warn "bad multipart boundary (missing trailing --)"
+  				else parts[-1][0..1] = ''
+  				end
+  				parts.each_with_index do |part, i|
+  					part =~ /^(\r?\n)?(.*?)(\r?\n)?\Z/m
+  					part.replace $2
+  					warn "bad multipart boundary" if (1...parts.length-1) === i and !($1 && $3)
+  				end
+  				@preamble = parts.shift
+  				@epilogue = parts.pop
+  				@parts = parts.map { |part| Mime.new part }
+  			end
+  		end
+  	end
+
+	# @return [Boolean]
+  	def multipart?
+  		@content_type && @content_type =~ /^multipart/ ? true : false
+  	end
+
+	# @return [String]
+	def inspect
+  		# add some extra here.
+  		"#<Mime content_type=#{@content_type.inspect}>"
+  	end
+
+	# @return [String]
+	def to_tree
+  		if multipart?
+  			str = "- #{inspect}\n"
+  			parts.each_with_index do |part, i|
+  				last = i == parts.length - 1
+  				part.to_tree.split(/\n/).each_with_index do |line, j|
+  					str << "  #{last ? (j == 0 ? "\\" : ' ') : '|'}" + line + "\n"
+  				end
+  			end
+  			str
+  		else
+  			"- #{inspect}\n"
+  		end
+  	end
+
+	# Compose rfc822 eml file
+	#
+	# @param opts [Hash]
+	# @return [String]
+  	def to_s opts={}
+  		opts = {:boundary_counter => 0}.merge opts
+
+		body_encoder = proc { |body| body.bytes.pack("C*") }
+
+  		if multipart?
+  			boundary = Mime.make_boundary opts[:boundary_counter] += 1, self
+  			@body = [preamble, parts.map { |part| "\r\n" + part.to_s(opts) + "\r\n" }, "--\r\n" + epilogue].
+  				flatten.join("\r\n--" + boundary)
+  			content_type, attrs = Mime.split_header @headers['Content-Type'][0]
+  			attrs['boundary'] = boundary
+  			@headers['Content-Type'] = [([content_type] + attrs.map { |key, val| %{#{key}="#{val}"} }).join('; ')]
+		else
+			raw_content_type = @headers['Content-Type'][0]
+			if raw_content_type
+				content_type, attrs = Mime.split_header raw_content_type
+				case content_type.split("/").first()
+				when "text"
+					attrs["charset"] = @body.encoding.name
+					@headers['Content-Type'] = [([content_type] + attrs.map { |key, val| %{#{key}="#{val}"} }).join('; ')]
+				end
+			end
+  		end
+
+		# ensure non ASCII chars are well encoded (like rfc2047) by upper source
+		value_encoder = Proc.new { |val| val.encode("ASCII") }
+
+  		str = ''
+  		@headers.each do |key, vals|
+			case vals
+			when String
+				val = vals
+				str << "#{key}: #{value_encoder.call(val)}\r\n"
+			when Array
+				vals.each { |val| str << "#{key}: #{value_encoder.call(val)}\r\n" }
+			end
+  		end
+  		str << "\r\n"
+		str << body_encoder.call(@body)
+  	end
+
+	# Compose encoded-word (rfc2047) for non-ASCII text
+	# 
+	# @param str [String, nil]
+	# @return [String, nil]
+	# @private
+	def self.to_encoded_word str
+		# We can assume that str can produce valid byte array in regardless of encoding.
+
+		# Check if non-printable characters (including CR/LF) are inside.
+		if str && str.bytes.any? {|byte| byte <= 31 || 127 <= byte}
+			sprintf("=?%s?B?%s?=", str.encoding.name, Base64.strict_encode64(str))
+		else
+			str
+		end
+	end
+
+	# @param header [String]
+	# @return [Array(String, Hash{String => String})]
+  	def self.split_header header
+  		# FIXME: haven't read standard. not sure what its supposed to do with " in the name, or if other
+  		# escapes are allowed. can't test on windows as " isn't allowed anyway. can be fixed with more
+  		# accurate parser later.
+  		# maybe move to some sort of Header class. but not all headers should be of it i suppose.
+  		# at least add a join_header then, taking name and {}. for use in Mime#to_s (for boundary
+  		# rewrite), and Attachment#to_mime, among others...
+  		attrs = {}
+  		header.scan(/;\s*([^\s=]+)\s*=\s*("[^"]*"|[^\s;]*)\s*/m).each do |key, value|
+  			if attrs[key]; warn "ignoring duplicate header attribute #{key.inspect}"
+  			else attrs[key] = value[/^"/] ? value[1..-2] : value
+  			end
+  		end
+
+  		[header[/^[^;]+/].strip, attrs]
+  	end
+
+  	# +i+ is some value that should be unique for all multipart boundaries for a given message
+	#
+	# @return [String]
+  	def self.make_boundary i, extra_obj = Mime
+  		"----_=_NextPart_#{'%03d' % i}_#{'%08x' % extra_obj.object_id}.#{'%08x' % Time.now}"
+  	end
+  end
+end
+
+=begin
+things to consider for header work.
+encoded words:
+Subject: =?iso-8859-1?q?p=F6stal?=
+
+and other mime funkyness:
+Content-Disposition: attachment;
+	filename*0*=UTF-8''09%20%D7%90%D7%A5;
+	filename*1*=%20%D7%A1%D7%91-;
+	filename*2*=%D7%A7%95%A5.wma
+Content-Transfer-Encoding: base64
+
+and another, doing a test with an embedded newline in an attachment name, I
+get this output from evolution. I get the feeling that this is probably a bug
+with their implementation though, they weren't expecting new lines in filenames.
+Content-Disposition: attachment; filename="asdf'b\"c
+d   efgh=i: ;\\j"
+d   efgh=i: ;\\j"; charset=us-ascii
+Content-Type: text/plain; name="asdf'b\"c"; charset=us-ascii
+
+=end
+
+