ruby-msg 1.3.1 → 1.4.0

data/lib/mapi/rtf.rb

@@ -0,0 +1,169 @@
+require 'stringio'
+require 'strscan'
+require 'rtf'
+
+module Mapi
+	#
+	# = Introduction
+	#
+	# The +RTF+ module contains a few helper functions for dealing with rtf
+	# in mapi messages: +rtfdecompr+, and <tt>rtf2html</tt>.
+	#
+	# Both were ported from their original C versions for simplicity's sake.
+	#
+	module RTF
+		RTF_PREBUF = 
+			"{\\rtf1\\ansi\\mac\\deff0\\deftab720{\\fonttbl;}" \
+			"{\\f0\\fnil \\froman \\fswiss \\fmodern \\fscript " \
+			"\\fdecor MS Sans SerifSymbolArialTimes New RomanCourier" \
+			"{\\colortbl\\red0\\green0\\blue0\n\r\\par " \
+			"\\pard\\plain\\f0\\fs20\\b\\i\\u\\tab\\tx"
+
+		# Decompresses compressed rtf +data+, as found in the mapi property
+		# +PR_RTF_COMPRESSED+. Code converted from my C version, which in turn
+		# I wrote from a Java source, in JTNEF I believe.
+		#
+		# C version was modified to use circular buffer for back references,
+		# instead of the optimization of the Java version to index directly into
+		# output buffer. This was in preparation to support streaming in a
+		# read/write neutral fashion.
+		def rtfdecompr data
+			io  = StringIO.new data
+			buf = RTF_PREBUF + "\x00" * (4096 - RTF_PREBUF.length)
+			wp  = RTF_PREBUF.length
+			rtf = ''
+
+			# get header fields (as defined in RTFLIB.H)
+			compr_size, uncompr_size, magic, crc32 = io.read(16).unpack 'V*'
+			#warn "compressed-RTF data size mismatch" unless io.size == data.compr_size + 4
+
+			# process the data
+			case magic
+			when 0x414c454d # "MELA" magic number that identifies the stream as a uncompressed stream
+				rtf = io.read uncompr_size
+			when 0x75465a4c # "LZFu" magic number that identifies the stream as a compressed stream
+				flag_count = -1
+				flags = nil
+				while rtf.length < uncompr_size and !io.eof?
+					# each flag byte flags 8 literals/references, 1 per bit
+					flags = ((flag_count += 1) % 8 == 0) ? io.getc : flags >> 1
+					if 1 == (flags & 1) # each flag bit is 1 for reference, 0 for literal
+						rp, l = io.getc, io.getc
+						# offset is a 12 byte number. 2^12 is 4096, so thats fine
+						rp = (rp << 4) | (l >> 4) # the offset relative to block start
+						l = (l & 0xf) + 2 # the number of bytes to copy
+						l.times do
+							rtf << buf[wp] = buf[rp]
+							wp = (wp + 1) % 4096
+							rp = (rp + 1) % 4096
+						end
+					else
+						rtf << buf[wp] = io.getc
+						wp = (wp + 1) % 4096
+					end
+				end
+			else # unknown magic number
+				raise "Unknown compression type (magic number 0x%08x)" % magic
+			end
+			
+			# not sure if its due to a bug in the above code. doesn't seem to be
+			# in my tests, but sometimes there's a trailing null. we chomp it here,
+			# which actually makes the resultant rtf smaller than its advertised
+			# size (+uncompr_size+).
+			rtf.chomp! 0.chr
+			rtf
+		end
+
+		# Note, this is a conversion of the original C code. Not great - needs tests and
+		# some refactoring, and an attempt to correct some inaccuracies. Hacky but works.
+		#
+		# Returns +nil+ if it doesn't look like an rtf encapsulated rtf.
+		#
+		# Some cases that the original didn't deal with have been patched up, eg from 
+		# this chunk, where there are tags outside of the htmlrtf ignore block.
+		#
+		# "{\\*\\htmltag116 <br />}\\htmlrtf \\line \\htmlrtf0 \\line {\\*\\htmltag84 <a href..."
+		#
+		# We take the approach of ignoring all rtf tags not explicitly handled. A proper
+		# parse tree would be nicer to work with. will need to look for ruby rtf library
+		#
+		# Some of the original comment to the c code is excerpted here:
+		#
+		# Sometimes in MAPI, the PR_BODY_HTML property contains the HTML of a message.
+		# But more usually, the HTML is encoded inside the RTF body (which you get in the
+		# PR_RTF_COMPRESSED property). These routines concern the decoding of the HTML
+		# from this RTF body.
+		#
+		# An encoded htmlrtf file is a valid RTF document, but which contains additional
+		# html markup information in its comments, and sometimes contains the equivalent
+		# rtf markup outside the comments. Therefore, when it is displayed by a plain
+		# simple RTF reader, the html comments are ignored and only the rtf markup has
+		# effect. Typically, this rtf markup is not as rich as the html markup would have been.
+		# But for an html-aware reader (such as the code below), we can ignore all the
+		# rtf markup, and extract the html markup out of the comments, and get a valid
+		# html document.
+		#
+		# There are actually two kinds of html markup in comments. Most of them are
+		# prefixed by "\*\htmltagNNN", for some number NNN. But sometimes there's one
+		# prefixed by "\*\mhtmltagNNN" followed by "\*\htmltagNNN". In this case,
+		# the two are equivalent, but the m-tag is for a MIME Multipart/Mixed Message
+		# and contains tags that refer to content-ids (e.g. img src="cid:072344a7")
+		# while the normal tag just refers to a name (e.g. img src="fred.jpg")
+		# The code below keeps the m-tag and discards the normal tag.
+		# If there are any m-tags like this, then the message also contains an
+		# attachment with a PR_CONTENT_ID property e.g. "072344a7". Actually,
+		# sometimes the m-tag is e.g. img src="http://outlook/welcome.html" and the
+		# attachment has a PR_CONTENT_LOCATION "http://outlook/welcome.html" instead
+		# of a PR_CONTENT_ID.
+		#
+		def rtf2html rtf
+			scan = StringScanner.new rtf
+			# require \fromhtml. is this worth keeping? apparently you see \\fromtext if it
+			# was converted from plain text. 
+			return nil unless rtf["\\fromhtml"]
+			html = ''
+			ignore_tag = nil
+			# skip up to the first htmltag. return nil if we don't ever find one
+			return nil unless scan.scan_until /(?=\{\\\*\\htmltag)/
+			until scan.empty?
+				if scan.scan /\{/
+				elsif scan.scan /\}/
+				elsif scan.scan /\\\*\\htmltag(\d+) ?/
+					#p scan[1]
+					if ignore_tag == scan[1]
+						scan.scan_until /\}/
+						ignore_tag = nil
+					end
+				elsif scan.scan /\\\*\\mhtmltag(\d+) ?/
+						ignore_tag = scan[1]
+				elsif scan.scan /\\par ?/
+					html << "\r\n"
+				elsif scan.scan /\\tab ?/
+					html << "\t"
+				elsif scan.scan /\\'([0-9A-Za-z]{2})/
+					html << scan[1].hex.chr
+				elsif scan.scan /\\pntext/
+					scan.scan_until /\}/
+				elsif scan.scan /\\htmlrtf/
+					scan.scan_until /\\htmlrtf0 ?/
+				# a generic throw away unknown tags thing.
+				# the above 2 however, are handled specially
+				elsif scan.scan /\\[a-z-]+(\d+)? ?/
+				#elsif scan.scan /\\li(\d+) ?/
+				#elsif scan.scan /\\fi-(\d+) ?/
+				elsif scan.scan /[\r\n]/
+				elsif scan.scan /\\([{}\\])/
+					html << scan[1]
+				elsif scan.scan /(.)/
+					html << scan[1]
+				else
+					p :wtf
+				end
+			end
+			html.strip.empty? ? nil : html
+		end
+
+		module_function :rtf2html, :rtfdecompr
+	end
+end
+

data/lib/mapi/types.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'ole/types'
+
+module Mapi
+	Log = Logger.new_with_callstack
+
+	module Types
+		#
+		# Mapi property types, taken from http://msdn2.microsoft.com/en-us/library/bb147591.aspx.
+		#
+		# The fields are [mapi name, variant name, description]. Maybe I should just make it a
+		# struct.
+		#
+		# seen some synonyms here, like PT_I8 vs PT_LONG. seen stuff like PT_SRESTRICTION, not
+		# sure what that is. look at `grep ' PT_' data/mapitags.yaml  | sort -u`
+		# also, it has stuff like PT_MV_BINARY, where _MV_ probably means multi value, and is
+		# likely just defined to | in 0x1000.
+		#
+		# Note that the last 2 are the only ones where the Mapi value differs from the Variant value
+		# for the corresponding variant type. Odd. Also, the last 2 are currently commented out here
+		# because of the clash.
+		#
+		# Note 2 - the strings here say VT_BSTR, but I don't have that defined in Ole::Types. Should
+		# maybe change them to match. I've also seen reference to PT_TSTRING, which is defined as some
+		# sort of get unicode first, and fallback to ansii or something.
+		#
+		DATA = {
+			0x0001 => ['PT_NULL', 'VT_NULL', 'Null (no valid data)'],
+			0x0002 => ['PT_SHORT', 'VT_I2', '2-byte integer (signed)'],
+			0x0003 => ['PT_LONG', 'VT_I4', '4-byte integer (signed)'],
+			0x0004 => ['PT_FLOAT', 'VT_R4', '4-byte real (floating point)'],
+			0x0005 => ['PT_DOUBLE', 'VT_R8', '8-byte real (floating point)'],
+			0x0006 => ['PT_CURRENCY', 'VT_CY', '8-byte integer (scaled by 10,000)'],
+			0x000a => ['PT_ERROR', 'VT_ERROR', 'SCODE value; 32-bit unsigned integer'],
+			0x000b => ['PT_BOOLEAN', 'VT_BOOL', 'Boolean'],
+			0x000d => ['PT_OBJECT', 'VT_UNKNOWN', 'Data object'],
+			0x001e => ['PT_STRING8', 'VT_BSTR', 'String'],
+			0x001f => ['PT_UNICODE', 'VT_BSTR', 'String'],
+			0x0040 => ['PT_SYSTIME', 'VT_DATE', '8-byte real (date in integer, time in fraction)'],
+			#0x0102 => ['PT_BINARY', 'VT_BLOB', 'Binary (unknown format)'],
+			#0x0102 => ['PT_CLSID', 'VT_CLSID', 'OLE GUID']
+		}
+
+		module Constants
+			DATA.each { |num, (mapi_name, variant_name, desc)| const_set mapi_name, num }
+		end
+
+		include Constants
+	end
+end
+

data/lib/rtf.rb

@@ -1,5 +1,3 @@
-#! /usr/bin/ruby -w
-
 require 'stringio'
 
 # this file is pretty crap, its just to ensure there is always something readable if
@@ -109,10 +107,3 @@ module RTF
 	end
 end
 
-if $0 == __FILE__
-	#str = File.read('test.rtf')
-	str = YAML.load(open('rtfs.yaml'))[2]
-	#puts str
-	puts text
-end
-

data/test/test_convert_contact.rb

@@ -0,0 +1,60 @@
+require 'test/unit'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'mapi'
+require 'mapi/convert'
+
+class TestMapiPropertySet < Test::Unit::TestCase
+	include Mapi
+
+	def test_contact_from_property_hash
+		make_key1 = proc { |id| PropertySet::Key.new id }
+		make_key2 = proc { |id| PropertySet::Key.new id, PropertySet::PSETID_Address }
+		store = {
+			make_key1[0x001a] => 'IPM.Contact',
+			make_key1[0x0037] => 'full name',
+			make_key1[0x3a06] => 'given name',
+			make_key1[0x3a08] => 'business telephone number',
+			make_key1[0x3a11] => 'surname',
+			make_key1[0x3a15] => 'postal address',
+			make_key1[0x3a16] => 'company name',
+			make_key1[0x3a17] => 'title',
+			make_key1[0x3a18] => 'department name',
+			make_key1[0x3a19] => 'office location',
+			make_key2[0x8005] => 'file under',
+			make_key2[0x801b] => 'business address',
+			make_key2[0x802b] => 'web page',
+			make_key2[0x8045] => 'business address street',
+			make_key2[0x8046] => 'business address city',
+			make_key2[0x8047] => 'business address state',
+			make_key2[0x8048] => 'business address postal code',
+			make_key2[0x8049] => 'business address country',
+			make_key2[0x804a] => 'business address post office box',
+			make_key2[0x8062] => 'im address',
+			make_key2[0x8082] => 'SMTP',
+			make_key2[0x8083] => 'email@address.com'
+		}
+		props = PropertySet.new store
+		message = Message.new props
+		assert_equal 'text/x-vcard', message.mime_type
+		vcard = message.to_vcard
+		assert_equal Vpim::Vcard, vcard.class
+		assert_equal <<-'end', vcard.to_s
+BEGIN:VCARD
+VERSION:3.0
+N:surname;given name;;;
+FN:full name
+ADR;TYPE=work:;;business address street;business address city\, business ad
+ dress state;;;
+X-EVOLUTION-FILE-AS:file under
+EMAIL:email@address.com
+ORG:company name
+END:VCARD
+		end
+	end
+
+	def test_contact_from_msg
+		# load some msg contacts and convert them...
+	end
+end
+

data/test/test_convert_note.rb

@@ -0,0 +1,66 @@
+require 'test/unit'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'mapi'
+require 'mapi/convert'
+
+class TestMapiPropertySet < Test::Unit::TestCase
+	include Mapi
+
+	def test_using_pseudo_properties
+		# load some compressed rtf data
+		data = File.read File.dirname(__FILE__) + '/test_rtf.data'
+		store = {
+			PropertySet::Key.new(0x0037) => 'Subject',
+			PropertySet::Key.new(0x0c1e) => 'SMTP',
+			PropertySet::Key.new(0x0c1f) => 'sender@email.com',
+			PropertySet::Key.new(0x1009) => StringIO.new(data)
+		}
+		props = PropertySet.new store 
+		msg = Message.new props
+		def msg.attachments
+			[]
+		end
+		def msg.recipients
+			[]
+		end
+		# the ignoring of \r here should change. its actually not output consistently currently.
+		assert_equal((<<-end), msg.to_mime.to_s.gsub(/NextPart[_0-9a-z\.]+/, 'NextPart_XXX').delete("\r"))
+From: sender@email.com
+Subject: Subject
+Content-Type: multipart/alternative; boundary="----_=_NextPart_XXX"
+
+This is a multi-part message in MIME format.
+
+------_=_NextPart_XXX
+Content-Type: text/plain
+
+
+I will be out of the office starting  15.02.2007 and will not return until
+27.02.2007.
+
+I will respond to your message when I return. For urgent enquiries please
+contact Motherine Jacson.
+
+
+
+------_=_NextPart_XXX
+Content-Type: text/html
+
+<html>
+<body>
+<br>I will be out of the office starting  15.02.2007 and will not return until
+<br>27.02.2007.
+<br>
+<br>I will respond to your message when I return. For urgent enquiries please
+<br>contact Motherine Jacson.
+<br>
+<br></body>
+</html>
+
+
+------_=_NextPart_XXX--
+		end
+	end
+end
+

data/test/test_mime.rb

@@ -14,9 +14,11 @@ class TestMime < Test::Unit::TestCase
 		assert_equal 'Body text.', mime.body
 		assert_equal false, mime.multipart?
 		assert_equal nil, mime.parts
-		# we get round trip conversion. this is mostly fluke, as orderedhash hasn't been
-		# added yet
 		assert_equal "Header1: Value1\r\nHeader2: Value2\r\n\r\nBody text.", mime.to_s
 	end
+	
+	def test_boundaries
+		assert_match(/^----_=_NextPart_001_/, Mime.make_boundary(1))
+	end
 end
 

data/test/test_msg.rb

@@ -0,0 +1,29 @@
+#! /usr/bin/ruby
+
+TEST_DIR = File.dirname __FILE__
+$: << "#{TEST_DIR}/../lib"
+
+require 'test/unit'
+require 'mapi/msg'
+
+class TestMsg < Test::Unit::TestCase
+	def test_blammo
+		Mapi::Msg.open "#{TEST_DIR}/test_Blammo.msg" do |msg|
+			assert_equal '"TripleNickel" <TripleNickel@mapi32.net>', msg.from
+			assert_equal 'BlammoBlammo', msg.subject
+			assert_equal 0, msg.recipients.length
+			assert_equal 0, msg.attachments.length
+			# this is all properties
+			assert_equal 66, msg.properties.raw.length
+			# this is unique named properties
+			assert_equal 48, msg.properties.to_h.length
+			# get the named property keys
+			keys = msg.properties.raw.keys.select { |key| String === key.code }
+			assert_equal '55555555-5555-5555-c000-000000000046', keys[0].guid.format
+			assert_equal 'Yippee555', msg.properties[keys[0]]
+			assert_equal '66666666-6666-6666-c000-000000000046', keys[1].guid.format
+			assert_equal 'Yippee666', msg.properties[keys[1]]
+		end
+	end
+end
+

data/test/test_property_set.rb

@@ -0,0 +1,116 @@
+require 'test/unit'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'mapi/property_set'
+
+class TestMapiPropertySet < Test::Unit::TestCase
+	include Mapi
+
+	def test_constants
+		assert_equal '00020328-0000-0000-c000-000000000046', PropertySet::PS_MAPI.format
+	end
+
+	def test_lookup
+		guid = Ole::Types::Clsid.parse '00020328-0000-0000-c000-000000000046'
+		assert_equal 'PS_MAPI', PropertySet::NAMES[guid]
+	end
+
+	def test_simple_key
+		key = PropertySet::Key.new 0x0037
+		assert_equal PropertySet::PS_MAPI, key.guid
+		hash = {key => 'hash lookup'}
+		assert_equal 'hash lookup', hash[PropertySet::Key.new(0x0037)]
+		assert_equal '0x0037', key.inspect
+		assert_equal :subject, key.to_sym
+	end
+
+	def test_complex_keys
+		key = PropertySet::Key.new 'Keywords', PropertySet::PS_PUBLIC_STRINGS
+		# note that the inspect string now uses symbolic guids
+		assert_equal '#<Key PS_PUBLIC_STRINGS/"Keywords">', key.inspect
+		# note that this isn't categories
+		assert_equal :keywords, key.to_sym
+		custom_guid = '00020328-0000-0000-c000-deadbeefcafe'
+		key = PropertySet::Key.new 0x8000, Ole::Types::Clsid.parse(custom_guid)
+		assert_equal "#<Key {#{custom_guid}}/0x8000>",  key.inspect
+		key = PropertySet::Key.new 0x8005, PropertySet::PSETID_Address
+		assert_equal 'file_under', key.to_s
+	end
+
+	def test_property_set_basics
+		# the propertystore can be mocked with a hash:
+		store = {
+			PropertySet::Key.new(0x0037) => 'the subject',
+			PropertySet::Key.new('Keywords', PropertySet::PS_PUBLIC_STRINGS) => ['some keywords'],
+			PropertySet::Key.new(0x8888) => 'un-mapped value'
+		}
+		props = PropertySet.new store
+		# can resolve subject
+		assert_equal PropertySet::Key.new(0x0037), props.resolve('subject')
+		# note that the way things are set up, you can't resolve body though. ie, only
+		# existent (not all-known) properties resolve. maybe this should be changed. it'll
+		# need to be, for <tt>props.body=</tt> to work as it should.
+		assert_equal nil, props.resolve('body')
+		assert_equal 'the subject', props.subject
+		assert_equal ['some keywords'], props.keywords
+		# other access methods
+		assert_equal 'the subject', props['subject']
+		assert_equal 'the subject', props[0x0037]
+		assert_equal 'the subject', props[0x0037, PropertySet::PS_MAPI]
+		# note that the store is accessible directly, as #raw currently (maybe i should rename)
+		assert_equal store, props.raw
+		# note that currently, props.each / props.to_h works with the symbolically
+		# mapped properties, so the above un-mapped value won't be in the list:
+		assert_equal({:subject => 'the subject', :keywords => ['some keywords']}, props.to_h)
+		assert_equal [:keywords, :subject], props.keys.sort_by(&:to_s)
+		assert_equal [['some keywords'], 'the subject'], props.values.sort_by(&:to_s)
+	end
+
+	# other things we could test - write support. duplicate key handling
+
+	def test_pseudo_properties
+		# load some compressed rtf data
+		data = File.read File.dirname(__FILE__) + '/test_rtf.data'
+		props = PropertySet.new PropertySet::Key.new(0x1009) => StringIO.new(data)
+		# all these get generated from the rtf. still need tests for the way the priorities work
+		# here, and also the html embedded in rtf stuff....
+		assert_equal((<<-'end').chomp.gsub(/\n/, "\n\r"), props.body_rtf)
+{\rtf1\ansi\ansicpg1252\fromtext \deff0{\fonttbl
+{\f0\fswiss Arial;}
+{\f1\fmodern Courier New;}
+{\f2\fnil\fcharset2 Symbol;}
+{\f3\fmodern\fcharset0 Courier New;}}
+{\colortbl\red0\green0\blue0;\red0\green0\blue255;}
+\uc1\pard\plain\deftab360 \f0\fs20 \par
+I will be out of the office starting  15.02.2007 and will not return until\par
+27.02.2007.\par
+\par
+I will respond to your message when I return. For urgent enquiries please\par
+contact Motherine Jacson.\par
+\par
+}
+		end
+		assert_equal <<-'end', props.body_html
+<html>
+<body>
+<br>I will be out of the office starting  15.02.2007 and will not return until
+<br>27.02.2007.
+<br>
+<br>I will respond to your message when I return. For urgent enquiries please
+<br>contact Motherine Jacson.
+<br>
+<br></body>
+</html>
+		end
+		assert_equal <<-'end', props.body
+
+I will be out of the office starting  15.02.2007 and will not return until
+27.02.2007.
+
+I will respond to your message when I return. For urgent enquiries please
+contact Motherine Jacson.
+
+		end
+	end
+end
+